release: update go.mod for v1.22.0

Service limits are per tenancy. vm-single-principal now builds in its own, so
the shape the other topologies share can have zero quota there, and the apply
then fails before anything about the instance principal is exercised.

Measured in ours: standard-e6-core-count is 0 available, against 13 for e5 and
10 for standard3. The first principal run died on 400-LimitExceeded for Core
Instance, which reads as a capacity problem and is a shape with no quota.

OCI_PRINCIPAL_SHAPE overrides it for that topology alone and defaults to the
shape every other topology uses, so nothing changes where quota already exists.

.github/actions/e2e-analyze/analyze.go

@@ -134,21 +134,22 @@ var cascadeMarkers = []string{
 	"Expected value not to be nil",
 }
 
-// teardownPatterns match the content a test logs from t.Cleanup or from the
-// harness's on-failure diagnostics — i.e. after it has already failed. They are
-// the LAST file:line lines in the failure body, so extractErrorLine's
-// keep-the-last rule picks them over the assertion unless they are skipped.
-// Anchored deliberately: a fragment loose enough to match an assertion would
-// blank out the one line the report exists to show.
-var teardownPatterns = []*regexp.Regexp{
+// nonFailurePatterns match content a test emits AFTER the assertion that failed
+// — t.Cleanup, harness diagnostics, or a t.Logf on a passing branch — which
+// extractErrorLine's keep-the-last rule would otherwise pick over the assertion.
+// Go emits t.Logf and t.Errorf identically, so anchor every entry: a fragment
+// loose enough to match an assertion blanks out the line the report exists to
+// show. `\bas expected\b` unanchored also matches "did not fail as expected".
+var nonFailurePatterns = []*regexp.Regexp{
 	regexp.MustCompile(`^DB instance \S+ is gone$`),
 	regexp.MustCompile(`^db diagnostics \S+:`),
+	regexp.MustCompile(`^\S+: management bridge \S+:\d+ refused as expected$`),
 }
 
-// isTeardownLine reports whether a candidate error line was emitted after the
-// failure it would otherwise be mistaken for.
-func isTeardownLine(content string) bool {
-	for _, re := range teardownPatterns {
+// isNonFailureLine reports whether a candidate error line was emitted after the
+// failure it would otherwise be mistaken for, or reports an expected outcome.
+func isNonFailureLine(content string) bool {
+	for _, re := range nonFailurePatterns {
 		if re.MatchString(content) {
 			return true
 		}
@@ -245,12 +246,12 @@ func extractErrorLine(body string) string {
 			// progress lines from the same file:line pattern before the
 			// failing assertion finally fires. Cleanup runs after it, though,
 			// so its lines are last of all and have to be skipped.
-			if content := strings.TrimSpace(m[1]); !isTeardownLine(content) {
+			if content := strings.TrimSpace(m[1]); !isNonFailureLine(content) {
 				goFileLine = content
 			}
 			continue
 		}
-		if fileHintRe.MatchString(l) && !isTeardownLine(l) {
+		if fileHintRe.MatchString(l) && !isNonFailureLine(l) {
 			lastTestLine = l
 		}
 	}
@@ -296,15 +297,23 @@ func extractFileHint(body string) string {
 	}
 	// Scanned line by line rather than across the body so a cleanup line's own
 	// file:line does not become the site the report points a reader at.
-	hint := ""
+	hint, filtered := "", ""
 	for _, raw := range strings.Split(body, "\n") {
 		l := strings.TrimSpace(raw)
-		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isTeardownLine(strings.TrimSpace(m[1])) {
+		all := fileHintRe.FindAllString(l, -1)
+		if len(all) == 0 {
 			continue
 		}
-		if all := fileHintRe.FindAllString(l, -1); len(all) > 0 {
-			hint = all[len(all)-1]
+		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isNonFailureLine(strings.TrimSpace(m[1])) {
+			filtered = all[len(all)-1]
+			continue
 		}
+		hint = all[len(all)-1]
+	}
+	// A body whose only file:line was allowlisted still gets a pointer: coarse
+	// beats none, and extractErrorLine has the same last-resort shape.
+	if hint == "" {
+		return filtered
 	}
 	return hint
 }

.github/actions/e2e-analyze/analyze_test.go

@@ -141,6 +141,46 @@ func TestExtractErrorLine_SkipsCleanupLoggedAfterTheFailure(t *testing.T) {
 	}
 }
 
+// A subtest looping over two subjects calls t.Errorf for the first and t.Logf
+// for the second, so the success line is last in the body. Nightly run
+// 34383800404 reported that log line as the failure.
+func TestExtractErrorLine_SkipsSuccessLoggedAfterTheFailure(t *testing.T) {
+	body := `    isolation_test.go:80: rds-e2e-iso-a-1788981456: management bridge 10.15.8.10:5432 could not be tested: want connection refused, got dial tcp 10.15.8.10:5432: i/o timeout
+    isolation_test.go:84: rds-e2e-iso-b-1788981456: management bridge 10.15.8.11:5432 refused as expected`
+	got := extractErrorLine(body)
+	if !strings.Contains(got, "could not be tested") {
+		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
+	}
+	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
+	}
+}
+
+// "as expected" reads the same in an affirmation and in its negation, so an
+// unanchored pattern suppresses the assertion it was meant to outrank and
+// promotes a harness progress line in its place.
+func TestExtractErrorLine_KeepsNegatedExpectationAssertions(t *testing.T) {
+	body := `    isolation_test.go:60: waiting for the db instance to become available
+    isolation_test.go:80: rds-e2e-iso-a: management bridge 10.15.8.10:5432 was not refused as expected: got a connection`
+	got := extractErrorLine(body)
+	if !strings.Contains(got, "was not refused as expected") {
+		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
+	}
+	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
+	}
+}
+
+// Every candidate allowlisted leaves extractFileHint with nothing to return,
+// and unlike extractErrorLine it has no last-resort branch. A coarse pointer
+// beats sending the reader to no file at all.
+func TestExtractFileHint_FallsBackWhenEveryCandidateFiltered(t *testing.T) {
+	body := `    isolation_test.go:84: rds-e2e-iso-b: management bridge 10.15.8.11:5432 refused as expected`
+	if hint := extractFileHint(body); hint != "isolation_test.go:84" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:84", hint)
+	}
+}
+
 func TestExtractFileHint_PicksLastMatch(t *testing.T) {
 	body := "ec2helpers.go:50: setup\n    vpc_test.go:227: Eventually: condition not met"
 	got := extractFileHint(body)

.github/assets/diagrams/exoscale-single-node.svg

@@ -0,0 +1,79 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="836" viewBox="0 0 1020 836" role="img" aria-label="Spinifex on a single Exoscale instance: the instance sits on the zone&#x27;s public network with one interface, eth0, holding its own address and the default route. Manual Elastic IPs are delivered to eth0 and routed by the host into OVN through the spx-nat transit veth, and an Exoscale block storage volume backs /var/lib/spinifex.">
+<rect x="0" y="0" width="1020" height="836" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — SINGLE NODE</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one instance · routed NAT · Elastic IPs</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">One Exoscale instance, single Spinifex node</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">AWS SDKs, SSH and guest egress all arrive here</text>
+<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
+<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="582" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1  ·  PUBLIC NETWORK 91.92.116.0/23</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">gateway 91.92.116.1 · security group open to all</text>
+<rect x="48" y="232" width="924" height="520" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="68" y="266" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">standard.huge</text>
+<text x="212" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">spinifex-node-01  ·  8 vCPU / 32 GiB  ·  Ubuntu 26.04</text>
+<text x="952" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one host = one Spinifex region</text>
+<line x1="68" y1="282" x2="952" y2="282" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="68" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">eth0 — PUBLIC INTERFACE</text>
+<text x="482" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">the only NIC</text>
+<text x="88" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">91.92.116.250/23  ·  holds the default route</text>
+<text x="88" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">advertise IP · API :9999 · SSH · host masquerade</text>
+<rect x="518" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="538" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">ELASTIC IPs — MANUAL</text>
+<text x="932" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">attached to the instance</text>
+<text x="538" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">89.145.163.1/32  →  delivered to eth0</text>
+<text x="538" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">the host routes each /32 into OVN, never owns it</text>
+<line x1="735" y1="406" x2="735" y2="422" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="424" width="884" height="40" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="88" y="449" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="600" text-anchor="start">spx-nat-host ⇄ spx-nat-ovs</text>
+<text x="296" y="449" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">transit 100.127.0.0/24  ·  EIP/32 routed in  ·  masquerade out eth0 for the rest</text>
+<line x1="510" y1="464" x2="510" y2="480" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="482" width="884" height="100" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">SPINIFEX CONTROL PLANE</text>
+<text x="932" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">every service is one binary: spx service &lt;name&gt;</text>
+<rect x="86" y="530" width="93.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="132.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw :9999</text>
+<rect x="188.0" y="530" width="58.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="217.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="255.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="277.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="308.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="330.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">NATS</text>
+<rect x="361.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="404.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="456.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="499.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="551.0" y="530" width="79.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="590.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">northstar</text>
+<rect x="639.0" y="530" width="72.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="675.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">ui :3000</text>
+<text x="86" y="568" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">SigV4 in, NATS in the middle, QEMU/KVM (nested) · OVN/OVS · Predastore · Viperblock underneath</text>
+<line x1="248" y1="582" x2="248" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="772" y1="582" x2="772" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<rect x="68" y="600" width="532" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">TENANT VPCs — OVN br-int</text>
+<text x="580" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">geneve-local</text>
+<rect x="86" y="644" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-a  172.31.0.0/16</text>
+<text x="300" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  172.31.0.4  →  89.145.163.1</text>
+<rect x="86" y="686" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-b  10.0.0.0/16</text>
+<text x="300" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  10.0.1.9  ·  private, egress only</text>
+<rect x="616" y="600" width="336" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="636" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">BLOCK STORAGE VOLUME</text>
+<text x="932" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">256 GiB+</text>
+<text x="636" y="658" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">virtio-blk  →  /dev/vdb</text>
+<text x="636" y="680" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">/var/lib/spinifex</text>
+<text x="636" y="700" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">found by-id, defaults,nofail</text>
+<text x="636" y="718" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">viperblock · predastore · JetStream</text>
+<text x="28" y="796" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">A guest&#x27;s public address is a manual Exoscale Elastic IP, delivered to eth0 addressed to itself — no private/public pair to translate.</text>
+<text x="28" y="814" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">The host routes it into OVN, so inbound and outbound are the same address and the guest sees its private IP only.</text>
+</svg>

.github/assets/diagrams/exoscale-three-node.svg

@@ -0,0 +1,123 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="796" viewBox="0 0 1020 796" role="img" aria-label="Spinifex across three Exoscale instances in one zone, spread by an anti-affinity group. Each has a public eth0 carrying its own Elastic IPs, a private-network eth1 for the cluster mesh, its own block storage volume and its share of the guests. Geneve, NATS JetStream, predastore erasure coding and the OVN raft join them east-west over the private network. This topology is not yet tested.">
+<rect x="0" y="0" width="1020" height="796" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — THREE NODES</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">anti-affinity group · one zone · not yet tested</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">Three Exoscale instances, redundant multi-node install</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one endpoint, whichever node answers</text>
+<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
+<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="440" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1  ·  PUBLIC NETWORK + PRIVATE NETWORK</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one security group · one anti-affinity group</text>
+<rect x="48" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="64" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-01</text>
+<text x="332" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
+<line x1="64" y1="272" x2="332" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="64" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="78" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0  91.92.116.250</text>
+<text x="78" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
+<rect x="64" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="78" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1  10.10.0.11</text>
+<text x="78" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
+<rect x="64" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="78" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP  89.145.163.1</text>
+<text x="78" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
+<rect x="64" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="89.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
+<rect x="124.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="152.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="190.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="212.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="64" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="86.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
+<rect x="117.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="159.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="210.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="252.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="64" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-23d2…  172.31.0.4 → 89.145.163.1</text>
+<rect x="64" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
+<rect x="360" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="376" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-02</text>
+<text x="644" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
+<line x1="376" y1="272" x2="644" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="376" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="390" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0  91.92.117.14</text>
+<text x="390" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
+<rect x="376" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="390" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1  10.10.0.12</text>
+<text x="390" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
+<rect x="376" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="390" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP  89.145.160.107</text>
+<text x="390" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
+<rect x="376" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="401.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
+<rect x="436.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="464.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="502.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="524.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="376" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="398.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
+<rect x="429.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="471.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="522.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="564.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="376" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="388" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-6b91…  172.31.0.9 → 89.145.160.107</text>
+<rect x="376" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="388" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
+<rect x="672" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="688" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-03</text>
+<text x="956" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
+<line x1="688" y1="272" x2="956" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="688" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0  91.92.116.88</text>
+<text x="702" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
+<rect x="688" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1  10.10.0.13</text>
+<text x="702" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
+<rect x="688" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="702" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP  89.145.162.131</text>
+<text x="702" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
+<rect x="688" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="713.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
+<rect x="748.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="776.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="814.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="836.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="688" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="710.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
+<rect x="741.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="783.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="834.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="876.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="688" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="700" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-0ae4…  172.31.1.6 → 89.145.162.131</text>
+<rect x="688" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="700" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
+<line x1="510" y1="628" x2="510" y2="644" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="646" width="964" height="70" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">EAST–WEST, OVER THE EXOSCALE PRIVATE NETWORK</text>
+<text x="972" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">no Elastic IP is involved · MTU 1500</text>
+<rect x="46" y="684" width="124.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="108.0" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Geneve UDP 6081</text>
+<rect x="179.0" y="684" width="137.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="247.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">NATS JetStream R3</text>
+<rect x="325.6" y="684" width="144.4" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="397.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">predastore RS(2,1)</text>
+<rect x="479.0" y="684" width="151.2" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="554.6" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVN Northbound raft</text>
+<rect x="639.2" y="684" width="56.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="667.2" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVSDB</text>
+<text x="28" y="744" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Guest-to-guest traffic rides Geneve between the nodes&#x27; private-network interfaces — it never touches eth0 or an Elastic IP.</text>
+<text x="28" y="762" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Each node carries the Elastic IPs of its own guests. Moving one between nodes is an Exoscale API call, not yet automated.</text>
+<text x="28" y="780" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Three nodes is the smallest cluster that tolerates losing one; the anti-affinity group keeps them on different hypervisors.</text>
+</svg>

.github/assets/diagrams/oci-single-node.svg

@@ -0,0 +1,80 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="836" viewBox="0 0 1020 836" role="img" aria-label="Spinifex on a single OCI instance: the instance sits inside the VCN public subnet 10.200.0.0/23; its primary VNIC carries the default route and the cluster mesh, its secondary VNIC is the br-wan bridge holding registered secondary private IPs paired with reserved public IPs; guests reach them through the spx-nat transit veth, and an iSCSI block volume backs /var/lib/spinifex.">
+<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
+<rect x="0" y="0" width="1020" height="836" rx="20" fill="#14181D" stroke="#2A2F36" stroke-width="1"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON OCI — SINGLE NODE</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one instance · routed NAT · real public addresses</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">One OCI instance, single Spinifex node</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">AWS SDKs, SSH and guest egress all arrive here</text>
+<rect x="46" y="126" width="272.7" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="182.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="327.7" y="126" width="177.2" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="416.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest public IP</text>
+<rect x="513.92" y="126" width="218.1" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="623.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510.0" y1="170" x2="510.0" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="582" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">OCI VCN 10.200.0.0/16  ·  PUBLIC SUBNET 10.200.0.0/23</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">virtual router 10.200.0.1 · internet gateway</text>
+<rect x="48" y="232" width="924" height="520" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="68" y="266" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">BM.Standard.E2.64</text>
+<text x="260" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">spinifex-node-01  ·  FAULT-DOMAIN-1</text>
+<text x="952" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one host = one Spinifex region</text>
+<line x1="68" y1="282" x2="952" y2="282" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="68" y="298" width="434.0" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">VNIC 0 — PRIMARY</text>
+<text x="482.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">enp0s9</text>
+<text x="88" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">10.200.0.252/23  ·  holds the default route</text>
+<text x="88" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">advertise IP · ovn-encap-ip · NATS · API :9999</text>
+<rect x="518.0" y="298" width="434.0" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="538.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">VNIC 1 — SECONDARY</text>
+<text x="932.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">br-wan (Linux bridge)</text>
+<text x="538.0" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">10.200.1.233/32  +  ip rule → table 200</text>
+<text x="538.0" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">secondary private IPs ⇄ reserved public IPs</text>
+<line x1="735.0" y1="406" x2="735.0" y2="422" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="424" width="884" height="40" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="88" y="449" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="600" text-anchor="start">spx-nat-host ⇄ spx-nat-ovs</text>
+<text x="296" y="449" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">transit 100.127.0.0/24  ·  nft chain spinifex-nat-egress masquerades out br-wan</text>
+<line x1="510.0" y1="464" x2="510.0" y2="480" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="482" width="884" height="100" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">SPINIFEX CONTROL PLANE</text>
+<text x="932" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">every service is one binary: spx service &lt;name&gt;</text>
+<rect x="86" y="530" width="109.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="140.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw :9999</text>
+<rect x="204.01999999999998" y="530" width="74.9" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="241.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="287.94" y="530" width="61.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="318.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="358.22" y="530" width="61.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="388.9" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">NATS</text>
+<rect x="428.5" y="530" width="102.2" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="479.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="539.7" y="530" width="102.2" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="590.8" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="650.9000000000001" y="530" width="95.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="698.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">northstar</text>
+<rect x="755.2800000000001" y="530" width="88.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="799.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">ui :3000</text>
+<text x="86" y="568" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">SigV4 in, NATS in the middle, QEMU/KVM · OVN/OVS · Predastore · Viperblock underneath</text>
+<line x1="248" y1="582" x2="248" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="772" y1="582" x2="772" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<rect x="68" y="600" width="532" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">TENANT VPCs — OVN br-int</text>
+<text x="580" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">geneve-local</text>
+<rect x="86" y="644" width="496" height="34" rx="9" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-a  10.0.0.0/16</text>
+<text x="310" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  10.0.1.4  →  150.230.13.131</text>
+<rect x="86" y="686" width="496" height="34" rx="9" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-b  172.31.0.0/16</text>
+<text x="310" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  172.31.4.9  ·  private, egress only</text>
+<rect x="616" y="600" width="336" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="636" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">BLOCK VOLUME — iSCSI</text>
+<text x="932" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">256 GB+</text>
+<text x="636" y="658" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">169.254.2.2:3260  →  /dev/sdb</text>
+<text x="636" y="680" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">/var/lib/spinifex</text>
+<text x="636" y="700" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">mounted by UUID, defaults,_netdev,nofail</text>
+<text x="636" y="718" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">viperblock · predastore · JetStream</text>
+<text x="28" y="796" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">A guest's public address is a real OCI reserved public IP, attached to a secondary private IP on br-wan.</text>
+<text x="28" y="814" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Inbound and outbound are therefore the same address, and OCI's per-VNIC source check passes.</text>
+</svg>

.github/assets/diagrams/oci-three-node.svg

@@ -0,0 +1,116 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="796" viewBox="0 0 1020 796" role="img" aria-label="Spinifex across three OCI instances: all three sit inside the VCN public subnet 10.200.0.0/23, one per fault domain. Each carries a primary VNIC for the cluster mesh and a br-wan secondary VNIC holding its reserved public addresses, its own iSCSI block volume, and its share of the guests. Geneve, NATS JetStream, predastore erasure coding and the OVN raft join them east-west.">
+<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
+<rect x="0" y="0" width="1020" height="796" rx="20" fill="#14181D" stroke="#2A2F36" stroke-width="1"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON OCI — THREE NODES</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">three fault domains · one region · one API endpoint</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">Three OCI instances, redundant multi-node install</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one endpoint, whichever node answers</text>
+<rect x="46" y="126" width="272.7" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="182.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="327.7" y="126" width="177.2" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="416.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest public IP</text>
+<rect x="513.92" y="126" width="218.1" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="623.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510.0" y1="170" x2="510.0" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="464" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">OCI VCN 10.200.0.0/16  ·  PUBLIC SUBNET 10.200.0.0/23</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">virtual router 10.200.0.1 · internet gateway</text>
+<rect x="48.0" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="66.0" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-01</text>
+<text x="327.3333333333333" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-1</text>
+<text x="66.0" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
+<line x1="66.0" y1="292" x2="327.3333333333333" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<text x="66.0" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0  10.200.0.252</text>
+<text x="66.0" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
+<text x="66.0" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1  br-wan  10.200.1.233/32</text>
+<text x="66.0" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
+<rect x="66.0" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="91.5" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
+<rect x="123.0" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="151.6" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="186.2" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="208.6" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="66.0" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="88.4" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
+<rect x="116.8" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="157.8" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="204.8" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="245.8" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="66.0" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76.0" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-23d2…  10.0.1.4 → 150.230.13.131</text>
+<rect x="66.0" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76.0" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
+<line x1="345.3333333333333" y1="364" x2="361.3333333333333" y2="364" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.5" stroke-dasharray="2 4"/>
+<rect x="361.3333333333333" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="379.3333333333333" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-02</text>
+<text x="640.6666666666666" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-2</text>
+<text x="379.3333333333333" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
+<line x1="379.3333333333333" y1="292" x2="640.6666666666666" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<text x="379.3333333333333" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0  10.200.0.118</text>
+<text x="379.3333333333333" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
+<text x="379.3333333333333" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1  br-wan  10.200.1.96/32</text>
+<text x="379.3333333333333" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
+<rect x="379.3333333333333" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="404.8" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
+<rect x="436.3333333333333" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="464.9" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="499.5333333333333" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="521.9" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="379.3333333333333" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="401.7" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
+<rect x="430.1333333333333" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="471.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="518.1333333333333" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="559.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="379.3333333333333" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="389.3333333333333" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-6b91…  10.0.1.9 → 137.23.9.125</text>
+<rect x="379.3333333333333" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="389.3333333333333" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
+<line x1="658.6666666666666" y1="364" x2="674.6666666666666" y2="364" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.5" stroke-dasharray="2 4"/>
+<rect x="674.6666666666666" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="692.6666666666666" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-03</text>
+<text x="954.0" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-3</text>
+<text x="692.6666666666666" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
+<line x1="692.6666666666666" y1="292" x2="954.0" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<text x="692.6666666666666" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0  10.200.0.41</text>
+<text x="692.6666666666666" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
+<text x="692.6666666666666" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1  br-wan  10.200.1.12/32</text>
+<text x="692.6666666666666" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
+<rect x="692.6666666666666" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="718.2" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
+<rect x="749.6666666666666" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="778.3" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="812.8666666666667" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="835.3" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="692.6666666666666" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="715.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
+<rect x="743.4666666666666" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="784.5" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="831.4666666666666" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="872.5" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="692.6666666666666" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702.6666666666666" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-0ae4…  10.0.2.6 → 150.230.4.88</text>
+<rect x="692.6666666666666" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702.6666666666666" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
+<rect x="48" y="524" width="924" height="104" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="68" y="554" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">EAST–WEST, INSIDE THE VCN</text>
+<text x="952" y="554" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">no public address is involved</text>
+<rect x="66" y="574" width="136.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="134.2" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">Geneve UDP 6081</text>
+<rect x="211.3" y="574" width="149.9" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="286.3" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">NATS JetStream R3</text>
+<rect x="370.24" y="574" width="156.8" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="448.6" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore RS(2,1)</text>
+<rect x="536.0" y="574" width="163.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="617.8" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">OVN Northbound raft</text>
+<rect x="708.58" y="574" width="68.1" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="742.6" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">OVSDB</text>
+<text x="66" y="612" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Guest-to-guest traffic rides Geneve between the nodes' primary VNICs — it never leaves the subnet and never touches br-wan.</text>
+<rect x="28" y="668" width="964" height="46" rx="12" fill="#EE4B12" fill-opacity="0.07" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.55"/>
+<text x="48" y="688" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13" fill="#F5F3EE" font-weight="700" text-anchor="start">A guest that moves takes its address with it.</text>
+<text x="48" y="705" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">The reserved public IP follows the secondary private IP to whichever node's br-wan the guest lands on — the tenant sees no change.</text>
+<text x="28" y="756" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Three nodes is the smallest cluster that tolerates losing one: NATS JetStream keeps quorum, predastore RS(2,1)</text>
+<text x="28" y="774" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">keeps the data readable, and the OVN Northbound raft keeps a leader. One node per fault domain is what makes that real.</text>
+</svg>

.github/assets/images/cisco-ucs-platform-benchmark/guest-storage-throughput.svg

@@ -0,0 +1,44 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
+<rect width="760" height="470" fill="#ffffff"/>
+<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Guest block storage throughput (Viperblock)</text>
+<text x="74" y="49" font-size="12.5" fill="#6b7280">fio, 4 jobs, iodepth 32, direct I/O, per-guest sequential runs — Spinifex v1.20.0</text>
+<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">0</text>
+<line x1="74" y1="305.0" x2="734" y2="305.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="309.0" font-size="12" text-anchor="end" fill="#6b7280">100</text>
+<line x1="74" y1="224.0" x2="734" y2="224.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="228.0" font-size="12" text-anchor="end" fill="#6b7280">200</text>
+<line x1="74" y1="143.0" x2="734" y2="143.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="147.0" font-size="12" text-anchor="end" fill="#6b7280">300</text>
+<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">400</text>
+<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">MiB/s</text>
+<rect x="111.4" y="81.4" width="45.4" height="304.6" rx="2.5" fill="#2563eb"/>
+<text x="134.1" y="75.4" font-size="11" text-anchor="middle" fill="#1f2937">376</text>
+<rect x="159.8" y="198.1" width="45.4" height="187.9" rx="2.5" fill="#0d9488"/>
+<text x="182.5" y="192.1" font-size="11" text-anchor="middle" fill="#1f2937">232</text>
+<rect x="208.2" y="299.3" width="45.4" height="86.7" rx="2.5" fill="#f59e0b"/>
+<text x="230.9" y="293.3" font-size="11" text-anchor="middle" fill="#1f2937">107</text>
+<text x="184.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">GPU</text>
+<rect x="331.4" y="85.5" width="45.4" height="300.5" rx="2.5" fill="#2563eb"/>
+<text x="354.1" y="79.5" font-size="11" text-anchor="middle" fill="#1f2937">371</text>
+<rect x="379.8" y="202.9" width="45.4" height="183.1" rx="2.5" fill="#0d9488"/>
+<text x="402.5" y="196.9" font-size="11" text-anchor="middle" fill="#1f2937">226</text>
+<rect x="428.2" y="321.2" width="45.4" height="64.8" rx="2.5" fill="#f59e0b"/>
+<text x="450.9" y="315.2" font-size="11" text-anchor="middle" fill="#1f2937">80</text>
+<text x="404.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU1</text>
+<rect x="551.4" y="84.7" width="45.4" height="301.3" rx="2.5" fill="#2563eb"/>
+<text x="574.1" y="78.7" font-size="11" text-anchor="middle" fill="#1f2937">372</text>
+<rect x="599.8" y="201.3" width="45.4" height="184.7" rx="2.5" fill="#0d9488"/>
+<text x="622.5" y="195.3" font-size="11" text-anchor="middle" fill="#1f2937">228</text>
+<rect x="648.2" y="321.2" width="45.4" height="64.8" rx="2.5" fill="#f59e0b"/>
+<text x="670.9" y="315.2" font-size="11" text-anchor="middle" fill="#1f2937">80</text>
+<text x="624.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU2</text>
+<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
+<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
+<text x="93" y="455" font-size="12.5" fill="#1f2937">128K random read</text>
+<rect x="216.8" y="444" width="13" height="13" rx="2.5" fill="#0d9488"/>
+<text x="235.8" y="455" font-size="12.5" fill="#1f2937">128K mixed read</text>
+<rect x="352.3" y="444" width="13" height="13" rx="2.5" fill="#f59e0b"/>
+<text x="371.3" y="455" font-size="12.5" fill="#1f2937">16K random read</text>
+</svg>

.github/assets/images/cisco-ucs-platform-benchmark/network-goodput-vs-streams.svg

@@ -0,0 +1,35 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
+<rect width="760" height="470" fill="#ffffff"/>
+<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Network throughput: overlay vs physical fabric</text>
+<text x="74" y="49" font-size="12.5" fill="#6b7280">iperf3 TCP receiver goodput, median across node pairs — Spinifex v1.20.0</text>
+<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">20</text>
+<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">25</text>
+<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">Gbit/s</text>
+<text x="140.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">1</text>
+<text x="272.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">2</text>
+<text x="404.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">4</text>
+<text x="536.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">8</text>
+<text x="668.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">16</text>
+<path d="M140.0,164.4 L272.0,160.5 L404.0,160.5 L536.0,160.5 L668.0,160.5" fill="none" stroke="#2563eb" stroke-width="3"/>
+<circle cx="140.0" cy="164.4" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="272.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="404.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="536.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="668.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<text x="664.0" y="149.5" font-size="11.5" text-anchor="end" font-weight="600" fill="#2563eb">23.48</text>
+<path d="M140.0,255.1 L272.0,222.1 L404.0,222.1 L536.0,222.1 L668.0,228.5" fill="none" stroke="#0d9488" stroke-width="3"/>
+<circle cx="140.0" cy="255.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="272.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="404.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="536.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="668.0" cy="228.5" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<text x="664.0" y="217.5" font-size="11.5" text-anchor="end" font-weight="600" fill="#0d9488">22.43</text>
+<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
+<text x="404.0" y="430.0" font-size="12.5" text-anchor="middle" fill="#6b7280">parallel TCP streams</text>
+<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
+<text x="93" y="455" font-size="12.5" fill="#1f2937">Host physical fabric (25GbE)</text>
+<rect x="304.4" y="444" width="13" height="13" rx="2.5" fill="#0d9488"/>
+<text x="323.4" y="455" font-size="12.5" fill="#1f2937">Guest overlay (OVN)</text>
+</svg>

.github/assets/images/cisco-ucs-platform-benchmark/predastore-s3-read-by-client.svg

@@ -0,0 +1,39 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
+<rect width="760" height="470" fill="#ffffff"/>
+<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Predastore S3 read throughput by client</text>
+<text x="74" y="49" font-size="12.5" fill="#6b7280">1 GiB objects, same presigned-curl client on every path — Spinifex v1.20.0</text>
+<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">0</text>
+<line x1="74" y1="278.0" x2="734" y2="278.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="282.0" font-size="12" text-anchor="end" fill="#6b7280">200</text>
+<line x1="74" y1="170.0" x2="734" y2="170.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="174.0" font-size="12" text-anchor="end" fill="#6b7280">400</text>
+<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">600</text>
+<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">MiB/s</text>
+<rect x="102.0" y="202.4" width="51.5" height="183.6" rx="2.5" fill="#2563eb"/>
+<text x="127.8" y="196.4" font-size="11" text-anchor="middle" fill="#1f2937">340</text>
+<rect x="156.5" y="136.5" width="51.5" height="249.5" rx="2.5" fill="#f59e0b"/>
+<text x="182.2" y="130.5" font-size="11" text-anchor="middle" fill="#1f2937">462</text>
+<text x="156.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">Host</text>
+<rect x="267.1" y="170.5" width="51.5" height="215.5" rx="2.5" fill="#2563eb"/>
+<text x="292.8" y="164.5" font-size="11" text-anchor="middle" fill="#1f2937">399</text>
+<rect x="321.5" y="174.9" width="51.5" height="211.1" rx="2.5" fill="#f59e0b"/>
+<text x="347.2" y="168.9" font-size="11" text-anchor="middle" fill="#1f2937">391</text>
+<text x="321.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">GPU guest</text>
+<rect x="432.1" y="102.5" width="51.5" height="283.5" rx="2.5" fill="#2563eb"/>
+<text x="457.8" y="96.5" font-size="11" text-anchor="middle" fill="#1f2937">525</text>
+<rect x="486.5" y="197.0" width="51.5" height="189.0" rx="2.5" fill="#f59e0b"/>
+<text x="512.2" y="191.0" font-size="11" text-anchor="middle" fill="#1f2937">350</text>
+<text x="486.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU1 guest</text>
+<rect x="597.0" y="106.8" width="51.5" height="279.2" rx="2.5" fill="#2563eb"/>
+<text x="622.8" y="100.8" font-size="11" text-anchor="middle" fill="#1f2937">517</text>
+<rect x="651.5" y="196.5" width="51.5" height="189.5" rx="2.5" fill="#f59e0b"/>
+<text x="677.2" y="190.5" font-size="11" text-anchor="middle" fill="#1f2937">351</text>
+<text x="651.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU2 guest</text>
+<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
+<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
+<text x="93" y="455" font-size="12.5" fill="#1f2937">Single-stream GET</text>
+<rect x="224.1" y="444" width="13" height="13" rx="2.5" fill="#f59e0b"/>
+<text x="243.1" y="455" font-size="12.5" fill="#1f2937">Chunked/concurrent GET</text>
+</svg>

.github/e2e-cells.json

@@ -130,7 +130,7 @@
     "runner": "ci-single",
     "extra": "rds",
     "suite": "rds",
-    "timeout_minutes": 100
+    "timeout_minutes": 120
   },
   {
     "cell": 29,
@@ -143,5 +143,52 @@
     "extra": "",
     "suite": "storagefault",
     "timeout_minutes": 100
+  },
+  {
+    "cell": 30,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "nat",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "nat-multi",
+    "suite": ""
+  },
+  {
+    "cell": 31,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "instancerecovery",
+    "timeout_minutes": 105
+  },
+  {
+    "cell": 32,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "lbrecovery",
+    "timeout_minutes": 90
+  },
+  {
+    "cell": 33,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "kvquorum",
+    "timeout_minutes": 55
   }
 ]

.github/scripts/suite-sets.sh

@@ -40,7 +40,24 @@
 E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
 
 # Suites runnable against a multi-node environment.
-E2E_SUITES_MULTI="multinode lb cert quota storagefault"
+#
+# instancerecovery is multi-only and needs three: recovery is inert below that,
+# because there is no survivor to move a guest to that is not the node that
+# failed. Like storagefault it takes a node away and freezes a cluster-wide
+# service, so it runs last and owns its environment.
+#
+# lbrecovery is multi-only and needs three for two reasons rather than one: a
+# survivor to move the load balancer to, and a node for its backends that is not
+# the load balancer's, so taking that node away does not take the backends with
+# it. It runs beside instancerecovery at the end because it also takes a node
+# away and owns its environment while it does.
+#
+# kvquorum is multi-only and needs three, for the same reason as those two: a
+# surviving majority. It is what proves the control plane's own state is
+# replicated rather than sitting on one node, so it runs before the pair that
+# take a node away — a cluster whose buckets are already wrong should report
+# that, not report a recovery failing.
+E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
@@ -64,6 +81,10 @@ E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
 # after a failed seal do not exist on one node, and they are the assertions
 # worth a nightly.
 E2E_SUITES_NIGHTLY_SINGLE="single cert iam"
+#
+# The multi-node nightly subset carries kvquorum's read-only half through
+# `multinode`, which sweeps every KV bucket rather than the three the daemon
+# owns. The node-loss half is too slow for a 35-minute cell and has its own.
 E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
 
 # How long a suite gets, for the same reason the lists above live here: the
@@ -77,8 +98,9 @@ e2e_suite_timeout() {
     # what the queue costs, not what the tests do: single tests logged 11m, 22m
     # and 26m waiting for budget. 17 tests at that concurrency is over an hour
     # before anything goes wrong, and the first pass to run deep into the suite
-    # still had three left at 60m.
-    rds) echo "90m" ;;
+    # still had three left at 60m. The loaded graceful-reboot case adds an
+    # instance that holds its slot through a pgbench load as well as a create.
+    rds) echo "105m" ;;
     # diskperf runs two 16 GiB fio profiles per repetition, each on a volume it
     # creates first. One pass is ~20 minutes on the bare-metal cell and a
     # baseline capture at SPINIFEX_DISKPERF_REPS=3 is three times that.
@@ -87,6 +109,23 @@ e2e_suite_timeout() {
     # SIGSTOPped for seven minutes to prove a guest survives it. A full pass on
     # dev-prod is ~63 minutes, so the 30m default truncated it mid-suite.
     storagefault) echo "90m" ;;
+    # instancerecovery is almost entirely waiting, and none of it is avoidable:
+    # a dead owner has to be seen stale twice, a guest has to boot, and the
+    # storage case waits for a pause measured at 4m47s and then holds four
+    # minutes to prove nothing moved. Three tests, each taking a node away and
+    # bringing it back, and the partition case adds a lease validity, a lease
+    # TTL and a boot on top of everything the other two wait for.
+    instancerecovery) echo "90m" ;;
+    # lbrecovery pays for a load balancer twice: once to stand one up with two
+    # backends placed around it, and once for the recovery itself — a stale
+    # owner, a lease, a boot, HAProxy's own start, and its health checks against
+    # the backends before it forwards anything. Then the node comes back and has
+    # to be shown not to answer for an address it no longer holds.
+    lbrecovery) echo "75m" ;;
+    # kvquorum audits every bucket in seconds and then spends the rest of its
+    # budget on a node stopping and starting: seven units down, a raft election,
+    # and the whole service stack back up with every stream caught up.
+    kvquorum) echo "40m" ;;
     *) echo "30m" ;;
   esac
 }

.github/workflows/apt-packages.yml

@@ -33,8 +33,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # The suites the nightly E2E matrix actually installs on. This matches
-        # what detect_os accepts: Debian 13+ and Ubuntu 24.04+.
         image:
           - ubuntu:24.04
           - ubuntu:26.04

.github/workflows/dev-release.yml

@@ -0,0 +1,188 @@
+name: Dev Prerelease
+
+# Publishes the dev channel that install.mulgadc.com already serves. setup.sh has
+# understood INSTALL_SPINIFEX_CHANNEL=dev since it was written and the install
+# Worker resolves that channel to the newest GitHub prerelease, but nothing has
+# ever produced one -- so /download/dev/amd64 returned 404. This is the producer.
+#
+# A separate workflow rather than conditionals in release.yml, because a dev build
+# is a different artifact and not a release with flags off: amd64 only, no ISO, no
+# AMI refresh, no bare-metal gate, and nothing that touches a self-hosted runner.
+on:
+  push:
+    branches: [dev]
+    # Nothing here changes the binary, and a prerelease per docs commit would
+    # churn the channel our contacts are installing from.
+    paths-ignore:
+      - "docs/**"
+      - "**/*.md"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+# Supersede rather than queue: on a run of merges to dev only the newest commit is
+# worth publishing, and the older build would land after it and become the channel.
+concurrency:
+  group: dev-release
+  cancel-in-progress: true
+
+jobs:
+  prerelease:
+    runs-on: ubuntu-26.04
+    steps:
+      # Full history and tags, because the tag name is git describe and a shallow
+      # clone has no tag to describe against.
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+          fetch-depth: 0
+
+      # dev, not each repository's default branch, which is what release.yml takes.
+      # A dev build of spinifex against main of the others is a pairing nobody runs.
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: dev
+          path: predastore
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: dev
+          path: viperblock
+
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: dev
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: dev
+          path: bluebottle
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: spinifex/go.mod
+
+      # The tag carries the describe string so the commit is recoverable from the
+      # version alone, and the -dev suffix is what release.yml's own convention
+      # already treats as a prerelease. VERSION must equal the tag: the install
+      # Worker derives the asset name from the tag it resolves.
+      - name: Resolve the version
+        id: version
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          # Describing against a dev tag would nest it, growing the name every release.
+          TAG="$(git describe --tags --always --exclude '*-dev')-dev"
+          case "$TAG" in
+              v*) ;;
+              *) echo "::error::git describe gave '$TAG', which is not a v-prefixed version"; exit 1 ;;
+          esac
+          if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+              echo "::notice::$TAG already published; nothing to do"
+              echo "exists=true" >> "$GITHUB_OUTPUT"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "Publishing $TAG"
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Build microVM artifacts (lb-agent + kernel + initramfs)
+        if: steps.version.outputs.exists != 'true'
+        run: cd spinifex && make build-microvm-image
+
+      - name: Build amd64 Tarball
+        if: steps.version.outputs.exists != 'true'
+        env:
+          VERSION: ${{ steps.version.outputs.tag }}
+        run: cd spinifex && make distro-amd64 VERSION="$VERSION"
+
+      # --prerelease on create, never a later promote step. release.yml sets it in
+      # a job that runs after its verification gates, which leaves a window where
+      # the release is not flagged; GitHub's releases/latest excludes prereleases
+      # only, so in that window a dev build is what every customer curl installs.
+      - name: Publish the prerelease
+        if: steps.version.outputs.exists != 'true'
+        working-directory: spinifex
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ steps.version.outputs.tag }}
+        run: |
+          set -euo pipefail
+          # A file, not an inline string: the notes are Markdown, and leading
+          # whitespace from indenting a shell literal turns them into code blocks.
+          cat > "$RUNNER_TEMP/notes.md" <<EOF
+          Automated dev-channel build of \`$GITHUB_SHA\`.
+
+          **Not a release.** amd64 only, no ISO and no system images, and it has not been through the release verification gates. Install it with:
+
+          \`\`\`
+          curl -sfL https://install.mulgadc.com | sudo bash -s -- --channel dev
+          \`\`\`
+          EOF
+          sed -i 's/^          //' "$RUNNER_TEMP/notes.md"
+          gh release create "$TAG" \
+              --repo "$GITHUB_REPOSITORY" \
+              --target "$GITHUB_SHA" \
+              --prerelease \
+              --title "$TAG" \
+              --notes-file "$RUNNER_TEMP/notes.md" \
+              "dist/spinifex-$TAG-linux-amd64.tar.gz" \
+              "dist/spinifex-$TAG-linux-amd64.tar.gz.sha256" \
+              "scripts/setup.sh"
+
+      # The assertion that matters. A prerelease must never become the latest
+      # release, because latest is what an unqualified curl | bash installs -- so
+      # this checks the published state rather than trusting the flag we passed.
+      - name: Assert the stable channel is untouched
+        if: steps.version.outputs.exists != 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ steps.version.outputs.tag }}
+        run: |
+          set -euo pipefail
+          latest="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq .tag_name)"
+          if [ "$latest" = "$TAG" ]; then
+              echo "::error::$TAG became the latest release: every customer curl | bash would install it"
+              gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --prerelease
+              exit 1
+          fi
+          echo "latest is still $latest"
+          dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=20" --jq 'map(select(.prerelease))[0].tag_name')"
+          [ "$dev" = "$TAG" ] || { echo "::error::the dev channel resolves to $dev, not $TAG"; exit 1; }
+
+      # Kept rather than unbounded: the Worker reads the first page of releases to
+      # resolve the channel, so enough prereleases would eventually push the real
+      # ones off it, and each build carries two assets.
+      #
+      # Twenty, not five. A pinned dev tag is how an external evaluator installs a
+      # specific build, and five is about a week of them -- short enough that the tag
+      # someone was given stops resolving while they are still using it.
+      - name: Prune old dev prereleases
+        if: steps.version.outputs.exists != 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \
+              --jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[20:] | .[].tag_name' \
+              > stale.txt
+          while read -r tag; do
+              [ -n "$tag" ] || continue
+              echo "deleting $tag"
+              gh release delete "$tag" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
+          done < stale.txt

.github/workflows/e2e-baremetal.yml

@@ -55,15 +55,12 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
-  # Same central ES/APM sink as the other four e2e workflows. Bare-metal had
-  # no default here before — this cell's telemetry never reached the sink at
-  # all (not just missing ci.cell), see bm-bootstrap.sh's telemetry.env step.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
 
 jobs:
   e2e_baremetal:
     name: Bare-Metal Single-Node E2E
-    runs-on: [ self-hosted, ci-baremetal ]
+    runs-on: [self-hosted, ci-baremetal]
     timeout-minutes: 240
     concurrency:
       group: e2e-baremetal-cluster
@@ -72,9 +69,6 @@ jobs:
       BM_ISO_CACHE: /var/cache/spinifex-iso-builder
       BM_CELL: ${{ inputs.cell || '20' }}
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
-      # Caller-supplied on workflow_call. A dispatch run has no such input, and
-      # leaving it empty cost this cell its ci.run_id: its telemetry reached the
-      # sink but could not be found by run, only by guessing a time window.
       SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
     steps:
       - name: Clean Workspace
@@ -129,7 +123,7 @@ jobs:
             INSTALLER_BIN="${RUNNER_TEMP}/bm-bins/spinifex-installer" \
             SPINIFEX_SRC="${GITHUB_WORKSPACE}" \
             VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
-            VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always 2>/dev/null || echo dev)"
+            VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always --exclude '*-dev' 2>/dev/null || echo dev)"
 
       - name: Write ePDU credentials
         run: |
@@ -190,8 +184,6 @@ jobs:
         run: |
           mkdir -p "${ARTIFACT_DIR}"
           set -o pipefail
-          # Per-suite budgets are declared once in suite-sets.sh. The script
-          # applies one timeout to every suite in the list, so the widest wins.
           . "${GITHUB_WORKSPACE}/.github/scripts/suite-sets.sh"
           BM_TEST_TIMEOUT=""
           for suite in ${BM_SUITES}; do

.github/workflows/e2e-cloudvendor-nightly.yml

@@ -0,0 +1,287 @@
+name: E2E Cloud Vendor Nightly
+
+# Dispatched by the same Cloudflare Worker that drives e2e-nightly.yml, once per
+# ref, rather than by a `schedule:` here. GitHub registers a schedule only from
+# the default branch, so a cron in this file could never run `dev` and a feature
+# branch as separate nightlies, and it would always execute the default branch's
+# copy of this file -- leaving a fix to the workflow itself untestable until it
+# had already shipped. A dispatch takes both the file and the source from its ref.
+on:
+  workflow_dispatch:
+    inputs:
+      topologies:
+        description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal). One name runs one topology."
+        required: false
+      workbooks:
+        description: "Space-separated workbooks (blank = the driver's default list)"
+        required: false
+      source:
+        description: "tree (build this ref) or release (the published installer)"
+        required: false
+        default: tree
+      channel:
+        description: "With source=release: latest, or dev for the newest prerelease"
+        required: false
+        default: latest
+      keep_on_fail:
+        description: "Leave a failed topology up to inspect. It keeps billing."
+        required: false
+        type: boolean
+        default: false
+      auth:
+        description: "How nodes reach the OCI API: instance_principal (needs the tenancy dynamic group) or api_key (needs the credential hook). The vm-single-principal topology always uses instance_principal."
+        required: false
+        default: instance_principal
+        type: choice
+        options: [instance_principal, api_key]
+      no_external_pool:
+        description: "Form without the OCI allocator. Skips every workbook needing a public address. Not applied to vm-single-principal."
+        required: false
+        type: boolean
+        default: false
+      runner:
+        description: "Runner label (blank = the OCI_E2E_RUNNER variable, else ubuntu-26.04)"
+        required: false
+
+permissions:
+  contents: read
+
+# Every topology is built in one OCI compartment under one set of resource names,
+# so two runs at once would each see the other's instances and the sweep in
+# e2e-oci.sh could destroy a cluster the other run was still testing. Queue the
+# second rather than cancelling it: a dispatch is usually a verification run.
+concurrency:
+  group: e2e-oci
+  cancel-in-progress: false
+
+jobs:
+  oci:
+    # Hosted by default: this job only drives Terraform and SSH, and the work
+    # happens on OCI, so nothing here needs a hypervisor. Overridable because the
+    # argument for a per-provider self-hosted runner is control, not capacity --
+    # a dedicated label keeps a two-hour cloud wait off the hypervisor labels and
+    # lifts the six-hour cap a hosted job carries.
+    runs-on: ${{ inputs.runner || vars.OCI_E2E_RUNNER || 'ubuntu-26.04' }}
+    # Three topologies including a bare-metal build, each with its own install,
+    # formation and workbook pass. The sweep still runs if this fires.
+    timeout-minutes: 300
+    steps:
+      # Set here rather than in a job-level env block: the runner context is not
+      # available at job level, only from a step onwards, and a workflow that
+      # references it there is rejected as invalid before any job starts.
+      #
+      # Outside github.workspace on purpose. The checkout wipes untracked files, so
+      # on a persistent runner it would delete the Terraform state of a killed run
+      # before the next sweep could read it, leaving instances nothing can find.
+      - name: Put the Terraform state outside the checkout
+        run: echo "OCI_STATE_ROOT=$RUNNER_WORKSPACE/oci-state" >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          # mulga alone. It is the only private repository of the six, and the
+          # five that build-install-artifacts.sh clones are public, so the builder
+          # needs no credential and this token is not widened to reach them.
+          repositories: mulga
+
+      # build-install-artifacts.sh lives in mulga. The workbooks and their driver
+      # are in spinifex, so only the one directory is needed here.
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          path: mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+
+      - uses: hashicorp/setup-terraform@v4
+        with:
+          terraform_wrapper: false
+
+      # Written to the runner's own filesystem and nowhere else. The OCI API
+      # credential is not here at all: it reaches Terraform as OCI_PRIVATE_KEY in
+      # the environment, so no key file is ever created.
+      - name: Install the throwaway SSH key
+        env:
+          OCI_E2E_SSH_PRIVATE_KEY: ${{ secrets.OCI_E2E_SSH_PRIVATE_KEY }}
+          OCI_E2E_SSH_PUBLIC_KEY: ${{ secrets.OCI_E2E_SSH_PUBLIC_KEY }}
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.ssh
+          printf '%s\n' "$OCI_E2E_SSH_PRIVATE_KEY" > ~/.ssh/oci-spx
+          printf '%s\n' "$OCI_E2E_SSH_PUBLIC_KEY" > ~/.ssh/oci-spx.pub
+          chmod 600 ~/.ssh/oci-spx
+          ssh-keygen -y -f ~/.ssh/oci-spx > /dev/null
+
+      - name: Run the OCI topologies
+        working-directory: spinifex/scripts/terraform/oci-spx
+        env:
+          OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
+          OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
+          OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
+          OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
+          OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
+          TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
+          MULGA_ROOT: ${{ github.workspace }}/mulga
+          SPX_REF: ${{ github.ref_name }}
+          OCI_TOPOLOGIES: ${{ inputs.topologies }}
+          OCI_SOURCE: ${{ inputs.source }}
+          OCI_CHANNEL: ${{ inputs.channel }}
+          # Independent of the pool, deliberately. These were one switch, which made
+          # the combination we actually need -- an API key with the allocator running
+          # -- impossible to ask for, so public-IP allocation went untested.
+          OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
+          # Skipping the pool records the allocator and every public-address workbook
+          # as SKIPPED, so a green run with this set is not a claim about guest
+          # networking. It exists for a formation-only check, not as a way past a
+          # missing credential.
+          OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
+          # Only an API-key run needs one: an instance principal is the node's own
+          # certificate, so there is nothing to hand over. Unset is not an error, and
+          # the allocator gate then fails rather than passing a node that cannot
+          # authenticate.
+          OCI_CREDENTIAL_HOOK: ${{ inputs.auth == 'api_key' && vars.OCI_CREDENTIAL_HOOK || '' }}
+          # vm-single-principal builds in its own tenancy. The dynamic group it needs
+          # is a tenancy-root resource, and the shared test tenancy grants us only
+          # compartment rights, so that topology can never pass there however correct
+          # the code is. Unset skips it rather than spending a provision to fail.
+          OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
+          OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
+          OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
+          OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
+          OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
+          OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
+          # Service limits are per tenancy, so the shape the other topologies use
+          # can have zero quota in this one. Ours has no E6 cores at all, and the
+          # apply then fails before anything about the principal is exercised.
+          OCI_PRINCIPAL_SHAPE: ${{ vars.OCI_PRINCIPAL_SHAPE }}
+          OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
+          OCI_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }}
+          # Through the environment, never interpolated into the script below: a
+          # dispatch input is attacker-controlled text as far as the shell is
+          # concerned, and a quote in it would otherwise end the string.
+          WORKBOOKS_INPUT: ${{ inputs.workbooks }}
+        run: |
+          set -uo pipefail
+          # WORKBOOKS is exported only when the dispatch named some: the harness
+          # distinguishes unset (the driver's default list) from empty (run none).
+          if [ -n "$WORKBOOKS_INPUT" ]; then export WORKBOOKS="$WORKBOOKS_INPUT"; fi
+          ./e2e-oci.sh 2>&1 | tee "$RUNNER_TEMP/e2e-oci.log"
+          exit "${PIPESTATUS[0]}"
+
+      # Before the sweep, which destroys the state the tables are read from, and
+      # always(): a run whose verdict is a failure is exactly the one somebody
+      # wants a table for. The successful path writes the same tables itself, so
+      # this step is for the run that did not get there.
+      #
+      # The markdown lives in the script rather than here. A heredoc in a YAML
+      # string is the one place in this repository nothing can lint or run.
+      - name: Publish the result tables
+        if: always()
+        working-directory: spinifex/scripts/terraform/oci-spx
+        env:
+          OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
+          OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
+          OCI_TOPOLOGIES: ${{ inputs.topologies }}
+          OCI_SOURCE: ${{ inputs.source }}
+          OCI_CHANNEL: ${{ inputs.channel }}
+          SPX_REF: ${{ github.ref_name }}
+          # The same two the run step passes. The header states what was and was not
+          # tested, so a summary rendered without them would claim more than the run.
+          OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
+          OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
+        run: ./e2e-oci.sh --summary-only
+
+      # The sweep below deletes the per-topology state directories, so anything
+      # still only in one of them at that point is unrecoverable -- and a failing
+      # workbook writes its assertion output to exactly there. Copying into the
+      # artifact directory first is what makes a failure diagnosable by whoever
+      # reads the run page, rather than only on the runner that produced it.
+      - name: Collect the per-topology logs
+        if: always()
+        run: |
+          set -uo pipefail
+          dest="$RUNNER_TEMP/e2e-oci"
+          mkdir -p "$dest"
+          # The topologies this run drove, taken from the per-topology log the
+          # harness writes rather than from the dispatch input: a run that names
+          # none gets the harness's own default, and reading that back here would
+          # mean a second copy of the list to disagree with the first. A state
+          # directory left by an earlier run is then not collected, so the artifact
+          # cannot show a topology this run never attempted.
+          topologies=""
+          for log in "$dest"/*.log; do
+              [ -f "$log" ] || continue
+              name="${log##*/}"; name="${name%.log}"
+              [ -d "$OCI_STATE_ROOT/.validate-$name" ] && topologies="$topologies $name"
+          done
+          echo "collecting:${topologies:- nothing}"
+          for topology in $topologies; do
+              for file in "$OCI_STATE_ROOT/.validate-$topology"/*.log \
+                          "$OCI_STATE_ROOT/.validate-$topology"/*.txt \
+                          "$OCI_STATE_ROOT/.validate-$topology"/*.tsv; do
+                  [ -f "$file" ] || continue
+                  # The hook handles a private key and the one thing known about an
+                  # operator's executable is that it is not ours to publish.
+                  case "${file##*/}" in credential-hook.log) continue ;; esac
+                  cp -- "$file" "$dest/$topology-${file##*/}"
+              done
+          done
+          ls -l "$dest"
+
+      # always(), and before the artifact upload: an OCI bare-metal host left
+      # running by a cancelled or timed-out job is the most expensive way this
+      # workflow can fail. The sweep works from the Terraform state files, so it
+      # can only destroy what this run created.
+      - name: Sweep anything left behind
+        if: always() && !inputs.keep_on_fail
+        working-directory: spinifex/scripts/terraform/oci-spx
+        env:
+          OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
+          OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
+          OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
+          OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
+          OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
+          TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
+          # The sweep destroys by state file, so it needs whichever tenancy each
+          # topology built in. Without these a leftover principal deployment is
+          # swept against a tenancy that never held it and reported clean.
+          OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
+          OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
+          OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
+          OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
+          OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
+          OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
+          OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci-sweep
+        run: ./e2e-oci.sh --sweep-only
+
+      # Explicit, not left to the runner's own cleanup: a hosted runner is
+      # discarded after the job, but a self-hosted one keeps $HOME, so the key
+      # would outlive the run that needed it and sit there between nightlies.
+      - name: Remove the throwaway SSH key
+        if: always()
+        run: rm -f ~/.ssh/oci-spx ~/.ssh/oci-spx.pub
+
+      # Named extensions rather than a directory glob: the artifact directory also
+      # holds the distro tarball and the test archive, which are hundreds of
+      # megabytes and reproducible from the ref. Everything here is a file the
+      # collect step above chose, so terraform.tfstate cannot reach an artifact.
+      - name: Upload the run
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: e2e-oci-${{ github.run_id }}
+          path: |
+            ${{ runner.temp }}/e2e-oci.log
+            ${{ runner.temp }}/e2e-oci/*.log
+            ${{ runner.temp }}/e2e-oci/verdict.txt
+            ${{ runner.temp }}/e2e-oci/summary.md
+            ${{ runner.temp }}/e2e-oci/*.txt
+            ${{ runner.temp }}/e2e-oci/*.tsv
+            ${{ runner.temp }}/e2e-oci-sweep/*.log
+          retention-days: 14

.github/workflows/e2e-ddil.yml

@@ -1,7 +1,5 @@
 name: E2E DDIL
 
-# This workflow is manually triggered while the DDIL scenarios are being
-# validated.
 on:
   workflow_dispatch:
     inputs:
@@ -10,37 +8,31 @@ on:
         required: false
         default: ""
 
-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
+# Shared with e2e-nightly.yml and e2e-suites.yml: every one of them provisions
+# many cells on the same runner pools and the same hypervisors, so two in
 # flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
+# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
+# is outside the group: one cluster per run is cheap enough to overlap.
 concurrency:
   group: e2e-hypervisors
   cancel-in-progress: false
 
 permissions:
   contents: read
-  # actions: read lets the otel-job-id action look up this job's numeric id via
-  # the Actions API — github.job is only the YAML key, no context exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
   ddil:
     name: DDIL Real Multi-Node
-    runs-on: [ self-hosted, ci-multi ]
+    runs-on: [self-hosted, ci-multi]
     timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
-      # Job-level env wins over the workflow-level default. ci.cell=ddil marks
-      # this workflow's telemetry so a DDIL run's partition-induced errors are
-      # never mistaken for a healthy multi-node run's in the sink.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=ddil
     steps:
       - name: Clean Workspace

.github/workflows/e2e-nightly.yml

@@ -1,19 +1,20 @@
 name: E2E Nightly
 
+# Triggered nightly by a Cloudflare Worker that dispatches this workflow once on
+# `dev` and once on `main`, rather than by a `schedule:` here. GitHub registers a
+# schedule only from the default branch, so a cron in this file can never run
+# `main` and `dev` as separate nightlies, and it would always execute `main`'s
+# copy of this file — leaving a fix to the workflow itself untestable until it
+# had already shipped. A dispatch takes both the file and the source from its ref.
 on:
-  schedule:
-    - cron: "0 14 * * 0-4" # 00:00 Mon-Fri AEST = 14:00 Sun-Thu UTC
   workflow_dispatch:
     inputs:
       cells:
-        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection"
+        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
         required: false
 
 permissions:
   contents: read
-  # actions: read lets the "Resolve CI job id" step below look up this job's
-  # numeric id via the Actions API — github.job is only the YAML key, no
-  # context exposes the id itself.
   actions: read
 
 # Every cell builds its cluster on the same five hypervisors, so two runs in
@@ -21,30 +22,21 @@ permissions:
 # second rather than cancelling it: a dispatch is usually a verification run
 # and discarding it loses the result someone is waiting on.
 #
-# The group is shared with e2e.yml, e2e-suites.yml and e2e-ddil.yml, which draw
-# on the same runner pools. A per-workflow group let a push-triggered E2E land
-# on top of a nightly cell and take libvirt down under both.
+# The group is shared with e2e-suites.yml and e2e-ddil.yml, which draw on the
+# same runner pools. e2e.yml is deliberately outside it: a single-cluster push
+# run is cheap enough to overlap, and grouping it stalled every push behind a
+# multi-hour nightly.
 concurrency:
   group: e2e-hypervisors
   cancel-in-progress: false
 
-# Scheduled runs are loaded from the default branch (main), so the schedule
-# trigger must stay in sync on main for cron to fire — and so must the
-# concurrency block above. main currently has none, which is why the nightly
-# cron runs ungrouped and a dispatch on any branch can land on top of it.
 env:
-  GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+  GIT_BRANCH: ${{ github.ref_name }}
   UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
-  # The matrix is built here rather than filtered inside each cell. A job whose
-  # steps all skip still reports success, so gating steps made a one-cell
-  # dispatch look like a green nineteen-job suite. A cell that was not selected
-  # now produces no job at all.
   select:
     runs-on: ubuntu-26.04
     outputs:
@@ -60,9 +52,6 @@ jobs:
       - id: pick
         env:
           CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
-          # Provision, install and teardown around the suite. Measured at ~6
-          # minutes single-node and ~10 multi-node, so this is the floor the
-          # job budget must carry on top of the suite's own.
           SETUP_ALLOWANCE_MIN: 10
         run: |
           if [ -z "$CELLS" ]; then
@@ -125,15 +114,11 @@ jobs:
       max-parallel: 3
       matrix:
         include: ${{ fromJSON(needs.select.outputs.cells) }}
-    runs-on: [ self-hosted, "${{ matrix.runner }}" ]
+    runs-on: [self-hosted, "${{ matrix.runner }}"]
     timeout-minutes: ${{ matrix.timeout_minutes || 50 }}
     env:
       TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
-      # Job-level env wins over the workflow-level default above, so telemetry
-      # from this cell carries ci.cell/ci.matrix. Twelve cells run under one
-      # run id and every cell names its hosts node1/node2/node3, so without
-      # this every cell's telemetry collapses into one unattributable bucket.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}
 
     steps:
@@ -293,8 +278,31 @@ jobs:
              rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
              if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
 
+      # natuplink proves a guest with a public IP reaches the internet. The two
+      # surfaces it never touches are a private subnet's isolation and its NAT
+      # gateway, and TestVPCEgressPaths is the authority on both — it skipped in
+      # nat mode until the fixture learned that routed NAT carries EIPs too.
+      - name: Run VPC Egress Paths in NAT mode
+        if: matrix.extra == 'nat-single'
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             echo '::group::suite single/TestVPCEgressPaths'; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-vpcegress-nat.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./single.test -test.v -test.timeout=30m -test.run '^TestVPCEgressPaths\$' \
+               2>&1 | tee '${ARTIFACT_DIR}/test-vpcegress-nat.log'; \
+             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
+             if [ \$rc -ne 0 ]; then echo 'vpcegress-nat: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
+
       - name: Run Multi Node E2E
-        if: matrix.topo == 'real-multi' && matrix.suite == ''
+        if: matrix.topo == 'real-multi' && matrix.suite == '' && matrix.extra == ''
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set -euo pipefail
@@ -329,6 +337,40 @@ jobs:
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
              done; exit \$FAIL"
 
+      # The same suite as the nat-single cell, run against three nodes: there it
+      # proves routed NAT works at all, here that each node delivers its own
+      # guests' public IPs and no others — which one node cannot show.
+      - name: Run NAT Uplink Multi E2E
+        if: matrix.extra == 'nat-multi'
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+
+          NODES_CSV=""
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
+          done
+
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
+             echo '::group::suite natuplink'; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+               SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
+               SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./natuplink.test -test.v -test.timeout=30m \
+               2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
+             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
+             if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
+
       # The dedicated suite cells only. Resolved once here rather than in each
       # step below, which would repeat the same three lines five times.
       - name: Resolve VM SSH details
@@ -690,6 +732,66 @@ jobs:
             > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
           echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
 
+      # Whether br-mgmt is a per-host island or one L2 segment spanning the
+      # cluster decides whether a system instance can reach the gateway address
+      # it was baked with. Nothing in a failing run records which one was built,
+      # and the environment is destroyed before it can be asked.
+      - name: Collect management-fabric topology
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          OUT="${ARTIFACT_DIR}/mgmt-fabric.txt"
+          INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")
+
+          declare -a WAN=() MGMT=()
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            WAN[$i]=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+          done
+
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            idx=$((i + 1))
+            {
+              echo "=== node${idx} (${WAN[$i]}) ==="
+              ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+                'sudo ovs-vsctl list-ports br-mgmt 2>&1 | sed "s/^/ports: /"; \
+                 ip -4 addr show br-mgmt 2>&1 | sed "s/^/addr:  /"; \
+                 sudo ovs-vsctl get Bridge br-mgmt fail_mode 2>&1 | sed "s/^/fail:  /"'
+              echo
+            } >> "$OUT" 2>&1
+            MGMT[$i]=$(ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+              "ip -4 -o addr show br-mgmt 2>/dev/null | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null)
+          done
+
+          # An island cluster reaches only its own address and that is correct;
+          # a spanned one must reach every peer. Either result is a finding, and
+          # a mixed result is the failure this matrix exists to catch.
+          {
+            echo "=== br-mgmt reachability (row = prober, column = target) ==="
+            printf '%-10s' "from\\to"
+            for i in $(seq 0 $((NODE_COUNT - 1))); do printf '%-18s' "node$((i+1)):${MGMT[$i]:-none}"; done
+            echo
+            for i in $(seq 0 $((NODE_COUNT - 1))); do
+              printf '%-10s' "node$((i+1))"
+              for j in $(seq 0 $((NODE_COUNT - 1))); do
+                if [ -z "${MGMT[$j]}" ]; then printf '%-18s' "-"; continue; fi
+                if ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+                     "ping -c1 -W1 ${MGMT[$j]} >/dev/null 2>&1"; then
+                  printf '%-18s' ok
+                else
+                  printf '%-18s' FAIL
+                fi
+              done
+              echo
+            done
+          } >> "$OUT" 2>&1
+
+          cat "$OUT"
+
       - name: Install go-junit-report
         if: always()
         run: |
@@ -775,12 +877,11 @@ jobs:
 
   # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
   # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
-  # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
-  # (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
+  # matrix cell ("cell 20"): blank cells is the whole nightly, or name 20 to run
+  # it alone. A reusable call needs its own ref, since it cannot read GIT_BRANCH.
   baremetal:
     name: cell-20 (baremetal/pxe/real-hw)
     if: >-
-      github.event_name == 'schedule' ||
       inputs.cells == '' ||
       contains(format(',{0},', inputs.cells), ',20,')
     permissions:
@@ -788,7 +889,7 @@ jobs:
       statuses: write
     uses: ./.github/workflows/e2e-baremetal.yml
     with:
-      ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+      ref: ${{ github.ref_name }}
       # A reusable-workflow call gets none of this workflow's context (env:
       # above, matrix.cell) for free — cell-20 needs its own attrs string.
       # There's no matrix leg here, just the one fixed baremetal cell.
@@ -808,7 +909,6 @@ jobs:
     needs: baremetal
     if: >-
       !cancelled() && (
-        github.event_name == 'schedule' ||
         inputs.cells == '' ||
         contains(format(',{0},', inputs.cells), ',28,')
       )
@@ -817,7 +917,7 @@ jobs:
       statuses: write
     uses: ./.github/workflows/e2e-baremetal.yml
     with:
-      ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+      ref: ${{ github.ref_name }}
       cell: "28"
       suites: diskperf
       otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=28,ci.matrix=baremetal/pxe/diskperf
@@ -834,7 +934,6 @@ jobs:
   iso-single:
     name: cell-21 (iso/dhcp/single)
     if: >-
-      github.event_name == 'schedule' ||
       inputs.cells == '' ||
       contains(format(',{0},', inputs.cells), ',21,')
     runs-on: [self-hosted, ci-single]
@@ -1089,7 +1188,6 @@ jobs:
     needs: [iso-single]
     if: >-
       !cancelled() && (
-      github.event_name == 'schedule' ||
       inputs.cells == '' ||
       contains(format(',{0},', inputs.cells), ',22,') )
     runs-on: [self-hosted, ci-multi]
@@ -1251,7 +1349,6 @@ jobs:
     needs: [matrix, baremetal, iso-single, iso-multi]
     if: >-
       !cancelled() && (
-      github.event_name == 'schedule' ||
       inputs.cells == '' ||
       contains(format(',{0},', inputs.cells), ',23,') )
     runs-on: [self-hosted, ci-single]
@@ -1282,7 +1379,7 @@ jobs:
       - name: Resolve sibling refs
         id: siblings
         env:
-          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+          BRANCH: ${{ github.ref_name }}
         run: bash .github/resolve-sibling-refs.sh
         working-directory: spinifex
 
@@ -1363,7 +1460,6 @@ jobs:
     needs: [matrix, baremetal, iso-single, iso-multi, performance]
     if: >-
       !cancelled() && (
-      github.event_name == 'schedule' ||
       inputs.cells == '' ||
       contains(format(',{0},', inputs.cells), ',24,') )
     runs-on: [self-hosted, ci-single]
@@ -1407,7 +1503,7 @@ jobs:
       - name: Resolve sibling refs
         id: siblings
         env:
-          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+          BRANCH: ${{ github.ref_name }}
         run: bash .github/resolve-sibling-refs.sh
         working-directory: spinifex
 
@@ -1446,8 +1542,16 @@ jobs:
 
       # Leftovers from a run killed before its cleanup trap: a surviving node
       # holds its ports and fails this run somewhere unrelated.
+      #
+      # Under the host benchmark lock, because this pattern matches every
+      # predastore node on the runner and the predastore repository runs the
+      # same harnesses on the same machine. Holding the lock is what makes
+      # "still running" mean "stale" rather than "somebody else's benchmark":
+      # on 2026-09-16 this step killed a predastore run 32s into its workload.
       - name: Clear stale nodes
-        run: pkill -KILL -f '/bin/s3d -config' || true
+        run: |
+          flock -w 3600 "${PREDA_BENCH_LOCK:-/var/lock/predastore-bench.lock}" \
+            pkill -KILL -f '/bin/s3d -config' || true
 
       - name: Run fault injection
         working-directory: predastore
@@ -1467,10 +1571,14 @@ jobs:
           if-no-files-found: warn
           retention-days: 90
 
+      # Under the lock for the same reason as "Clear stale nodes": by the time
+      # this runs the harness has released it, so another repository's benchmark
+      # can already have a cluster of its own up.
       - name: Clean up
         if: always()
         run: |
-          pkill -KILL -f '/bin/s3d -config' || true
+          flock -w 3600 "${PREDA_BENCH_LOCK:-/var/lock/predastore-bench.lock}" \
+            pkill -KILL -f '/bin/s3d -config' || true
           rm -rf "$STRESS_RESULTS_ROOT"
 
   # Reading a nightly meant opening every cell job, and the Actions UI does not
@@ -1478,7 +1586,17 @@ jobs:
   # every cell so this runs last; always() so a red run is the one it describes.
   summary:
     name: run summary
-    needs: [select, matrix, baremetal, diskperf, iso-single, iso-multi, performance, stress]
+    needs:
+      [
+        select,
+        matrix,
+        baremetal,
+        diskperf,
+        iso-single,
+        iso-multi,
+        performance,
+        stress,
+      ]
     if: always()
     runs-on: ubuntu-26.04
     steps:
@@ -1488,7 +1606,7 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - name: Collect junit
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
         with:
           pattern: junit-cell-*-${{ github.run_id }}
           path: junit

.github/workflows/e2e-suites.yml

@@ -43,26 +43,22 @@ on:
         required: false
         default: ""
 
-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
+# Shared with e2e-nightly.yml and e2e-ddil.yml: every one of them provisions
+# many cells on the same runner pools and the same hypervisors, so two in
 # flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
+# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
+# is outside the group: one cluster per run is cheap enough to overlap.
 concurrency:
   group: e2e-hypervisors
   cancel-in-progress: false
 
 permissions:
   contents: read
-  # actions: read lets the otel-job-id action look up a provision job's numeric
-  # id via the Actions API — github.job is only the YAML key, no context
-  # exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   TEST_RUN: ${{ inputs.test_run || '' }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
@@ -81,12 +77,6 @@ jobs:
       matrix: ${{ steps.suites.outputs.matrix }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
-      # This job (not single_suite) is where "Bootstrap via Binary Install"
-      # bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env, before the
-      # suite matrix even exists — single_suite's legs share this one
-      # already-booted VM, so daemon telemetry can't be scoped any finer
-      # than "single env" per suite leg. ci.cell=single distinguishes it
-      # from multi_provision's VMs within the same run id.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
     steps:
       - name: Reap Orphan Procs

.github/workflows/e2e.yml

@@ -24,27 +24,15 @@ on:
         required: false
         default: ""
 
-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
-# flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
-concurrency:
-  group: e2e-hypervisors
-  cancel-in-progress: false
-
 permissions:
   contents: read
   statuses: write
-  # actions: read lets the otel-job-id action look up a job's numeric id via
-  # the Actions API — github.job is only the YAML key, no context exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam e2e-partialblock' }}
   TEST_RUN: ${{ inputs.test_run || '' }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 

.github/workflows/gpu-e2e.yml

@@ -56,15 +56,7 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  # OTLP export to the central Elastic sink (wattle is the sink itself — see
-  # Part A.2). Repo variable overrides; carried for correlation even though
-  # the GPU suite's own daemon-side spans are emitted by wattle's already-
-  # configured obs-agent, not by this job.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
-  # ci.cell=gpu keeps this workflow's telemetry separable from the tofu-env
-  # workflows'. Nothing in this job bakes these attrs into a daemon (see the
-  # note above), so the label only takes effect once wattle's obs-agent is
-  # given them; there is deliberately no ci.job_id lookup for the same reason.
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=gpu
   WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
   WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
@@ -78,8 +70,6 @@ jobs:
     env:
       ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
     steps:
-      # Manual-work mutex gate — see setup note 4 above. Checked BEFORE any
-      # SSH to wattle so a paused run touches the box zero times.
       - name: Manual-work mutex gate
         id: gate
         env:
@@ -122,8 +112,6 @@ jobs:
           echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
           echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
 
-      # Healthcheck: confirm wattle's persistent AWS-compat control plane is
-      # actually up before spending time building/shipping the test binary.
       - name: Healthcheck — describe-regions
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -131,18 +119,6 @@ jobs:
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
             'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
 
-      # Ensure the base EC2 GPU AMI is present — the raw-EC2 GPU passthrough
-      # test is the always-on baseline, so CI self-provisions it rather than
-      # depending on a manual import. `--name` selects the catalog entry (pulls
-      # the prebuilt image from iso.mulgadc.com); `--ami-name` is REQUIRED to
-      # name the AMI ubuntu-26.04-nvidia-gpu-x86_64 — without it the import
-      # falls back to the generic ami-<distro>-<version>-<arch>, which the
-      # tests/e2e/gpu filter would never match. Idempotent — skipped once
-      # wattle's persistent loop-fs already has the named AMI (the 4.4G source
-      # qcow2 is cached in the images dir, so a re-import wouldn't re-download).
-      # `spx admin images import` writes local spinifex storage, so it needs
-      # the `spinifex` group active (sg); aws-CLI calls hit the gateway over
-      # HTTPS and don't.
       - name: Ensure base GPU AMI present (import-if-absent)
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -171,11 +147,6 @@ jobs:
           '
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
 
-      # Preflight — the base GPU AMI is REQUIRED: the setup step above just
-      # ensured it, so a miss here means the import silently failed. Hard-fail
-      # rather than run a false-green all-skipped suite. ECS/EKS node AMIs are
-      # opt-in tiers (heavier — node AMIs + cluster spin-up); their tests
-      # self-skip when absent, so only warn.
       - name: Preflight — verify GPU AMIs
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -205,10 +176,6 @@ jobs:
           '
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
 
-      # Same build command CI's other e2e workflows use to cross-compile a
-      # suite into a shippable binary (tests/e2e/Makefile e2e-build target,
-      # SUITES_BUILD scoped to just gpu). Building on the runner keeps go
-      # toolchain / module cache off wattle entirely.
       - name: Build GPU e2e test binary on runner
         if: steps.gate.outputs.paused != 'true'
         working-directory: tests/e2e
@@ -231,12 +198,6 @@ jobs:
           ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
           scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
 
-      # Same invocation shape e2e.yml uses to run a pre-built suite binary
-      # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
-      # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
-      # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
-      # the advertised gateway, so the override is required here. The harness
-      # should prefer spinifex.toml's advertised bridge.
       - name: Run GPU e2e suite on wattle
         id: e2e
         if: steps.gate.outputs.paused != 'true'
@@ -287,8 +248,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "GPU E2E results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
       - name: Ship test results to Elasticsearch
         if: always() && steps.gate.outputs.paused != 'true'
         continue-on-error: true
@@ -313,14 +272,6 @@ jobs:
           if-no-files-found: warn
           retention-days: 14
 
-      # Cleanup is NEVER a host teardown: it never runs setup-ovn.sh,
-      # dev-reset, or cluster-reset. It only reconciles EC2-API-level state —
-      # terminate any GPU-typed instance still pending/running (the Go
-      # suite's own t.Cleanup already does this on normal pass/fail exit;
-      # this is the backstop for a hard `-test.timeout` kill or a workflow
-      # cancellation, which skip deferred Go cleanups). Termination triggers
-      # gpuManager.Release() on wattle, returning the A1000 to the pool for
-      # the next queued run or manual test.
       - name: Cleanup — terminate leftover GPU instances, release GPU
         if: always() && steps.gate.outputs.paused != 'true'
         run: |

.github/workflows/image-scripts.yml

@@ -1,9 +1,5 @@
 name: Image Scripts
 
-# Separate from spinifex.yml on purpose: scripts/images/ and images/ are shell
-# and mkosi assets baked into system images, not Go, so they get their own
-# path-filtered gate instead of running on every Go contributor's push (see
-# ui.yml for the same pattern applied to the frontend).
 on:
   push:
     branches:
@@ -38,15 +34,6 @@ jobs:
       - name: Run image script tests + shellcheck
         run: make test-images
 
-  # The mkosi EKS image is what customers run, and until now nothing built it:
-  # the composition existed in mkosi-build.sh but had no caller, so a change
-  # that broke it was only found by hand. Cell-25 still boots the Alpine image,
-  # so this is the only gate the Ubuntu one has.
-  #
-  # Self-hosted because the build needs Docker and several GB of disk, matching
-  # publish-system-ami.yml. Its own concurrency group, not e2e-hypervisors: the
-  # build provisions no libvirt guest, and sharing that group would queue image
-  # builds behind whole nightlies.
   eks_mkosi_image:
     name: Build mkosi EKS image
     runs-on: [self-hosted, ci-single]
@@ -58,9 +45,6 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v7
 
-      # The profile binaries (eks-token-webhook, eks-gateway-publish, ...) are
-      # built on the host, not in the builder container, so Go is needed here
-      # even though the toolchain image carries mkosi itself.
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
@@ -69,7 +53,5 @@ jobs:
           # disk, and cache-save would re-upload hundreds of MB for nothing.
           cache: false
 
-      # stage_profile_binaries() sets GOWORK=off, so the sub-repo resolves its
-      # own go.mod and the sibling monorepo checkouts are not needed here.
       - name: Build the image
         run: make build-eks-node-mkosi-image

.github/workflows/perf-3node.yml

@@ -0,0 +1,415 @@
+name: Predastore 3-Node Performance
+
+on:
+  workflow_dispatch:
+    inputs:
+      perf_preset:
+        description: "Warp workload preset: smoke (30s) or compare (2m)"
+        required: false
+        default: compare
+        type: choice
+        options:
+          - smoke
+          - compare
+      cluster_ref:
+        description: "Git ref to build the cluster from, e.g. v1.18.0. Empty uses this branch."
+        required: false
+        default: ""
+        type: string
+
+# Shared with e2e.yml and e2e-nightly.yml: this job provisions on the same
+# hypervisors they do, so a run in flight there and here would compete for
+# disks and libvirt guests. Queued, not cancelled, same as those workflows.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  GIT_BRANCH: ${{ github.ref_name }}
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+
+jobs:
+  perf_3node:
+    name: Predastore Throughput (3-node)
+    # ci-multi is a systemd runner on the LAN, not the ephemeral pool. A pool
+    # VM sits behind the geneve overlay at MTU 1408, which would put the
+    # runner's own network path inside the thing being measured.
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 90
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-perf3node-${{ github.run_id }}.log
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=perf-3node
+    steps:
+      - name: Reap Orphan Procs
+        run: |
+          set +e
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM
+          sleep 3
+          reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
+      - name: Clean Workspace
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      # ci-multi and ci-single resolve to the same physical host, so the
+      # nvme path cell-23 uses for its throughput run is already there.
+      # Pointing TMPDIR at it keeps warp's payloads off a small /tmp tmpfs.
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-3node-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Re-enable Go toolchain auto-download
+        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
+
+      # Same-named branch when one exists, dev otherwise. Matches how the
+      # e2e path picks siblings, so this measures the same predastore
+      # commit a cross-repo change would be built and tested against.
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ env.GIT_BRANCH }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - uses: actions/checkout@v7
+        id: predastore
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # The cluster's ref is a knob and the harness's is not. Measuring an old
+      # release with a new harness is the only way its number can be set beside
+      # a current one; rebuilding both would change the ruler and the thing.
+      - name: Build Install Artifacts
+        working-directory: mulga/scripts/tofu-cluster
+        env:
+          CLUSTER_REF: ${{ inputs.cluster_ref }}
+        run: |
+          ./build-install-artifacts.sh "${CLUSTER_REF:-$GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
+
+      # The libvirt provider dials the hypervisors by bare hostname. A pool VM
+      # sits inside a VPC and cannot resolve them; a LAN runner already can, so
+      # this is self-gating on the lookup rather than on the runner's labels.
+      - name: Map hypervisor names
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          if getent hosts banksia >/dev/null; then
+            echo "hypervisor names already resolve"
+          else
+            sudo tee -a /etc/hosts < network/hosts >/dev/null
+            echo "added $(grep -c '^[0-9]' network/hosts) entries to /etc/hosts"
+          fi
+          # Pinned rather than StrictHostKeyChecking=no: these hops reach the
+          # hypervisors themselves, so an unverified key is the one place in
+          # this job where accepting anything would matter.
+          if ssh-keygen -F banksia >/dev/null 2>&1; then
+            echo "hypervisor host keys already known"
+          else
+            install -d -m 700 ~/.ssh
+            cat network/known_hosts >> ~/.ssh/known_hosts
+            chmod 600 ~/.ssh/known_hosts
+            echo "pinned $(grep -c '^[a-z]' network/known_hosts) host keys"
+          fi
+
+      - name: Provision VMs
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
+          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
+          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ok "3-node cluster provisioned"
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
+          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Resolve cluster SSH details
+        id: cluster
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
+          REGION=$(echo "$INVENTORY" | jq -r '.region')
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          NODES_CSV=""
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
+          done
+          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
+          echo "ssh_user=${SSH_USER}" >> "$GITHUB_OUTPUT"
+          echo "region=${REGION}" >> "$GITHUB_OUTPUT"
+          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
+
+      # PERF_EXTERNAL_HOSTS wants host:port, and the gate listens on 8443 on
+      # every node. run-info.txt from this run is then comparable with the
+      # nightly's loopback one, since the harness itself never changes.
+      - name: Resolve PERF_EXTERNAL_HOSTS
+        id: perf_hosts
+        run: |
+          set -euo pipefail
+          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
+          HOSTS=""
+          for ip in "${NODE_LIST[@]}"; do
+            [ -z "$ip" ] && continue
+            if [ -z "$HOSTS" ]; then HOSTS="$ip:8443"; else HOSTS="$HOSTS,$ip:8443"; fi
+          done
+          echo "hosts=${HOSTS}" >> "$GITHUB_OUTPUT"
+
+      # A slow bootstrap should fail here, legibly, rather than inside warp
+      # with a wall of connection-refused errors. Bounded at 5 minutes per
+      # gate so a genuinely dead node still fails the job in finite time.
+      - name: Wait for gates to answer on :8443
+        run: |
+          set -euo pipefail
+          IFS=',' read -ra GATES <<< "${{ steps.perf_hosts.outputs.hosts }}"
+          for gate in "${GATES[@]}"; do
+            echo "waiting for https://$gate"
+            up=0
+            for i in $(seq 1 60); do
+              if curl -k -sS --max-time 5 -o /dev/null "https://$gate/"; then
+                up=1
+                break
+              fi
+              sleep 5
+            done
+            [ "$up" -eq 1 ] || { echo "::error::gate $gate never answered HTTPS on :8443"; exit 1; }
+            echo "gate $gate is up"
+          done
+
+      # Warp reads whole objects, so no workload here touches a ranged read and
+      # the throughput figures stay healthy while ranged GETs are broken. This
+      # probe is the only thing in CI that looks at them.
+      - name: Ranged GET probe
+        env:
+          GATE: ${{ steps.cluster.outputs.node1_ip }}
+          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
+          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
+          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
+          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
+        run: |
+          set -euo pipefail
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
+          AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          echo "::add-mask::$SK"
+          B="ranged-probe-${GITHUB_RUN_ID}"
+          AWSX=(--region "$CLUSTER_REGION" --endpoint-url "https://${GATE}:8443" --no-verify-ssl)
+          export AWS_ACCESS_KEY_ID="$AK" AWS_SECRET_ACCESS_KEY="$SK"
+          aws s3 mb "s3://$B" "${AWSX[@]}"
+          trap 'aws s3 rb "s3://$B" --force "${AWSX[@]}" >/dev/null 2>&1 || true' EXIT
+          # Two sizes: read amplification makes a boundary-crossing range cost
+          # the whole object, so the fault shows as time tracking object size.
+          for mb in 64 256; do
+            head -c $((mb*1024*1024)) /dev/urandom > "${TMPDIR}/probe-${mb}.bin"
+            aws s3 cp "${TMPDIR}/probe-${mb}.bin" "s3://$B/probe-${mb}.bin" --only-show-errors "${AWSX[@]}"
+            rm -f "${TMPDIR}/probe-${mb}.bin"
+          done
+          t() {
+            curl -sk --aws-sigv4 "aws:amz:${CLUSTER_REGION}:s3" -u "$AK:$SK" \
+              -H "Range: bytes=0-$1" -o /dev/null -w '%{time_total}' \
+              --url "https://${GATE}:8443/${B}/probe-${2}.bin"
+          }
+          # 4194304 is the last byte inside the first 4 MiB block; one more
+          # crosses into the second and leaves the single-shard fast path.
+          IN64=$(t 4194303 64);  OUT64=$(t 4194304 64)
+          IN256=$(t 4194303 256); OUT256=$(t 4194304 256)
+          printf '%-10s in-block=%ss  crossing=%ss\n' "64MiB" "$IN64" "$OUT64"
+          printf '%-10s in-block=%ss  crossing=%ss\n' "256MiB" "$IN256" "$OUT256"
+          # The 4x object-size ratio is the signal, and it is compared against
+          # the crossing reads alone so a slow runner cancels out of both sides.
+          awk -v a="$OUT64" -v b="$OUT256" 'BEGIN{
+            r = (a > 0) ? b/a : 0
+            printf "crossing-range cost ratio 256MiB/64MiB = %.2f (object size ratio is 4.00)\n", r
+            if (r > 2.0) {
+              print "::error::a boundary-crossing ranged GET scales with object size, so it is reading the whole object"
+              exit 1
+            }
+          }'
+          echo "ranged GET does not scale with object size"
+
+      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
+      # free-space watermark, so this asserts the same headroom cell-23 does.
+      # tmpfs is rejected outright: that would measure RAM, not the gates.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          case "$fstype" in
+            tmpfs|ramfs)
+              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
+              exit 1 ;;
+          esac
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
+            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
+            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      - name: Install warp
+        working-directory: predastore
+        run: make warp-install
+
+      # PERF_EXTERNAL_HOSTS skips profile selection, port offsetting, and
+      # cluster start/stop, and measures these three gates directly. Same
+      # workloads and analysis as the nightly loopback run, so the numbers
+      # from the two are directly comparable.
+      #
+      # The cluster's credentials are read and used inside this one step, the
+      # way e2e-ddil.yml does it: a step-level env: block is echoed verbatim
+      # into the run log, and a shell variable is not.
+      - name: Run 3-node throughput benchmark
+        working-directory: predastore
+        env:
+          PERF_EXTERNAL_HOSTS: ${{ steps.perf_hosts.outputs.hosts }}
+          PERF_CONFIGS: spinifex-3node
+          PERF_PRESET: ${{ inputs.perf_preset || 'compare' }}
+          # What is deployed, not what the harness was checked out at. With a
+          # cluster_ref those differ, and run-info.txt records the cluster.
+          PERF_EXTERNAL_SHA: ${{ inputs.cluster_ref || steps.predastore.outputs.commit }}
+          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
+          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
+          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
+          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
+        run: |
+          set -euo pipefail
+          # spx admin init mints a random pair at bootstrap and writes it to
+          # node1's ~/.aws/credentials. The benchmark's built-in key is the one
+          # the loopback configs trust, which this cluster has never heard of.
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
+          PERF_ACCESS_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          PERF_SECRET_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          if [ -z "$PERF_ACCESS_KEY" ] || [ -z "$PERF_SECRET_KEY" ]; then
+            echo "::error::could not read spinifex AWS credentials from ${NODE1_IP}"
+            exit 1
+          fi
+          echo "::add-mask::$PERF_SECRET_KEY"
+          mkdir -p "$PERF_RESULTS_ROOT"
+          PERF_ACCESS_KEY="$PERF_ACCESS_KEY" \
+          PERF_SECRET_KEY="$PERF_SECRET_KEY" \
+          PERF_REGION="$CLUSTER_REGION" \
+            make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"
+
+      # Report-only, matching cell-23: the baseline comparison and any
+      # regression gate are future work once run-to-run spread is known from
+      # real bare-metal runs.
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-perf-3node-${{ github.run_id }}
+          path: ${{ env.PERF_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up results
+        if: always()
+        run: rm -rf "${PERF_RESULTS_ROOT}"
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM
+          sleep 2
+          reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
+          ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-perf3node-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/publish-system-ami.yml

@@ -60,7 +60,7 @@ concurrency:
 jobs:
   matrix:
     name: Resolve image matrix
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
     outputs:
       images: ${{ steps.gen.outputs.images }}
     steps:
@@ -84,7 +84,7 @@ jobs:
   publish:
     name: Build + publish ${{ matrix.name }}
     needs: matrix
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
     strategy:
       fail-fast: false
       # ci-single is one runner and each GPU build needs ~16G of scratch disk;
@@ -131,9 +131,7 @@ jobs:
         with:
           go-version-file: "spinifex/go.mod"
           # Self-hosted runner: the Go module + build cache already persists on
-          # local disk between runs. setup-go's cache-save ("Post Setup Go")
-          # otherwise re-uploads hundreds of MB to the Actions cache backend,
-          # taking longer than the build itself.
+          # local disk between runs.
           cache: false
 
       - name: Setup Go Workspace
@@ -160,7 +158,6 @@ jobs:
       - name: Build + publish ${{ matrix.name }} to R2
         working-directory: spinifex
         env:
-          # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
           AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
           R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}

.github/workflows/release.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.27.0"
+          go-version-file: spinifex/go.mod
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image

.golangci.yml

@@ -1,5 +1,8 @@
 version: "2"
 
+run:
+  allow-serial-runners: true
+
 issues:
   max-same-issues: 10
 
@@ -56,7 +59,6 @@ linters:
     - unconvert # removes unnecessary type conversions
     - usestdlibvars # detects hardcoded values that should use stdlib constants
     - usetesting # catches os.TempDir() etc. in tests — should use t.TempDir()
-    - wastedassign # finds assignments to variables that are never subsequently read
     - whitespace # detects leading and trailing whitespace in functions
 
   settings:
@@ -143,15 +145,11 @@ linters:
     staticcheck:
       checks:
         - all
-        - -ST1000 # package comment
         - -ST1003 # naming conventions (Id→ID etc.) — too noisy to retrofit
         - -ST1012 # error var naming — errname covers this + error type suffixes
         - -ST1013 # HTTP status constants — usestdlibvars is broader
         - -ST1016 # receiver name consistency (too noisy to retrofit)
-        - -ST1020 # godoc on exported types — not enforcing yet
-        - -ST1021 # godoc on exported type groups — not enforcing yet
-        - -ST1022 # godoc on exported vars/consts — not enforcing yet
-        - -SA4006 # value overwritten — ineffassign + wastedassign cover this better
+        - -SA4006 # value overwritten — ineffassign catches strictly more of these
         - -SA9005 # marshaling structs with no exported fields — intended {} for empty API-response types
 
     testifylint:

CONTRIBUTING.md

@@ -0,0 +1,96 @@
+# How to Contribute
+
+We welcome contributions to Spinifex!
+
+Report security vulnerabilities privately through [SECURITY.md](SECURITY.md), not as a public issue.
+
+**AWS parity is the design rule.** When Spinifex's behaviour is in question, the answer is what AWS does. If you believe Spinifex diverges from AWS, state what AWS does and show it with a test or a reproduction against a running cluster.
+
+## Understanding the Codebase
+
+Every AWS call enters through a SigV4-authenticated gateway, is published to NATS, and is answered by the daemon that claims it. Start with these:
+
+| Doc | Covers |
+|---|---|
+| [`docs/DESIGN.md`](docs/DESIGN.md) | Architecture, request flow, daemons, storage integration, file layout. |
+| [`docs/COMMANDS.md`](docs/COMMANDS.md) | The `spx admin` CLI and the systemd units it manages. |
+| [`docs/coverage/`](docs/coverage/README.md) | Which AWS API operations each service implements. |
+| [`scripts/README.md`](scripts/README.md) | Install, dev-environment, and image-build scripts. |
+
+User-facing docs are published at [docs.mulgadc.com](https://docs.mulgadc.com).
+
+## Development Setup
+
+These steps build Spinifex from source and run it as a single-node cluster, following the [Source Install](https://docs.mulgadc.com/docs/install-source) guide.
+
+### Prerequisites
+
+- **Ubuntu 26.04** or **Debian 13**, on x86_64 or aarch64
+- A host with hardware virtualisation (KVM) and `sudo` access
+- The host's WAN interface enslaved to a Linux bridge named `br-wan`, with the host IP, default route and DHCP on the bridge rather than the bare NIC (see [VPC Networking → Bridge Setup](https://docs.mulgadc.com/docs/vpc-networking#bridge-setup-physical-network-wiring))
+
+Check the bridge before continuing:
+
+```bash
+ip -br link show br-wan   # bridge exists and is UP
+ip route                  # default route's dev is br-wan
+```
+
+### Install and Run
+
+```bash
+mkdir -p ~/Development/mulga && cd ~/Development/mulga
+git clone https://github.com/mulgadc/spinifex.git
+sudo make -C spinifex quickinstall        # system packages, Go 1.27, AWS CLI v2
+export PATH=$PATH:/usr/local/go/bin
+
+cd spinifex
+./scripts/clone-deps.sh                   # sibling repos: viperblock, predastore, northstar, bluebottle
+./scripts/dev-install.sh                  # build, install, init a one-node cluster, start services
+
+AWS_PROFILE=spinifex aws ec2 describe-instance-types
+```
+
+A list of instance types means the cluster is up.
+
+`make preflight` also needs `shellcheck` and `golangci-lint` (CI pins v2.13.0):
+
+```bash
+sudo apt-get install -y shellcheck
+curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v2.13.0
+```
+
+The web console additionally needs Node.js 24 and `pnpm`.
+
+## Development Workflow
+
+| Command | Use it to |
+|---|---|
+| `make deploy` | Rebuild, install `spx`, and restart `spinifex.target` after a code change. |
+| `make reinstall` | Re-run `dev-install.sh` after changing systemd units, helper scripts, or logrotate config. |
+| `./scripts/reset-dev-env.sh` | Wipe all cluster state and reinstall from scratch, keeping your network topology. |
+| `make test` | Run unit tests (`make test-race` for the race detector). |
+| `make test-integration` | Run the real gateway router against embedded NATS JetStream. |
+| `make preflight` | Run the pre-commit gate: lint, `govulncheck`, coverage, and manifest and script checks. Must pass. |
+
+Follow the services with `journalctl -u 'spinifex-*' -f`.
+
+The web console lives in `spinifex/services/spinifexui/frontend/` (`pnpm install`, `pnpm dev`, `pnpm lint`, `pnpm test`). `spx` embeds its checked-in `dist/`, so run `make build-ui` and commit the rebuilt `dist/` with your change.
+
+## Creating a Pull Request
+
+1. Fork [mulgadc/spinifex](https://github.com/mulgadc/spinifex) and clone your fork into the same parent directory as the sibling repositories, branching off `dev`:
+
+   ```bash
+   cd ~/Development/mulga
+   git clone git@github.com:YOUR_USERNAME/spinifex.git && cd spinifex
+   git remote add upstream git@github.com:mulgadc/spinifex.git
+   git fetch upstream && git checkout -b feat/my-feature upstream/dev
+   ```
+
+2. Make your change, then verify it:
+   - `make preflight` passes.
+   - `go mod tidy` leaves only intended changes in `go.mod` and `go.sum`, with none of the `clone-deps.sh` `replace` lines.
+   - Runtime changes are exercised on your dev cluster with `make deploy`.
+
+3. Commit using [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `refactor:`, `docs:`, `chore:`), push, and open a pull request against `dev`, filling in the template.

Makefile

@@ -57,7 +57,7 @@ build-ui:
 	cd spinifex/services/spinifexui/frontend && pnpm build
 
 # GO commands
-VERSION ?= $(shell git describe --tags --always --dirty)
+VERSION ?= $(shell git describe --tags --always --dirty --exclude '*-dev')
 COMMIT  ?= $(shell git rev-parse --short HEAD)
 LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSION) -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=$(COMMIT)
 
@@ -149,21 +149,39 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 
 # the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
 # integration and race tests skipped to keep quick, they run in CI.
+#
+# -j2 overlaps the only two heavy gates, lint and test-cover: alone each leaves
+# most of the machine idle, so pairing them cuts a cold run by about a quarter.
+# -Otarget keeps each gate's output in one block instead of interleaving them.
+# manifest-check manifest-lint test-package-check removed, can add back later
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
+	@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 lint govulncheck test-cover diff-coverage test-harness test-build-scripts
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
 # Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
 # ship on every node (unlike scripts/images/, kept in preflight: a wrong
 # restart decision here is an outage, not asset churn).
+#
+# NODE_SCRIPTS are the four that setup.sh installs into
+# /usr/local/share/spinifex on every node. They are covered here for the same
+# reason: between them they install the software, wire OVN, erase the state and
+# remove the install, so a shellcheck warning in one is an outage or a wrong
+# deletion, not a style note.
+NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh
+
 test-build-scripts:
 	@echo -e "\n....Running build/scripts/**/*_test.sh...."
 	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
 		echo "-- $$t"; \
 		sh "$$t" || exit 1; \
 	done
-	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh...."
-	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort)
+	@echo -e "\n....Running scripts/*_test.sh...."
+	@for t in $$(find scripts -maxdepth 1 -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		sh "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
+	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
 	@echo "  test-build-scripts ok"
 
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
@@ -181,24 +199,25 @@ test-harness:
 # JetStream, with only the daemon-side NATS subjects stubbed.
 AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
 AWS_MODEL_CONFORMANCE_MODE ?= fail
-AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
+AWS_MODEL_OPERATION_COVERAGE_DIR ?= $(CURDIR)/docs/coverage
 # -count=1 is load-bearing: the conformance report is written by the test binary,
 # so a cached pass skips the run, leaves no report, and the cat below fails the
 # target. It also keeps the conformance gate honest — a cached result would mean
-# the check never ran against this commit.
+# the check never ran against this commit. The generated request sweep adds
+# about 15s, most of it EC2 fan-outs waiting out their window for instances
+# no daemon-lite owns.
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
-	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 120s ./tests/integration/... $(_RACEQ)
 	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
 	@$(MAKE) --no-print-directory generate-aws-model-coverage
-	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
+	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_DIR)"
 
 generate-aws-model-coverage:
-	@mkdir -p $(dir $(AWS_MODEL_OPERATION_COVERAGE_REPORT))
-	@go run ./cmd/aws-model-coverage > $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+	@go run ./cmd/aws-model-coverage -out $(AWS_MODEL_OPERATION_COVERAGE_DIR) -json $(AWS_MODEL_OPERATION_COVERAGE_DIR)/coverage.json
 
 aws-model-coverage: generate-aws-model-coverage
-	@cat $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+	@go run ./cmd/aws-model-coverage
 
 # Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
 # checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
@@ -243,6 +262,17 @@ test-cover:
 	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
+# Refresh the README coverage badge. Measured as awesome-go documents
+# (-coverpkg=./... over ./...), with cmd/ entrypoints left out of the total.
+coverage-badge:
+	@echo -e "\n....Measuring whole-module coverage for the README badge...."
+	LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 600s -covermode=atomic -coverpkg=./... -coverprofile=coverage-badge.out ./...
+	@grep -v '/cmd/' coverage-badge.out > coverage-badge-nocmd.out
+	@pct=$$(go tool cover -func=coverage-badge-nocmd.out | tail -1 | awk '{print $$NF}' | tr -d '%'); \
+	color=$$(awk -v p="$$pct" 'BEGIN { print (p >= 80) ? "brightgreen" : (p >= 70) ? "yellow" : "red" }'); \
+	sed -i -E "s|badge/coverage-[0-9.]+%25-[a-z]+|badge/coverage-$${pct}%25-$${color}|" README.md; \
+	echo "Coverage $${pct}% ($${color}); README badge updated"
+
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
@@ -299,6 +329,17 @@ bench:
 	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
 
+# Fuzz every Fuzz* target for FUZZTIME each; -fuzz takes one target per run.
+# Plain `go test` already replays the seeds and testdata/fuzz on preflight.
+FUZZTIME ?= 30s
+fuzz:
+	@set -e; grep -rlE '^func Fuzz\w+\(' --include='*_test.go' spinifex cmd internal | while read -r file; do \
+		for name in $$(grep -oE '^func Fuzz\w+' "$$file" | cut -d' ' -f2); do \
+			echo -e "\n....Fuzzing $$name ($$(dirname $$file)) for $(FUZZTIME)...."; \
+			LOG_IGNORE=1 go test -run='^$$' -fuzz="^$$name$$" -fuzztime=$(FUZZTIME) ./$$(dirname $$file); \
+		done; \
+	done
+
 # Fast iteration: build + install binary + restart all services.
 # Microvm artifacts are reinstalled when they already exist on disk — the rule's
 # input timestamps drive a rebuild only if anything actually changed. On a fresh
@@ -434,7 +475,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
+.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover coverage-badge test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -4,8 +4,11 @@
 
 <p align="center">
   <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://pkg.go.dev/github.com/mulgadc/spinifex"><img src="https://img.shields.io/badge/pkg.go.dev-reference-007D9C?style=flat-square&logo=go&logoColor=white" alt="Go Reference"></a>
+  <a href="https://github.com/mulgadc/spinifex/blob/main/Makefile"><img src="https://img.shields.io/badge/coverage-82.9%25-brightgreen?style=flat-square" alt="Coverage"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
   <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
+  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
 </p>
 
 <p align="center">
@@ -79,7 +82,7 @@ Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all wo
 | **ECR** | Container Registry | Available |
 | **ECS** | Container Service | Available |
 | **RDS** | Databases | Available |
-| **Bedrock** | AI Deployment | Q3 2026 |
+| **Bedrock** | AI Deployment | Q4 2026 |
 
 Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 

cmd/aws-model-coverage/main.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"os"
 
+	"github.com/mulgadc/predastore/s3api"
 	"github.com/mulgadc/spinifex/internal/awsmodel"
 	"github.com/mulgadc/spinifex/spinifex/gateway"
 
@@ -11,6 +13,36 @@ import (
 )
 
 func main() {
+	outputDir := flag.String("out", "", "write one publishable page per service into this directory")
+	jsonOutput := flag.String("json", "", "write a machine-readable coverage inventory to this file")
+	flag.Parse()
+
+	coverages, err := compareAll()
+	if err != nil {
+		fail(err)
+	}
+	if *outputDir == "" && *jsonOutput == "" {
+		fmt.Print(awsmodel.RenderCoverageSummary(coverages))
+		return
+	}
+	if *outputDir != "" {
+		if err := awsmodel.WritePages(*outputDir, coverages); err != nil {
+			fail(err)
+		}
+	}
+	if *jsonOutput != "" {
+		if err := awsmodel.WriteCoverageJSON(*jsonOutput, coverages); err != nil {
+			fail(err)
+		}
+	}
+}
+
+func fail(err error) {
+	fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
+	os.Exit(1)
+}
+
+func compareAll() ([]awsmodel.OperationCoverage, error) {
 	dispatch := gateway.AWSOperationInventory()
 	coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
 	for _, service := range awsmodel.Services() {
@@ -23,23 +55,19 @@ func main() {
 				Stubbed:     inventory.Stubbed,
 				Unsupported: inventory.Unsupported,
 			}
+		// Predastore serves the S3 REST surface, and names the operations each
+		// of its routes answers, so the comparison is the same one.
 		case service == awsmodel.S3:
-			modelInventory = awsmodel.DispatchInventory{
-				Opaque: true,
-				Note:   "Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.",
-			}
+			modelInventory = awsmodel.DispatchInventory{Registered: s3api.Operations()}
 		default:
-			fmt.Fprintf(os.Stderr, "aws-model-coverage: no gateway inventory for %s\n", service)
-			os.Exit(1)
+			return nil, fmt.Errorf("no gateway inventory for %s", service)
 		}
 
 		coverage, err := awsmodel.CompareOperations(service, modelInventory)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
-			os.Exit(1)
+			return nil, err
 		}
 		coverages = append(coverages, coverage)
 	}
-
-	fmt.Print(awsmodel.RenderCoverageMarkdown(coverages))
+	return coverages, nil
 }

cmd/ecs-agent/agent.go

@@ -30,10 +30,13 @@ type Agent struct {
 	cfg      config
 	id       identity
 	cp       controlPlane
-	puller   ctrruntime.ImagePuller
-	runner   ctrruntime.Runner
 	resolver ctrruntime.Resolver
 
+	// rt holds the container runtime, which is resolved by a loop rather than
+	// once at boot. Empty until the containerd socket answers, and emptied again
+	// if it stops answering.
+	rt *runtimeGate
+
 	reg *registrar
 	hb  *heartbeater
 
@@ -48,26 +51,34 @@ type Agent struct {
 	// nvidia-smi discovery.
 	gpu *gpuLedger
 
+	// dialRuntime connects to the container runtime. nil in unit tests, which
+	// drive the gate directly.
+	dialRuntime func() (ctrruntime.Runtime, error)
+
 	closers []func() error
 }
 
 // newAgent assembles an Agent from already-built seams. Tests use this directly
-// with fakes; New builds the production seams and delegates here. runner may be
-// nil when containerd is unavailable; the assign path then reports the task
-// STOPPED rather than crashing.
-func newAgent(cfg config, id identity, cp controlPlane, puller ctrruntime.ImagePuller, runner ctrruntime.Runner, resolver ctrruntime.Resolver) *Agent {
-	return &Agent{
+// with fakes; New builds the production seams and delegates here. rt may be nil
+// when containerd is unavailable; the assign path then reports the task STOPPED
+// rather than crashing, and the watch loop keeps dialling for it.
+func newAgent(cfg config, id identity, cp controlPlane, rt ctrruntime.Runtime, resolver ctrruntime.Resolver) *Agent {
+	gate := newRuntimeGate(rt)
+	a := &Agent{
 		cfg:      cfg,
 		id:       id,
 		cp:       cp,
-		puller:   puller,
-		runner:   runner,
 		resolver: resolver,
-		reg:      newRegistrar(cp, id),
-		hb:       newHeartbeater(cp, id, cfg.Heartbeat),
+		rt:       gate,
 		netns:    newTaskNetns(execNetRunner{}),
 		gpu:      newGPULedger(id.Capacity.GPUIDs),
 	}
+	// Registration reports the host's capacity only while a runtime is there to
+	// use it, so an agent that cannot run a container advertises none and takes
+	// no placement.
+	a.reg = newRegistrar(cp, id, gate.available)
+	a.hb = newHeartbeater(cp, id, cfg.Heartbeat, gate.available)
+	return a
 }
 
 // New builds an Agent from config: it resolves the host identity from IMDS,
@@ -89,6 +100,7 @@ func New(cfg config) (*Agent, error) {
 		ClusterName:  cfg.ClusterName,
 		InstanceID:   meta.InstanceID,
 		AZ:           meta.AZ,
+		InstanceType: meta.InstanceType,
 		Hostname:     host,
 		Capacity:     detectCapacity(discoverNvidiaGPUs(execCommandRunner)),
 		AgentVersion: version,
@@ -97,13 +109,12 @@ func New(cfg config) (*Agent, error) {
 	creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
 	resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
 
-	var puller ctrruntime.ImagePuller
-	var runner ctrruntime.Runner
-	if rt, perr := ctrruntime.New(cfg.ContainerdSocket); perr != nil {
-		slog.Warn("ecs-agent: containerd unavailable at boot, image pulls disabled", "err", perr)
-	} else {
-		puller = rt
-		runner = rt
+	// A dial that fails here is not fatal and not final: the node starts the
+	// container daemon and the agent back to back, so the socket is often not
+	// accepting yet. Run's watch loop keeps dialling.
+	rt, perr := ctrruntime.New(cfg.ContainerdSocket)
+	if perr != nil {
+		slog.Warn("ecs-agent: containerd unavailable at boot, will keep retrying", "err", perr)
 	}
 
 	cp, err := newGatewayControlPlane(cfg, creds)
@@ -111,12 +122,10 @@ func New(cfg config) (*Agent, error) {
 		return nil, fmt.Errorf("build gateway control-plane: %w", err)
 	}
 
-	a := newAgent(cfg, id, cp, puller, runner, resolver)
+	a := newAgent(cfg, id, cp, rt, resolver)
 	a.cred = newCredEndpoint(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA,
 		cfg.CredEndpointIP, cfg.CredEndpointPort, execNetRunner{})
-	if puller != nil {
-		a.closers = append(a.closers, puller.Close)
-	}
+	a.dialRuntime = func() (ctrruntime.Runtime, error) { return ctrruntime.New(cfg.ContainerdSocket) }
 	return a, nil
 }
 
@@ -137,6 +146,12 @@ func (a *Agent) Run(ctx context.Context) error {
 
 	go a.hb.Run(ctx)
 
+	// Keeps the runtime current for the life of the process: dials while there
+	// is none, and drops one that stops answering.
+	if a.dialRuntime != nil {
+		go a.rt.watch(ctx, a.dialRuntime, a.cfg.RuntimeRetry)
+	}
+
 	// Re-adopt containers still running from before this restart, then poll with
 	// their tasks pre-seeded so re-delivered assignments are acked, not re-run.
 	go a.pollAssignments(ctx, a.reconcile(ctx))
@@ -156,6 +171,7 @@ func (a *Agent) Stop() error {
 			firstErr = err
 		}
 	}
+	a.rt.clear()
 	return firstErr
 }
 

cmd/ecs-agent/agent_test.go

@@ -12,7 +12,7 @@ func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
 	cp := &fakeCP{}
 	cfg := config{Heartbeat: 5 * time.Millisecond, PollInterval: 5 * time.Millisecond}
 	puller := &ctrruntime.FakePuller{}
-	a := newAgent(cfg, testIdentity(), cp, puller, puller, nil)
+	a := newAgent(cfg, testIdentity(), cp, puller, nil)
 	a.closers = append(a.closers, puller.Close)
 
 	ctx, cancel := context.WithCancel(context.Background())

cmd/ecs-agent/config.go

@@ -33,6 +33,7 @@ type config struct {
 	ContainerdSocket string
 	Heartbeat        time.Duration
 	PollInterval     time.Duration
+	RuntimeRetry     time.Duration
 	CredEndpointIP   string
 	CredEndpointPort int
 }
@@ -75,6 +76,12 @@ func loadConfig(envFile string) config {
 			cfg.PollInterval = d
 		}
 	}
+	cfg.RuntimeRetry = defaultRuntimeRetry
+	if v := get("ECS_RUNTIME_RETRY"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.RuntimeRetry = d
+		}
+	}
 	cfg.CredEndpointIP = get("ECS_CRED_ENDPOINT_IP")
 	if cfg.CredEndpointIP == "" {
 		cfg.CredEndpointIP = defaultCredEndpointIP

cmd/ecs-agent/controlplane.go

@@ -76,6 +76,10 @@ func (g *gatewayControlPlane) Register(id identity) error {
 			{Name: aws.String("MEMORY"), Type: aws.String("INTEGER"), IntegerValue: aws.Int64(int64(id.Capacity.MemoryMiB))},
 		},
 		VersionInfo: &ecs.VersionInfo{AgentVersion: aws.String(id.AgentVersion)},
+		// The built-in container-instance attributes AWS's own agent reports.
+		// The scheduler sizes the host's awsvpc ENI capacity from the instance
+		// type, so a registration without one is not ENI-gated.
+		Attributes: instanceAttributes(id),
 	}
 	if len(id.Capacity.GPUIDs) > 0 {
 		in.TotalResources = append(in.TotalResources, &ecs.Resource{

cmd/ecs-agent/heartbeat.go

@@ -14,18 +14,23 @@ type heartbeater struct {
 	cp       controlPlane
 	id       identity
 	interval time.Duration
+	// runtimeReady reports whether a container runtime is resolved, so a beat
+	// advertises capacity only while there is something to run it with. nil
+	// means always ready.
+	runtimeReady func() bool
 }
 
-func newHeartbeater(cp controlPlane, id identity, interval time.Duration) *heartbeater {
+func newHeartbeater(cp controlPlane, id identity, interval time.Duration, runtimeReady func() bool) *heartbeater {
 	if interval <= 0 {
 		interval = defaultHeartbeat
 	}
-	return &heartbeater{cp: cp, id: id, interval: interval}
+	return &heartbeater{cp: cp, id: id, interval: interval, runtimeReady: runtimeReady}
 }
 
 // beat re-registers the instance once, refreshing its LastSeen.
 func (h *heartbeater) beat() error {
-	return h.cp.Register(h.id)
+	ready := h.runtimeReady == nil || h.runtimeReady()
+	return h.cp.Register(advertisedIdentity(h.id, ready))
 }
 
 // Run beats every interval until ctx is cancelled. A failed beat is logged and

cmd/ecs-agent/metadata.go

@@ -13,9 +13,10 @@ import (
 
 // instanceMetadata is the IMDS-derived identity of the host the agent runs on.
 type instanceMetadata struct {
-	AccountID  string
-	InstanceID string
-	AZ         string
+	AccountID    string
+	InstanceID   string
+	AZ           string
+	InstanceType string
 }
 
 // fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2
@@ -39,10 +40,14 @@ func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata,
 		return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
 	}
 
+	// The instance type comes off the identity document already in hand rather
+	// than a further meta-data read; the scheduler sizes the host's awsvpc ENI
+	// capacity from it.
 	return instanceMetadata{
-		AccountID:  doc.AccountID,
-		InstanceID: strings.TrimSpace(instanceID),
-		AZ:         strings.TrimSpace(az),
+		AccountID:    doc.AccountID,
+		InstanceID:   strings.TrimSpace(instanceID),
+		AZ:           strings.TrimSpace(az),
+		InstanceType: strings.TrimSpace(doc.InstanceType),
 	}, nil
 }
 

cmd/ecs-agent/reconcile.go

@@ -24,10 +24,11 @@ import (
 // agent falls back to its pre-reconcile behaviour (poll + run).
 func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 	adopted := map[string]bool{}
-	if a.runner == nil {
+	runner := a.rt.get()
+	if runner == nil {
 		return adopted
 	}
-	containers, err := a.runner.List(ctx)
+	containers, err := runner.List(ctx)
 	if err != nil {
 		slog.Warn("ecs-agent: reconcile list failed", "err", err)
 		return adopted
@@ -96,9 +97,13 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 // is STOPPED, so the scheduler stops counting a task this instance is not
 // running and its service can place a replacement.
 func (a *Agent) reportStoppedTask(ctx context.Context, as *bus.Assign, dead []ctrruntime.Container) {
+	runner := a.rt.get()
 	statuses := make([]bus.ContainerStatus, 0, len(dead))
 	for _, c := range dead {
-		if err := a.runner.Remove(ctx, c.ID); err != nil {
+		if runner == nil {
+			break
+		}
+		if err := runner.Remove(ctx, c.ID); err != nil {
 			slog.Warn("ecs-agent: reconcile remove failed", "task", as.TaskID, "container", c.ID, "err", err)
 		}
 		statuses = append(statuses, bus.ContainerStatus{

cmd/ecs-agent/reconcile_test.go

@@ -55,7 +55,7 @@ func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
 			adoptedContainer("t-001", "web", "default", "t-001", testTaskRole, "52:54:00:de:ad:01", true),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, nil)
 	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
 
 	adopted := a.reconcile(context.Background())
@@ -84,7 +84,7 @@ func TestReconcile_ReportsTaskWhoseContainerDied(t *testing.T) {
 			adoptedContainer("t-dead", "web", "default", "t-dead", testTaskRole, "52:54:00:de:ad:02", false),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, nil)
 	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
 
 	adopted := a.reconcile(context.Background())
@@ -116,7 +116,7 @@ func TestReconcile_PartiallyDeadTaskStaysRunning(t *testing.T) {
 			adoptedContainer("t-mixed", "sidecar", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", false),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, nil)
 	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
 
 	a.reconcile(context.Background())
@@ -145,7 +145,7 @@ func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
 		nil,
 	}}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go a.pollAssignments(ctx, map[string]bool{"t-001": true})
@@ -197,7 +197,7 @@ func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
 			adoptedContainer("t-ok", "web", "default", "", "", "", true),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, nil)
 
 	adopted := a.reconcile(context.Background())
 
@@ -213,13 +213,13 @@ func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
 
 // A nil runner or a List error degrades to an empty set without panicking.
 func TestReconcile_DegradesGracefully(t *testing.T) {
-	noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil, nil)
+	noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil)
 	if got := noRunner.reconcile(context.Background()); len(got) != 0 {
 		t.Errorf("nil runner: adopted = %v, want empty", got)
 	}
 
 	rt := &ctrruntime.FakePuller{ListErr: errors.New("boom")}
-	listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+	listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, nil)
 	if got := listErr.reconcile(context.Background()); len(got) != 0 {
 		t.Errorf("list error: adopted = %v, want empty", got)
 	}

cmd/ecs-agent/registration.go

@@ -3,6 +3,9 @@ package main
 import (
 	"fmt"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ecs"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
@@ -14,25 +17,61 @@ type identity struct {
 	ClusterName  string
 	InstanceID   string
 	AZ           string
+	InstanceType string
 	Hostname     string
 	Capacity     bus.InstanceCapacity
 	AgentVersion string
 }
 
+// instanceAttributes builds the built-in container-instance attributes from the
+// host's identity. Only attributes whose value was actually discovered are
+// reported, so an empty one is absent rather than present and blank.
+func instanceAttributes(id identity) []*ecs.Attribute {
+	var attrs []*ecs.Attribute
+	add := func(name, value string) {
+		if value == "" {
+			return
+		}
+		attrs = append(attrs, &ecs.Attribute{Name: aws.String(name), Value: aws.String(value)})
+	}
+	add(handlers_ecs.AttrInstanceType, id.InstanceType)
+	add(handlers_ecs.AttrAvailabilityZone, id.AZ)
+	return attrs
+}
+
+// advertisedIdentity is the identity to register with, given whether a
+// container runtime is available. A host with no runtime can run nothing, so it
+// advertises no capacity: the scheduler places nothing on it, and the control
+// plane reads the zero as an involuntary drain. The identity is otherwise
+// unchanged, so the instance stays visible rather than disappearing.
+func advertisedIdentity(id identity, runtimeReady bool) identity {
+	if runtimeReady {
+		return id
+	}
+	id.Capacity = bus.InstanceCapacity{}
+	return id
+}
+
 // registrar registers the host as a container instance through the gateway.
 type registrar struct {
 	cp controlPlane
 	id identity
+	// runtimeReady reports whether a container runtime is resolved. nil means
+	// always ready, which is what a test with no gate wants.
+	runtimeReady func() bool
 }
 
-func newRegistrar(cp controlPlane, id identity) *registrar {
-	return &registrar{cp: cp, id: id}
+func newRegistrar(cp controlPlane, id identity, runtimeReady func() bool) *registrar {
+	return &registrar{cp: cp, id: id, runtimeReady: runtimeReady}
 }
 
+// ready reports the runtime state, defaulting to ready when no gate is wired.
+func (r *registrar) ready() bool { return r.runtimeReady == nil || r.runtimeReady() }
+
 // Register calls the gateway's RegisterContainerInstance. The scheduler records
 // (or refreshes) the container instance on receipt.
 func (r *registrar) Register() error {
-	if err := r.cp.Register(r.id); err != nil {
+	if err := r.cp.Register(advertisedIdentity(r.id, r.ready())); err != nil {
 		return fmt.Errorf("register instance %s: %w", r.id.InstanceID, err)
 	}
 	return nil

cmd/ecs-agent/registration_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"slices"
 	"sync"
 	"testing"
 	"time"
@@ -17,14 +18,16 @@ import (
 type fakeCP struct {
 	mu sync.Mutex
 
-	registers    int
-	states       []bus.TaskState
-	gpuReports   []gpuReport
-	pollAcks     [][]string
-	pollStopAcks [][]string
-	pollReplies  [][]bus.Assign
-	stopReplies  [][]bus.StopDirective
-	pollCalls    int
+	registers      int
+	lastRegistered identity
+	states         []bus.TaskState
+	gpuReports     []gpuReport
+	pollAcks       [][]string
+	pollStopAcks   [][]string
+	pollReplies    [][]bus.Assign
+	stopReplies    [][]bus.StopDirective
+	stopPending    []bus.StopDirective
+	pollCalls      int
 
 	registerErr bool
 	submitErr   bool
@@ -39,16 +42,24 @@ type gpuReport struct {
 
 var _ controlPlane = (*fakeCP)(nil)
 
-func (f *fakeCP) Register(_ identity) error {
+func (f *fakeCP) Register(id identity) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	if f.registerErr {
 		return errors.New("register boom")
 	}
 	f.registers++
+	f.lastRegistered = id
 	return nil
 }
 
+// registered returns the identity of the most recent Register call.
+func (f *fakeCP) registered() identity {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.lastRegistered
+}
+
 func (f *fakeCP) SubmitTaskState(st bus.TaskState) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -72,6 +83,12 @@ func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]
 	if f.pollCalls < len(f.stopReplies) {
 		stops = f.stopReplies[f.pollCalls]
 	}
+	// stopPending is the durable inbox the gateway keeps: an acked directive is
+	// deleted, and everything still in it is redelivered on every poll.
+	f.stopPending = slices.DeleteFunc(f.stopPending, func(sd bus.StopDirective) bool {
+		return slices.Contains(ackStops, sd.TaskID)
+	})
+	stops = slices.Concat(stops, f.stopPending)
 	f.pollCalls++
 	return out, stops, nil
 }
@@ -142,7 +159,7 @@ func testIdentityWithGPUs(uuids ...string) identity {
 
 func TestRegistrar_Register(t *testing.T) {
 	cp := &fakeCP{}
-	if err := newRegistrar(cp, testIdentity()).Register(); err != nil {
+	if err := newRegistrar(cp, testIdentity(), nil).Register(); err != nil {
 		t.Fatalf("Register: %v", err)
 	}
 	if cp.registerCount() != 1 {
@@ -152,14 +169,14 @@ func TestRegistrar_Register(t *testing.T) {
 
 func TestRegistrar_RegisterError(t *testing.T) {
 	cp := &fakeCP{registerErr: true}
-	if err := newRegistrar(cp, testIdentity()).Register(); err == nil {
+	if err := newRegistrar(cp, testIdentity(), nil).Register(); err == nil {
 		t.Fatal("expected register error")
 	}
 }
 
 func TestHeartbeater_Beat(t *testing.T) {
 	cp := &fakeCP{}
-	if err := newHeartbeater(cp, testIdentity(), time.Second).beat(); err != nil {
+	if err := newHeartbeater(cp, testIdentity(), time.Second, nil).beat(); err != nil {
 		t.Fatalf("beat: %v", err)
 	}
 	if cp.registerCount() != 1 {
@@ -168,7 +185,7 @@ func TestHeartbeater_Beat(t *testing.T) {
 }
 
 func TestHeartbeater_DefaultInterval(t *testing.T) {
-	h := newHeartbeater(&fakeCP{}, testIdentity(), 0)
+	h := newHeartbeater(&fakeCP{}, testIdentity(), 0, nil)
 	if h.interval != defaultHeartbeat {
 		t.Errorf("interval = %v, want default", h.interval)
 	}
@@ -176,7 +193,7 @@ func TestHeartbeater_DefaultInterval(t *testing.T) {
 
 func TestHeartbeater_RunStopsOnContext(t *testing.T) {
 	cp := &fakeCP{}
-	h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond)
+	h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond, nil)
 	ctx, cancel := context.WithCancel(context.Background())
 
 	done := make(chan struct{})
@@ -193,3 +210,48 @@ func TestHeartbeater_RunStopsOnContext(t *testing.T) {
 		t.Error("expected at least one re-register before cancel")
 	}
 }
+
+// TestRegistrar_NoRuntimeAdvertisesNoCapacity is the placement half of the boot
+// bug: an agent with no container runtime kept registering its full CPU and
+// memory, so the scheduler placed tasks it could only fail to start. Capacity is
+// advertised only while a runtime is there to use it.
+func TestRegistrar_NoRuntimeAdvertisesNoCapacity(t *testing.T) {
+	cp := &fakeCP{}
+	ready := false
+	r := newRegistrar(cp, testIdentity(), func() bool { return ready })
+
+	if err := r.Register(); err != nil {
+		t.Fatalf("register: %v", err)
+	}
+	if got := cp.registered().Capacity; got.CPU != 0 || got.MemoryMiB != 0 {
+		t.Fatalf("advertised capacity with no runtime: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
+	}
+
+	ready = true
+	if err := r.Register(); err != nil {
+		t.Fatalf("register: %v", err)
+	}
+	want := testIdentity().Capacity
+	if got := cp.registered().Capacity; got.CPU != want.CPU || got.MemoryMiB != want.MemoryMiB {
+		t.Fatalf("capacity not restored once the runtime resolved: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
+	}
+}
+
+// TestHeartbeater_NoRuntimeAdvertisesNoCapacity is the same rule on the beat,
+// which is how a runtime lost after boot reaches the control plane.
+func TestHeartbeater_NoRuntimeAdvertisesNoCapacity(t *testing.T) {
+	cp := &fakeCP{}
+	h := newHeartbeater(cp, testIdentity(), time.Second, func() bool { return false })
+
+	if err := h.beat(); err != nil {
+		t.Fatalf("beat: %v", err)
+	}
+	if got := cp.registered().Capacity; got.CPU != 0 || got.MemoryMiB != 0 {
+		t.Fatalf("beat advertised capacity with no runtime: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
+	}
+	// The instance still identifies itself, so it stays visible as drained rather
+	// than vanishing from the cluster.
+	if cp.registered().InstanceID != testIdentity().InstanceID {
+		t.Error("beat dropped the instance identity")
+	}
+}

cmd/ecs-agent/runtime/containerd.go

@@ -3,6 +3,7 @@ package runtime
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"
 
 	containerd "github.com/containerd/containerd/v2/client"
@@ -22,6 +23,11 @@ const servingProbeTimeout = 3 * time.Second
 // containerdPuller drives a real containerd daemon over its unix socket.
 type containerdPuller struct {
 	client *containerd.Client
+
+	// stdinDone holds one channel per interactive container, closed when that
+	// container is stopped or removed so its stdin reader returns EOF.
+	mu        sync.Mutex
+	stdinDone map[string]chan struct{}
 }
 
 var _ ImagePuller = (*containerdPuller)(nil)

cmd/ecs-agent/runtime/containerd_run.go

@@ -3,9 +3,14 @@ package runtime
 import (
 	"context"
 	"fmt"
+	"io"
 	"sort"
+	"strings"
+	"syscall"
+	"time"
 
 	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/core/containers"
 	"github.com/containerd/containerd/v2/pkg/cdi"
 	"github.com/containerd/containerd/v2/pkg/cio"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
@@ -41,6 +46,106 @@ func cdiSpecOpts(gpuIDs []string) []oci.SpecOpts {
 	return []oci.SpecOpts{cdi.WithCDIDevices(devices...)}
 }
 
+// ociCapName renders an ECS Linux capability (Docker's bare name, e.g.
+// "SYS_ADMIN") as the OCI runtime-spec name ("CAP_SYS_ADMIN") capabilities.Add/
+// Drop expect.
+func ociCapName(name string) string {
+	if strings.HasPrefix(name, "CAP_") {
+		return name
+	}
+	return "CAP_" + name
+}
+
+// ociCapNames maps a list of ECS capability names to their OCI form.
+func ociCapNames(names []string) []string {
+	if len(names) == 0 {
+		return nil
+	}
+	out := make([]string, len(names))
+	for i, n := range names {
+		out[i] = ociCapName(n)
+	}
+	return out
+}
+
+// withSysctls sets Linux.Sysctl on the OCI spec from systemControls namespace/
+// value pairs. containerd has no built-in SpecOpts for sysctls.
+func withSysctls(ctls []SystemControl) oci.SpecOpts {
+	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
+		if len(ctls) == 0 {
+			return nil
+		}
+		if s.Linux == nil {
+			s.Linux = &specs.Linux{}
+		}
+		if s.Linux.Sysctl == nil {
+			s.Linux.Sysctl = map[string]string{}
+		}
+		for _, c := range ctls {
+			s.Linux.Sysctl[c.Namespace] = c.Value
+		}
+		return nil
+	}
+}
+
+// heldStdin backs an interactive container's stdin: reads block instead of
+// hitting an immediate EOF, which is the "stdin held open" behaviour
+// interactive=true asks for, until done closes and the read reports EOF.
+type heldStdin struct{ done <-chan struct{} }
+
+func (h heldStdin) Read([]byte) (int, error) {
+	<-h.done
+	return 0, io.EOF
+}
+
+// ioStreamOpts describes the task IO for a container. A terminal container needs
+// a terminal FIFO set: oci.WithTTY alone puts Terminal in the OCI spec while the
+// shim creates no console socket, and runc refuses that pair outright. A
+// terminal carries no separate stderr, so that stream stays nil and the shim
+// merges it into the console.
+func ioStreamOpts(stdin io.Reader, terminal bool) []cio.Opt {
+	if terminal {
+		return []cio.Opt{cio.WithStreams(stdin, io.Discard, nil), cio.WithTerminal}
+	}
+	return []cio.Opt{cio.WithStreams(stdin, io.Discard, io.Discard)}
+}
+
+// containerIOCreator returns cio.NullIO for a container needing neither stdin
+// nor a terminal, else a creator carrying whichever it asked for. An interactive
+// container's stdin is held open until releaseStdin; containerd copies stdin on
+// a goroutine that exits only once the read returns, so a reader that never
+// returned would park it for the life of the agent. A terminal container that is
+// not interactive gets no stdin stream at all, which is what a nil reader means.
+func (p *containerdPuller) containerIOCreator(containerID string, interactive, terminal bool) cio.Creator {
+	if !interactive && !terminal {
+		return cio.NullIO
+	}
+	var stdin io.Reader
+	if interactive {
+		done := make(chan struct{})
+		p.mu.Lock()
+		if p.stdinDone == nil {
+			p.stdinDone = make(map[string]chan struct{})
+		}
+		p.stdinDone[containerID] = done
+		p.mu.Unlock()
+		stdin = heldStdin{done: done}
+	}
+	return cio.NewCreator(ioStreamOpts(stdin, terminal)...)
+}
+
+// releaseStdin closes a held stdin so containerd's copier goroutine exits. Safe
+// for a container that never held one, and safe to call more than once.
+func (p *containerdPuller) releaseStdin(containerID string) {
+	p.mu.Lock()
+	done, ok := p.stdinDone[containerID]
+	delete(p.stdinDone, containerID)
+	p.mu.Unlock()
+	if ok {
+		close(done)
+	}
+}
+
 var _ Runner = (*containerdPuller)(nil)
 
 // Run creates a container from an already-pulled image and starts its task. The
@@ -81,6 +186,26 @@ func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (st
 		specOpts = append(specOpts, oci.WithEnv(envSlice(spec.Env)))
 	}
 	specOpts = append(specOpts, cdiSpecOpts(spec.GPUIDs)...)
+	if spec.User != "" {
+		specOpts = append(specOpts, oci.WithUser(spec.User))
+	}
+	if spec.ReadonlyRootFilesystem != nil && *spec.ReadonlyRootFilesystem {
+		specOpts = append(specOpts, oci.WithRootFSReadonly())
+	}
+	if spec.Privileged != nil && *spec.Privileged {
+		specOpts = append(specOpts, oci.WithPrivileged)
+	}
+	terminal := spec.PseudoTerminal != nil && *spec.PseudoTerminal
+	if terminal {
+		specOpts = append(specOpts, oci.WithTTY)
+	}
+	specOpts = append(specOpts, withSysctls(spec.SystemControls))
+	if len(spec.CapAdd) > 0 {
+		specOpts = append(specOpts, oci.WithAddedCapabilities(ociCapNames(spec.CapAdd)))
+	}
+	if len(spec.CapDrop) > 0 {
+		specOpts = append(specOpts, oci.WithDroppedCapabilities(ociCapNames(spec.CapDrop)))
+	}
 
 	container, err := p.client.NewContainer(ctx, id,
 		containerd.WithNewSnapshot(id+"-snapshot", image),
@@ -91,12 +216,15 @@ func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (st
 		return "", fmt.Errorf("create container %s: %w", id, err)
 	}
 
-	task, err := container.NewTask(ctx, cio.NullIO)
+	interactive := spec.Interactive != nil && *spec.Interactive
+	task, err := container.NewTask(ctx, p.containerIOCreator(id, interactive, terminal))
 	if err != nil {
+		p.releaseStdin(id)
 		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
 		return "", fmt.Errorf("create task %s: %w", id, err)
 	}
 	if err := task.Start(ctx); err != nil {
+		p.releaseStdin(id)
 		_, _ = task.Delete(ctx)
 		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
 		return "", fmt.Errorf("start task %s: %w", id, err)
@@ -148,9 +276,35 @@ func (p *containerdPuller) List(ctx context.Context) ([]Container, error) {
 	return out, nil
 }
 
+// Stop enforces a container's stopTimeout: SIGTERM the task, wait up to
+// timeout for it to exit on its own, then fall through to Remove's forced
+// kill+delete regardless of whether it exited in time. A container with no
+// live task (already gone) or that fails to accept the signal goes straight to
+// Remove, matching Remove's own already-gone tolerance.
+func (p *containerdPuller) Stop(ctx context.Context, containerID string, timeout time.Duration) error {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	container, err := p.client.LoadContainer(ctx, containerID)
+	if err != nil {
+		return nil //nolint:nilerr // load failure means the container is already gone
+	}
+	task, terr := container.Task(ctx, nil)
+	if terr != nil {
+		return p.Remove(ctx, containerID)
+	}
+	statusC, werr := task.Wait(ctx)
+	if werr == nil && task.Kill(ctx, syscall.SIGTERM) == nil {
+		select {
+		case <-statusC:
+		case <-time.After(timeout):
+		}
+	}
+	return p.Remove(ctx, containerID)
+}
+
 // Remove kills and deletes the container's task, then the container + snapshot.
 func (p *containerdPuller) Remove(ctx context.Context, containerID string) error {
 	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	p.releaseStdin(containerID)
 	container, err := p.client.LoadContainer(ctx, containerID)
 	if err != nil {
 		return nil //nolint:nilerr // load failure means the container is already gone

cmd/ecs-agent/runtime/containerd_run_test.go

@@ -1,8 +1,15 @@
 package runtime
 
 import (
+	"context"
+	"errors"
+	"io"
 	"reflect"
 	"testing"
+	"time"
+
+	"github.com/containerd/containerd/v2/pkg/cio"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
 // cdiDeviceNames maps a container's pinned GPU UUIDs to CDI device names
@@ -45,3 +52,166 @@ func TestCDISpecOpts_Wiring(t *testing.T) {
 		t.Error("GPU container: spec opt is nil")
 	}
 }
+
+// TestOCICapNames verifies ECS's bare capability names (Docker convention,
+// e.g. "SYS_PTRACE") are prefixed to the OCI runtime-spec form the process
+// capability sets expect, and a name already carrying the prefix passes
+// through unchanged.
+func TestOCICapNames(t *testing.T) {
+	cases := []struct {
+		name string
+		in   []string
+		want []string
+	}{
+		{name: "empty", in: nil, want: nil},
+		{name: "bare", in: []string{"SYS_PTRACE"}, want: []string{"CAP_SYS_PTRACE"}},
+		{name: "already prefixed", in: []string{"CAP_NET_ADMIN"}, want: []string{"CAP_NET_ADMIN"}},
+		{name: "mixed", in: []string{"SYS_PTRACE", "CAP_NET_ADMIN"}, want: []string{"CAP_SYS_PTRACE", "CAP_NET_ADMIN"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := ociCapNames(tc.in)
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("ociCapNames(%v) = %v, want %v", tc.in, got, tc.want)
+			}
+		})
+	}
+}
+
+// TestWithSysctls verifies systemControls namespace/value pairs land on
+// Linux.Sysctl, and an empty list leaves the spec's Linux section untouched
+// rather than allocating an empty map (the "requests nothing" case).
+func TestWithSysctls(t *testing.T) {
+	s := &specs.Spec{}
+	if err := withSysctls(nil)(context.Background(), nil, nil, s); err != nil {
+		t.Fatalf("empty ctls: %v", err)
+	}
+	if s.Linux != nil {
+		t.Errorf("empty ctls: want no Linux section, got %+v", s.Linux)
+	}
+
+	ctls := []SystemControl{{Namespace: "net.core.somaxconn", Value: "1024"}}
+	if err := withSysctls(ctls)(context.Background(), nil, nil, s); err != nil {
+		t.Fatalf("with ctls: %v", err)
+	}
+	if s.Linux == nil || s.Linux.Sysctl["net.core.somaxconn"] != "1024" {
+		t.Errorf("want sysctl net.core.somaxconn=1024, got %+v", s.Linux)
+	}
+}
+
+// An interactive container's stdin read blocks until the container is released,
+// then reports EOF so containerd's copier goroutine exits instead of parking
+// for the life of the agent.
+func TestHeldStdin_ReadsEOFOnceReleased(t *testing.T) {
+	p := &containerdPuller{}
+	p.containerIOCreator("c1", true, false)
+
+	p.mu.Lock()
+	done := p.stdinDone["c1"]
+	p.mu.Unlock()
+	if done == nil {
+		t.Fatal("interactive container registered no stdin channel")
+	}
+
+	read := make(chan error, 1)
+	go func() {
+		_, err := heldStdin{done: done}.Read(make([]byte, 1))
+		read <- err
+	}()
+
+	select {
+	case err := <-read:
+		t.Fatalf("read returned before release: %v", err)
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	p.releaseStdin("c1")
+
+	select {
+	case err := <-read:
+		if !errors.Is(err, io.EOF) {
+			t.Fatalf("want io.EOF after release, got %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("read did not return after release")
+	}
+}
+
+// A non-interactive container uses cio.NullIO and registers nothing to release.
+func TestContainerIOCreator_NonInteractiveRegistersNothing(t *testing.T) {
+	p := &containerdPuller{}
+	p.containerIOCreator("c1", false, false)
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if len(p.stdinDone) != 0 {
+		t.Fatalf("want no registered stdin channels, got %d", len(p.stdinDone))
+	}
+}
+
+// applyIOOpts resolves the cio options onto a Streams the way cio.NewCreator
+// does, which is the only way to observe what the creator was built with.
+func applyIOOpts(opts []cio.Opt) cio.Streams {
+	var s cio.Streams
+	for _, o := range opts {
+		o(&s)
+	}
+	return s
+}
+
+// A terminal container needs the terminal flag set on its FIFO set, else the
+// shim creates no console socket and runc refuses to start the container at all.
+// The terminal carries no separate stderr stream.
+func TestIOStreamOpts_TerminalRequestsAConsole(t *testing.T) {
+	s := applyIOOpts(ioStreamOpts(nil, true))
+	if !s.Terminal {
+		t.Error("terminal container: want Terminal set")
+	}
+	if s.Stderr != nil {
+		t.Errorf("terminal container: want no stderr stream, got %#v", s.Stderr)
+	}
+	if s.Stdout == nil {
+		t.Error("terminal container: want a stdout stream")
+	}
+}
+
+// A container with stdin but no terminal keeps both output streams and asks for
+// no console.
+func TestIOStreamOpts_NonTerminalKeepsStderr(t *testing.T) {
+	s := applyIOOpts(ioStreamOpts(heldStdin{}, false))
+	if s.Terminal {
+		t.Error("non-terminal container: want Terminal unset")
+	}
+	if s.Stderr == nil {
+		t.Error("non-terminal container: want a stderr stream")
+	}
+	if s.Stdin == nil {
+		t.Error("non-terminal interactive container: want a stdin stream")
+	}
+}
+
+// A terminal container that is not interactive still gets a creator rather than
+// cio.NullIO, and registers no stdin to release.
+func TestContainerIOCreator_TerminalWithoutInteractive(t *testing.T) {
+	p := &containerdPuller{}
+	if c := p.containerIOCreator("c1", false, true); c == nil {
+		t.Fatal("terminal container: want a creator")
+	}
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if len(p.stdinDone) != 0 {
+		t.Fatalf("want no registered stdin channels, got %d", len(p.stdinDone))
+	}
+}
+
+// Remove calls releaseStdin unconditionally and Stop falls through to Remove,
+// so an unknown container and a second release must both be no-ops.
+func TestReleaseStdin_UnknownAndRepeatedAreSafe(t *testing.T) {
+	p := &containerdPuller{}
+	p.releaseStdin("never-registered")
+
+	p.containerIOCreator("c1", true, false)
+	p.releaseStdin("c1")
+	p.releaseStdin("c1")
+}

cmd/ecs-agent/runtime/runtime.go

@@ -4,7 +4,10 @@
 // scheduler-only build run against a fake.
 package runtime
 
-import "context"
+import (
+	"context"
+	"time"
+)
 
 // PullSpec describes one image to pull.
 type PullSpec struct {
@@ -35,6 +38,12 @@ type ImagePuller interface {
 	Close() error
 }
 
+// SystemControl is a sysctl namespace/value pair applied to the OCI spec.
+type SystemControl struct {
+	Namespace string
+	Value     string
+}
+
 // RunSpec describes a single container the agent must create and start. v1 runs
 // the container in the host network namespace (bridge/CNI task networking lands
 // in a later sprint); Labels carry the mulga.ecs.* task identity for the reboot
@@ -54,6 +63,22 @@ type RunSpec struct {
 	// per GPU requested. Empty for non-GPU containers or if pinning fell short.
 	// The runner injects these as CDI devices into the OCI spec.
 	GPUIDs []string
+	// User is the OCI process user (oci.WithUser). Empty runs as the image
+	// default.
+	User string
+	// ReadonlyRootFilesystem, Privileged, PseudoTerminal and Interactive are
+	// pointers so the runner can tell "not requested" (nil) from an explicit
+	// false, which still needs echoing but applies nothing.
+	ReadonlyRootFilesystem *bool
+	Privileged             *bool
+	PseudoTerminal         *bool
+	Interactive            *bool
+	// SystemControls are sysctls set on the OCI spec's Linux.Sysctl map.
+	SystemControls []SystemControl
+	// CapAdd / CapDrop are Linux capabilities added/dropped from the OCI
+	// spec's process capability sets (linuxParameters.capabilities).
+	CapAdd  []string
+	CapDrop []string
 }
 
 // RunStatus is a finished container's outcome.
@@ -71,12 +96,16 @@ type Container struct {
 }
 
 // Runner creates and starts containers from already-pulled images. id is a
-// caller-unique container ID; Wait blocks until the container exits; Remove
-// tears down the container + its task; List enumerates known containers so the
-// reboot reconciler can re-adopt the ones still running.
+// caller-unique container ID; Wait blocks until the container exits; Stop
+// requests a graceful exit (SIGTERM, then a forced kill+remove once timeout
+// elapses) — the agent's enforcement of a container's stopTimeout; Remove
+// tears down the container + its task immediately, with no grace period; List
+// enumerates known containers so the reboot reconciler can re-adopt the ones
+// still running.
 type Runner interface {
 	Run(ctx context.Context, id string, spec RunSpec) (containerID string, err error)
 	Wait(ctx context.Context, containerID string) (RunStatus, error)
+	Stop(ctx context.Context, containerID string, timeout time.Duration) error
 	Remove(ctx context.Context, containerID string) error
 	List(ctx context.Context) ([]Container, error)
 }

cmd/ecs-agent/runtime/runtime_fake.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 )
 
 // FakePuller is an in-memory ImagePuller for tests and the scheduler-only build.
@@ -26,11 +27,21 @@ type FakePuller struct {
 	Removed  []string
 	OnRun    func(string, RunSpec) (string, error)
 
-	// List bookkeeping: Containers is replayed by List; ListErr forces a failure;
-	// Listed records that List was called.
+	// Stop bookkeeping: StopTimeouts pairs the container ID with the timeout it
+	// was stopped with, in call order, so a test can assert both the target and
+	// the enforced stopTimeout without a fake containerd.
+	Stopped      []string
+	StopTimeouts []time.Duration
+	StopErr      error
+
+	// List bookkeeping: Containers is replayed by List; ListErr forces a failure,
+	// limited to the first ListErrFor calls when that is non-zero, so a test can
+	// model a runtime that recovers. Listed and ListCalls record the calls.
 	Containers []Container
 	ListErr    error
+	ListErrFor int
 	Listed     bool
+	ListCalls  int
 }
 
 var (
@@ -98,6 +109,15 @@ func (f *FakePuller) Wait(_ context.Context, containerID string) (RunStatus, err
 	return RunStatus{ExitCode: f.WaitCode}, f.WaitErr
 }
 
+// Stop records the container ID and the timeout it was stopped with.
+func (f *FakePuller) Stop(_ context.Context, containerID string, timeout time.Duration) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Stopped = append(f.Stopped, containerID)
+	f.StopTimeouts = append(f.StopTimeouts, timeout)
+	return f.StopErr
+}
+
 // Remove records the container ID.
 func (f *FakePuller) Remove(_ context.Context, containerID string) error {
 	f.mu.Lock()
@@ -113,13 +133,36 @@ func (f *FakePuller) Waits() []string {
 	return append([]string(nil), f.Waited...)
 }
 
+// RunCalls returns a copy of the specs passed to Run, for assertions made while
+// the agent's goroutines may still be live.
+func (f *FakePuller) RunCalls() []RunSpec {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]RunSpec(nil), f.Runs...)
+}
+
+// Reaped returns a copy of the container IDs passed to Stop.
+func (f *FakePuller) Reaped() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.Stopped...)
+}
+
 // List replays the programmed container set (or error).
 func (f *FakePuller) List(_ context.Context) ([]Container, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	f.Listed = true
-	if f.ListErr != nil {
+	f.ListCalls++
+	if f.ListErr != nil && (f.ListErrFor == 0 || f.ListCalls <= f.ListErrFor) {
 		return nil, f.ListErr
 	}
 	return f.Containers, nil
 }
+
+// Lists reports how many times List was called.
+func (f *FakePuller) Lists() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.ListCalls
+}

cmd/ecs-agent/runtimegate.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"sync"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+)
+
+// runtimeProbeTimeout bounds the liveness probe against a runtime the gate is
+// already holding, so a wedged socket cannot stall the watch loop.
+const runtimeProbeTimeout = 5 * time.Second
+
+// defaultRuntimeRetry is how often the gate dials a runtime it does not have,
+// and probes one it does. The node starts containerd and the agent back to
+// back, so the first few dials are expected to fail.
+const defaultRuntimeRetry = 10 * time.Second
+
+// runtimeGate holds the container runtime the agent has resolved, or nothing
+// when the socket has not answered. The runtime is resolved by a loop rather
+// than once at boot: the node's service manager reports the container daemon
+// started when the process spawned, not when its socket began accepting, so a
+// dial that fails in the first second of boot is a transient condition.
+//
+// Every read of the runtime goes through the gate, so a runtime that arrives
+// after boot is picked up on the next task without restarting the agent.
+type runtimeGate struct {
+	mu sync.RWMutex
+	rt ctrruntime.Runtime
+}
+
+// newRuntimeGate returns a gate holding rt, which may be nil.
+func newRuntimeGate(rt ctrruntime.Runtime) *runtimeGate {
+	return &runtimeGate{rt: rt}
+}
+
+// get returns the current runtime, or nil when none is resolved.
+func (g *runtimeGate) get() ctrruntime.Runtime {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+	return g.rt
+}
+
+// available reports whether a runtime is resolved. It is what registration
+// consults to decide whether the host has any capacity to advertise.
+func (g *runtimeGate) available() bool { return g.get() != nil }
+
+// set replaces the held runtime, closing the one it displaces.
+func (g *runtimeGate) set(rt ctrruntime.Runtime) {
+	g.mu.Lock()
+	old := g.rt
+	g.rt = rt
+	g.mu.Unlock()
+	if old != nil && old != rt {
+		_ = old.Close()
+	}
+}
+
+// clear drops and closes the held runtime, returning whether it held one.
+func (g *runtimeGate) clear() bool {
+	g.mu.Lock()
+	old := g.rt
+	g.rt = nil
+	g.mu.Unlock()
+	if old == nil {
+		return false
+	}
+	_ = old.Close()
+	return true
+}
+
+// watch keeps the gate current until ctx is cancelled: it dials while the gate
+// is empty, and probes while it is full, clearing the gate when the probe fails
+// so the next tick re-dials. A containerd stopped and restarted underneath a
+// running agent is therefore picked up the same way a slow boot is.
+func (g *runtimeGate) watch(ctx context.Context, dial func() (ctrruntime.Runtime, error), interval time.Duration) {
+	if interval <= 0 {
+		interval = defaultRuntimeRetry
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			g.step(ctx, dial)
+		}
+	}
+}
+
+// step performs one pass of the watch loop: dial when empty, probe when full.
+func (g *runtimeGate) step(ctx context.Context, dial func() (ctrruntime.Runtime, error)) {
+	if rt := g.get(); rt != nil {
+		probeCtx, cancel := context.WithTimeout(ctx, runtimeProbeTimeout)
+		// A container list is what reconcile already asks the runtime at boot and
+		// the cheapest thing it answers, so it doubles as the liveness probe.
+		_, err := rt.List(probeCtx)
+		cancel()
+		if err != nil && ctx.Err() == nil {
+			slog.Warn("ecs-agent: container runtime stopped answering, will re-dial", "err", err)
+			g.clear()
+		}
+		return
+	}
+
+	rt, err := dial()
+	if err != nil {
+		slog.Debug("ecs-agent: container runtime still unavailable", "err", err)
+		return
+	}
+	slog.Info("ecs-agent: container runtime available")
+	g.set(rt)
+}

cmd/ecs-agent/runtimegate_test.go

@@ -0,0 +1,124 @@
+//test:in-package — package main cannot be imported by an external test package.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+)
+
+// TestRuntimeGate_DialsUntilTheSocketAnswers is the boot case the agent used to
+// lose: the node starts containerd and the agent together, so the first dials
+// fail. Resolving once left the agent runtime-less for the life of the process.
+func TestRuntimeGate_DialsUntilTheSocketAnswers(t *testing.T) {
+	g := newRuntimeGate(nil)
+	if g.available() {
+		t.Fatal("empty gate reported a runtime")
+	}
+
+	rt := &ctrruntime.FakePuller{}
+	dials := 0
+	dial := func() (ctrruntime.Runtime, error) {
+		dials++
+		if dials < 3 {
+			return nil, errors.New("dial containerd: no such file or directory")
+		}
+		return rt, nil
+	}
+
+	for range 2 {
+		g.step(t.Context(), dial)
+		if g.available() {
+			t.Fatalf("gate resolved a runtime after %d failed dials", dials)
+		}
+	}
+
+	g.step(t.Context(), dial)
+	if g.get() != rt {
+		t.Fatalf("gate did not pick up the runtime once the socket answered (dials=%d)", dials)
+	}
+}
+
+// TestRuntimeGate_DropsARuntimeThatStopsAnswering covers a containerd restarted
+// underneath a live agent: the held runtime stops answering, so the gate clears
+// it and the next pass re-dials rather than handing out a dead client forever.
+func TestRuntimeGate_DropsARuntimeThatStopsAnswering(t *testing.T) {
+	dead := &ctrruntime.FakePuller{ListErr: errors.New("connection refused")}
+	fresh := &ctrruntime.FakePuller{}
+	g := newRuntimeGate(dead)
+
+	dial := func() (ctrruntime.Runtime, error) { return fresh, nil }
+
+	g.step(t.Context(), dial)
+	if g.available() {
+		t.Fatal("gate kept a runtime whose probe failed")
+	}
+	if !dead.Closed {
+		t.Error("displaced runtime was not closed")
+	}
+
+	g.step(t.Context(), dial)
+	if g.get() != fresh {
+		t.Fatal("gate did not re-dial after clearing")
+	}
+}
+
+// TestRuntimeGate_KeepsAHealthyRuntime guards the other direction: a probe that
+// succeeds must not churn the runtime, or every pass would drop a working client.
+func TestRuntimeGate_KeepsAHealthyRuntime(t *testing.T) {
+	rt := &ctrruntime.FakePuller{}
+	g := newRuntimeGate(rt)
+
+	dial := func() (ctrruntime.Runtime, error) {
+		t.Error("dialled while holding a healthy runtime")
+		return nil, errors.New("unexpected dial")
+	}
+	g.step(t.Context(), dial)
+
+	if g.get() != rt {
+		t.Fatal("gate dropped a runtime whose probe succeeded")
+	}
+	if rt.Closed {
+		t.Error("healthy runtime was closed")
+	}
+}
+
+// TestRuntimeGate_ClearIsIdempotent keeps Stop safe to call on an agent that
+// never resolved a runtime.
+func TestRuntimeGate_ClearIsIdempotent(t *testing.T) {
+	g := newRuntimeGate(nil)
+	if g.clear() {
+		t.Error("clear reported dropping a runtime the gate never held")
+	}
+
+	rt := &ctrruntime.FakePuller{}
+	g.set(rt)
+	if !g.clear() {
+		t.Error("clear did not report dropping the held runtime")
+	}
+	if !rt.Closed {
+		t.Error("cleared runtime was not closed")
+	}
+	if g.clear() {
+		t.Error("second clear reported dropping a runtime")
+	}
+}
+
+// TestRuntimeGate_WatchStopsWithContext keeps the watch loop from outliving the
+// agent it belongs to.
+func TestRuntimeGate_WatchStopsWithContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(t.Context())
+	g := newRuntimeGate(nil)
+
+	done := make(chan struct{})
+	go func() {
+		g.watch(ctx, func() (ctrruntime.Runtime, error) { return nil, errors.New("no socket") }, 0)
+		close(done)
+	}()
+
+	cancel()
+	<-done
+}

cmd/ecs-agent/taskrun.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"log/slog"
 	"maps"
+	"strconv"
+	"sync"
 	"time"
 
 	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
@@ -12,6 +14,10 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
+// defaultStopTimeout is applied when a container carries no explicit
+// stopTimeout, matching the EC2 launch-type default ECS itself uses.
+const defaultStopTimeout = 30 * time.Second
+
 // pollAssignments drains the instance's assignment inbox on a ticker until ctx is
 // cancelled. Each assign is dispatched to runTask exactly once; the taskIDs seen
 // on a poll are acked on the next poll so the gateway can drop them — a crash
@@ -23,7 +29,7 @@ func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool)
 	ticker := time.NewTicker(a.cfg.PollInterval)
 	defer ticker.Stop()
 
-	stopping := map[string]bool{}
+	reaps := newStopLedger()
 	var ackAssigns, ackStops []string
 	for {
 		select {
@@ -40,17 +46,18 @@ func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool)
 			ackStops = ackStops[:0]
 			for i := range stops {
 				sd := stops[i]
-				if !stopping[sd.TaskID] {
-					stopping[sd.TaskID] = true
+				switch {
+				case reaps.done(sd.TaskID):
+					ackStops = append(ackStops, sd.TaskID)
+				case reaps.begin(sd.TaskID):
 					slog.Info("ecs-agent: task stop requested", "task", sd.TaskID, "reason", sd.Reason)
-					go a.stopTask(ctx, sd)
+					go func() { reaps.end(sd.TaskID, a.stopTask(ctx, sd)) }()
 				}
-				ackStops = append(ackStops, sd.TaskID)
 			}
 			ackAssigns = ackAssigns[:0]
 			for i := range assigns {
 				as := assigns[i]
-				if !dispatched[as.TaskID] && !stopping[as.TaskID] {
+				if !dispatched[as.TaskID] && !reaps.seen(as.TaskID) {
 					dispatched[as.TaskID] = true
 					slog.Info("ecs-agent: task assigned", "task", as.TaskID, "containers", len(as.Containers))
 					go a.runTask(ctx, &as)
@@ -61,27 +68,83 @@ func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool)
 	}
 }
 
-// stopTask reaps a task's containers on a control-plane stop directive: it lists
-// the runtime's containers, removes the ones labeled with this task (kill +
-// delete, releasing host ports/netns), then reports the task STOPPED with the
-// directive's reason. Idempotent — a task with no live containers still reports
-// STOPPED so the scheduler releases its capacity.
-func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) {
-	if a.runner == nil {
-		return
+// stopLedger tracks reaps in flight and reaps that reported STOPPED. The two
+// are separate because only the second may be acked: acking the first deletes a
+// directive that was never performed, and nothing re-posts it.
+type stopLedger struct {
+	mu       sync.Mutex
+	inflight map[string]bool
+	stopped  map[string]bool
+}
+
+func newStopLedger() *stopLedger {
+	return &stopLedger{inflight: map[string]bool{}, stopped: map[string]bool{}}
+}
+
+// begin claims a task for a reap, reporting false when one is already running.
+func (l *stopLedger) begin(taskID string) bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.inflight[taskID] {
+		return false
 	}
-	containers, err := a.runner.List(ctx)
+	l.inflight[taskID] = true
+	return true
+}
+
+// end releases the claim, recording the task as stopped only if the reap got
+// that far. A failed reap leaves nothing behind, so the next poll redelivers
+// the directive and dispatches it again.
+func (l *stopLedger) end(taskID string, reported bool) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	delete(l.inflight, taskID)
+	if reported {
+		l.stopped[taskID] = true
+	}
+}
+
+// done reports whether the task's reap reported STOPPED, which is the only
+// condition under which its directive may be acked.
+func (l *stopLedger) done(taskID string) bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.stopped[taskID]
+}
+
+// seen reports whether a stop for the task is in flight or complete, so an
+// assignment arriving alongside its own stop is never started.
+func (l *stopLedger) seen(taskID string) bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.inflight[taskID] || l.stopped[taskID]
+}
+
+// stopTask reaps a task's containers on a control-plane stop directive: it
+// lists the runtime's containers, gracefully stops (SIGTERM, wait up to the
+// container's stopTimeout, then force kill + delete) the ones labeled with
+// this task, then reports the task STOPPED with the directive's reason.
+// Idempotent — a task with no live containers still reports STOPPED so the
+// scheduler releases its capacity. Reports whether it got as far as saying
+// STOPPED: a caller may only ack the directive when it did.
+func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) bool {
+	runner := a.rt.get()
+	if runner == nil {
+		slog.Warn("ecs-agent: stop deferred, no container runtime", "task", sd.TaskID)
+		return false
+	}
+	containers, err := runner.List(ctx)
 	if err != nil {
 		slog.Warn("ecs-agent: stop list failed", "task", sd.TaskID, "err", err)
-		return
+		return false
 	}
 	statuses := make([]bus.ContainerStatus, 0)
 	for _, c := range containers {
 		if c.Labels[labelTaskID] != sd.TaskID {
 			continue
 		}
-		if rerr := a.runner.Remove(ctx, c.ID); rerr != nil {
-			slog.Warn("ecs-agent: stop remove failed", "task", sd.TaskID, "container", c.ID, "err", rerr)
+		if rerr := runner.Stop(ctx, c.ID, containerStopTimeout(c.Labels)); rerr != nil {
+			slog.Warn("ecs-agent: stop failed", "task", sd.TaskID, "container", c.ID, "err", rerr)
 		}
 		statuses = append(statuses, bus.ContainerStatus{
 			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
@@ -93,13 +156,15 @@ func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) {
 	}
 	a.reportTaskState(&bus.Assign{TaskID: sd.TaskID}, bus.TaskStatusStopped, reason, statuses)
 	slog.Info("ecs-agent: task reaped", "task", sd.TaskID, "containers", len(statuses))
+	return true
 }
 
 // runTask pulls each container image, starts the containers, and reports task
 // state on the bus. A missing runtime or any per-container failure reports the
 // task STOPPED with a reason; success reports RUNNING and waits for exit.
 func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
-	if a.puller == nil || a.runner == nil {
+	rt := a.rt.get()
+	if rt == nil {
 		a.reportTaskState(as, bus.TaskStatusStopped, "containerd unavailable on agent", nil)
 		return
 	}
@@ -117,7 +182,12 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 
 	statuses := make([]bus.ContainerStatus, 0, len(as.Containers))
 	for _, c := range as.Containers {
-		if _, err := a.puller.Pull(ctx, ctrruntime.PullSpec{Ref: c.Image}, resolver); err != nil {
+		// startTimeout bounds pull-through-start as a unit: real ECS measures a
+		// container's time-to-RUNNING from the same starting point.
+		runCtx, cancel := containerStartCtx(ctx, c.StartTimeout)
+
+		if _, err := rt.Pull(runCtx, ctrruntime.PullSpec{Ref: c.Image}, resolver); err != nil {
+			cancel()
 			slog.Error("ecs-agent: pull failed", "task", as.TaskID, "image", c.Image, "err", err)
 			a.teardownTaskNetns(as)
 			a.reportTaskState(as, bus.TaskStatusStopped, "image pull failed: "+err.Error(), statuses)
@@ -126,16 +196,29 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 
 		cid := containerID(as.TaskID, c.Name)
 		gpuIDs := a.pinContainerGPUs(as.TaskID, c.Name, c.GPU)
-		spec := ctrruntime.RunSpec{
-			Image:     c.Image,
-			Command:   c.Command,
-			Env:       withCredEnv(c.Environment, credID),
-			Labels:    taskLabels(as, c.Name),
-			NetnsPath: netnsPath,
-			GPU:       c.GPU,
-			GPUIDs:    gpuIDs,
+		labels := taskLabels(as, c.Name)
+		if c.StopTimeout != nil {
+			labels[labelStopTimeout] = strconv.FormatInt(*c.StopTimeout, 10)
 		}
-		id, err := a.runner.Run(ctx, cid, spec)
+		spec := ctrruntime.RunSpec{
+			Image:                  c.Image,
+			Command:                c.Command,
+			Env:                    withCredEnv(c.Environment, credID),
+			Labels:                 labels,
+			NetnsPath:              netnsPath,
+			GPU:                    c.GPU,
+			GPUIDs:                 gpuIDs,
+			User:                   c.User,
+			ReadonlyRootFilesystem: c.ReadonlyRootFilesystem,
+			Privileged:             c.Privileged,
+			PseudoTerminal:         c.PseudoTerminal,
+			Interactive:            c.Interactive,
+			SystemControls:         toRuntimeSystemControls(c.SystemControls),
+			CapAdd:                 c.CapAdd,
+			CapDrop:                c.CapDrop,
+		}
+		id, err := rt.Run(runCtx, cid, spec)
+		cancel()
 		if err != nil {
 			slog.Error("ecs-agent: run failed", "task", as.TaskID, "container", c.Name, "err", err)
 			a.teardownTaskNetns(as)
@@ -155,7 +238,12 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 // waitContainer blocks until a container exits, then reports the task STOPPED
 // with the exit code. v1 stops the whole task when its first container exits.
 func (a *Agent) waitContainer(ctx context.Context, as *bus.Assign, name, containerID string) {
-	status, err := a.runner.Wait(ctx, containerID)
+	runner := a.rt.get()
+	if runner == nil {
+		slog.Warn("ecs-agent: cannot wait on container, no container runtime", "task", as.TaskID, "container", name)
+		return
+	}
+	status, err := runner.Wait(ctx, containerID)
 	if err != nil {
 		slog.Warn("ecs-agent: wait container failed", "task", as.TaskID, "container", name, "err", err)
 		return
@@ -240,6 +328,44 @@ func withCredEnv(env map[string]string, credID string) map[string]string {
 	return out
 }
 
+// containerStartCtx bounds pull+run by a container's startTimeout, if set. A
+// nil startTimeout (the common case) returns ctx unchanged with a no-op
+// cancel, matching today's unbounded behaviour.
+func containerStartCtx(ctx context.Context, startTimeout *int64) (context.Context, context.CancelFunc) {
+	if startTimeout == nil {
+		return ctx, func() {}
+	}
+	return context.WithTimeout(ctx, time.Duration(*startTimeout)*time.Second)
+}
+
+// containerStopTimeout reads the stopTimeout stamped on a container's labels
+// by runTask, falling back to defaultStopTimeout when absent or unparseable.
+func containerStopTimeout(labels map[string]string) time.Duration {
+	v, ok := labels[labelStopTimeout]
+	if !ok {
+		return defaultStopTimeout
+	}
+	secs, err := strconv.ParseInt(v, 10, 64)
+	if err != nil {
+		return defaultStopTimeout
+	}
+	return time.Duration(secs) * time.Second
+}
+
+// toRuntimeSystemControls maps the assign's sysctl pairs to the runtime
+// package's own type, keeping cmd/ecs-agent/runtime decoupled from the ecs
+// handlers' bus package.
+func toRuntimeSystemControls(in []bus.SystemControl) []ctrruntime.SystemControl {
+	if len(in) == 0 {
+		return nil
+	}
+	out := make([]ctrruntime.SystemControl, len(in))
+	for i, sc := range in {
+		out[i] = ctrruntime.SystemControl{Namespace: sc.Namespace, Value: sc.Value}
+	}
+	return out
+}
+
 // pinContainerGPUs reserves n device UUIDs for a container from the local
 // ledger, before the container is started so the runner can inject them as
 // CDI devices. A short ledger (fewer free devices than requested, e.g.
@@ -317,6 +443,9 @@ const (
 	labelCredID        = "mulga.ecs.credID"
 	labelTaskRoleARN   = "mulga.ecs.taskRoleArn"
 	labelENIMac        = "mulga.ecs.eniMac"
+	// labelStopTimeout carries a container's stopTimeout (seconds) so stopTask
+	// can read it back from List's labels; it has no assign to consult there.
+	labelStopTimeout = "mulga.ecs.stopTimeout"
 )
 
 // taskLabels are the mulga.ecs.* labels stamped on a container. The cred/role/MAC

cmd/ecs-agent/taskrun_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"slices"
 	"testing"
 	"time"
 
@@ -24,13 +25,13 @@ func testAssign() *bus.Assign {
 }
 
 func newRunAgent(cp controlPlane, rt *ctrruntime.FakePuller) *Agent {
-	return newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	return newAgent(config{}, testIdentity(), cp, rt, nil)
 }
 
 // runTask with no runtime reports the task STOPPED instead of crashing.
 func TestRunTask_NoRuntimeReportsStopped(t *testing.T) {
 	cp := &fakeCP{}
-	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
+	a := newAgent(config{}, testIdentity(), cp, nil, nil)
 	a.runTask(context.Background(), testAssign())
 
 	st := cp.taskStates()
@@ -70,6 +71,157 @@ func TestRunTask_PullRunReportsRunning(t *testing.T) {
 	}
 }
 
+// Each enforced field maps onto RunSpec exactly, asserted through the fake
+// runtime — the mapping this change exists to wire, without a real containerd.
+func TestRunTask_RuntimeFieldsCarriedToRunSpec(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	ro := true
+	priv := false
+	tty := true
+	interactive := true
+	as.Containers[0].User = "1000:1000"
+	as.Containers[0].ReadonlyRootFilesystem = &ro
+	as.Containers[0].Privileged = &priv
+	as.Containers[0].PseudoTerminal = &tty
+	as.Containers[0].Interactive = &interactive
+	as.Containers[0].SystemControls = []bus.SystemControl{{Namespace: "net.core.somaxconn", Value: "1024"}}
+	as.Containers[0].CapAdd = []string{"SYS_PTRACE"}
+	as.Containers[0].CapDrop = []string{"NET_RAW"}
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 1 {
+		t.Fatalf("expected one run, got %+v", rt.Runs)
+	}
+	got := rt.Runs[0]
+	if got.User != "1000:1000" {
+		t.Errorf("User = %q, want 1000:1000", got.User)
+	}
+	if got.ReadonlyRootFilesystem == nil || !*got.ReadonlyRootFilesystem {
+		t.Errorf("ReadonlyRootFilesystem = %v, want true", got.ReadonlyRootFilesystem)
+	}
+	if got.Privileged == nil || *got.Privileged {
+		t.Errorf("Privileged = %v, want false (present, not applied)", got.Privileged)
+	}
+	if got.PseudoTerminal == nil || !*got.PseudoTerminal {
+		t.Errorf("PseudoTerminal = %v, want true", got.PseudoTerminal)
+	}
+	if got.Interactive == nil || !*got.Interactive {
+		t.Errorf("Interactive = %v, want true", got.Interactive)
+	}
+	if len(got.SystemControls) != 1 || got.SystemControls[0].Namespace != "net.core.somaxconn" {
+		t.Errorf("SystemControls = %+v, want one net.core.somaxconn entry", got.SystemControls)
+	}
+	if len(got.CapAdd) != 1 || got.CapAdd[0] != "SYS_PTRACE" {
+		t.Errorf("CapAdd = %+v, want [SYS_PTRACE]", got.CapAdd)
+	}
+	if len(got.CapDrop) != 1 || got.CapDrop[0] != "NET_RAW" {
+		t.Errorf("CapDrop = %+v, want [NET_RAW]", got.CapDrop)
+	}
+}
+
+// An assign carrying none of the new fields leaves RunSpec's pointer fields
+// nil rather than defaulting to a false indistinguishable from "explicitly
+// not requested".
+func TestRunTask_NilRuntimeFieldsLeaveRunSpecUnset(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	if len(rt.Runs) != 1 {
+		t.Fatalf("expected one run, got %+v", rt.Runs)
+	}
+	got := rt.Runs[0]
+	if got.ReadonlyRootFilesystem != nil || got.Privileged != nil || got.PseudoTerminal != nil || got.Interactive != nil {
+		t.Errorf("want all runtime-field pointers nil, got %+v", got)
+	}
+	if len(got.SystemControls) != 0 || len(got.CapAdd) != 0 || len(got.CapDrop) != 0 {
+		t.Errorf("want no sysctls/capabilities, got %+v", got)
+	}
+}
+
+// TestRunTask_StampsStopTimeoutLabel verifies runTask stamps a container's
+// stopTimeout as a label — the only way stopTask (which only has List's
+// labels, not the original assign) can recover it later.
+func TestRunTask_StampsStopTimeoutLabel(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	stop := int64(45)
+	as.Containers[0].StopTimeout = &stop
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 1 {
+		t.Fatalf("expected one run, got %+v", rt.Runs)
+	}
+	if got := rt.Runs[0].Labels[labelStopTimeout]; got != "45" {
+		t.Errorf("stopTimeout label = %q, want 45", got)
+	}
+}
+
+// TestRunTask_NoStopTimeoutLabelWhenUnset verifies a container with no
+// stopTimeout carries no label, so stopTask falls back to the default rather
+// than reading a stray "0".
+func TestRunTask_NoStopTimeoutLabelWhenUnset(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	if len(rt.Runs) != 1 {
+		t.Fatalf("expected one run, got %+v", rt.Runs)
+	}
+	if _, ok := rt.Runs[0].Labels[labelStopTimeout]; ok {
+		t.Errorf("want no stopTimeout label, got %+v", rt.Runs[0].Labels)
+	}
+}
+
+// TestStopTask_UsesContainerStopTimeout verifies that stopTask reads a
+// container's stopTimeout label and calls the runner's graceful Stop (SIGTERM
+// + bounded wait) with it, instead of the hard kill+delete used before this
+// fix.
+func TestStopTask_UsesContainerStopTimeout(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{{
+		ID: "t-001-web",
+		Labels: map[string]string{
+			labelTaskID: "t-001", labelContainerName: "web", labelStopTimeout: "45",
+		},
+	}}}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
+		t.Fatalf("want t-001-web stopped, got %+v", rt.Stopped)
+	}
+	if len(rt.StopTimeouts) != 1 || rt.StopTimeouts[0] != 45*time.Second {
+		t.Fatalf("want stopTimeout 45s, got %+v", rt.StopTimeouts)
+	}
+}
+
+// TestStopTask_DefaultStopTimeoutWhenUnset verifies a container with no
+// stopTimeout label gets the EC2 launch-type default rather than an unbounded
+// or zero grace period.
+func TestStopTask_DefaultStopTimeoutWhenUnset(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{{
+		ID:     "t-001-web",
+		Labels: map[string]string{labelTaskID: "t-001", labelContainerName: "web"},
+	}}}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(rt.StopTimeouts) != 1 || rt.StopTimeouts[0] != defaultStopTimeout {
+		t.Fatalf("want default stopTimeout %v, got %+v", defaultStopTimeout, rt.StopTimeouts)
+	}
+}
+
 // TestRunTask_GPUCarriedToRunSpec verifies that the AssignContainer's GPU count
 // reaches the runtime RunSpec unchanged; a
 // container with no GPU request keeps RunSpec.GPU at zero (regression).
@@ -100,7 +252,7 @@ func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
 func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
 	cp := &fakeCP{}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, nil)
 	as := testAssign()
 	as.Containers = append(as.Containers, bus.AssignContainer{
 		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
@@ -144,7 +296,7 @@ func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
 func TestRunTask_GPUIDsReachRunSpec(t *testing.T) {
 	cp := &fakeCP{}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, nil)
 	as := testAssign()
 	as.Containers = append(as.Containers, bus.AssignContainer{
 		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
@@ -209,7 +361,7 @@ func TestStopTask_ReleasesGPUsBackToLedger(t *testing.T) {
 			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "trainer",
 		}},
 	}}
-	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, nil)
 	if _, err := a.gpu.Pin(gpuKey("t-001", "trainer"), 1); err != nil {
 		t.Fatalf("seed pin: %v", err)
 	}
@@ -287,7 +439,7 @@ func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
 		nil,   // poll 3: gateway dropped it after the ack
 	}}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go a.pollAssignments(ctx, map[string]bool{})
@@ -342,8 +494,8 @@ func TestStopTask_ReapsLabeledContainersAndReportsStopped(t *testing.T) {
 
 	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
 
-	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
-		t.Fatalf("want only t-001-web removed, got %+v", rt.Removed)
+	if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
+		t.Fatalf("want only t-001-web stopped, got %+v", rt.Stopped)
 	}
 	st := cp.taskStates()
 	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
@@ -381,12 +533,12 @@ func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
 	as := testAssign()
 	cp := &fakeCP{
 		pollReplies: [][]bus.Assign{{*as}, {*as}, nil},
-		stopReplies: [][]bus.StopDirective{{{TaskID: as.TaskID, Reason: "bye"}}},
+		stopPending: []bus.StopDirective{{TaskID: as.TaskID, Reason: "bye"}},
 	}
 	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
 		{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
 	}}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go a.pollAssignments(ctx, map[string]bool{})
@@ -425,10 +577,126 @@ func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
 	}
 }
 
+// countStopAcks reports how many polls carried taskID in their stop-ack list.
+func countStopAcks(acks [][]string, taskID string) int {
+	n := 0
+	for _, ack := range acks {
+		if slices.Contains(ack, taskID) {
+			n++
+		}
+	}
+	return n
+}
+
+// A reap that cannot run is never acked, so the directive stays in the gateway's
+// inbox and is redelivered and retried until the runtime comes back.
+func TestPollAssignments_UnperformableStopIsNotAckedAndRetried(t *testing.T) {
+	cp := &fakeCP{stopPending: []bus.StopDirective{{TaskID: "t-001", Reason: "bye"}}}
+	rt := &ctrruntime.FakePuller{ListErr: errors.New("containerd is down")}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	deadline := time.After(time.Second)
+	for rt.Lists() < 3 {
+		select {
+		case <-deadline:
+			t.Fatalf("failed reap was not retried; List calls=%d", rt.Lists())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	cancel()
+
+	if n := countStopAcks(cp.stopAcks(), "t-001"); n != 0 {
+		t.Errorf("a reap that never ran was acked %d times; stopAcks=%v", n, cp.stopAcks())
+	}
+	if stopped := countStatus(cp.taskStates(), "t-001", bus.TaskStatusStopped); stopped != 0 {
+		t.Errorf("a reap that never ran reported STOPPED %d times", stopped)
+	}
+}
+
+// Once the runtime recovers, the redelivered directive is reaped, reported
+// STOPPED and acked exactly once, which deletes it from the inbox.
+func TestPollAssignments_RecoveredStopIsAckedExactlyOnce(t *testing.T) {
+	cp := &fakeCP{stopPending: []bus.StopDirective{{TaskID: "t-001", Reason: "bye"}}}
+	rt := &ctrruntime.FakePuller{
+		ListErr:    errors.New("containerd is down"),
+		ListErrFor: 1,
+		Containers: []ctrruntime.Container{
+			{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
+		},
+	}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	deadline := time.After(2 * time.Second)
+	for countStopAcks(cp.stopAcks(), "t-001") == 0 {
+		select {
+		case <-deadline:
+			t.Fatalf("recovered reap was never acked; states=%+v stopAcks=%v", cp.taskStates(), cp.stopAcks())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(30 * time.Millisecond)
+	cancel()
+
+	if n := countStopAcks(cp.stopAcks(), "t-001"); n != 1 {
+		t.Errorf("stop acked %d times, want exactly 1; stopAcks=%v", n, cp.stopAcks())
+	}
+	if stopped := countStatus(cp.taskStates(), "t-001", bus.TaskStatusStopped); stopped != 1 {
+		t.Errorf("task reported STOPPED %d times, want exactly 1", stopped)
+	}
+	if reaped := rt.Reaped(); len(reaped) != 1 || reaped[0] != "t-001-web" {
+		t.Errorf("want t-001-web reaped once, got %+v", reaped)
+	}
+}
+
+// A task whose reap completed is never started again by an assign the gateway
+// is still redelivering.
+func TestPollAssignments_StoppedTaskIsNotRerunByRedeliveredAssign(t *testing.T) {
+	as := testAssign()
+	cp := &fakeCP{
+		pollReplies: [][]bus.Assign{{*as}, {*as}, {*as}, {*as}, {*as}, {*as}},
+		stopPending: []bus.StopDirective{{TaskID: as.TaskID, Reason: "bye"}},
+	}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
+	}}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	deadline := time.After(time.Second)
+	for countStopAcks(cp.stopAcks(), as.TaskID) == 0 {
+		select {
+		case <-deadline:
+			t.Fatalf("stop never completed; states=%+v", cp.taskStates())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	// Let the remaining scripted assigns arrive after the reap has finished.
+	time.Sleep(40 * time.Millisecond)
+	cancel()
+
+	if runs := rt.RunCalls(); len(runs) != 0 {
+		t.Errorf("stopped task was started by a redelivered assign: %+v", runs)
+	}
+	if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 0 {
+		t.Errorf("stopped task reported RUNNING %d times", running)
+	}
+}
+
 // stopTask with no runtime is a no-op: nothing to reap, no state reported.
 func TestStopTask_NilRunnerNoOp(t *testing.T) {
 	cp := &fakeCP{}
-	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
+	a := newAgent(config{}, testIdentity(), cp, nil, nil)
 	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
 	if len(cp.taskStates()) != 0 {
 		t.Fatalf("nil runner should report no state, got %+v", cp.taskStates())
@@ -459,8 +727,8 @@ func TestStopTask_EmptyReasonReapsWithDefault(t *testing.T) {
 	a := newRunAgent(cp, rt)
 	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001"})
 
-	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
-		t.Fatalf("want container t-001-web reaped, got %+v", rt.Removed)
+	if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
+		t.Fatalf("want container t-001-web reaped, got %+v", rt.Stopped)
 	}
 	st := cp.taskStates()
 	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
@@ -539,7 +807,7 @@ func (sentinelResolver) Authorize(context.Context, string) (string, string, stri
 // exec-role resolver is built.
 func TestPullResolver(t *testing.T) {
 	fake := sentinelResolver{}
-	a := newAgent(config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, nil, fake)
+	a := newAgent(config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, fake)
 
 	if got := a.pullResolver(&bus.Assign{TaskID: "t1"}); got != fake {
 		t.Fatalf("no execution role: want instance resolver fallback")

cmd/eks-token-webhook/cert.go

@@ -33,12 +33,12 @@ func serverTLSConfig(cert tls.Certificate) *tls.Config {
 // valid even if the webhook process is restarted. Returns the parsed cert and
 // the certificate PEM (for embedding as the kubeconfig CA bundle).
 func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error) {
-	if certPEM, keyPEM, ok := readCertPair(certPath, keyPath); ok {
-		cert, err := tls.X509KeyPair(certPEM, keyPEM)
-		if err == nil {
+	// Fall through to regenerate on a missing/corrupt/incompatible persisted
+	// pair. The cert PEM is re-read because tls.Certificate keeps only DER.
+	if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
+		if certPEM, err := os.ReadFile(certPath); err == nil {
 			return cert, certPEM, nil
 		}
-		// Fall through to regenerate on a corrupt/incompatible persisted pair.
 	}
 
 	certPEM, keyPEM, err := generateSelfSigned()
@@ -58,18 +58,6 @@ func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error
 	return cert, certPEM, nil
 }
 
-func readCertPair(certPath, keyPath string) (certPEM, keyPEM []byte, ok bool) {
-	c, err := os.ReadFile(certPath)
-	if err != nil {
-		return nil, nil, false
-	}
-	k, err := os.ReadFile(keyPath)
-	if err != nil {
-		return nil, nil, false
-	}
-	return c, k, true
-}
-
 func generateSelfSigned() (certPEM, keyPEM []byte, err error) {
 	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {

cmd/installer/install/install.go

@@ -870,7 +870,7 @@ func fireInstallCallback() {
 		return
 	}
 	client := &http.Client{Timeout: 10 * time.Second}
-	resp, err := client.Get(url) //nolint:noctx // installer has no context; best-effort fire-and-forget
+	resp, err := client.Get(url)
 	if err != nil {
 		slog.Warn("fireInstallCallback: request failed", "url", url, "err", err)
 		return

cmd/rds-agent/agent.go

@@ -220,7 +220,8 @@ func newAgent(cfg config, cp controlPlane, probe *engineProbe) (*Agent, error) {
 	return a, nil
 }
 
-// Does not wait for IMDS: the register loop rides out a datapath still coming up.
+// New builds the agent from cfg, failing if no gateway URL is set. It does not
+// wait for IMDS: the register loop rides out a datapath still coming up.
 func New(cfg config) (*Agent, error) {
 	if cfg.GatewayURL == "" {
 		return nil, fmt.Errorf("no gateway URL configured (RDS_GATEWAY_URL)")
@@ -365,16 +366,16 @@ func unescapeMountField(field string) string {
 
 // Boot-retry bounds. Tight to start, since the usual cause is a control plane
 // still creating this instance's record, and capped so an outage does not turn
-// a booting fleet into a retry storm.
-const (
+// a booting fleet into a retry storm. Vars so tests can shorten the wait.
+var (
 	retryMin = 1 * time.Second
 	retryMax = 30 * time.Second
-
-	// With the production backoff, attempt five is about 15 seconds into the
-	// failure and still precedes the first 30-second heartbeat.
-	retryErrorAttempt = 5
 )
 
+// With the production backoff, attempt five is about 15 seconds into the
+// failure and still precedes the first 30-second heartbeat.
+const retryErrorAttempt = 5
+
 // Wrapping an error in this stops retryObserved. For the failures the control
 // plane decides from its own record, no retry can change the answer, and looping
 // on one only keeps the caller blocked.

cmd/rds-agent/engine_mariadb_test.go

@@ -948,8 +948,8 @@ func TestMariaDBProbe_LiveButUnreachableEngineResetsTheRollbackDeadline(t *testi
 	recovery := &fakeRecovery{restored: true}
 	guard := newTestGuard(recovery, probe)
 	// Bounded so the deadline-resetting loop cannot run forever if it stops
-	// resetting; the assertion is that the rollback was never reached.
-	ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+	// resetting; four rollback budgets, so a missed reset would roll back.
+	ctx, cancel := context.WithTimeout(t.Context(), 80*time.Millisecond)
 	defer cancel()
 	guard.Run(ctx)
 

cmd/rds-agent/main_test.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"os"
+	"testing"
+	"time"
+)
+
+// TestMain shortens the boot-retry backoff for the whole package: the tests
+// assert which calls are retried, not how long the agent waits between them.
+func TestMain(m *testing.M) {
+	retryMin, retryMax = time.Millisecond, time.Millisecond
+	os.Exit(m.Run())
+}

cmd/rds-agent/paramrecovery_test.go

@@ -460,8 +460,8 @@ func TestParamGuard_LeavesARecoveringEngineAlone(t *testing.T) {
 	engine := &fakeRecovery{restored: true}
 	guard := newTestGuard(engine, stubProbe(t, 1, nil))
 	// Bounded so the deadline-resetting loop cannot run forever if it stops
-	// resetting; the assertion is that the rollback was never reached.
-	ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+	// resetting; four rollback budgets, so a missed reset would roll back.
+	ctx, cancel := context.WithTimeout(t.Context(), 80*time.Millisecond)
 	defer cancel()
 	guard.Run(ctx)
 

cmd/spinifex/cmd/admin.go

@@ -30,6 +30,7 @@ import (
 	"github.com/mulgadc/bluebottle/pkg/masterkey"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/mulgadc/spinifex/spinifex/clustersize"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
@@ -200,6 +201,15 @@ takes effect immediately. Prompts for confirmation unless --yes is passed.`,
 	Run: runimagesPromoteCmd,
 }
 
+var imagesDescribeCmd = &cobra.Command{
+	Use:   "describe",
+	Short: "Show an AMI's metadata and source digest",
+	Long: `Print an AMI's control-plane metadata, including the digest of the
+artifact it was imported from and who verified it. The digest describes the
+imported source file, not the volume's current contents.`,
+	Run: runimagesDescribeCmd,
+}
+
 var volumesCmd = &cobra.Command{
 	Use:   "volumes",
 	Short: "Inspect block storage volumes",
@@ -294,7 +304,7 @@ spx admin images import --name debian-13-x86_64
 spx admin images list
 
 // Manually import a path
-spx admin images import --file /path/to/image --distro debian --version 13 --arch x86_64
+spx admin images import --file /path/to/image --checksum /path/to/SHA512SUMS --distro debian --version 13 --arch x86_64
 
 -> x <-
 */
@@ -328,6 +338,7 @@ func init() {
 	imagesCmd.AddCommand(imagesListCmd)
 	imagesCmd.AddCommand(imagesRemoveCmd)
 	imagesCmd.AddCommand(imagesPromoteCmd)
+	imagesCmd.AddCommand(imagesDescribeCmd)
 
 	adminCmd.AddCommand(volumesCmd)
 	volumesCmd.AddCommand(volumesOrphansCmd)
@@ -358,10 +369,11 @@ func init() {
 
 	// Flags for admin init
 	adminInitCmd.Flags().Bool("force", false, "Force re-initialization (overwrites existing config)")
+	adminInitCmd.Flags().Bool("discard-jetstream", true, "With --nodes >= 2, remove this node's existing JetStream streams once every node has joined; =false refuses to form over a non-empty store instead. Ignored with --nodes 1")
 	adminInitCmd.Flags().String("region", "ap-southeast-2", "Mulga region to create")
 	adminInitCmd.Flags().String("az", "ap-southeast-2a", "Mulga AZ to create")
 	adminInitCmd.Flags().String("node", "node1", "Node name, increment for additional nodes (default, node1)")
-	adminInitCmd.Flags().Int("nodes", 3, "Number of nodes to expect for cluster")
+	adminInitCmd.Flags().Int("nodes", 3, "Number of nodes to expect for cluster (1, or 3 and up)")
 	adminInitCmd.Flags().String("host", "", "Leader node to join (if not specified, tries multicast discovery)")
 	adminInitCmd.Flags().Int("port", 4432, "Port to bind cluster services on")
 	adminInitCmd.Flags().String("bind", "0.0.0.0", "IP address to bind services to (e.g., 10.11.12.1 for multi-node). Default 0.0.0.0 listens on all interfaces.")
@@ -413,7 +425,7 @@ func init() {
 	adminJoinCmd.MarkFlagRequired("host")
 	adminJoinCmd.MarkFlagRequired("token")
 
-	imagesImportCmd.Flags().String("tmp-dir", os.TempDir(), "Temporary directory for image import processing")
+	imagesImportCmd.Flags().String("tmp-dir", "", "Scratch directory for extracting the image (default: <spinifex-dir>/images); must be disk-backed, not tmpfs")
 
 	imagesImportCmd.Flags().String("name", "", "Import specified image by name")
 	imagesImportCmd.Flags().String("ami-name", "", "Override the registered AMI name (DescribeImages name). Defaults to ami-{distro}-{version}-{arch}. Use for locally-built appliances (e.g. --ami-name spinifex-eks-node).")
@@ -426,6 +438,7 @@ func init() {
 	imagesImportCmd.Flags().StringSlice("tag", nil, "Tag to apply to the imported AMI as key=value (repeatable; e.g. --tag spinifex:managed-by=elbv2)")
 	imagesImportCmd.Flags().Bool("force", false, "Force command execution (overwrites existing files)")
 	imagesImportCmd.Flags().Bool("skip-verify", false, "Skip catalog-image checksum verification (INSECURE; operator assumes integrity responsibility)")
+	imagesImportCmd.Flags().String("checksum", "", "Sums file to verify a --file image against (SHA256SUMS, SHA512SUMS, <image>.sha256/.sha512)")
 
 	imagesRemoveCmd.Flags().String("image-id", "", "AMI ID to remove (required)")
 	imagesRemoveCmd.Flags().Bool("force", false, "Bypass dependency, ownership and config-corrupt checks (salvage mode)")
@@ -435,6 +448,9 @@ func init() {
 	imagesPromoteCmd.Flags().String("image-id", "", "AMI ID to promote to system image (required)")
 	imagesPromoteCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
 	_ = imagesPromoteCmd.MarkFlagRequired("image-id")
+
+	imagesDescribeCmd.Flags().String("image-id", "", "AMI ID to describe (required)")
+	_ = imagesDescribeCmd.MarkFlagRequired("image-id")
 }
 
 const bytesPerGiB = 1024 * 1024 * 1024
@@ -463,11 +479,112 @@ func amiVolumeSizeGiB(sizeBytes int64) uint64 {
 	return safecast.Int64ToUint64((sizeBytes + bytesPerGiB - 1) / bytesPerGiB)
 }
 
+// validateChecksumFlags rejects --checksum combinations that would otherwise be silently ignored.
+// An explicitly empty --checksum (e.g. an unset shell variable) must not fall through to an unverified import.
+func validateChecksumFlags(checksumPath string, checksumSet bool, localFile string, skipVerify bool) error {
+	if !checksumSet {
+		return nil
+	}
+	if strings.TrimSpace(checksumPath) == "" {
+		return errors.New("--checksum requires a sums file path")
+	}
+	if localFile == "" {
+		return errors.New("--checksum requires --file")
+	}
+	if skipVerify {
+		return errors.New("--checksum and --skip-verify are mutually exclusive")
+	}
+	return nil
+}
+
+// catalogPinnedDigestSource records that the expected digest came from this
+// binary's catalog, so a recorded digest never reads as a fetched sums file.
+const catalogPinnedDigestSource = "catalog:pinned"
+
+// importSource is the file an import is about to extract, and what it is to be verified against.
+type importSource struct {
+	imageFile, imageName, localFile            string
+	image                                      utils.Images
+	checksumPath, checksumAlgo, expectedDigest string
+	skipVerify                                 bool
+}
+
+// resolveDigest verifies or hashes the source before extract and returns the digest to record.
+// Upstream digests are of the artifact as supplied. Failures are reported to errOut before
+// returning; the file is left in place, and a catalog download recovers with --force.
+func (src importSource) resolveDigest(out, errOut io.Writer) (ebsmetadata.ImageDigest, error) {
+	digest := ebsmetadata.ImageDigest{Filename: filepath.Base(src.imageFile)}
+	switch sourceDigestMode(src.localFile, src.checksumPath, src.skipVerify) {
+	case ebsmetadata.DigestCatalog:
+		if src.image.ChecksumType == "" || (src.image.Checksum == "" && src.image.ChecksumDigest == "") {
+			fmt.Fprintf(errOut, "Catalog entry %q is missing Checksum/ChecksumDigest/ChecksumType; refusing import.\n", src.imageName)
+			return digest, errors.New("catalog entry has no checksum")
+		}
+		// A pinned digest is checked against this file rather than a sums file
+		// fetched from the same host that served the image.
+		if src.image.ChecksumDigest != "" {
+			actual, err := utils.VerifyImageDigest(src.imageFile, src.image.ChecksumType, src.image.ChecksumDigest)
+			if err != nil {
+				printChecksumError(errOut, src.imageFile, src.imageName, src.image, err)
+				return digest, err
+			}
+			fmt.Fprintf(out, "✅ Verified image checksum (%s) against the pinned catalog digest\n", src.image.ChecksumType)
+			digest.Algorithm, digest.Value, digest.Source = src.image.ChecksumType, actual, catalogPinnedDigestSource
+			digest.Verification = ebsmetadata.DigestCatalog
+			return digest, nil
+		}
+		actual, err := utils.VerifyImageChecksum(src.imageFile, src.image.Checksum, src.image.ChecksumType)
+		if err != nil {
+			printChecksumError(errOut, src.imageFile, src.imageName, src.image, err)
+			return digest, err
+		}
+		fmt.Fprintf(out, "✅ Verified image checksum (%s)\n", src.image.ChecksumType)
+		digest.Algorithm, digest.Value, digest.Source = src.image.ChecksumType, actual, src.image.Checksum
+		digest.Verification = ebsmetadata.DigestCatalog
+	case ebsmetadata.DigestOperator:
+		actual, err := utils.VerifyImageDigest(src.localFile, src.checksumAlgo, src.expectedDigest)
+		if err != nil {
+			fmt.Fprintf(errOut, "Image integrity verification failed: %v\n", err)
+			fmt.Fprintf(errOut, "  file:     %s\n", src.localFile)
+			fmt.Fprintf(errOut, "  checksum: %s\n", src.checksumPath)
+			return digest, err
+		}
+		fmt.Fprintf(out, "✅ Verified image checksum (%s) against %s\n", src.checksumAlgo, src.checksumPath)
+		digest.Algorithm, digest.Value, digest.Source = src.checksumAlgo, actual, filepath.Base(src.checksumPath)
+		digest.Verification = ebsmetadata.DigestOperator
+	default:
+		if src.localFile != "" {
+			fmt.Fprintf(errOut, "Importing %s without checksum validation (pass --checksum <sums-file> to verify)\n", src.localFile)
+		} else {
+			fmt.Fprintf(errOut, "⚠️  --skip-verify set: checksum verification skipped for %s\n", src.imageName)
+		}
+		actual, err := utils.HashImageFile(src.imageFile, "sha256")
+		if err != nil {
+			fmt.Fprintf(errOut, "Could not hash image: %v\n", err)
+			return digest, err
+		}
+		digest.Algorithm, digest.Value = "sha256", actual
+		digest.Verification = ebsmetadata.DigestUnverified
+	}
+	return digest, nil
+}
+
+// sourceDigestMode picks how an import's source digest is obtained.
+func sourceDigestMode(localFile, checksumPath string, skipVerify bool) string {
+	switch {
+	case localFile != "" && checksumPath != "":
+		return ebsmetadata.DigestOperator
+	case localFile != "" || skipVerify:
+		return ebsmetadata.DigestUnverified
+	default:
+		return ebsmetadata.DigestCatalog
+	}
+}
+
 func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	var image utils.Images
 
 	var imageFile string
-	var imageStat os.FileInfo
 	var err error
 
 	forceCmd, _ := cmd.Flags().GetBool("force")
@@ -502,6 +619,20 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	checksumPath, _ := cmd.Flags().GetString("checksum")
+	if err := validateChecksumFlags(checksumPath, cmd.Flags().Changed("checksum"), localFile, skipVerify); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	var checksumAlgo, expectedDigest string
+	if checksumPath != "" {
+		checksumAlgo, expectedDigest, err = utils.ReadExpectedDigest(checksumPath, filepath.Base(localFile))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Could not read checksum from %s: %v\n", checksumPath, err)
+			os.Exit(1)
+		}
+	}
+
 	if imageName != "" {
 		var exists bool
 		image, exists = utils.AvailableImages[imageName]
@@ -588,8 +719,8 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Next, if the file is selected to download, fetch it, extract disk image, and save to path
-	if imageName != "" && localFile == "" {
+	// Next, if the file is selected to download, fetch it and save to path
+	if localFile == "" {
 		// Download the file to the image path
 		filename := path.Base(image.URL)
 		imageFile = fmt.Sprintf("%s/%s", imagePath, filename)
@@ -605,45 +736,75 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 				os.Exit(1)
 			}
 		}
+	}
 
-		// Verify before extract: the catalog digest is of the artifact as it
-		// sits on the mirror (.tar.xz/.img/.raw). Also catches a poisoned
-		// cache on the re-run path — failure leaves the file on disk so the
-		// operator can inspect; recover with --force.
-		if skipVerify {
-			fmt.Fprintf(os.Stderr, "⚠️  --skip-verify set: checksum verification skipped for %s\n", imageName)
-		} else {
-			if image.Checksum == "" || image.ChecksumType == "" {
-				fmt.Fprintf(os.Stderr, "Catalog entry %q is missing Checksum/ChecksumType; refusing import.\n", imageName)
-				os.Exit(1)
-			}
-			if err := utils.VerifyImageChecksum(imageFile, image.Checksum, image.ChecksumType); err != nil {
-				printChecksumError(os.Stderr, imageFile, imageName, image, err)
-				os.Exit(1)
-			}
-			fmt.Printf("✅ Verified image checksum (%s)\n", image.ChecksumType)
+	sourceDigest, err := importSource{
+		imageFile:      imageFile,
+		imageName:      imageName,
+		localFile:      localFile,
+		image:          image,
+		checksumPath:   checksumPath,
+		checksumAlgo:   checksumAlgo,
+		expectedDigest: expectedDigest,
+		skipVerify:     skipVerify,
+	}.resolveDigest(os.Stdout, os.Stderr)
+	if err != nil {
+		os.Exit(1)
+	}
+
+	// An empty --tmp-dir scratches beside the catalog: disk-backed, unlike a
+	// tmpfs /tmp, and already root:spinifex 0770.
+	scratchRoot := ostmpDir
+	if scratchRoot == "" {
+		scratchRoot = imageDir
+	}
+
+	extractedImagePath, cleanupScratch, err := extractToScratch(imageFile, scratchRoot)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	// os.Exit skips deferred calls, so scratch is removed before any exit.
+	volumeId, err := registerExtractedImage(image, amiNameOverride, imagePath, extractedImagePath, sourceDigest)
+	cleanupScratch()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
+	fmt.Printf("✅ Source digest: %s:%s (%s)\n", sourceDigest.Algorithm, sourceDigest.Value, sourceDigest.Verification)
+}
+
+// extractToScratch extracts imageFile into a fresh hidden directory under
+// scratchRoot and returns the extracted path with a cleanup that removes that
+// directory. On error the directory is already removed.
+func extractToScratch(imageFile, scratchRoot string) (string, func(), error) {
+	scratchDir, err := os.MkdirTemp(scratchRoot, ".import-*")
+	if err != nil {
+		return "", nil, fmt.Errorf("create scratch dir under %s: %w", scratchRoot, err)
+	}
+	cleanup := func() {
+		if err := os.RemoveAll(scratchDir); err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: could not remove scratch dir %s: %v\n", scratchDir, err)
 		}
 	}
 
-	// Next, validate if the image is raw, tar, gz, xv, etc. We need to upload the raw image
-	tmpDir, err := os.MkdirTemp(ostmpDir, "spinifex-image-tmp-*")
-
+	extracted, err := utils.ExtractDiskImageFromFile(imageFile, scratchDir)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not create temp dir: %v\n", err)
-		os.Exit(1)
+		cleanup()
+		return "", nil, fmt.Errorf("extract image: %w", err)
 	}
+	return extracted, cleanup, nil
+}
 
-	extractedImagePath, err := utils.ExtractDiskImageFromFile(imageFile, imagePath)
+// registerExtractedImage writes the AMI manifest into imagePath, imports
+// extractedImagePath into storage and registers the AMI, returning its ID.
+func registerExtractedImage(image utils.Images, amiNameOverride, imagePath, extractedImagePath string, sourceDigest ebsmetadata.ImageDigest) (string, error) {
+	imageStat, err := os.Stat(extractedImagePath)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not extract image: %v\n", err)
-		os.Exit(1)
-	}
-
-	imageStat, err = os.Stat(extractedImagePath)
-
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not stat image: %v\n", err)
-		os.Exit(1)
+		return "", fmt.Errorf("stat image: %w", err)
 	}
 
 	// Describe the image/AMI as the control-plane document it will become.
@@ -659,7 +820,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		Description:     fmt.Sprintf("%s cloud image prepared for Spinifex", amiName),
 		Architecture:    image.Arch,
 		PlatformDetails: image.Platform,
-		CreationDate:    time.Now(),
+		CreationDate:    time.Now().UTC(),
 		RootDeviceType:  "ebs",
 		Virtualization:  "hvm",
 		ImageOwnerAlias: "system",
@@ -668,6 +829,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		BootMode:        image.BootMode,
 		Distro:          image.Distro,
 		DistroFamily:    utils.DistroFamily(image.Distro),
+		SourceDigest:    &sourceDigest,
 	}
 
 	// Copy catalog-provided tags (e.g. spinifex:managed-by for system AMIs)
@@ -681,24 +843,19 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	// Save as JSON
 	jsonData, err := json.Marshal(ami)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not marshal manifest: %v\n", err)
-		os.Exit(1)
+		return "", fmt.Errorf("marshal manifest: %w", err)
 	}
 
 	manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, ami.Name)
 	// Write to file
 	err = os.WriteFile(manifestFilename, jsonData, 0600)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not write manifest: %v\n", err)
-		os.Exit(1)
+		return "", fmt.Errorf("write manifest: %w", err)
 	}
 
-	defer os.RemoveAll(tmpDir)
-
 	appConfig, nc, err := loadConfigAndConnect()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
-		os.Exit(1)
+		return "", fmt.Errorf("connect to the cluster: %w", err)
 	}
 	defer nc.Close()
 	node := appConfig.Nodes[appConfig.Node]
@@ -715,8 +872,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		Progress:         os.Stdout,
 	})
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not import image: %v\n", err)
-		os.Exit(1)
+		return "", fmt.Errorf("import image: %w", err)
 	}
 
 	// admin.ImportImage only wrote the provider's half of the snapshot (its
@@ -724,8 +880,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	// DescribeSnapshots and GetAMISourceVolumeID cannot resolve the import.
 	metaStore := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
 	if err := registerImportedAMISnapshot(metaStore, node.Predastore.Bucket, ami, node.AZ, node.Viperblock.EncryptionKeyFile != ""); err != nil {
-		fmt.Fprintf(os.Stderr, "Imported %s but could not register its snapshot: %v\n", volumeId, err)
-		os.Exit(1)
+		return "", fmt.Errorf("imported %s but could not register its snapshot: %w", volumeId, err)
 	}
 
 	// The document is what DescribeImages enumerates, so it is written last:
@@ -733,11 +888,10 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	// partway leaves no half-registered image behind.
 	ami.State = "available"
 	if err := ebsmetadata.NewStore(metaStore, node.Predastore.Bucket).PutAMI(context.Background(), ami); err != nil {
-		fmt.Fprintf(os.Stderr, "Imported %s but could not register it: %v\n", volumeId, err)
-		os.Exit(1)
+		return "", fmt.Errorf("imported %s but could not register it: %w", volumeId, err)
 	}
 
-	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
+	return volumeId, nil
 }
 
 // registerImportedAMISnapshot writes the EC2 control plane's snapshot document
@@ -1000,6 +1154,62 @@ func runimagesPromoteCmd(cmd *cobra.Command, args []string) {
 	fmt.Printf("✅ Promoted %s to system image (owner: %s).\n", imageID, admin.SystemOwnerAlias)
 }
 
+func runimagesDescribeCmd(cmd *cobra.Command, args []string) {
+	imageID, _ := cmd.Flags().GetString("image-id")
+
+	cfgFile, _ := cmd.Flags().GetString("config")
+	if cfgFile == "" {
+		cfgFile = DefaultConfigFile()
+	}
+
+	appConfig, err := config.LoadConfig(cfgFile)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Error loading config file:", err)
+		os.Exit(1)
+	}
+
+	node := appConfig.Nodes[appConfig.Node]
+	store := objectstore.NewS3ObjectStoreFromConfig(
+		node.Predastore.Host,
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+
+	meta, err := admin.GetAMIMetadata(store, node.Predastore.Bucket, imageID)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Failed to describe AMI:", err)
+		os.Exit(1)
+	}
+
+	printAMIDescription(os.Stdout, meta)
+}
+
+// printAMIDescription writes an AMI's metadata and source digest for an operator.
+func printAMIDescription(w io.Writer, meta ebsmetadata.AMI) {
+	fmt.Fprintf(w, "Image ID:       %s\n", meta.ImageID)
+	fmt.Fprintf(w, "Name:           %s\n", meta.Name)
+	fmt.Fprintf(w, "Owner:          %s\n", meta.ImageOwnerAlias)
+	if !meta.CreationDate.IsZero() {
+		fmt.Fprintf(w, "Created:        %s\n", meta.CreationDate.UTC().Format("2006-01-02T15:04:05Z"))
+	}
+	fmt.Fprintf(w, "Boot mode:      %s\n", meta.BootMode)
+	fmt.Fprintf(w, "Architecture:   %s\n", meta.Architecture)
+
+	d := meta.SourceDigest
+	if d == nil {
+		fmt.Fprintln(w, "Source digest:  not recorded (imported before digest recording)")
+		return
+	}
+	fmt.Fprintf(w, "Source digest:  %s:%s\n", d.Algorithm, d.Value)
+	if d.Source != "" {
+		fmt.Fprintf(w, "Verification:   %s (%s)\n", d.Verification, d.Source)
+	} else {
+		fmt.Fprintf(w, "Verification:   %s\n", d.Verification)
+	}
+	fmt.Fprintf(w, "Source file:    %s\n", d.Filename)
+}
+
 // List remote images available.
 func runimagesListCmd(cmd *cobra.Command, args []string) {
 	//fmt.Println(availableImages)
@@ -1027,6 +1237,26 @@ func runimagesListCmd(cmd *cobra.Command, args []string) {
 	pterm.Println("spx admin images import --name <image-name>")
 }
 
+// validateNodeCount accepts the cluster sizes Spinifex supports: one server, or
+// three and up.
+//
+// Two is rejected rather than allowed and warned about. On two servers OVN runs
+// standalone, the storage metadata quorum has no majority to lose, and the
+// control plane's own KV state has no replica count that works — one replica
+// disappears with its node, and two stop accepting writes when either goes. A
+// cluster that cannot survive the event it was built for is worth refusing at the
+// point someone asks for it.
+func validateNodeCount(nodes int) error {
+	if nodes < 1 {
+		return fmt.Errorf("a cluster needs at least one node, got %d", nodes)
+	}
+	if nodes > 1 && nodes < clustersize.MinHANodes {
+		return fmt.Errorf("%d nodes is not a supported cluster size: use 1 for a single server, or %d and up for a cluster that can survive losing one",
+			nodes, clustersize.MinHANodes)
+	}
+	return nil
+}
+
 // TODO: Move all logic to a module, use minimal application logic in viper commands.
 func runAdminInit(cmd *cobra.Command, args []string) {
 	if os.Getuid() != 0 {
@@ -1058,6 +1288,15 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	clusterName, _ := cmd.Flags().GetString("cluster-name")
 	services, _ := cmd.Flags().GetStringSlice("services")
 
+	// Rejected before anything is written, because a two-node cluster has no
+	// correct answer rather than a worse one: no layer of the stack has a
+	// majority to lose, and no replica count for its state both survives a node
+	// going away and still accepts writes.
+	if err := validateNodeCount(nodes); err != nil {
+		fmt.Fprintf(os.Stderr, "--nodes: %v\n", err)
+		os.Exit(1)
+	}
+
 	// Optional operator email — validated up-front so a bad address fails
 	// before we touch any config state. Empty is allowed here; reset / repeat
 	// inits on a box that already has an email in /etc/spinifex/spinifex.toml
@@ -1115,7 +1354,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	detected, err := admin.DetectNetwork()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "⚠️  Network auto-detection failed: %v\n", err)
-		fmt.Fprintf(os.Stderr, "   Use --external-mode=nat for outbound-only VMs on a non-bridgeable uplink, or specify --external-* flags manually.\n")
+		fmt.Fprintf(os.Stderr, "   Use --external-mode=nat on a non-bridgeable uplink, or specify --external-* flags manually.\n")
 	} else {
 		detectedNet = detected
 
@@ -1153,7 +1392,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			if externalMode == "" && !cmd.Flags().Changed("external-mode") {
 				if isNonBridgeableUplink(detected.WAN.Name) {
 					fmt.Fprintf(os.Stderr, "\n❌ Detected WAN interface %s cannot be bridged (WiFi/cellular/PPP).\n", detected.WAN.Name)
-					fmt.Fprintf(os.Stderr, "   Use routed NAT mode instead (outbound-only VM networking):\n")
+					fmt.Fprintf(os.Stderr, "   Use routed NAT mode instead:\n")
 					fmt.Fprintf(os.Stderr, "     ./scripts/setup-ovn.sh --management --nat-uplink\n")
 					fmt.Fprintf(os.Stderr, "     spx admin init --external-mode=nat\n")
 					os.Exit(1)
@@ -1175,9 +1414,14 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	// the transit segment claims externalGateway below.
 	natPublicGateway := externalGateway
 	if externalMode == "nat" {
+		// Verified on three OCI nodes 2026-09-26: each node delivers its own
+		// instances' public IPs, inbound and outbound. Still flagged because no
+		// nightly cell covers multi-node nat yet.
 		if nodes >= 2 {
-			fmt.Fprintf(os.Stderr, "❌ Error: --external-mode=nat is single-node only (v1); use --nodes=1\n")
-			os.Exit(1)
+			fmt.Fprintf(os.Stderr, "⚠️  --external-mode=nat on %d nodes is NEW.\n", nodes)
+			fmt.Fprintf(os.Stderr, "   Public IPs are delivered by the node running the instance; default egress stays on the VPC's gateway chassis.\n")
+			fmt.Fprintf(os.Stderr, "   Verified by hand on three nodes, but not yet covered by a nightly cell.\n")
+			fmt.Fprintf(os.Stderr, "   Check with: ip route show dev spx-nat-host — exactly one node should hold each public address.\n")
 		}
 		if !natPublicPool && (externalBindBridge != "" || gatewayIP != "") {
 			fmt.Fprintf(os.Stderr, "❌ Error: --external-bind-bridge/--gateway-ip require --external-pool or --external-source in --external-mode=nat\n")
@@ -1313,6 +1557,15 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(0)
 	}
 
+	if spxRoot == "" {
+		spxRoot = DefaultDataDir()
+	}
+	discardJetStream, _ := cmd.Flags().GetBool("discard-jetstream")
+	if err := checkInitJetStreamStore(jetStreamStoreDir(spxRoot), nodes, discardJetStream); err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+
 	// Preserve the previously-captured operator email across --force re-inits
 	// when --email is omitted (e.g. reset-dev-env.sh workflows). Without this
 	// a reset would silently blank the address.
@@ -1429,7 +1682,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 
 	// Generate SSL certificates (with bind IP in SANs for multi-node support)
-	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP, region, config.DefaultAWSInternalSuffix)
+	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP, region, config.DefaultAWSServicesDomain)
 
 	// Generate per-node IPsec peer cert when cluster-wide IPsec is enabled
 	// (default true). Reuses the cluster CA — no intermediate strongSwan PKI.
@@ -1498,14 +1751,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			bootstrapSubnetId := utils.GenerateResourceID("subnet")
 			bootstrapIgwId := utils.GenerateResourceID("igw")
 			networkConfig.ExternalMode = externalMode
-			networkConfig.PoolName = "wan"
-			networkConfig.PoolSource = externalSource
-			networkConfig.PoolBindBridge = externalBindBridge
-			networkConfig.PoolStart = poolStart
-			networkConfig.PoolEnd = poolEnd
-			networkConfig.PoolGateway = externalGateway
-			networkConfig.PoolGatewayIP = gatewayIP
-			networkConfig.PoolPrefixLen = externalPrefixLen
+			networkConfig.Pools = externalPools
 			networkConfig.PoolDNSServers = dnsServers
 			networkConfig.BootstrapAccountId = admin.DefaultAccountID()
 			networkConfig.BootstrapVpcId = bootstrapVpcId
@@ -1658,7 +1904,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 				fmt.Printf("  Public pool:   dhcp via %s\n", externalBindBridge)
 			}
 		} else {
-			fmt.Printf("\n📡 External networking: nat (routed, outbound-only — no public IPs/EIPs)\n")
+			fmt.Printf("\n📡 External networking: nat (routed; no public pool configured yet)\n")
 		}
 		fmt.Printf("  Transit:       %s via %s (host masquerades out any uplink)\n", host.NATTransitCIDR, host.NATTransitHostEnd)
 		fmt.Printf("  Host setup:    ./scripts/setup-ovn.sh --nat-uplink (run before starting services)\n")
@@ -1845,6 +2091,17 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	fmt.Printf("✅ All %d nodes joined!\n", expectedNodes)
 
+	// The pre-start check refused a store with streams unless discard was asked
+	// for, so anything left here is removed now that the cluster is committed.
+	if discard, _ := cmd.Flags().GetBool("discard-jetstream"); discard {
+		storeDir := jetStreamStoreDir(spxRoot)
+		if err := discardJetStreamStore(storeDir); err != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("✅ Discarded this node's pre-formation JetStream store: %s\n", storeDir)
+	}
+
 	// Build cluster topology from formation data
 	allNodes := fs.Nodes()
 	clusterRoutes := formation.BuildClusterRoutes(allNodes)
@@ -2002,19 +2259,27 @@ func joinRetryable(err error, statusCode int) bool {
 }
 
 // checkJoinPreconditions rejects a join that would silently destroy this node's
-// existing cluster identity. Joining adopts the primary's CA and master key,
-// overwriting whatever is here: correct on a freshly installed node, and on one
-// that has been in service it orphans every fragment and volume sealed under the
-// old key. runAdminInit guards the same way before re-initializing.
-func checkJoinPreconditions(configDir string, force bool) error {
+// identity or JetStream data; --force accepts that loss. A running NATS service
+// is refused even with --force, because the store cannot be removed under it.
+func checkJoinPreconditions(configDir, storeDir string, force bool) error {
+	if err := checkStoreDiscardable(); err != nil {
+		return err
+	}
 	if force {
 		return nil
 	}
 	tomlPath := filepath.Join(configDir, "spinifex.toml")
-	if !admin.FileExists(tomlPath) {
-		return nil
+	if admin.FileExists(tomlPath) {
+		return fmt.Errorf("this node is already initialized: %s", tomlPath)
 	}
-	return fmt.Errorf("this node is already initialized: %s", tomlPath)
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return err
+	}
+	if len(streams) > 0 {
+		return fmt.Errorf("this node's JetStream store %s holds %d stream(s)", storeDir, len(streams))
+	}
+	return nil
 }
 
 // joinDiscardsIdentityMsg spells out what a forced join throws away. Kept out of
@@ -2023,6 +2288,8 @@ const joinDiscardsIdentityMsg = `Joining will discard this node's own cluster id
   - CA certificate and key
   - master key, and any data sealed under it
   - viperblock key, and any volumes encrypted under it
+  - JetStream store (<data-dir>/nats/jetstream): every stream and KV bucket
+    this node holds, so it takes the cluster's copy instead of diverging from it
 
 That is safe on a freshly installed node — an ISO install initializes a
 single-node cluster at first boot, and nothing has been sealed under these keys
@@ -2103,7 +2370,11 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	// Checked before any network call so a node that will not join says so
 	// immediately. Unlike init this exits non-zero: a node that did not join
 	// must not look like success to a provisioning script.
-	if err := checkJoinPreconditions(configDir, force); err != nil {
+	if dataDir == "" {
+		dataDir = DefaultDataDir()
+	}
+	storeDir := jetStreamStoreDir(dataDir)
+	if err := checkJoinPreconditions(configDir, storeDir, force); err != nil {
 		fmt.Fprintf(os.Stderr, "⚠️  %v\n\n%s\n", err, joinDiscardsIdentityMsg)
 		os.Exit(1)
 	}
@@ -2113,11 +2384,6 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		clusterBind = bindIP
 	}
 
-	// Set default data directory
-	if dataDir == "" {
-		dataDir = DefaultDataDir()
-	}
-
 	fmt.Println("🚀 Joining Spinifex cluster...")
 	fmt.Printf("Node: %s\n", node)
 	fmt.Printf("Leader: %s\n", leaderHost)
@@ -2298,6 +2564,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	// Removed only once formation has succeeded, so a failed join leaves the
+	// node's single-node cluster intact.
+	if err := discardJetStreamStore(storeDir); err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("✅ Discarded this node's pre-formation JetStream store: %s\n", storeDir)
+
 	// Set up config directory
 	if configDir == "" {
 		configDir = DefaultConfigDir()
@@ -2385,7 +2659,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}
 
 	// Generate server cert signed by CA with this node's bind IP
-	if err := admin.GenerateServerCertOnly(configDir, bindIP, region, config.DefaultAWSInternalSuffix); err != nil {
+	if err := admin.GenerateServerCertOnly(configDir, bindIP, region, config.DefaultAWSServicesDomain); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
 		os.Exit(1)
 	}
@@ -2604,7 +2878,7 @@ func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterCo
 
 	// Background: this runs at CLI top level, where there is no request to
 	// inherit a deadline from and the process exits after the one command.
-	svc, err := handlers_iam.NewIAMServiceImpl(context.Background(), nc, masterKey, len(cfg.Nodes))
+	svc, err := handlers_iam.NewIAMServiceImpl(context.Background(), nc, masterKey)
 	if err != nil {
 		nc.Close()
 		return nil, nil, nil, nil, fmt.Errorf("init IAM service: %w", err)
@@ -2720,7 +2994,6 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 			endpointHost = h
 		}
 	}
-	endpointURL := "https://" + net.JoinHostPort(endpointHost, "9999")
 
 	credPath := filepath.Join(homeDir, ".aws", "credentials")
 	configPath := filepath.Join(homeDir, ".aws", "config")
@@ -2735,12 +3008,7 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	if region == "" {
 		region = "ap-southeast-2"
 	}
-	if err := admin.UpdateAWSINIFile(configPath, "profile "+profileName, map[string]string{
-		"region":       region,
-		"endpoint_url": endpointURL,
-		"ca_bundle":    certPath,
-		"output":       "json",
-	}); err != nil {
+	if err := admin.WriteAWSProfileConfig(configPath, profileName, region, certPath, endpointHost); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not update AWS config: %v\n", err)
 	}
 
@@ -2994,18 +3262,11 @@ func applyNetworkConfig(settings *admin.ConfigSettings, nc *formation.NetworkCon
 	settings.IPSecEnabled = nc.IPSecEnabled
 	settings.ExternalMode = nc.ExternalMode
 	settings.PoolDNSServers = nc.PoolDNSServers
+	// Derived from the mode rather than carried in NetworkConfig, so the leader
+	// and every joiner pin the same value from the same rule as single-node init.
+	settings.BridgeMode = bridgeModeFor(nc.ExternalMode)
 	if nc.ExternalMode != "" {
-		settings.Pools = []admin.PoolData{{
-			Name:       nc.PoolName,
-			Source:     nc.PoolSource,
-			BindBridge: nc.PoolBindBridge,
-			Start:      nc.PoolStart,
-			End:        nc.PoolEnd,
-			Gateway:    nc.PoolGateway,
-			GatewayIP:  nc.PoolGatewayIP,
-			PrefixLen:  nc.PoolPrefixLen,
-			DNSServers: nc.PoolDNSServers,
-		}}
+		settings.Pools = nc.Pools
 	}
 
 	settings.BootstrapAccountId = nc.BootstrapAccountId
@@ -3256,11 +3517,11 @@ func resolvePublicPoolFlags(source, poolRange, bindBridge, gateway, defaultBindB
 		if gateway == "" {
 			return "", "", "", "", fmt.Errorf("--external-gateway is required with --external-source=static")
 		}
-		parts := strings.SplitN(poolRange, "-", 2)
-		if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil {
+		var ok bool
+		start, end, ok = strings.Cut(poolRange, "-")
+		if !ok || net.ParseIP(start) == nil || net.ParseIP(end) == nil {
 			return "", "", "", "", fmt.Errorf("--external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s", poolRange)
 		}
-		start, end = parts[0], parts[1]
 	default:
 		return "", "", "", "", fmt.Errorf("--external-source must be 'static' or 'dhcp', got: %s", source)
 	}
@@ -3706,7 +3967,12 @@ func resolveIfaceIP(iface string) string {
 func printChecksumError(w io.Writer, imageFile, imageName string, image utils.Images, err error) {
 	fmt.Fprintf(w, "Image integrity verification failed: %v\n", err)
 	fmt.Fprintf(w, "  file:     %s\n", imageFile)
-	fmt.Fprintf(w, "  source:   %s\n", image.Checksum)
+	if image.ChecksumDigest != "" {
+		fmt.Fprintf(w, "  source:   %s (%s)\n", catalogPinnedDigestSource, image.ChecksumDigest)
+		fmt.Fprintf(w, "  url:      %s\n", image.URL)
+	} else {
+		fmt.Fprintf(w, "  source:   %s\n", image.Checksum)
+	}
 	fmt.Fprintln(w)
 	fmt.Fprintln(w, "The cached file was left in place. To re-download and retry:")
 	fmt.Fprintf(w, "  spx admin images import --name %s --force\n", imageName)

cmd/spinifex/cmd/admin_account_quota.go

@@ -3,7 +3,8 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"sort"
+	"maps"
+	"slices"
 	"strconv"
 
 	"github.com/mulgadc/spinifex/spinifex/gateway"
@@ -166,11 +167,7 @@ func printAccountQuota(resp *gateway.AccountQuotaResponse) {
 	fmt.Printf("%-16s %10s  %s\n", "DIMENSION", "LIMIT", "SOURCE")
 	fmt.Printf("%-16s %10s  %s\n", "---------", "-----", "------")
 
-	names := make([]string, 0, len(resp.Limits))
-	for name := range resp.Limits {
-		names = append(names, name)
-	}
-	sort.Strings(names)
+	names := slices.Sorted(maps.Keys(resp.Limits))
 	for _, name := range names {
 		limit := strconv.Itoa(resp.Limits[name])
 		if resp.Limits[name] == handlers_quota.Unlimited {

cmd/spinifex/cmd/admin_images_import_test.go

@@ -0,0 +1,117 @@
+package cmd
+
+import (
+	"bytes"
+	"compress/gzip"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func readRawDiskFixture(t *testing.T) []byte {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join("..", "..", "..", "tests", "unit-test-disk-image.raw"))
+	require.NoError(t, err)
+	return data
+}
+
+func gzipData(t *testing.T, data []byte) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	gw := gzip.NewWriter(&buf)
+	_, err := gw.Write(data)
+	require.NoError(t, err)
+	require.NoError(t, gw.Close())
+	return buf.Bytes()
+}
+
+func qcow2Data(t *testing.T, raw []byte) []byte {
+	t.Helper()
+	if _, err := exec.LookPath("qemu-img"); err != nil {
+		t.Skip("qemu-img not found")
+	}
+	dir := t.TempDir()
+	rawPath := filepath.Join(dir, "src.raw")
+	qcowPath := filepath.Join(dir, "src.qcow2")
+	require.NoError(t, os.WriteFile(rawPath, raw, 0o600))
+	out, err := exec.Command("qemu-img", "convert", "-f", "raw", "-O", "qcow2", rawPath, qcowPath).CombinedOutput()
+	require.NoError(t, err, string(out))
+	data, err := os.ReadFile(qcowPath)
+	require.NoError(t, err)
+	return data
+}
+
+func dirNames(t *testing.T, dir string) []string {
+	t.Helper()
+	entries, err := os.ReadDir(dir)
+	require.NoError(t, err)
+	names := make([]string, 0, len(entries))
+	for _, e := range entries {
+		names = append(names, e.Name())
+	}
+	sort.Strings(names)
+	return names
+}
+
+// The catalog dir doubles as the download cache, so the source lives there
+// as it does for a --name import, with the images dir as the scratch root.
+func TestExtractToScratch_LeavesNoIntermediateInCatalog(t *testing.T) {
+	raw := readRawDiskFixture(t)
+
+	cases := []struct {
+		name    string
+		source  string
+		payload func(t *testing.T) []byte
+		wantErr string
+	}{
+		{
+			name:    "gzip raw is decompressed into scratch",
+			source:  "disk.raw.gz",
+			payload: func(t *testing.T) []byte { return gzipData(t, raw) },
+		},
+		{
+			name:    "qcow2 is converted into scratch",
+			source:  "disk.qcow2",
+			payload: func(t *testing.T) []byte { return qcow2Data(t, raw) },
+		},
+		{
+			name:    "intermediate that fails the disk sniff is removed",
+			source:  "disk.raw.gz",
+			payload: func(t *testing.T) []byte { return gzipData(t, []byte("not a disk image")) },
+			wantErr: "extract image",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			imagesDir := t.TempDir()
+			catalogDir := filepath.Join(imagesDir, "debian", "13", "x86_64")
+			require.NoError(t, os.MkdirAll(catalogDir, 0o770))
+			source := filepath.Join(catalogDir, tc.source)
+			require.NoError(t, os.WriteFile(source, tc.payload(t), 0o600))
+
+			extracted, cleanup, err := extractToScratch(source, imagesDir)
+
+			if tc.wantErr != "" {
+				require.ErrorContains(t, err, tc.wantErr)
+				assert.Nil(t, cleanup)
+			} else {
+				require.NoError(t, err)
+				scratchDir := filepath.Dir(extracted)
+				assert.Equal(t, imagesDir, filepath.Dir(scratchDir))
+				assert.True(t, strings.HasPrefix(filepath.Base(scratchDir), ".import-"), "extracted into %s", extracted)
+				assert.FileExists(t, extracted)
+				cleanup()
+			}
+
+			assert.Equal(t, []string{"debian"}, dirNames(t, imagesDir), "scratch dir must be removed")
+			assert.Equal(t, []string{tc.source}, dirNames(t, catalogDir), "catalog must hold only the source")
+		})
+	}
+}

cmd/spinifex/cmd/admin_jsprobe_test.go

@@ -61,6 +61,7 @@ func TestCanaryRoundTrip_WritesADistinctValueEachRun(t *testing.T) {
 // A deleted bucket stands in for the unreachable-store case: the probe must
 // report an error rather than treating a failed write as success.
 func TestCanaryRoundTrip_FailsWhenBucketIsGone(t *testing.T) {
+	t.Parallel()
 	kv, js := newCanaryBucket(t)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

cmd/spinifex/cmd/admin_kv.go

@@ -0,0 +1,448 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"maps"
+	"os"
+	"path/filepath"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/highwayhash"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/spf13/cobra"
+)
+
+var kvCmd = &cobra.Command{
+	Use:   "kv",
+	Short: "Inspect JetStream KV state",
+}
+
+var kvDigestCmd = &cobra.Command{
+	Use:   "digest",
+	Short: "Print a digest of every stream held by this node's JetStream replica",
+	Long: `Read this node's own copy of every JetStream stream and print one line per
+stream: name, first and last sequence, message count, and a SHA-256 over every
+stored message (sequence, subject, timestamp, headers, data).
+
+Run it on every node and compare the output. Replicas of one stream must
+produce identical lines; any difference means the replicas have diverged, and
+--seqs prints a hash per sequence to find where. For a check that tells a
+diverged replica from one that is merely behind, run it with --json --seqs on
+every node and pass the results to "spx admin kv compare".
+
+It never touches the live store. A direct get over NATS may be answered by any
+replica, so instead the streams tree is copied to --work-dir and read by a
+private in-process server with no listener, which is removed afterwards. The
+copy's stream config is set to one replica with no expiry so it can be loaded.
+
+Writes landing during the copy, or applied but not yet flushed to disk, can
+make a replica's tail differ, so gate API writes and re-run before calling a
+tail difference divergence.`,
+	Run: runKVDigest,
+}
+
+func init() {
+	adminCmd.AddCommand(kvCmd)
+	kvCmd.AddCommand(kvDigestCmd)
+	kvDigestCmd.Flags().String("store-dir", "", "JetStream store to read (default <spinifex-dir>/nats/jetstream)")
+	kvDigestCmd.Flags().String("work-dir", os.TempDir(), "Directory to hold the temporary copy")
+	kvDigestCmd.Flags().StringSlice("stream", nil, "Only these streams (repeatable)")
+	kvDigestCmd.Flags().Bool("seqs", false, "Also print a hash per sequence")
+	kvDigestCmd.Flags().Bool("json", false, "Print one JSON document, the input to \"spx admin kv compare\"")
+}
+
+// nodeDigest is one node's digest report, the unit "kv compare" reads.
+type nodeDigest struct {
+	Host     string         `json:"host"`
+	Store    string         `json:"store"`
+	CopiedAt time.Time      `json:"copied_at"`
+	Skipped  []string       `json:"skipped,omitempty"`
+	Streams  []streamDigest `json:"streams"`
+}
+
+// streamDigest is one stream's content as held by one replica.
+type streamDigest struct {
+	streamMeta
+
+	Name     string      `json:"name"`
+	FirstSeq uint64      `json:"first_seq"`
+	LastSeq  uint64      `json:"last_seq"`
+	Msgs     uint64      `json:"msgs"`
+	Digest   string      `json:"sha256"`
+	Seqs     []seqDigest `json:"seqs,omitempty"`
+}
+
+// streamMeta is the part of the live stream config that decides which replica
+// differences are legitimate. It is read before the copy is rewritten.
+type streamMeta struct {
+	Replicas    int           `json:"replicas,omitempty"`
+	Retention   string        `json:"retention,omitempty"`
+	MaxAge      time.Duration `json:"max_age_ns,omitempty"`
+	AllowMsgTTL bool          `json:"allow_msg_ttl,omitempty"`
+	MaxMsgs     int64         `json:"max_msgs,omitempty"`
+	MaxBytes    int64         `json:"max_bytes,omitempty"`
+}
+
+// seqDigest is the hash of one stored sequence, or "deleted" when the replica
+// has no message at that sequence. Subject, time and TTL are empty then.
+type seqDigest struct {
+	Seq     uint64        `json:"seq"`
+	Hash    string        `json:"hash"`
+	Subject string        `json:"subject,omitempty"`
+	Time    time.Time     `json:"time,omitzero"`
+	TTL     time.Duration `json:"ttl_ns,omitempty"`
+}
+
+// deletedHash marks a sequence the replica holds no message for.
+const deletedHash = "deleted"
+
+func runKVDigest(cmd *cobra.Command, _ []string) {
+	storeDir, _ := cmd.Flags().GetString("store-dir")
+	if storeDir == "" {
+		spxRoot, _ := cmd.Flags().GetString("spinifex-dir")
+		if spxRoot == "" {
+			spxRoot = DefaultDataDir()
+		}
+		storeDir = jetStreamStoreDir(spxRoot)
+	}
+	workParent, _ := cmd.Flags().GetString("work-dir")
+	only, _ := cmd.Flags().GetStringSlice("stream")
+	withSeqs, _ := cmd.Flags().GetBool("seqs")
+	asJSON, _ := cmd.Flags().GetBool("json")
+
+	work, err := os.MkdirTemp(workParent, "spx-kv-digest-")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		os.Exit(1)
+	}
+	defer os.RemoveAll(work)
+
+	copiedAt := time.Now().UTC()
+	skipped, metas, err := copyStreamTree(storeDir, filepath.Join(work, "jetstream"))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		os.RemoveAll(work)
+		os.Exit(1)
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Minute)
+	defer cancel()
+	digests, err := digestStore(ctx, work, only, withSeqs, metas)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		cancel()
+		os.RemoveAll(work)
+		os.Exit(1)
+	}
+
+	host, _ := os.Hostname()
+	if asJSON {
+		enc := json.NewEncoder(cmd.OutOrStdout())
+		if err := enc.Encode(nodeDigest{Host: host, Store: storeDir, CopiedAt: copiedAt, Skipped: skipped, Streams: digests}); err != nil {
+			fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+			os.Exit(1)
+		}
+		return
+	}
+	writeDigests(cmd.OutOrStdout(), host, storeDir, copiedAt, skipped, digests)
+}
+
+// writeDigests prints the report. Only the "#" lines carry node-specific
+// values, so `grep -v '^#'` output from two replicas diffs cleanly.
+func writeDigests(w io.Writer, host, storeDir string, copiedAt time.Time, skipped []string, digests []streamDigest) {
+	fmt.Fprintf(w, "# host=%s store=%s copied_at=%s\n", host, storeDir, copiedAt.Format(time.RFC3339))
+	for _, s := range skipped {
+		fmt.Fprintf(w, "# skipped account stream (not read): %s\n", s)
+	}
+	for _, d := range digests {
+		fmt.Fprintf(w, "%s\tfirst=%d\tlast=%d\tmsgs=%d\tsha256=%s\n", d.Name, d.FirstSeq, d.LastSeq, d.Msgs, d.Digest)
+		for _, s := range d.Seqs {
+			fmt.Fprintf(w, "%s\tseq=%d\t%s\n", d.Name, s.Seq, s.Hash)
+		}
+	}
+}
+
+// globalAccountDir is the account Spinifex's token-authenticated clients use.
+// Streams under any other account are listed but not read.
+const globalAccountDir = "$G"
+
+// copyStreamTree copies <storeDir>/$G/streams to <dst>/$G/streams, leaving out
+// consumer state (obs). It returns streams found under other accounts and each
+// copied stream's config as the live server held it, keyed by stream name.
+func copyStreamTree(storeDir, dst string) (skipped []string, metas map[string]streamMeta, err error) {
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return nil, nil, err
+	}
+	if len(streams) == 0 {
+		return nil, nil, fmt.Errorf("no streams found under %s", storeDir)
+	}
+	for _, s := range streams {
+		if !strings.HasPrefix(s, globalAccountDir+"/") {
+			skipped = append(skipped, s)
+		}
+	}
+
+	src := filepath.Join(storeDir, globalAccountDir, "streams")
+	out := filepath.Join(dst, globalAccountDir, "streams")
+	err = filepath.WalkDir(src, func(path string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		rel, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		// <stream>/obs holds consumers, which are not stream content.
+		if parts := strings.Split(rel, string(filepath.Separator)); d.IsDir() && len(parts) == 2 && parts[1] == "obs" {
+			return filepath.SkipDir
+		}
+		target := filepath.Join(out, rel)
+		if d.IsDir() {
+			return os.MkdirAll(target, 0o700)
+		}
+		if !d.Type().IsRegular() {
+			return nil
+		}
+		return copyFile(path, target)
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("copy streams from %s: %w", src, err)
+	}
+	entries, err := os.ReadDir(out)
+	if err != nil {
+		return nil, nil, err
+	}
+	metas = make(map[string]streamMeta, len(entries))
+	for _, e := range entries {
+		if e.IsDir() {
+			meta, err := rewriteStreamMetaStandalone(filepath.Join(out, e.Name()))
+			if err != nil {
+				return nil, nil, err
+			}
+			metas[e.Name()] = meta
+		}
+	}
+	return skipped, metas, nil
+}
+
+// rewriteStreamMetaStandalone makes a copied stream loadable by a standalone
+// server, which refuses replicas > 1 and would expire aged messages on
+// recovery. meta.sum is highwayhash64 keyed by sha256 of the directory name.
+// It returns the settings as they were before the rewrite.
+func rewriteStreamMetaStandalone(streamDir string) (streamMeta, error) {
+	metaPath := filepath.Join(streamDir, "meta.inf")
+	raw, err := os.ReadFile(metaPath)
+	if errors.Is(err, os.ErrNotExist) {
+		return streamMeta{}, nil
+	}
+	if err != nil {
+		return streamMeta{}, err
+	}
+	var orig struct {
+		Replicas    int           `json:"num_replicas"`
+		Retention   string        `json:"retention"`
+		MaxAge      time.Duration `json:"max_age"`
+		AllowMsgTTL bool          `json:"allow_msg_ttl"`
+		MaxMsgs     int64         `json:"max_msgs"`
+		MaxBytes    int64         `json:"max_bytes"`
+	}
+	if err := json.Unmarshal(raw, &orig); err != nil {
+		return streamMeta{}, fmt.Errorf("parse %s: %w", metaPath, err)
+	}
+	kept := streamMeta{
+		Replicas: orig.Replicas, Retention: orig.Retention, MaxAge: orig.MaxAge,
+		AllowMsgTTL: orig.AllowMsgTTL, MaxMsgs: orig.MaxMsgs, MaxBytes: orig.MaxBytes,
+	}
+	dec := json.NewDecoder(bytes.NewReader(raw))
+	dec.UseNumber()
+	var meta map[string]any
+	if err := dec.Decode(&meta); err != nil {
+		return streamMeta{}, fmt.Errorf("parse %s: %w", metaPath, err)
+	}
+	meta["num_replicas"] = 1
+	meta["max_age"] = 0
+	delete(meta, "allow_msg_ttl")
+	delete(meta, "subject_delete_marker_ttl")
+	delete(meta, "placement")
+	out, err := json.Marshal(meta)
+	if err != nil {
+		return streamMeta{}, err
+	}
+	key := sha256.Sum256([]byte(filepath.Base(streamDir)))
+	hh, err := highwayhash.New64(key[:])
+	if err != nil {
+		return streamMeta{}, err
+	}
+	hh.Write(out)
+	if err := os.WriteFile(metaPath, out, 0o600); err != nil {
+		return streamMeta{}, err
+	}
+	return kept, os.WriteFile(filepath.Join(streamDir, "meta.sum"), []byte(hex.EncodeToString(hh.Sum(nil))), 0o600)
+}
+
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(out, in); err != nil {
+		out.Close()
+		return err
+	}
+	return out.Close()
+}
+
+// digestStore opens storeParent (holding jetstream/) in a standalone server
+// that accepts in-process connections only, and digests each stream.
+func digestStore(ctx context.Context, storeParent string, only []string, withSeqs bool, metas map[string]streamMeta) ([]streamDigest, error) {
+	ns, err := server.NewServer(&server.Options{
+		ServerName: "spx-kv-digest",
+		JetStream:  true,
+		StoreDir:   storeParent,
+		DontListen: true,
+		NoLog:      true,
+		NoSigs:     true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("create reader server: %w", err)
+	}
+	ns.Start()
+	defer func() {
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}()
+	if !ns.ReadyForConnections(30 * time.Second) {
+		return nil, errors.New("reader server did not become ready")
+	}
+
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	if err != nil {
+		return nil, fmt.Errorf("connect to reader server: %w", err)
+	}
+	defer nc.Close()
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return nil, err
+	}
+
+	var names []string
+	lister := js.StreamNames(ctx)
+	for name := range lister.Name() {
+		if len(only) == 0 || slices.Contains(only, name) {
+			names = append(names, name)
+		}
+	}
+	if err := lister.Err(); err != nil {
+		return nil, fmt.Errorf("list streams: %w", err)
+	}
+	sort.Strings(names)
+
+	digests := make([]streamDigest, 0, len(names))
+	for _, name := range names {
+		d, err := digestStream(ctx, js, name, withSeqs)
+		if err != nil {
+			return nil, err
+		}
+		d.streamMeta = metas[name]
+		digests = append(digests, d)
+	}
+	return digests, nil
+}
+
+func digestStream(ctx context.Context, js jetstream.JetStream, name string, withSeqs bool) (streamDigest, error) {
+	s, err := js.Stream(ctx, name)
+	if err != nil {
+		return streamDigest{}, fmt.Errorf("open stream %s: %w", name, err)
+	}
+	info, err := s.Info(ctx)
+	if err != nil {
+		return streamDigest{}, fmt.Errorf("stream info %s: %w", name, err)
+	}
+	st := info.State
+	d := streamDigest{Name: name, FirstSeq: st.FirstSeq, LastSeq: st.LastSeq, Msgs: st.Msgs}
+
+	whole := sha256.New()
+	for seq := st.FirstSeq; st.Msgs > 0 && seq <= st.LastSeq; seq++ {
+		msg, err := s.GetMsg(ctx, seq)
+		sd := seqDigest{Seq: seq}
+		switch {
+		case errors.Is(err, jetstream.ErrMsgNotFound):
+			sd.Hash = deletedHash
+		case err != nil:
+			return streamDigest{}, fmt.Errorf("read %s seq %d: %w", name, seq, err)
+		default:
+			sd.Hash = hashStoredMsg(msg)
+			sd.Subject = msg.Subject
+			sd.Time = msg.Time.UTC()
+			sd.TTL = msgTTL(msg.Header.Get(jetstream.MsgTTLHeader))
+		}
+		fmt.Fprintf(whole, "%d:%s\n", seq, sd.Hash)
+		if withSeqs {
+			d.Seqs = append(d.Seqs, sd)
+		}
+	}
+	d.Digest = hex.EncodeToString(whole.Sum(nil))
+	return d, nil
+}
+
+// hashStoredMsg hashes everything a replica stores for one sequence. Header
+// keys are sorted so the hash does not depend on map iteration order.
+func hashStoredMsg(m *jetstream.RawStreamMsg) string {
+	h := sha256.New()
+	var n [8]byte
+	field := func(b []byte) {
+		binary.BigEndian.PutUint64(n[:], uint64(len(b)))
+		h.Write(n[:])
+		h.Write(b)
+	}
+	binary.BigEndian.PutUint64(n[:], m.Sequence)
+	h.Write(n[:])
+	field([]byte(m.Subject))
+	binary.BigEndian.PutUint64(n[:], uint64(m.Time.UnixNano()))
+	h.Write(n[:])
+	keys := slices.Sorted(maps.Keys(m.Header))
+	for _, k := range keys {
+		field([]byte(k))
+		for _, v := range m.Header[k] {
+			field([]byte(v))
+		}
+	}
+	field(m.Data)
+	return hex.EncodeToString(h.Sum(nil))[:16]
+}
+
+// msgTTL parses a Nats-TTL header: a Go duration or whole seconds. "never", an
+// absent header or an unparseable one yield zero, meaning no expiry.
+func msgTTL(v string) time.Duration {
+	if v == "" {
+		return 0
+	}
+	if d, err := time.ParseDuration(v); err == nil && d > 0 {
+		return d
+	}
+	if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
+		return time.Duration(n) * time.Second
+	}
+	return 0
+}

cmd/spinifex/cmd/admin_kv_compare.go

@@ -0,0 +1,357 @@
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"maps"
+	"os"
+	"slices"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+var kvCompareCmd = &cobra.Command{
+	Use:   "compare <digest.json>...",
+	Short: "Compare the kv digests of every node and report replicas that have diverged",
+	Long: `Read the "spx admin kv digest --json --seqs" output of every node in the
+cluster, one file per node ("-" reads any number of them from stdin), and
+check that every stream's replicas hold the same content.
+
+A replica that is merely behind is not divergent, so only these fail:
+
+  - the same sequence holds different content on two replicas;
+  - a message held by one replica is missing from another, and nothing on
+    that replica explains it: no later write to the same subject, no expiry
+    under max_age or a message TTL, and no stream limit dropping its head;
+  - a stream is held by fewer nodes than its replica count, or by more;
+  - replicas disagree on the replica count;
+  - no sequence range is common to every replica, so nothing can be checked.
+
+Sequences past the slowest replica's last one are not compared. Digests are
+taken from live stores a few seconds apart, so re-run before trusting a
+failure that could be a write landing between two copies.
+
+Exits 0 when every stream is consistent, 1 on divergence, 2 on bad input.`,
+	Args: cobra.MinimumNArgs(1),
+	Run:  runKVCompare,
+}
+
+func init() {
+	kvCmd.AddCommand(kvCompareCmd)
+}
+
+func runKVCompare(cmd *cobra.Command, args []string) {
+	var nodes []nodeDigest
+	for _, arg := range args {
+		var r io.Reader
+		if arg == "-" {
+			r = cmd.InOrStdin()
+		} else {
+			f, err := os.Open(arg)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+				os.Exit(2)
+			}
+			defer f.Close()
+			r = f
+		}
+		got, err := loadNodeDigests(r, arg)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+			os.Exit(2)
+		}
+		nodes = append(nodes, got...)
+	}
+	res, err := compareNodeDigests(nodes)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+		os.Exit(2)
+	}
+	writeCompareResult(cmd.OutOrStdout(), nodes, res)
+	if res.divergent() {
+		os.Exit(1)
+	}
+}
+
+// loadNodeDigests reads every JSON digest document in r.
+func loadNodeDigests(r io.Reader, source string) ([]nodeDigest, error) {
+	dec := json.NewDecoder(r)
+	var out []nodeDigest
+	for {
+		var d nodeDigest
+		err := dec.Decode(&d)
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("%s: document %d: %w", source, len(out)+1, err)
+		}
+		if d.Host == "" {
+			return nil, fmt.Errorf("%s: document %d has no host; is it \"kv digest --json\" output?", source, len(out)+1)
+		}
+		out = append(out, d)
+	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("%s: no digest documents", source)
+	}
+	return out, nil
+}
+
+// maxFindingsPerStream caps what one stream prints; the rest are counted.
+const maxFindingsPerStream = 5
+
+type streamVerdict struct {
+	Name     string
+	Holders  []string
+	Problems []string
+	Notes    []string
+}
+
+type kvCompareResult struct {
+	Streams []streamVerdict
+}
+
+func (r kvCompareResult) divergent() bool {
+	return slices.ContainsFunc(r.Streams, func(v streamVerdict) bool { return len(v.Problems) > 0 })
+}
+
+// replicaView is one node's copy of one stream, indexed for the per-sequence rules.
+type replicaView struct {
+	host          string
+	copiedAt      time.Time
+	d             *streamDigest
+	bySeq         map[uint64]seqDigest
+	lastBySubject map[string]uint64
+}
+
+func compareNodeDigests(nodes []nodeDigest) (kvCompareResult, error) {
+	if len(nodes) < 2 {
+		return kvCompareResult{}, fmt.Errorf("need the digests of at least two nodes, got %d", len(nodes))
+	}
+	hosts := make([]string, 0, len(nodes))
+	byName := map[string][]*replicaView{}
+	for i := range nodes {
+		n := &nodes[i]
+		if slices.Contains(hosts, n.Host) {
+			return kvCompareResult{}, fmt.Errorf("two digests are from host %s", n.Host)
+		}
+		hosts = append(hosts, n.Host)
+		for j := range n.Streams {
+			d := &n.Streams[j]
+			v := &replicaView{host: n.Host, copiedAt: n.CopiedAt, d: d,
+				bySeq: make(map[uint64]seqDigest, len(d.Seqs)), lastBySubject: map[string]uint64{}}
+			for _, s := range d.Seqs {
+				v.bySeq[s.Seq] = s
+				if s.Hash != deletedHash && s.Seq > v.lastBySubject[s.Subject] {
+					v.lastBySubject[s.Subject] = s.Seq
+				}
+			}
+			byName[d.Name] = append(byName[d.Name], v)
+		}
+	}
+	names := slices.Sorted(maps.Keys(byName))
+
+	var res kvCompareResult
+	for _, name := range names {
+		res.Streams = append(res.Streams, compareStream(name, hosts, byName[name]))
+	}
+	return res, nil
+}
+
+func compareStream(name string, hosts []string, views []*replicaView) streamVerdict {
+	v := streamVerdict{Name: name}
+	for _, r := range views {
+		v.Holders = append(v.Holders, r.host)
+	}
+
+	counts := map[int][]string{}
+	replicas := 0
+	for _, r := range views {
+		if n := r.d.Replicas; n > 0 {
+			counts[n] = append(counts[n], r.host)
+			replicas = max(replicas, n)
+		}
+	}
+	if len(counts) > 1 {
+		var parts []string
+		for n, hs := range counts {
+			parts = append(parts, fmt.Sprintf("%d on %s", n, strings.Join(hs, ",")))
+		}
+		sort.Strings(parts)
+		v.Problems = append(v.Problems, "replicas disagree on the replica count: "+strings.Join(parts, "; "))
+	}
+	switch {
+	case replicas > 0 && len(views) < replicas:
+		missing := slices.DeleteFunc(slices.Clone(hosts), func(h string) bool { return slices.Contains(v.Holders, h) })
+		msg := fmt.Sprintf("held by %d of %d replicas; missing from %s", len(views), replicas, strings.Join(missing, ","))
+		if len(hosts) < replicas {
+			msg += fmt.Sprintf(" (only %d digests given)", len(hosts))
+		}
+		v.Problems = append(v.Problems, msg)
+	case replicas > 0 && len(views) > replicas:
+		v.Problems = append(v.Problems, fmt.Sprintf("held by %d nodes but configured for %d replicas; a node outside the stream's group still holds a copy", len(views), replicas))
+	}
+	if len(views) < 2 {
+		v.Notes = append(v.Notes, "single copy, nothing to compare")
+		return v
+	}
+
+	identical := true
+	for _, r := range views[1:] {
+		if r.d.Digest != views[0].d.Digest || r.d.FirstSeq != views[0].d.FirstSeq || r.d.LastSeq != views[0].d.LastSeq {
+			identical = false
+			break
+		}
+	}
+	if identical {
+		v.Notes = append(v.Notes, fmt.Sprintf("identical, seqs %d-%d", views[0].d.FirstSeq, views[0].d.LastSeq))
+		return v
+	}
+	for _, r := range views {
+		if r.d.Msgs > 0 && len(r.d.Seqs) == 0 {
+			v.Problems = append(v.Problems, "contents differ and "+r.host+"'s digest has no per-sequence detail; re-run digest with --seqs")
+			return v
+		}
+	}
+
+	lo, hi := views[0].d.FirstSeq, views[0].d.LastSeq
+	seqSet := map[uint64]struct{}{}
+	for _, r := range views {
+		lo, hi = max(lo, r.d.FirstSeq), min(hi, r.d.LastSeq)
+		for s := range r.bySeq {
+			seqSet[s] = struct{}{}
+		}
+	}
+	if lo > hi {
+		v.Problems = append(v.Problems, "no sequence range is common to every replica, so nothing could be compared")
+		return v
+	}
+	seqs := make([]uint64, 0, len(seqSet))
+	for s := range seqSet {
+		if s <= hi {
+			seqs = append(seqs, s)
+		}
+	}
+	slices.Sort(seqs)
+
+	var problems []string
+	explained := 0
+	for _, seq := range seqs {
+		var present, absent []*replicaView
+		for _, r := range views {
+			if e, ok := r.bySeq[seq]; ok && e.Hash != deletedHash {
+				present = append(present, r)
+			} else {
+				absent = append(absent, r)
+			}
+		}
+		if len(present) == 0 {
+			continue
+		}
+		byHash := map[string][]string{}
+		for _, r := range present {
+			h := r.bySeq[seq].Hash
+			byHash[h] = append(byHash[h], r.host)
+		}
+		if len(byHash) > 1 {
+			var parts []string
+			for h, hs := range byHash {
+				parts = append(parts, strings.Join(hs, ",")+"="+h)
+			}
+			sort.Strings(parts)
+			problems = append(problems, fmt.Sprintf("seq %d %s holds different content: %s", seq, present[0].bySeq[seq].Subject, strings.Join(parts, " ")))
+			continue
+		}
+		msg := present[0].bySeq[seq]
+		for _, r := range absent {
+			if absenceExplained(r, msg) {
+				explained++
+				continue
+			}
+			problems = append(problems, fmt.Sprintf("seq %d %s is on %s but missing from %s, with no later write to that subject, expiry or limit to explain it",
+				seq, msg.Subject, hostsOf(present), r.host))
+		}
+	}
+	if len(problems) > maxFindingsPerStream {
+		problems = append(problems[:maxFindingsPerStream], fmt.Sprintf("... and %d more", len(problems)-maxFindingsPerStream))
+	}
+	v.Problems = append(v.Problems, problems...)
+
+	note := fmt.Sprintf("seqs %d-%d compared", lo, hi)
+	if explained > 0 {
+		note += fmt.Sprintf(", %d superseded or expired messages missing from some replicas", explained)
+	}
+	var ahead []string
+	for _, r := range views {
+		if r.d.LastSeq > hi {
+			ahead = append(ahead, fmt.Sprintf("%s +%d", r.host, r.d.LastSeq-hi))
+		}
+	}
+	if len(ahead) > 0 {
+		note += ", tail not compared: " + strings.Join(ahead, " ")
+	}
+	v.Notes = append(v.Notes, note)
+	return v
+}
+
+// absenceExplained reports whether a replica lacking msg removed it legitimately:
+// the subject was written again later, the message aged out before that replica
+// was copied, or a stream limit dropped it from a head the replica has moved past.
+func absenceExplained(r *replicaView, msg seqDigest) bool {
+	if msg.Subject == "" {
+		return false
+	}
+	if r.lastBySubject[msg.Subject] > msg.Seq {
+		return true
+	}
+	meta := r.d.streamMeta
+	if meta.Retention != "" && meta.Retention != "limits" {
+		return true
+	}
+	if !msg.Time.IsZero() {
+		if meta.MaxAge > 0 && msg.Time.Add(meta.MaxAge).Before(r.copiedAt) {
+			return true
+		}
+		if msg.TTL > 0 && msg.Time.Add(msg.TTL).Before(r.copiedAt) {
+			return true
+		}
+	}
+	return msg.Seq < r.d.FirstSeq && (meta.MaxMsgs > 0 || meta.MaxBytes > 0)
+}
+
+func hostsOf(views []*replicaView) string {
+	hs := make([]string, len(views))
+	for i, r := range views {
+		hs[i] = r.host
+	}
+	return strings.Join(hs, ",")
+}
+
+func writeCompareResult(w io.Writer, nodes []nodeDigest, res kvCompareResult) {
+	fmt.Fprintf(w, "compared %d digests:", len(nodes))
+	for _, n := range nodes {
+		fmt.Fprintf(w, " %s@%s", n.Host, n.CopiedAt.Format(time.TimeOnly))
+	}
+	fmt.Fprintln(w)
+	bad := 0
+	for _, v := range res.Streams {
+		status := "ok"
+		if len(v.Problems) > 0 {
+			status = "DIVERGED"
+			bad++
+		}
+		fmt.Fprintf(w, "%-9s %s  [%s]\n", status, v.Name, strings.Join(v.Holders, ","))
+		for _, p := range v.Problems {
+			fmt.Fprintf(w, "    ! %s\n", p)
+		}
+		for _, n := range v.Notes {
+			fmt.Fprintf(w, "      %s\n", n)
+		}
+	}
+	fmt.Fprintf(w, "%d streams: %d consistent, %d diverged\n", len(res.Streams), len(res.Streams)-bad, bad)
+}

cmd/spinifex/cmd/admin_kv_compare_test.go

@@ -0,0 +1,233 @@
+package cmd_test
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var t0 = time.Date(2026, 9, 15, 10, 0, 0, 0, time.UTC)
+
+type msg struct {
+	seq     uint64
+	subject string
+	hash    string // "deleted" for a gap
+	at      time.Time
+}
+
+type stream struct {
+	name     string
+	replicas int
+	maxAge   time.Duration
+	maxMsgs  int64
+	msgs     []msg
+	noSeqs   bool
+}
+
+// doc renders one node's digest the way "kv digest --json --seqs" prints it.
+func doc(t *testing.T, host string, copiedAt time.Time, streams ...stream) string {
+	t.Helper()
+	var out []map[string]any
+	for _, s := range streams {
+		var seqs []map[string]any
+		var first, last, count uint64
+		digest := host + ":" // only equality matters to compare
+		for _, m := range s.msgs {
+			if first == 0 {
+				first = m.seq
+			}
+			last = m.seq
+			e := map[string]any{"seq": m.seq, "hash": m.hash}
+			if m.hash != "deleted" {
+				count++
+				e["subject"] = m.subject
+				at := m.at
+				if at.IsZero() {
+					at = t0
+				}
+				e["time"] = at
+			}
+			digest += m.hash + ","
+			seqs = append(seqs, e)
+		}
+		sd := map[string]any{"name": s.name, "first_seq": first, "last_seq": last, "msgs": count, "sha256": digest}
+		if s.replicas > 0 {
+			sd["replicas"] = s.replicas
+		}
+		if s.maxAge > 0 {
+			sd["max_age_ns"] = s.maxAge
+		}
+		if s.maxMsgs > 0 {
+			sd["max_msgs"] = s.maxMsgs
+		}
+		if !s.noSeqs {
+			sd["seqs"] = seqs
+		}
+		out = append(out, sd)
+	}
+	raw, err := json.Marshal(map[string]any{"host": host, "store": "/s", "copied_at": copiedAt, "streams": out})
+	require.NoError(t, err)
+	return string(raw)
+}
+
+func kvs(msgs ...msg) []msg { return msgs }
+
+func TestKVCompareIdentical(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 3, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0})}
+	// The digest field embeds the host, so this exercises the per-seq path too.
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, s), doc(t, "n3", t0, s))
+	require.NoError(t, err)
+	assert.False(t, divergent, report)
+	assert.Contains(t, report, "1 streams: 1 consistent, 0 diverged")
+}
+
+// The prod failure: each node wrote its own copy before formation, so the same
+// sequence holds different content on different replicas.
+func TestKVCompareSameSeqDifferentContent(t *testing.T) {
+	a := stream{name: "KV_acct", replicas: 2, msgs: kvs(msg{1, "$KV.acct.000000000002", "aaaa", t0})}
+	b := stream{name: "KV_acct", replicas: 2, msgs: kvs(msg{1, "$KV.acct.000000000002", "bbbb", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "node1", t0, a), doc(t, "node2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "DIVERGED  KV_acct")
+	assert.Contains(t, report, "seq 1 $KV.acct.000000000002 holds different content: node1=aaaa node2=bbbb")
+}
+
+func TestKVCompareMissingMessage(t *testing.T) {
+	full := kvs(msg{1, "k", "h1", t0}, msg{2, "j", "h2", t0}, msg{3, "k", "h3", t0})
+
+	t.Run("superseded by a later write to the subject", func(t *testing.T) {
+		lagging := stream{name: "KV_a", replicas: 2, msgs: full}
+		purged := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "", "deleted", time.Time{}}, full[1], full[2])}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, lagging), doc(t, "n2", t0, purged))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+		assert.Contains(t, report, "1 superseded or expired")
+	})
+
+	t.Run("nothing explains it", func(t *testing.T) {
+		have := stream{name: "KV_a", replicas: 2, msgs: full}
+		lost := stream{name: "KV_a", replicas: 2, msgs: kvs(full[0], msg{2, "", "deleted", time.Time{}}, full[2])}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, lost))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "seq 2 j is on n1 but missing from n2")
+	})
+
+	t.Run("aged out under max_age before the replica was copied", func(t *testing.T) {
+		old := t0.Add(-time.Hour)
+		have := stream{name: "KV_lease", replicas: 2, maxAge: time.Minute, msgs: kvs(msg{1, "l", "h1", old}, msg{2, "m", "h2", t0})}
+		gone := stream{name: "KV_lease", replicas: 2, maxAge: time.Minute, msgs: kvs(msg{2, "m", "h2", t0})}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, gone))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+
+		have.maxAge, gone.maxAge = 0, 0
+		divergent, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, gone))
+		require.NoError(t, err)
+		assert.True(t, divergent, "without max_age the same gap is unexplained")
+	})
+
+	t.Run("head dropped by a stream limit", func(t *testing.T) {
+		behind := stream{name: "S", replicas: 2, maxMsgs: 2, msgs: kvs(msg{1, "a", "h1", t0}, msg{2, "b", "h2", t0})}
+		ahead := stream{name: "S", replicas: 2, maxMsgs: 2, msgs: kvs(msg{2, "b", "h2", t0}, msg{3, "c", "h3", t0})}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, behind), doc(t, "n2", t0, ahead))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+	})
+}
+
+func TestKVCompareTailLag(t *testing.T) {
+	behind := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0})}
+	ahead := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0}, msg{3, "k", "h3", t0}, msg{4, "k", "h4", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, behind), doc(t, "n2", t0, ahead))
+	require.NoError(t, err)
+	assert.False(t, divergent, report)
+	assert.Contains(t, report, "tail not compared: n2 +2")
+}
+
+func TestKVCompareNoCommonRange(t *testing.T) {
+	a := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0})}
+	b := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{5, "k", "h5", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, a), doc(t, "n2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "no sequence range is common")
+}
+
+func TestKVCompareReplicaPlacement(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 3, msgs: kvs(msg{1, "k", "h1", t0})}
+	other := stream{name: "KV_b", replicas: 1, msgs: kvs(msg{1, "k", "h1", t0})}
+
+	t.Run("fewer holders than replicas", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, s), doc(t, "n3", t0, other))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "held by 2 of 3 replicas; missing from n3")
+	})
+
+	t.Run("more holders than replicas", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, other), doc(t, "n2", t0, other))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "held by 2 nodes but configured for 1 replicas")
+	})
+
+	t.Run("replica counts disagree", func(t *testing.T) {
+		r1 := s
+		r1.replicas = 1
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, r1))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "replicas disagree on the replica count")
+	})
+
+	t.Run("single replica stream on one node", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, other), doc(t, "n2", t0))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+	})
+}
+
+func TestKVCompareNeedsSeqs(t *testing.T) {
+	a := stream{name: "KV_a", replicas: 2, noSeqs: true, msgs: kvs(msg{1, "k", "h1", t0})}
+	b := stream{name: "KV_a", replicas: 2, noSeqs: true, msgs: kvs(msg{1, "k", "h2", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, a), doc(t, "n2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "re-run digest with --seqs")
+}
+
+func TestKVCompareInput(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0})}
+
+	// Several documents in one stream, as "-" reads them from stdin.
+	divergent, _, err := cmd.CompareKVDigests(doc(t, "n1", t0, s) + "\n" + doc(t, "n2", t0, s))
+	require.NoError(t, err)
+	assert.False(t, divergent)
+
+	_, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, s))
+	assert.ErrorContains(t, err, "at least two nodes")
+
+	_, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n1", t0, s))
+	assert.ErrorContains(t, err, "two digests are from host n1")
+
+	_, _, err = cmd.CompareKVDigests(`KV_a	first=1	last=1`)
+	assert.Error(t, err, "text digest output is rejected")
+
+	_, _, err = cmd.CompareKVDigests(`{"streams":[]}`)
+	assert.ErrorContains(t, err, "no host")
+
+	_, _, err = cmd.CompareKVDigests("  ")
+	assert.ErrorContains(t, err, "no digest documents")
+}
+
+func TestAdminInitDiscardsJetStreamByDefault(t *testing.T) {
+	assert.Equal(t, "true", cmd.AdminInitFlagDefault("discard-jetstream"))
+	assert.True(t, strings.HasPrefix(cmd.AdminInitFlagDefault("ipsec"), "true"))
+}

cmd/spinifex/cmd/admin_kv_replicas.go

@@ -0,0 +1,201 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/mulgadc/spinifex/spinifex/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/spf13/cobra"
+)
+
+var kvReplicasCmd = &cobra.Command{
+	Use:   "replicas",
+	Short: "Report how many nodes each KV bucket is replicated across",
+	Long: `Print one line per JetStream KV bucket: how many replicas it has, how many the
+cluster's size entitles it to, and which nodes currently hold it.
+
+A bucket with fewer replicas than the cluster has nodes lives on fewer nodes
+than exist, and every node depends on those. Losing one of them makes the
+bucket unreadable and unwritable from every node at once, which is a
+cluster-wide outage caused by a single-node event.
+
+A bucket also has to be able to serve, which is a different question from how
+many replicas it was configured for. ONLINE counts the replicas that are caught
+up and reachable; a bucket whose ONLINE is not a majority of REPLICAS cannot
+accept a write no matter what its replica count says.
+
+--repair raises any under-replicated bucket to the cluster's replica count,
+capped at JetStream's maximum of five. It changes nothing else about a bucket
+and never lowers a replica count, so it is safe to run on a healthy cluster and
+safe to run twice.
+
+Exit status separates the two problems, so this can gate a deployment without
+parsing the output:
+
+  0  every bucket is at the cluster's replica count and can serve
+  1  at least one bucket is under-replicated — --repair is the answer
+  2  at least one bucket has no quorum, or the report could not be produced —
+     a node needs looking at, and --repair will not help`,
+	Run: runKVReplicas,
+}
+
+func init() {
+	kvCmd.AddCommand(kvReplicasCmd)
+	kvReplicasCmd.Flags().Bool("repair", false, "raise every under-replicated bucket to the cluster's node count")
+	kvReplicasCmd.Flags().Bool("json", false, "emit the report as JSON")
+}
+
+func runKVReplicas(cmd *cobra.Command, _ []string) {
+	repair, _ := cmd.Flags().GetBool("repair")
+	asJSON, _ := cmd.Flags().GetBool("json")
+
+	// Anything that stops the report being produced exits unreachable rather
+	// than under-replicated: an empty answer is not a cluster with no buckets,
+	// and --repair is not the response to it.
+	_, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "connect to cluster: %v\n", err)
+		os.Exit(exitUnreachable)
+	}
+	defer nc.Close()
+
+	js, err := jetstream.New(nc)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "get JetStream context: %v\n", err)
+		os.Exit(exitUnreachable)
+	}
+
+	// Background: this runs at CLI top level, where there is no request to
+	// inherit a deadline from and the process exits after the one command.
+	ctx := context.Background()
+	reports, err := kvutil.AuditBucketReplicas(ctx, js)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "audit KV bucket replicas: %v\n", err)
+		os.Exit(exitUnreachable)
+	}
+
+	if repair {
+		repairReplicaReports(ctx, js, reports, os.Stderr)
+	}
+
+	if err := writeReplicaReport(os.Stdout, reports, asJSON); err != nil {
+		fmt.Fprintf(os.Stderr, "write report: %v\n", err)
+		os.Exit(exitUnreachable)
+	}
+
+	os.Exit(replicaReportExit(reports))
+}
+
+// Exit statuses, documented in the command's own help because a deployment gate
+// reads them rather than the output.
+const (
+	exitHealthy = 0
+	// exitUnderReplicated is the repairable problem: the buckets serve, but on
+	// fewer nodes than the cluster has.
+	exitUnderReplicated = 1
+	// exitUnreachable is the problem repair cannot touch — a bucket that cannot
+	// accept a write, or a report that could not be produced at all.
+	exitUnreachable = 2
+)
+
+// replicaReportExit is the exit status for a set of reports.
+//
+// No quorum outranks under-replicated because they need different responses: one
+// is fixed by raising a count, the other by finding out why a node is not
+// answering. A bucket that is both should send the operator to the node.
+func replicaReportExit(reports []kvutil.BucketReport) int {
+	status := exitHealthy
+	for _, r := range reports {
+		if !r.HasQuorum() {
+			return exitUnreachable
+		}
+		if r.UnderReplicated() {
+			status = exitUnderReplicated
+		}
+	}
+	return status
+}
+
+// repairReplicaReports raises every under-replicated bucket and updates reports
+// in place for the ones that were raised.
+//
+// One bucket that cannot be placed must not stop the others being repaired, so a
+// failure is named on errOut and the sweep continues. The bucket it failed on
+// stays UNDER-REPLICATED in the report, which is already what makes the exit
+// status non-zero, so nothing is lost by not returning an error here.
+func repairReplicaReports(ctx context.Context, js jetstream.JetStream, reports []kvutil.BucketReport, errOut io.Writer) {
+	for i, r := range reports {
+		if !r.UnderReplicated() {
+			continue
+		}
+		if err := kvutil.RaiseBucketReplicas(ctx, js, r.Bucket, r.Want); err != nil {
+			fmt.Fprintf(errOut, "repair %s: %v\n", r.Bucket, err)
+			continue
+		}
+		reports[i].Replicas = r.Want
+	}
+}
+
+// underReplicatedCount is how many buckets are below the cluster's replica count.
+func underReplicatedCount(reports []kvutil.BucketReport) int {
+	under := 0
+	for _, r := range reports {
+		if r.UnderReplicated() {
+			under++
+		}
+	}
+	return under
+}
+
+// noQuorumCount is how many buckets cannot currently accept a write.
+func noQuorumCount(reports []kvutil.BucketReport) int {
+	stuck := 0
+	for _, r := range reports {
+		if !r.HasQuorum() {
+			stuck++
+		}
+	}
+	return stuck
+}
+
+// replicaStatus is the one word for a bucket's condition.
+//
+// No quorum wins over under-replicated when both are true, matching the exit
+// status: a bucket that cannot serve is the more urgent of the two and is not
+// what --repair is for.
+func replicaStatus(r kvutil.BucketReport) string {
+	switch {
+	case !r.HasQuorum():
+		return "NO-QUORUM"
+	case r.UnderReplicated():
+		return "UNDER-REPLICATED"
+	default:
+		return "ok"
+	}
+}
+
+// writeReplicaReport renders reports as a table, or as JSON when asJSON.
+func writeReplicaReport(out io.Writer, reports []kvutil.BucketReport, asJSON bool) error {
+	if asJSON {
+		return json.NewEncoder(out).Encode(reports)
+	}
+
+	w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "BUCKET\tREPLICAS\tWANT\tONLINE\tSTATUS\tLEADER\tHELD BY")
+	for _, r := range reports {
+		fmt.Fprintf(w, "%s\t%d\t%d\t%d\t%s\t%s\t%s\n",
+			r.Bucket, r.Replicas, r.Want, r.Online, replicaStatus(r), r.Leader, strings.Join(r.Peers, ","))
+	}
+	if err := w.Flush(); err != nil {
+		return err
+	}
+	_, err := fmt.Fprintf(out, "\n%d buckets, %d under-replicated, %d without quorum\n",
+		len(reports), underReplicatedCount(reports), noQuorumCount(reports))
+	return err
+}

cmd/spinifex/cmd/admin_kv_replicas_test.go

@@ -0,0 +1,133 @@
+//test:in-package — it tests this command's unexported rendering and repair
+// helpers, which exist only to be exercised without a live cluster.
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/clustersize"
+	"github.com/mulgadc/spinifex/spinifex/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/testutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWriteReplicaReport_NamesEveryUnderReplicatedBucket(t *testing.T) {
+	reports := []kvutil.BucketReport{
+		{Bucket: "healthy", Replicas: 3, Want: 3, Online: 3, Leader: "a", Peers: []string{"a", "b", "c"}},
+		{Bucket: "short", Replicas: 1, Want: 3, Online: 1, Leader: "a", Peers: []string{"a"}},
+		{Bucket: "stuck", Replicas: 3, Want: 3, Online: 1, Peers: []string{"a", "b", "c"}},
+	}
+
+	var out bytes.Buffer
+	require.NoError(t, writeReplicaReport(&out, reports, false))
+	got := out.String()
+
+	assert.Contains(t, got, "BUCKET")
+	assert.Contains(t, got, "HELD BY")
+	assert.Regexp(t, `healthy\s+3\s+3\s+3\s+ok\s+a\s+a,b,c`, got)
+	assert.Regexp(t, `short\s+1\s+3\s+1\s+UNDER-REPLICATED\s+a\s+a`, got)
+	// A bucket that cannot serve reads as NO-QUORUM even though its configured
+	// count is right, because raising a count is not the response to it.
+	assert.Regexp(t, `stuck\s+3\s+3\s+1\s+NO-QUORUM`, got)
+	assert.Contains(t, got, "3 buckets, 1 under-replicated, 1 without quorum")
+}
+
+// TestReplicaReportExit pins the three exit classes a deployment gate reads. They
+// are separate because the responses are: a raise fixes an under-replicated
+// bucket, and nothing the command can do fixes one that will not answer.
+func TestReplicaReportExit(t *testing.T) {
+	healthy := kvutil.BucketReport{Bucket: "a", Replicas: 3, Want: 3, Online: 3}
+	short := kvutil.BucketReport{Bucket: "b", Replicas: 1, Want: 3, Online: 1}
+	stuck := kvutil.BucketReport{Bucket: "c", Replicas: 3, Want: 3, Online: 1}
+
+	assert.Equal(t, exitHealthy, replicaReportExit(nil))
+	assert.Equal(t, exitHealthy, replicaReportExit([]kvutil.BucketReport{healthy}))
+	assert.Equal(t, exitUnderReplicated, replicaReportExit([]kvutil.BucketReport{healthy, short}))
+	assert.Equal(t, exitUnreachable, replicaReportExit([]kvutil.BucketReport{healthy, stuck}))
+	// Both problems at once sends the operator to the node rather than to --repair.
+	assert.Equal(t, exitUnreachable, replicaReportExit([]kvutil.BucketReport{short, stuck}))
+}
+
+// TestWriteReplicaReport_JSONIsWhatTheE2ESuiteDecodes keeps the machine-readable
+// form honest: the e2e suite and anything gating a deployment parse these field
+// names, so a rename here breaks them silently.
+func TestWriteReplicaReport_JSONIsWhatTheE2ESuiteDecodes(t *testing.T) {
+	var out bytes.Buffer
+	require.NoError(t, writeReplicaReport(&out, []kvutil.BucketReport{
+		{Bucket: "short", Replicas: 1, Want: 3, Cluster: "spinifex", Online: 1, Leader: "a", Peers: []string{"a"}},
+	}, true))
+
+	assert.Contains(t, out.String(), `"bucket":"short"`)
+
+	var decoded []kvutil.BucketReport
+	require.NoError(t, json.Unmarshal(out.Bytes(), &decoded))
+	require.Len(t, decoded, 1)
+	assert.Equal(t, "short", decoded[0].Bucket)
+	assert.Equal(t, 1, decoded[0].Replicas)
+	assert.Equal(t, 3, decoded[0].Want)
+	assert.Equal(t, 1, decoded[0].Online)
+	assert.Equal(t, "a", decoded[0].Leader)
+	assert.True(t, decoded[0].UnderReplicated())
+	assert.True(t, decoded[0].HasQuorum())
+}
+
+func TestUnderReplicatedCount(t *testing.T) {
+	assert.Equal(t, 0, underReplicatedCount(nil))
+	assert.Equal(t, 2, underReplicatedCount([]kvutil.BucketReport{
+		{Bucket: "a", Replicas: 1, Want: 3},
+		{Bucket: "b", Replicas: 3, Want: 3},
+		{Bucket: "c", Replicas: 2, Want: 3},
+	}))
+}
+
+// TestRepairReplicaReports_KeepsGoingPastABucketItCannotPlace is the behaviour
+// that makes --repair usable mid-growth: the buckets it can raise are raised,
+// and the one it cannot is named rather than aborting the run.
+func TestRepairReplicaReports_KeepsGoingPastABucketItCannotPlace(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	defer nc.Close()
+
+	for _, b := range []string{"repair-alpha", "repair-beta"} {
+		_, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: b, History: 1})
+		require.NoError(t, err)
+	}
+
+	// One embedded server cannot hold two replicas, so every raise fails — and
+	// every bucket must still have been attempted and named.
+	reports := []kvutil.BucketReport{
+		{Bucket: "repair-alpha", Replicas: 1, Want: 2},
+		{Bucket: "repair-beta", Replicas: 1, Want: 2},
+	}
+	var errOut bytes.Buffer
+	repairReplicaReports(t.Context(), js, reports, &errOut)
+
+	assert.Contains(t, errOut.String(), "repair repair-alpha")
+	assert.Contains(t, errOut.String(), "repair repair-beta")
+	assert.Equal(t, 2, underReplicatedCount(reports), "a failed raise must not be reported as repaired")
+}
+
+// TestRepairReplicaReports_RaisesWhatItCan covers the success side against a real
+// server, so the in-place report update is proved rather than assumed.
+func TestRepairReplicaReports_RaisesWhatItCan(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	defer nc.Close()
+	clustersize.DeclareForTest(t, 1)
+
+	_, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "repair-ok", History: 1})
+	require.NoError(t, err)
+
+	// Want 1 against a bucket already at 1: the raise is a no-op that must still
+	// leave the report reading as healthy rather than as repaired-and-wrong.
+	reports := []kvutil.BucketReport{{Bucket: "repair-ok", Replicas: 1, Want: 1}}
+	var errOut bytes.Buffer
+	repairReplicaReports(t.Context(), js, reports, &errOut)
+
+	assert.Empty(t, strings.TrimSpace(errOut.String()))
+	assert.Equal(t, 0, underReplicatedCount(reports))
+}

cmd/spinifex/cmd/admin_kv_test.go

@@ -0,0 +1,327 @@
+//test:in-package: exercises the unexported digest helpers against a real clustered
+// store, which the cobra command only reaches through a live node.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// kvServer runs a standalone JetStream server on storeParent and returns a KV
+// bucket on it plus a stop function.
+func kvServer(t *testing.T, storeParent, bucket string) (jetstream.KeyValue, func()) {
+	t.Helper()
+	ns, err := server.NewServer(&server.Options{
+		ServerName: "kv-test", JetStream: true, StoreDir: storeParent,
+		DontListen: true, NoLog: true, NoSigs: true,
+	})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: bucket, History: 5})
+	require.NoError(t, err)
+	return kv, func() {
+		nc.Close()
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}
+}
+
+func digestOf(t *testing.T, storeDir string, withSeqs bool) []streamDigest {
+	t.Helper()
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(storeDir, filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	d, err := digestStore(context.Background(), work, nil, withSeqs, metas)
+	require.NoError(t, err)
+	return d
+}
+
+func TestKVDigest(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	kv, stop := kvServer(t, parent, "test")
+	for _, kvp := range [][2]string{{"a", "1"}, {"b", "2"}, {"a", "3"}} {
+		_, err := kv.PutString(ctx, kvp[0], kvp[1])
+		require.NoError(t, err)
+	}
+	require.NoError(t, kv.Delete(ctx, "b"))
+	store := filepath.Join(parent, "jetstream")
+
+	// Copied while the server is live, exactly as it runs on a node.
+	live := digestOf(t, store, true)
+	stop()
+
+	require.Len(t, live, 1)
+	assert.Equal(t, "KV_test", live[0].Name)
+	assert.Equal(t, uint64(1), live[0].FirstSeq)
+	assert.Equal(t, uint64(4), live[0].LastSeq)
+	assert.Equal(t, uint64(4), live[0].Msgs)
+	require.Len(t, live[0].Seqs, 4)
+
+	t.Run("stable across reads and a stopped server", func(t *testing.T) {
+		again := digestOf(t, store, false)
+		assert.Equal(t, live[0].Digest, again[0].Digest)
+	})
+
+	t.Run("source store is left untouched", func(t *testing.T) {
+		_, err := os.Stat(filepath.Join(store, "$G", "streams", "KV_test"))
+		assert.NoError(t, err)
+	})
+
+	// Same keys and values written separately differ in timestamps, which is
+	// exactly what separates two nodes' pre-formation writes.
+	t.Run("independently written store differs", func(t *testing.T) {
+		other := t.TempDir()
+		kv2, stop2 := kvServer(t, other, "test")
+		for _, kvp := range [][2]string{{"a", "1"}, {"b", "2"}, {"a", "3"}} {
+			_, err := kv2.PutString(ctx, kvp[0], kvp[1])
+			require.NoError(t, err)
+		}
+		require.NoError(t, kv2.Delete(ctx, "b"))
+		stop2()
+		d := digestOf(t, filepath.Join(other, "jetstream"), true)
+		assert.NotEqual(t, live[0].Digest, d[0].Digest)
+
+		// Two stores adopted as replicas of one stream: same subjects and
+		// sequences, different content, which compare must call divergent.
+		res, err := compareNodeDigests([]nodeDigest{
+			{Host: "node1", CopiedAt: time.Now(), Streams: live},
+			{Host: "node2", CopiedAt: time.Now(), Streams: d},
+		})
+		require.NoError(t, err)
+		require.True(t, res.divergent())
+		require.Len(t, res.Streams, 1)
+		assert.Contains(t, strings.Join(res.Streams[0].Problems, "\n"), "seq 1 $KV.test.a holds different content")
+	})
+
+	t.Run("report lines", func(t *testing.T) {
+		var buf bytes.Buffer
+		writeDigests(&buf, "node1", store, time.Unix(0, 0), []string{"ACC/S"}, live)
+		out := buf.String()
+		assert.Contains(t, out, "# host=node1 ")
+		assert.Contains(t, out, "# skipped account stream (not read): ACC/S")
+		assert.Contains(t, out, "KV_test\tfirst=1\tlast=4\tmsgs=4\tsha256="+live[0].Digest)
+		assert.Equal(t, 4, strings.Count(out, "KV_test\tseq="))
+	})
+}
+
+func TestKVDigestStreamFilterAndDeletedSeq(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	// The KV bucket is only there to be excluded by the stream filter.
+	_, stop := kvServer(t, parent, "other")
+	stop()
+
+	// KV streams deny message deletes, so an interior gap needs a plain stream.
+	ns, err := server.NewServer(&server.Options{JetStream: true, StoreDir: parent, DontListen: true, NoLog: true, NoSigs: true})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	s, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ONE", Subjects: []string{"one.>"}})
+	require.NoError(t, err)
+	for _, v := range []string{"v1", "v2", "v3"} {
+		_, err = js.Publish(ctx, "one.k", []byte(v))
+		require.NoError(t, err)
+	}
+	require.NoError(t, s.DeleteMsg(ctx, 2))
+	nc.Close()
+	ns.Shutdown()
+	ns.WaitForShutdown()
+
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(filepath.Join(parent, "jetstream"), filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	d, err := digestStore(ctx, work, []string{"ONE"}, true, metas)
+	require.NoError(t, err)
+	require.Len(t, d, 1)
+	assert.Equal(t, "ONE", d[0].Name)
+	assert.Equal(t, uint64(2), d[0].Msgs)
+	require.Len(t, d[0].Seqs, 3)
+	assert.Equal(t, seqDigest{Seq: 2, Hash: "deleted"}, d[0].Seqs[1])
+	for _, i := range []int{0, 2} {
+		assert.Equal(t, "one.k", d[0].Seqs[i].Subject)
+		assert.False(t, d[0].Seqs[i].Time.IsZero())
+	}
+	assert.Equal(t, "limits", d[0].Retention)
+	assert.Equal(t, 1, d[0].Replicas)
+}
+
+func freePort(t *testing.T) int {
+	t.Helper()
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	defer l.Close()
+	return l.Addr().(*net.TCPAddr).Port
+}
+
+// A replicated stream's meta.inf carries num_replicas > 1, which a standalone
+// server refuses to load; this is the case every production node presents.
+func TestKVDigestClusteredReplicas(t *testing.T) {
+	ctx := context.Background()
+	const n = 3
+	ports := make([]int, n)
+	routes := make([]string, n)
+	for i := range ports {
+		ports[i] = freePort(t)
+		routes[i] = fmt.Sprintf("nats://127.0.0.1:%d", ports[i])
+	}
+	stores := make([]string, n)
+	servers := make([]*server.Server, n)
+	for i := range servers {
+		stores[i] = t.TempDir()
+		ns, err := server.NewServer(&server.Options{
+			ServerName: fmt.Sprintf("s%d", i+1), JetStream: true, StoreDir: stores[i],
+			Host: "127.0.0.1", Port: -1, NoLog: true, NoSigs: true,
+			Cluster: server.ClusterOpts{Name: "digest", Host: "127.0.0.1", Port: ports[i]},
+			Routes:  server.RoutesFromStr(strings.Join(routes, ",")),
+		})
+		require.NoError(t, err)
+		ns.Start()
+		servers[i] = ns
+	}
+	t.Cleanup(func() {
+		for _, ns := range servers {
+			ns.Shutdown()
+			ns.WaitForShutdown()
+		}
+	})
+
+	nc, err := nats.Connect("", nats.InProcessServer(servers[0]))
+	require.NoError(t, err)
+	defer nc.Close()
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+
+	// Each attempt gets its own deadline: before a meta leader is elected a
+	// create blocks for the client's default timeout instead of failing fast.
+	var kv jetstream.KeyValue
+	require.Eventually(t, func() bool {
+		attemptCtx, cancel := context.WithTimeout(ctx, 250*time.Millisecond)
+		defer cancel()
+		kv, err = js.CreateKeyValue(attemptCtx, jetstream.KeyValueConfig{Bucket: "rep", History: 3, Replicas: n, TTL: time.Hour})
+		return err == nil
+	}, 30*time.Second, 20*time.Millisecond, "clustered KV create: %v", err)
+	for i := range 5 {
+		_, err := kv.PutString(ctx, fmt.Sprintf("k%d", i%2), fmt.Sprintf("v%d", i))
+		require.NoError(t, err)
+	}
+	s, err := js.Stream(ctx, "KV_rep")
+	require.NoError(t, err)
+	require.Eventually(t, func() bool {
+		info, err := s.Info(ctx)
+		if err != nil || info.Cluster == nil || len(info.Cluster.Replicas) != n-1 {
+			return false
+		}
+		for _, r := range info.Cluster.Replicas {
+			if !r.Current || r.Lag != 0 {
+				return false
+			}
+		}
+		return true
+	}, 30*time.Second, 250*time.Millisecond)
+
+	// Current means applied, not flushed; shutdown flushes every block to disk.
+	nc.Close()
+	for _, ns := range servers {
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}
+
+	var first string
+	var nodes []nodeDigest
+	for i, store := range stores {
+		d := digestOf(t, filepath.Join(store, "jetstream"), true)
+		require.Len(t, d, 1, "replica %d", i+1)
+		assert.Equal(t, uint64(5), d[0].Msgs, "replica %d", i+1)
+		assert.Equal(t, n, d[0].Replicas, "live replica count survives the standalone rewrite")
+		assert.Equal(t, time.Hour, d[0].MaxAge)
+		nodes = append(nodes, nodeDigest{Host: fmt.Sprintf("s%d", i+1), CopiedAt: time.Now(), Streams: d})
+		if i == 0 {
+			first = d[0].Digest
+			continue
+		}
+		assert.Equal(t, first, d[0].Digest, "replica %d matches replica 1", i+1)
+	}
+	res, err := compareNodeDigests(nodes)
+	require.NoError(t, err)
+	assert.False(t, res.divergent(), "%+v", res)
+}
+
+// Recovery applies MaxAge, so without the rewrite an expired bucket copy
+// would read as empty and mask whatever the replica holds.
+func TestKVDigestIgnoresExpiry(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	ns, err := server.NewServer(&server.Options{JetStream: true, StoreDir: parent, DontListen: true, NoLog: true, NoSigs: true})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	const ttl = 200 * time.Millisecond
+	kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "ttl", TTL: ttl})
+	require.NoError(t, err)
+	_, err = kv.PutString(ctx, "k", "v")
+	require.NoError(t, err)
+	putAt := time.Now()
+
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(filepath.Join(parent, "jetstream"), filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	nc.Close()
+	ns.Shutdown()
+	ns.WaitForShutdown()
+
+	// The server stamped the message before the put returned, so it is now
+	// older than the TTL.
+	time.Sleep(time.Until(putAt.Add(ttl + 50*time.Millisecond)))
+	d, err := digestStore(ctx, work, nil, false, metas)
+	require.NoError(t, err)
+	require.Len(t, d, 1)
+	assert.Equal(t, uint64(1), d[0].Msgs)
+}
+
+func TestCopyStreamTree(t *testing.T) {
+	store := seedStore(t, "KV_a")
+	require.NoError(t, os.MkdirAll(filepath.Join(store, "$G", "streams", "KV_a", "obs", "c1"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(store, "$G", "streams", "KV_a", "obs", "c1", "o.dat"), []byte("x"), 0o600))
+	require.NoError(t, os.MkdirAll(filepath.Join(store, "ACME", "streams", "ORDERS"), 0o755))
+
+	dst := filepath.Join(t.TempDir(), "jetstream")
+	skipped, _, err := copyStreamTree(store, dst)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"ACME/ORDERS"}, skipped)
+
+	_, err = os.Stat(filepath.Join(dst, "$G", "streams", "KV_a", "msgs", "1.blk"))
+	assert.NoError(t, err, "message blocks are copied")
+	_, err = os.Stat(filepath.Join(dst, "$G", "streams", "KV_a", "obs"))
+	assert.True(t, os.IsNotExist(err), "consumer state is left out")
+
+	_, _, err = copyStreamTree(filepath.Join(t.TempDir(), "absent"), dst)
+	assert.Error(t, err)
+}

cmd/spinifex/cmd/admin_node_count_test.go

@@ -0,0 +1,39 @@
+//test:in-package — validateNodeCount is an unexported guard on this command's
+// flags, and exporting it would widen the package's API for a test's convenience.
+
+package cmd
+
+import (
+	"strings"
+	"testing"
+)
+
+// TestValidateNodeCount pins the supported cluster sizes. Two is rejected rather
+// than warned about: on two servers no layer of the stack has a majority to lose,
+// and the control plane's own state has no replica count that both survives a node
+// going away and still accepts writes.
+func TestValidateNodeCount(t *testing.T) {
+	for _, nodes := range []int{1, 3, 4, 5, 9} {
+		if err := validateNodeCount(nodes); err != nil {
+			t.Errorf("validateNodeCount(%d) = %v, want nil", nodes, err)
+		}
+	}
+
+	for _, nodes := range []int{0, -1} {
+		if err := validateNodeCount(nodes); err == nil {
+			t.Errorf("validateNodeCount(%d) returned no error", nodes)
+		}
+	}
+
+	err := validateNodeCount(2)
+	if err == nil {
+		t.Fatal("validateNodeCount(2) returned no error")
+	}
+	// The message has to say what to do instead, because the operator typed a
+	// number and needs to know which one to type next.
+	for _, want := range []string{"2", "1", "3"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("error %q does not mention %s", err, want)
+		}
+	}
+}

cmd/spinifex/cmd/admin_ochre.go

@@ -438,14 +438,10 @@ func parseWeightsS3URI(uri string) (bucket, prefix string, err error) {
 	if trimmed == uri {
 		return "", "", fmt.Errorf("invalid --s3-uri %q: expected s3://bucket/prefix", uri)
 	}
-	parts := strings.SplitN(trimmed, "/", 2)
-	bucket = parts[0]
+	bucket, prefix, _ = strings.Cut(trimmed, "/")
 	if bucket == "" {
 		return "", "", fmt.Errorf("invalid --s3-uri %q: missing bucket", uri)
 	}
-	if len(parts) == 2 {
-		prefix = parts[1]
-	}
 	if prefix != "" && !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}
@@ -854,7 +850,7 @@ func runOchreWeightsStage(cmd *cobra.Command, _ []string) {
 		ochreExit(1)
 		return
 	}
-	weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
+	weightsStore := gateway_bedrock.NewWeightsStore(js)
 	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
 
 	provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout)
@@ -930,7 +926,7 @@ func runOchreWeightsList(_ *cobra.Command, _ []string) {
 		ochreExit(1)
 		return
 	}
-	weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
+	weightsStore := gateway_bedrock.NewWeightsStore(js)
 	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
 	checkSnapshotLive := newWeightsSnapshotChecker(store, node.Predastore.Bucket)
 
@@ -967,7 +963,7 @@ func removeWeights(ctx context.Context, weightsStore *gateway_bedrock.WeightsSto
 func runOchreWeightsRemove(cmd *cobra.Command, _ []string) {
 	modelID, _ := cmd.Flags().GetString("model-id")
 
-	appConfig, nc, err := loadConfigAndConnectFn()
+	_, nc, err := loadConfigAndConnectFn()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 		ochreExit(1)
@@ -981,7 +977,7 @@ func runOchreWeightsRemove(cmd *cobra.Command, _ []string) {
 		ochreExit(1)
 		return
 	}
-	weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
+	weightsStore := gateway_bedrock.NewWeightsStore(js)
 
 	entry, err := removeWeights(context.Background(), weightsStore, modelID)
 	if err != nil {
@@ -1014,7 +1010,7 @@ func resolveHFToken(ctx context.Context, cmd *cobra.Command, cfg *config.Cluster
 	if err != nil {
 		return ""
 	}
-	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil)
+	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, nil)
 	token, ok, err := credStore.Resolve(ctx, utils.GlobalAccountID, vendorHuggingFace)
 	if err != nil || !ok {
 		return ""
@@ -1088,7 +1084,7 @@ func ochreCredentialsStore() (*gateway_bedrock.CredentialStore, func(), error) {
 		nc.Close()
 		return nil, nil, fmt.Errorf("jetstream context: %w", err)
 	}
-	return gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil), func() { nc.Close() }, nil
+	return gateway_bedrock.NewCredentialStore(js, masterKey, nil), func() { nc.Close() }, nil
 }
 
 func runOchreCredentialsSet(cmd *cobra.Command, _ []string) {
@@ -1186,7 +1182,7 @@ func init() {
 // ochreAccessStore connects to the cluster and returns the grant store along
 // with a cleanup that closes the connection.
 func ochreAccessStore() (*gateway_bedrock.ModelAccessStore, func(), error) {
-	cfg, nc, err := loadConfigAndConnectFn()
+	_, nc, err := loadConfigAndConnectFn()
 	if err != nil {
 		return nil, nil, fmt.Errorf("connect to cluster: %w", err)
 	}
@@ -1195,7 +1191,7 @@ func ochreAccessStore() (*gateway_bedrock.ModelAccessStore, func(), error) {
 		nc.Close()
 		return nil, nil, fmt.Errorf("jetstream context: %w", err)
 	}
-	return gateway_bedrock.NewModelAccessStore(js, len(cfg.Nodes)), func() { nc.Close() }, nil
+	return gateway_bedrock.NewModelAccessStore(js), func() { nc.Close() }, nil
 }
 
 // ochreTargetModels resolves the --model-id / --all-models pair into the model

cmd/spinifex/cmd/admin_ochre_pull_test.go

@@ -470,7 +470,7 @@ func TestResolveHFToken_FallsBackToStoredPlatformCredential(t *testing.T) {
 	require.NoError(t, os.MkdirAll(filepath.Join(baseDir, "config"), 0750))
 	require.NoError(t, handlers_iam.SaveMasterKey(filepath.Join(baseDir, "config", "master.key"), masterKey))
 
-	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, 1, nil)
+	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, nil)
 	require.NoError(t, credStore.PutCredential(context.Background(), utils.GlobalAccountID, vendorHuggingFace, "hf_stored_token"))
 
 	cmd := newOchreWeightsPullTestCmd(t, selfHostModelID, testHFRepo, "main", "", "")
@@ -621,7 +621,7 @@ func TestRunOchreCredentialsSet_RoundTripsAndDefaultsToPlatformAccount(t *testin
 	defer verifyConn.Close()
 	verifyJS := testutil.NewJetStream(t, verifyConn)
 
-	credStore := gateway_bedrock.NewCredentialStore(verifyJS, masterKey, 1, nil)
+	credStore := gateway_bedrock.NewCredentialStore(verifyJS, masterKey, nil)
 	token, ok, err := credStore.Resolve(context.Background(), utils.GlobalAccountID, vendorHuggingFace)
 	require.NoError(t, err)
 	require.True(t, ok)

cmd/spinifex/cmd/admin_ochre_test.go

@@ -42,7 +42,7 @@ const (
 func newWeightsStoreForTest(t *testing.T) *gateway_bedrock.WeightsStore {
 	t.Helper()
 	_, nc, _ := testutil.StartTestJetStream(t)
-	return gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, nc), 1)
+	return gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, nc))
 }
 
 // explodingObjectStore fails the test immediately if any of its methods is
@@ -775,7 +775,7 @@ func TestRunOchreWeightsList_PrintsStagedModels(t *testing.T) {
 	}
 	stubConnect(t, cfg, nc, nil)
 
-	seedStore := gateway_bedrock.NewWeightsStore(js, 1)
+	seedStore := gateway_bedrock.NewWeightsStore(js)
 	require.NoError(t, seedStore.PutWeights(context.Background(), selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001"))
 
 	out := captureStdout(t, func() { runOchreWeightsList(nil, nil) })
@@ -822,7 +822,7 @@ func TestRunOchreWeightsRemove_RoundTripPrintsSnapshotAndDropsEntry(t *testing.T
 	ns, nc, js := testutil.StartTestJetStream(t)
 	stubConnect(t, fakeClusterConfig(), nc, nil)
 
-	seedStore := gateway_bedrock.NewWeightsStore(js, 1)
+	seedStore := gateway_bedrock.NewWeightsStore(js)
 	require.NoError(t, seedStore.PutWeights(context.Background(), selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001"))
 
 	cmd := newOchreWeightsRemoveTestCmd(t, selfHostModelID)
@@ -835,7 +835,7 @@ func TestRunOchreWeightsRemove_RoundTripPrintsSnapshotAndDropsEntry(t *testing.T
 	verifyNC, err := nats.Connect(ns.ClientURL())
 	require.NoError(t, err)
 	t.Cleanup(func() { verifyNC.Close() })
-	verifyStore := gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, verifyNC), 1)
+	verifyStore := gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, verifyNC))
 
 	_, ok, err := verifyStore.GetWeights(context.Background(), selfHostModelID)
 	require.NoError(t, err)

cmd/spinifex/cmd/admin_ochre_vector.go

@@ -283,11 +283,9 @@ func truncateForTable(s string, maxRunes int) string {
 }
 
 // formatQueryResults renders 'ochre vector query' output: a truncated table
-// by default, or the full response as indented JSON when asJSON is set
-// (D10's "full text via source ref" -- here, via --json).
+// by default, or the full response as indented JSON when asJSON is set.
 func formatQueryResults(results []handlers_ochrevector.QueryResult, asJSON bool) (string, error) {
 	if asJSON {
-		//nolint:musttag // QueryResult is an internal, non-wire Go type (predates this CLI); it round-trips fine on its default field names.
 		data, err := json.MarshalIndent(results, "", "  ")
 		if err != nil {
 			return "", fmt.Errorf("encode results as JSON: %w", err)
@@ -317,7 +315,6 @@ func parseFilterFlag(raw string) (*handlers_ochrevector.Filter, error) {
 		return nil, nil
 	}
 	var f handlers_ochrevector.Filter
-	//nolint:musttag // Filter is the existing filter-AST type (D9); it round-trips fine on its default field names.
 	if err := json.Unmarshal([]byte(raw), &f); err != nil {
 		return nil, fmt.Errorf("decode --filter: %w", err)
 	}

cmd/spinifex/cmd/admin_principal.go

@@ -188,13 +188,14 @@ func describePrincipal(svc handlers_iam.IAMService, accountID, userName string)
 }
 
 // principalPolicyActions returns the actions an inline policy allows, shortened
-// to the method name. An undecodable document reports nothing rather than
-// guessing, and the listing shows the user with no grants.
+// to the method name. A NotAction grant is one entry naming what it excludes.
+// An undecodable document reports nothing rather than guessing.
 func principalPolicyActions(document string) []string {
 	var doc struct {
 		Statement []struct {
-			Effect string                   `json:"Effect"`
-			Action handlers_iam.StringOrArr `json:"Action"`
+			Effect    string                   `json:"Effect"`
+			Action    handlers_iam.StringOrArr `json:"Action"`
+			NotAction handlers_iam.StringOrArr `json:"NotAction"`
 		} `json:"Statement"`
 	}
 	if err := json.Unmarshal([]byte(document), &doc); err != nil {
@@ -209,6 +210,13 @@ func principalPolicyActions(document string) []string {
 		for _, action := range stmt.Action {
 			actions = append(actions, strings.TrimPrefix(action, adminPrincipalService+":"))
 		}
+		if len(stmt.NotAction) > 0 {
+			excluded := make([]string, 0, len(stmt.NotAction))
+			for _, action := range stmt.NotAction {
+				excluded = append(excluded, strings.TrimPrefix(action, adminPrincipalService+":"))
+			}
+			actions = append(actions, "NotAction("+strings.Join(excluded, ", ")+")")
+		}
 	}
 	sort.Strings(actions)
 	return actions

cmd/spinifex/cmd/admin_principal_test.go

@@ -102,6 +102,18 @@ func TestPrincipalPolicyActionsRoundTrip(t *testing.T) {
 		cmd.PrincipalPolicyActions(cmd.PrincipalPolicyDocument(grants)))
 }
 
+// A NotAction grant covers everything but its list, so rendering only Action
+// would show the principal as holding nothing.
+func TestPrincipalPolicyActionsRenderNotAction(t *testing.T) {
+	document := `{"Version":"2012-10-17","Statement":[
+		{"Effect":"Allow","NotAction":["spinifex:DeleteAccount","spinifex:CreateAccount"],"Resource":"*"},
+		{"Effect":"Allow","Action":"spinifex:ListAccounts","Resource":"*"},
+		{"Effect":"Deny","NotAction":"spinifex:ListAccounts","Resource":"*"}]}`
+
+	assert.Equal(t, []string{"ListAccounts", "NotAction(DeleteAccount, CreateAccount)"},
+		cmd.PrincipalPolicyActions(document))
+}
+
 // A document nobody can parse is reported as no grants rather than as a guess.
 func TestPrincipalPolicyActionsIgnoreAnUndecodableDocument(t *testing.T) {
 	assert.Empty(t, cmd.PrincipalPolicyActions("{not json"))

cmd/spinifex/cmd/admin_test.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"net/netip"
@@ -9,11 +11,13 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/mulgadc/northstar/pkg/backend"
 	nsconfig "github.com/mulgadc/northstar/pkg/config"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	"github.com/mulgadc/spinifex/spinifex/formation"
 	handlers_dns "github.com/mulgadc/spinifex/spinifex/handlers/dns"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
@@ -734,21 +738,45 @@ func TestSpinifexTomlTemplate_ExternalPoolStaticSource(t *testing.T) {
 	assert.Contains(t, content, `gateway     = "192.168.1.1"`)
 }
 
-// Formation joiners must receive PoolBindBridge from the init node so the
+// Formation joiners must receive BindBridge from the init node so the
 // cluster-wide config reaches every chassis intact.
 func TestApplyNetworkConfig_PropagatesPoolBindBridge(t *testing.T) {
 	settings := &admin.ConfigSettings{}
 	nc := &formation.NetworkConfig{
-		ExternalMode:   "pool",
-		PoolName:       "wan",
-		PoolSource:     "dhcp",
-		PoolBindBridge: "br-wan",
-		PoolPrefixLen:  24,
+		ExternalMode: "pool",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "dhcp", BindBridge: "br-wan", PrefixLen: 24,
+		}},
 	}
 	applyNetworkConfig(settings, nc)
 	require.Len(t, settings.Pools, 1)
 	assert.Equal(t, "dhcp", settings.Pools[0].Source)
 	assert.Equal(t, "br-wan", settings.Pools[0].BindBridge)
+	assert.Empty(t, settings.BridgeMode, "bridged modes stay auto-detected, only nat is pinned")
+}
+
+// Routed NAT renders two pools, and the transit one carries a gateway-LRP range
+// that pool mode has no equivalent for. A joiner that received only the first,
+// or lost the range, would allocate VPC gateway addresses from nowhere.
+func TestApplyNetworkConfig_PropagatesEveryPool(t *testing.T) {
+	settings := &admin.ConfigSettings{}
+	nc := &formation.NetworkConfig{
+		ExternalMode: "nat",
+		Pools: []admin.PoolData{
+			{Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
+				GwLrpRangeStart: "100.127.0.16", GwLrpRangeEnd: "100.127.0.254"},
+			{Name: "wan", Source: "static", Start: "192.168.1.150", End: "192.168.1.200",
+				Gateway: "192.168.1.1", PrefixLen: 24},
+		},
+	}
+	applyNetworkConfig(settings, nc)
+	require.Len(t, settings.Pools, 2)
+	assert.Equal(t, "nat", settings.BridgeMode,
+		"a joiner that loses bridge_mode falls back to autodetection for a mode init pinned")
+	assert.Equal(t, "nat-transit", settings.Pools[0].Name)
+	assert.Equal(t, "100.127.0.16", settings.Pools[0].GwLrpRangeStart)
+	assert.Equal(t, "100.127.0.254", settings.Pools[0].GwLrpRangeEnd)
+	assert.Equal(t, "wan", settings.Pools[1].Name)
 }
 
 func renderSingleNodePredastore(t *testing.T, settings admin.ConfigSettings) string {
@@ -1288,7 +1316,218 @@ func TestAMIVolumeSizeGiB(t *testing.T) {
 	}
 }
 
+func TestValidateChecksumFlags(t *testing.T) {
+	tests := []struct {
+		name       string
+		checksum   string
+		unset      bool
+		file       string
+		skipVerify bool
+		wantErr    string
+	}{
+		{name: "no checksum", unset: true, file: "img.raw"},
+		{name: "checksum with file", checksum: "SHA256SUMS", file: "img.raw"},
+		{name: "empty checksum value", checksum: "", file: "img.raw", wantErr: "--checksum requires a sums file path"},
+		{name: "blank checksum value", checksum: "  ", file: "img.raw", wantErr: "--checksum requires a sums file path"},
+		{name: "checksum without file", checksum: "SHA256SUMS", wantErr: "--checksum requires --file"},
+		{name: "checksum with skip-verify", checksum: "SHA256SUMS", file: "img.raw", skipVerify: true,
+			wantErr: "mutually exclusive"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateChecksumFlags(tt.checksum, !tt.unset, tt.file, tt.skipVerify)
+			if tt.wantErr == "" {
+				assert.NoError(t, err)
+				return
+			}
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tt.wantErr)
+		})
+	}
+}
+
+func TestSourceDigestMode(t *testing.T) {
+	tests := []struct {
+		name       string
+		localFile  string
+		checksum   string
+		skipVerify bool
+		want       string
+	}{
+		{name: "file with checksum", localFile: "img.raw", checksum: "SHA256SUMS", want: ebsmetadata.DigestOperator},
+		{name: "catalog download verified", want: ebsmetadata.DigestCatalog},
+		{name: "catalog download skip-verify", skipVerify: true, want: ebsmetadata.DigestUnverified},
+		{name: "file without checksum", localFile: "img.raw", want: ebsmetadata.DigestUnverified},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, sourceDigestMode(tt.localFile, tt.checksum, tt.skipVerify))
+		})
+	}
+}
+
+func TestImportSourceResolveDigest(t *testing.T) {
+	imgBytes := []byte("import-source-fixture")
+	sum := sha256.Sum256(imgBytes)
+	imgHex := hex.EncodeToString(sum[:])
+	dir := t.TempDir()
+	img := filepath.Join(dir, "image.raw")
+	require.NoError(t, os.WriteFile(img, imgBytes, 0o600))
+	sums := filepath.Join(dir, "SHA256SUMS")
+
+	t.Run("operator verified", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		got, err := importSource{
+			imageFile: img, localFile: img,
+			checksumPath: sums, checksumAlgo: "sha256", expectedDigest: imgHex,
+		}.resolveDigest(&out, &errOut)
+		require.NoError(t, err)
+		assert.Equal(t, ebsmetadata.ImageDigest{
+			Algorithm: "sha256", Value: imgHex, Verification: ebsmetadata.DigestOperator,
+			Source: "SHA256SUMS", Filename: "image.raw",
+		}, got)
+		assert.Contains(t, out.String(), "Verified image checksum (sha256)")
+		assert.Empty(t, errOut.String())
+	})
+
+	t.Run("operator mismatch names the file and sums file", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		_, err := importSource{
+			imageFile: img, localFile: img,
+			checksumPath: sums, checksumAlgo: "sha256", expectedDigest: strings.Repeat("0", 64),
+		}.resolveDigest(&out, &errOut)
+		require.ErrorIs(t, err, utils.ErrChecksumMismatch)
+		assert.Contains(t, errOut.String(), "file:     "+img)
+		assert.Contains(t, errOut.String(), "checksum: "+sums)
+		assert.Contains(t, errOut.String(), imgHex)
+		assert.FileExists(t, img, "the operator's file must be left in place")
+	})
+
+	t.Run("local file without checksum is recorded unverified", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		got, err := importSource{imageFile: img, localFile: img}.resolveDigest(&out, &errOut)
+		require.NoError(t, err)
+		assert.Equal(t, ebsmetadata.ImageDigest{
+			Algorithm: "sha256", Value: imgHex, Verification: ebsmetadata.DigestUnverified, Filename: "image.raw",
+		}, got)
+		assert.Equal(t, "Importing "+img+" without checksum validation (pass --checksum <sums-file> to verify)\n",
+			errOut.String())
+	})
+
+	t.Run("catalog skip-verify is recorded unverified", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		got, err := importSource{imageFile: img, imageName: "debian-13-x86_64", skipVerify: true}.resolveDigest(&out, &errOut)
+		require.NoError(t, err)
+		assert.Equal(t, ebsmetadata.DigestUnverified, got.Verification)
+		assert.Equal(t, imgHex, got.Value)
+		assert.Contains(t, errOut.String(), "--skip-verify set")
+	})
+
+	t.Run("catalog entry without checksum is refused", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		_, err := importSource{imageFile: img, imageName: "debian-13-x86_64"}.resolveDigest(&out, &errOut)
+		require.Error(t, err)
+		assert.Contains(t, errOut.String(), "missing Checksum/ChecksumDigest/ChecksumType")
+	})
+
+	// Oracle publishes its digests inline on a web page and ships no sums file,
+	// so a pinned digest is the only way its entries can import verified.
+	t.Run("catalog pinned digest verifies without fetching a sums file", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		got, err := importSource{
+			imageFile: img, imageName: "oracle-10.1-x86_64",
+			image: utils.Images{ChecksumType: "sha256", ChecksumDigest: imgHex},
+		}.resolveDigest(&out, &errOut)
+		require.NoError(t, err)
+		assert.Equal(t, ebsmetadata.ImageDigest{
+			Algorithm: "sha256", Value: imgHex, Source: catalogPinnedDigestSource,
+			Verification: ebsmetadata.DigestCatalog, Filename: "image.raw",
+		}, got)
+		assert.Contains(t, out.String(), "pinned catalog digest")
+	})
+
+	t.Run("catalog pinned digest mismatch is refused and names the url", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		wrong := strings.Repeat("a", 64)
+		_, err := importSource{
+			imageFile: img, imageName: "oracle-10.1-x86_64",
+			image: utils.Images{ChecksumType: "sha256", ChecksumDigest: wrong, URL: "https://example.invalid/ol.qcow2"},
+		}.resolveDigest(&out, &errOut)
+		require.ErrorIs(t, err, utils.ErrChecksumMismatch)
+		assert.Contains(t, errOut.String(), catalogPinnedDigestSource)
+		assert.Contains(t, errOut.String(), "https://example.invalid/ol.qcow2")
+		assert.FileExists(t, img, "the downloaded file must be left in place")
+	})
+
+	t.Run("unreadable image fails the unverified hash", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		missing := filepath.Join(dir, "missing.raw")
+		_, err := importSource{imageFile: missing, localFile: missing}.resolveDigest(&out, &errOut)
+		require.Error(t, err)
+		assert.Contains(t, errOut.String(), "Could not hash image")
+	})
+}
+
+func TestImagesDescribeCmd_FlagSchema(t *testing.T) {
+	imageIDFlag := imagesDescribeCmd.Flags().Lookup("image-id")
+	require.NotNil(t, imageIDFlag, "--image-id must be defined")
+	assert.Equal(t, []string{"true"}, imageIDFlag.Annotations[cobraRequiredAnnotation],
+		"--image-id must be marked required")
+}
+
+func TestPrintAMIDescription(t *testing.T) {
+	base := ebsmetadata.AMI{
+		ImageID:         "ami-1",
+		Name:            "ami-debian-13-x86_64",
+		ImageOwnerAlias: "system",
+		CreationDate:    time.Date(2026, 9, 23, 1, 2, 3, 0, time.UTC),
+		BootMode:        "uefi",
+		Architecture:    "x86_64",
+	}
+
+	t.Run("digest recorded", func(t *testing.T) {
+		meta := base
+		meta.SourceDigest = &ebsmetadata.ImageDigest{
+			Algorithm: "sha512", Value: "abc123", Verification: ebsmetadata.DigestOperator,
+			Source: "SHA512SUMS", Filename: "debian-13-generic-amd64.tar.xz",
+		}
+		var buf bytes.Buffer
+		printAMIDescription(&buf, meta)
+		out := buf.String()
+		assert.Contains(t, out, "Image ID:       ami-1")
+		assert.Contains(t, out, "Created:        2026-09-23T01:02:03Z")
+		assert.Contains(t, out, "Boot mode:      uefi")
+		assert.Contains(t, out, "Source digest:  sha512:abc123")
+		assert.Contains(t, out, "Verification:   operator (SHA512SUMS)")
+		assert.Contains(t, out, "Source file:    debian-13-generic-amd64.tar.xz")
+	})
+
+	t.Run("unverified digest has no source", func(t *testing.T) {
+		meta := base
+		meta.SourceDigest = &ebsmetadata.ImageDigest{
+			Algorithm: "sha256", Value: "def456", Verification: ebsmetadata.DigestUnverified,
+			Filename: "image.raw",
+		}
+		var buf bytes.Buffer
+		printAMIDescription(&buf, meta)
+		assert.Contains(t, buf.String(), "Verification:   unverified\n")
+	})
+
+	t.Run("digest not recorded", func(t *testing.T) {
+		var buf bytes.Buffer
+		printAMIDescription(&buf, base)
+		out := buf.String()
+		assert.Contains(t, out, "Source digest:  not recorded (imported before digest recording)")
+		assert.NotContains(t, out, "Verification:")
+	})
+}
+
 func TestCheckJoinPreconditions(t *testing.T) {
+	withFakeProc(t, nil)
+	noStore := filepath.Join(t.TempDir(), "nats", "jetstream")
+
 	writeToml := func(t *testing.T) string {
 		t.Helper()
 		dir := t.TempDir()
@@ -1299,13 +1538,13 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	}
 
 	t.Run("uninitialized node joins", func(t *testing.T) {
-		if err := checkJoinPreconditions(t.TempDir(), false); err != nil {
+		if err := checkJoinPreconditions(t.TempDir(), noStore, false); err != nil {
 			t.Fatalf("want nil, got %v", err)
 		}
 	})
 
 	t.Run("initialized node is refused", func(t *testing.T) {
-		err := checkJoinPreconditions(writeToml(t), false)
+		err := checkJoinPreconditions(writeToml(t), noStore, false)
 		if err == nil {
 			t.Fatal("want refusal, got nil")
 		}
@@ -1317,7 +1556,7 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	// The guidance is the whole point of the guard: an operator must be able to
 	// tell what a forced join would destroy without reading the source.
 	t.Run("guidance names what is discarded", func(t *testing.T) {
-		for _, want := range []string{"CA certificate", "master key", "viperblock key", "--force"} {
+		for _, want := range []string{"CA certificate", "master key", "viperblock key", "JetStream store", "--force"} {
 			if !strings.Contains(joinDiscardsIdentityMsg, want) {
 				t.Errorf("guidance missing %q", want)
 			}
@@ -1325,10 +1564,27 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	})
 
 	t.Run("force overrides", func(t *testing.T) {
-		if err := checkJoinPreconditions(writeToml(t), true); err != nil {
+		if err := checkJoinPreconditions(writeToml(t), noStore, true); err != nil {
 			t.Fatalf("want nil with force, got %v", err)
 		}
 	})
+
+	// A store with streams and no config is still data the join would destroy.
+	t.Run("uninitialized node with streams is refused", func(t *testing.T) {
+		err := checkJoinPreconditions(t.TempDir(), seedStore(t, "KV_a"), false)
+		if err == nil || !strings.Contains(err.Error(), "JetStream store") {
+			t.Fatalf("want JetStream refusal, got %v", err)
+		}
+	})
+
+	// --force accepts the loss of data, never a wipe under a live server.
+	t.Run("running NATS is refused even with force", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"4242": "/usr/local/bin/spx\x00service\x00nats\x00start\x00"})
+		err := checkJoinPreconditions(writeToml(t), seedStore(t, "KV_a"), true)
+		if !errors.Is(err, errNATSRunning) {
+			t.Fatalf("want errNATSRunning, got %v", err)
+		}
+	})
 }
 
 func TestJoinRetryable(t *testing.T) {

cmd/spinifex/cmd/cluster.go

@@ -167,7 +167,7 @@ func runClusterShutdown(cmd *cobra.Command, args []string) {
 	fmt.Printf("Timeout per phase: %s\n\n", timeout)
 
 	// Write cluster shutdown marker to KV
-	jsm, err := daemon.NewJetStreamManager(nc, 1)
+	jsm, err := daemon.NewJetStreamManager(nc)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: failed to create JetStream manager: %v\n", err)
 	} else {

cmd/spinifex/cmd/cluster_test.go

@@ -35,26 +35,29 @@ func fakeDrainResponder(t *testing.T, nc *nats.Conn, released int, errMsg string
 }
 
 func TestDrainDHCPLeasesSumsReleased(t *testing.T) {
+	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	fakeDrainResponder(t, nc, 7, "")
 
-	released, responders := drainDHCPLeases(nc, 2*time.Second)
+	released, responders := drainDHCPLeases(nc, 250*time.Millisecond)
 	assert.Equal(t, 7, released)
 	assert.Equal(t, 1, responders)
 }
 
 func TestDrainDHCPLeasesReportsResponderError(t *testing.T) {
+	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	fakeDrainResponder(t, nc, 0, "list leases for drain: boom")
 
 	// An erroring responder still counts as a responder but contributes
 	// zero released leases.
-	released, responders := drainDHCPLeases(nc, 2*time.Second)
+	released, responders := drainDHCPLeases(nc, 250*time.Millisecond)
 	assert.Equal(t, 0, released)
 	assert.Equal(t, 1, responders)
 }
 
 func TestDrainDHCPLeasesNoResponders(t *testing.T) {
+	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
 
 	// No vpcd subscribed: the collection window elapses with zero replies.
@@ -281,7 +284,7 @@ func captureSlog(t *testing.T, fn func()) string {
 // at WARN level, and names only the running instance, not the stopped one.
 func TestWarnIfGuestsLeftRunningNamesRunningGuests(t *testing.T) {
 	origTimeout := guestEnumerationTimeout
-	guestEnumerationTimeout = 200 * time.Millisecond
+	guestEnumerationTimeout = 50 * time.Millisecond
 	t.Cleanup(func() { guestEnumerationTimeout = origTimeout })
 
 	_, nc, _ := testutil.StartTestJetStream(t)
@@ -312,7 +315,7 @@ func TestWarnIfGuestsLeftRunningNamesRunningGuests(t *testing.T) {
 // running guests produces no warning.
 func TestWarnIfGuestsLeftRunningNoGuestsIsSilent(t *testing.T) {
 	origTimeout := guestEnumerationTimeout
-	guestEnumerationTimeout = 200 * time.Millisecond
+	guestEnumerationTimeout = 50 * time.Millisecond
 	t.Cleanup(func() { guestEnumerationTimeout = origTimeout })
 
 	_, nc, _ := testutil.StartTestJetStream(t)
@@ -510,7 +513,7 @@ func TestCollectLocalShutdownACKExhaustedReturnsWithinBudget(t *testing.T) {
 // must log at ERROR, not WARN, and still name only the running instance.
 func TestReportGuestsLeftRunningSevereLogsErrorAndNamesGuests(t *testing.T) {
 	origTimeout := guestEnumerationTimeout
-	guestEnumerationTimeout = 200 * time.Millisecond
+	guestEnumerationTimeout = 50 * time.Millisecond
 	t.Cleanup(func() { guestEnumerationTimeout = origTimeout })
 
 	_, nc, _ := testutil.StartTestJetStream(t)

cmd/spinifex/cmd/export_test.go

@@ -4,8 +4,10 @@
 package cmd
 
 import (
+	"bytes"
 	"errors"
 	"strconv"
+	"strings"
 	"time"
 
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
@@ -27,6 +29,13 @@ func SetFirewallApplyHelper(path string) func() {
 // InitCLILogger exposes initCLILogger for testing.
 var InitCLILogger = initCLILogger
 
+// OutputStylingEnabled, IsTerminal and SetOutputStyling expose the spx colour guard.
+var (
+	OutputStylingEnabled = outputStylingEnabled
+	IsTerminal           = isTerminal
+	SetOutputStyling     = setOutputStyling
+)
+
 // CLILogLevel exposes the CLI logger's level var for testing.
 var CLILogLevel = cliLogLevel
 
@@ -146,3 +155,28 @@ var PrintAccountTable = printAccountTable
 
 // AccountSummaries exposes accountSummaries for testing.
 var AccountSummaries = accountSummaries
+
+// CompareKVDigests runs "kv compare" over digest JSON documents and returns
+// whether any stream diverged, plus the printed report.
+func CompareKVDigests(docs ...string) (bool, string, error) {
+	var nodes []nodeDigest
+	for i, doc := range docs {
+		got, err := loadNodeDigests(strings.NewReader(doc), "doc"+strconv.Itoa(i))
+		if err != nil {
+			return false, "", err
+		}
+		nodes = append(nodes, got...)
+	}
+	res, err := compareNodeDigests(nodes)
+	if err != nil {
+		return false, "", err
+	}
+	var buf bytes.Buffer
+	writeCompareResult(&buf, nodes, res)
+	return res.divergent(), buf.String(), nil
+}
+
+// AdminInitFlagDefault returns the registered default of an admin init flag.
+func AdminInitFlagDefault(name string) string {
+	return adminInitCmd.Flags().Lookup(name).DefValue
+}

cmd/spinifex/cmd/jetstream_store.go

@@ -0,0 +1,139 @@
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// jetStreamStoreDir is where the NATS service keeps its JetStream data under a
+// node's data directory. The nats.conf template sets store_dir to
+// <data_dir>/nats/, and nats-server appends "jetstream" itself.
+func jetStreamStoreDir(dataDir string) string {
+	return filepath.Join(dataDir, "nats", "jetstream")
+}
+
+// localStreams lists the streams a JetStream store holds on disk, as
+// "<account>/<stream>". A missing store holds nothing.
+func localStreams(storeDir string) ([]string, error) {
+	accounts, err := os.ReadDir(storeDir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read JetStream store %s: %w", storeDir, err)
+	}
+	var streams []string
+	for _, acc := range accounts {
+		if !acc.IsDir() {
+			continue
+		}
+		entries, err := os.ReadDir(filepath.Join(storeDir, acc.Name(), "streams"))
+		if errors.Is(err, os.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read streams for account %s: %w", acc.Name(), err)
+		}
+		for _, e := range entries {
+			if e.IsDir() {
+				streams = append(streams, acc.Name()+"/"+e.Name())
+			}
+		}
+	}
+	sort.Strings(streams)
+	return streams, nil
+}
+
+// procRoot is overridable in tests so the process scan can run against a fake
+// /proc tree.
+var procRoot = "/proc"
+
+// natsRunningLocally reports whether this host runs the Spinifex NATS service,
+// found as a process whose command line is `spx service nats ...`. Removing a
+// store under a live server races JetStream, which rewrites its index files.
+func natsRunningLocally() (bool, error) {
+	entries, err := os.ReadDir(procRoot)
+	if err != nil {
+		return false, fmt.Errorf("scan processes: %w", err)
+	}
+	for _, e := range entries {
+		if !e.IsDir() || strings.TrimLeft(e.Name(), "0123456789") != "" {
+			continue
+		}
+		raw, err := os.ReadFile(filepath.Join(procRoot, e.Name(), "cmdline"))
+		if err != nil || len(raw) == 0 {
+			continue
+		}
+		if isSpxNATSService(bytes.Split(bytes.TrimRight(raw, "\x00"), []byte{0})) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// isSpxNATSService matches `spx service nats` with any leading path on spx.
+func isSpxNATSService(argv [][]byte) bool {
+	return len(argv) >= 3 &&
+		filepath.Base(string(argv[0])) == "spx" &&
+		string(argv[1]) == "service" &&
+		string(argv[2]) == "nats"
+}
+
+// errNATSRunning is returned whenever a store would be discarded with the NATS
+// service still up. No flag overrides it: the wipe would race the live server.
+var errNATSRunning = errors.New("the NATS service is still running on this node; stop it first (sudo systemctl stop spinifex.target, then confirm `pgrep -af 'spx service'` prints nothing)")
+
+// checkStoreDiscardable fails if the NATS service is running locally.
+func checkStoreDiscardable() error {
+	running, err := natsRunningLocally()
+	if err != nil {
+		return err
+	}
+	if running {
+		return errNATSRunning
+	}
+	return nil
+}
+
+// discardJetStreamStore removes a node's JetStream store before it enters a
+// multi-node cluster. nats-server adopts a same-named local stream and raft
+// catch-up never repairs the prefix it already holds.
+func discardJetStreamStore(storeDir string) error {
+	if err := checkStoreDiscardable(); err != nil {
+		return err
+	}
+	if err := os.RemoveAll(storeDir); err != nil {
+		return fmt.Errorf("remove JetStream store %s: %w", storeDir, err)
+	}
+	if _, err := os.Stat(storeDir); !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("JetStream store %s still present after removal", storeDir)
+	}
+	return nil
+}
+
+// checkInitJetStreamStore refuses a multi-node init over a store holding streams
+// when discard was turned off. Keeping it as a seed is not safe: leadership
+// ignores who holds data, and unassigned local streams are deleted as orphans.
+func checkInitJetStreamStore(storeDir string, nodes int, discard bool) error {
+	if nodes < 2 {
+		return nil
+	}
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return err
+	}
+	if len(streams) == 0 {
+		return nil
+	}
+	if !discard {
+		return fmt.Errorf("this node's JetStream store %s holds %d stream(s) from before formation; "+
+			"a multi-node cluster cannot adopt them consistently. "+
+			"re-run without --discard-jetstream=false to remove them once every node has joined", storeDir, len(streams))
+	}
+	return checkStoreDiscardable()
+}

cmd/spinifex/cmd/jetstream_store_test.go

@@ -0,0 +1,140 @@
+//test:in-package: swaps the unexported procRoot for a fake /proc tree and asserts on
+// the store helpers the init and join commands call.
+
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// withFakeProc points the process scan at a temp tree holding the given
+// pid -> cmdline entries (NUL-separated argv) for the duration of the test.
+func withFakeProc(t *testing.T, procs map[string]string) {
+	t.Helper()
+	root := t.TempDir()
+	for pid, cmdline := range procs {
+		dir := filepath.Join(root, pid)
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "cmdline"), []byte(cmdline), 0o644))
+	}
+	prev := procRoot
+	procRoot = root
+	t.Cleanup(func() { procRoot = prev })
+}
+
+// seedStore builds a JetStream store with the named streams under account $G,
+// laid out the way nats-server writes it, and returns the store path.
+func seedStore(t *testing.T, streams ...string) string {
+	t.Helper()
+	store := filepath.Join(t.TempDir(), "nats", "jetstream")
+	for _, s := range streams {
+		dir := filepath.Join(store, "$G", "streams", s, "msgs")
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "1.blk"), []byte("x"), 0o600))
+	}
+	return store
+}
+
+func TestJetStreamStoreDir(t *testing.T) {
+	assert.Equal(t, "/var/lib/spinifex/nats/jetstream", jetStreamStoreDir("/var/lib/spinifex"))
+}
+
+func TestLocalStreams(t *testing.T) {
+	t.Run("missing store holds nothing", func(t *testing.T) {
+		got, err := localStreams(filepath.Join(t.TempDir(), "absent"))
+		require.NoError(t, err)
+		assert.Empty(t, got)
+	})
+
+	t.Run("lists streams across accounts, sorted", func(t *testing.T) {
+		store := seedStore(t, "KV_b", "KV_a")
+		require.NoError(t, os.MkdirAll(filepath.Join(store, "$SYS", "_js_", "_meta_"), 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(store, "$G", "streams", "stray-file"), nil, 0o600))
+
+		got, err := localStreams(store)
+		require.NoError(t, err)
+		assert.Equal(t, []string{"$G/KV_a", "$G/KV_b"}, got)
+	})
+}
+
+func TestNATSRunningLocally(t *testing.T) {
+	tests := []struct {
+		name  string
+		procs map[string]string
+		want  bool
+	}{
+		{"no processes", nil, false},
+		{"installed service", map[string]string{"10": "/usr/local/bin/spx\x00service\x00nats\x00start\x00"}, true},
+		{"bare spx on PATH", map[string]string{"10": "spx\x00service\x00nats\x00start\x00"}, true},
+		// The join itself is an spx process and must not count.
+		{"spx admin join", map[string]string{"10": "spx\x00admin\x00join\x00--force\x00"}, false},
+		{"other spx service", map[string]string{"10": "spx\x00service\x00predastore\x00start\x00"}, false},
+		{"nats-server binary", map[string]string{"10": "nats-server\x00service\x00nats\x00"}, false},
+		{"kernel thread, empty cmdline", map[string]string{"2": ""}, false},
+		{"non-pid entry ignored", map[string]string{"self": "spx\x00service\x00nats\x00"}, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withFakeProc(t, tt.procs)
+			got, err := natsRunningLocally()
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestDiscardJetStreamStore(t *testing.T) {
+	t.Run("removes the store", func(t *testing.T) {
+		withFakeProc(t, nil)
+		store := seedStore(t, "KV_a")
+		require.NoError(t, discardJetStreamStore(store))
+		_, err := os.Stat(store)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+	})
+
+	t.Run("missing store is not an error", func(t *testing.T) {
+		withFakeProc(t, nil)
+		assert.NoError(t, discardJetStreamStore(filepath.Join(t.TempDir(), "absent")))
+	})
+
+	t.Run("refuses under a running NATS and leaves the store", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"7": "spx\x00service\x00nats\x00start\x00"})
+		store := seedStore(t, "KV_a")
+		assert.ErrorIs(t, discardJetStreamStore(store), errNATSRunning)
+		_, err := os.Stat(store)
+		assert.NoError(t, err)
+	})
+}
+
+func TestCheckInitJetStreamStore(t *testing.T) {
+	withFakeProc(t, nil)
+
+	t.Run("single-node init keeps its store", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 1, false))
+	})
+
+	t.Run("multi-node init over an empty store", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(filepath.Join(t.TempDir(), "absent"), 3, false))
+	})
+
+	t.Run("multi-node init over streams is refused without the flag", func(t *testing.T) {
+		err := checkInitJetStreamStore(seedStore(t, "KV_a", "KV_b"), 4, false)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "2 stream(s)")
+		assert.Contains(t, err.Error(), "--discard-jetstream")
+	})
+
+	t.Run("multi-node init over streams proceeds with the flag", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 4, true))
+	})
+
+	t.Run("flag does not override a running NATS", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"7": "spx\x00service\x00nats\x00start\x00"})
+		assert.ErrorIs(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 4, true), errNATSRunning)
+	})
+}

cmd/spinifex/cmd/nats_server_name_test.go

@@ -0,0 +1,44 @@
+//test:in-package — the embedded nats.conf template it checks is unexported,
+// and exporting it would widen the package's API for a test's convenience.
+
+package cmd
+
+import (
+	"strings"
+	"testing"
+
+	natssvc "github.com/mulgadc/spinifex/spinifex/services/nats"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestNATSConfServerNameMatchesTheHelper ties the rendered server_name to
+// nats.ServerName. JetStream reports stream peers by server name, so anything
+// mapping a replica back to the node holding it reads the prefix from that
+// helper — and a template that drifted from it would break the mapping silently,
+// reporting a healthy cluster as one whose buckets name no node at all.
+func TestNATSConfServerNameMatchesTheHelper(t *testing.T) {
+	var line string
+	for l := range strings.SplitSeq(natsConfTemplate, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(l), "server_name:") {
+			line = strings.TrimSpace(l)
+			break
+		}
+	}
+	require.NotEmpty(t, line, "nats.conf no longer sets server_name")
+
+	got := strings.TrimSpace(strings.TrimPrefix(line, "server_name:"))
+	assert.Equal(t, natssvc.ServerName("{{.Node}}"), got)
+
+	node, ok := natssvc.NodeFromServerName(natssvc.ServerName("node2"))
+	assert.True(t, ok)
+	assert.Equal(t, "node2", node)
+
+	_, ok = natssvc.NodeFromServerName("node2")
+	assert.False(t, ok, "a bare node name is not a server name")
+
+	// The bare prefix is a name we could not have generated, and mapping it to the
+	// empty node would put a phantom peer into anything counting nodes.
+	_, ok = natssvc.NodeFromServerName(natssvc.ServerNamePrefix)
+	assert.False(t, ok, "the prefix with no node after it is not a server name")
+}

cmd/spinifex/cmd/root.go

@@ -1,13 +1,18 @@
+// Package cmd defines the spx command-line interface: the cobra commands that
+// install, configure, start and administer a Spinifex node and its cluster.
 package cmd
 
 import (
 	"fmt"
+	"io"
 	"log/slog"
 	"os"
 
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"golang.org/x/term"
 )
 
 var (
@@ -50,6 +55,8 @@ func Execute() {
 	cliLogLevel.Set(slog.LevelError)
 	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: cliLogLevel})))
 
+	setOutputStyling(outputStylingEnabled(os.Getenv("NO_COLOR"), os.Getenv("TERM"), isTerminal(os.Stdout)))
+
 	err := rootCmd.Execute()
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
@@ -57,6 +64,28 @@ func Execute() {
 	}
 }
 
+// outputStylingEnabled reports whether spx may emit ANSI colour and text styling:
+// only on a terminal that is not TERM=dumb, with NO_COLOR unset or empty.
+func outputStylingEnabled(noColor, termName string, stdoutIsTerminal bool) bool {
+	return noColor == "" && termName != "dumb" && stdoutIsTerminal
+}
+
+// isTerminal reports whether w is a file attached to a terminal.
+func isTerminal(w io.Writer) bool {
+	f, ok := w.(*os.File)
+	return ok && term.IsTerminal(int(f.Fd()))
+}
+
+// setOutputStyling switches pterm's colour and bold/underline styling for every
+// printer at once. pterm forces colour on at init regardless of the terminal.
+func setOutputStyling(enabled bool) {
+	if enabled {
+		pterm.EnableStyling()
+	} else {
+		pterm.DisableStyling()
+	}
+}
+
 // initCLILogger retunes the CLI logger from SPX_LOG_LEVEL, then --verbose so an
 // explicit flag wins over the environment. There is no root --debug flag: four
 // service subcommands already define one with a service-specific meaning, and a

cmd/spinifex/cmd/root_output_test.go

@@ -0,0 +1,149 @@
+package cmd_test
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/pterm/pterm"
+)
+
+func TestOutputStylingEnabled(t *testing.T) {
+	tests := []struct {
+		name     string
+		noColor  string
+		term     string
+		terminal bool
+		want     bool
+	}{
+		{name: "terminal keeps colour", term: "xterm-256color", terminal: true, want: true},
+		{name: "unset TERM on a terminal keeps colour", terminal: true, want: true},
+		{name: "TERM=dumb drops colour on a terminal", term: "dumb", terminal: true, want: false},
+		{name: "pipe drops colour", terminal: false, want: false},
+		{name: "NO_COLOR drops colour on a terminal", noColor: "1", terminal: true, want: false},
+		{name: "any non-empty NO_COLOR counts, even 0", noColor: "0", terminal: true, want: false},
+		{name: "NO_COLOR on a pipe", noColor: "1", terminal: false, want: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := cmd.OutputStylingEnabled(tt.noColor, tt.term, tt.terminal); got != tt.want {
+				t.Fatalf("OutputStylingEnabled(%q, %q, %v) = %v, want %v", tt.noColor, tt.term, tt.terminal, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsTerminalFalseForNonTTYWriters(t *testing.T) {
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+	defer w.Close()
+
+	f, err := os.CreateTemp(t.TempDir(), "out")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	for name, wr := range map[string]io.Writer{"pipe": w, "regular file": f, "buffer": &bytes.Buffer{}} {
+		if cmd.IsTerminal(wr) {
+			t.Errorf("IsTerminal(%s) = true, want false", name)
+		}
+	}
+}
+
+// renderNodesTable renders a table shaped like `spx get nodes` through the same
+// pterm.DefaultTable the CLI uses, after applying the styling decision.
+func renderNodesTable(t *testing.T, noColor, termName string, terminal bool) string {
+	t.Helper()
+	t.Cleanup(func() { cmd.SetOutputStyling(true) })
+	cmd.SetOutputStyling(cmd.OutputStylingEnabled(noColor, termName, terminal))
+
+	var buf bytes.Buffer
+	data := pterm.TableData{
+		{"NAME", "STATUS", "IP"},
+		{"node1", "Ready", "10.0.0.1"},
+		{"node-long-name", "NotReady", "10.0.0.22"},
+	}
+	if err := pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(data).WithWriter(&buf).Render(); err != nil {
+		t.Fatalf("render: %v", err)
+	}
+	return buf.String()
+}
+
+func TestTableOutputPlainWhenNotTerminal(t *testing.T) {
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+	defer w.Close()
+
+	out := renderNodesTable(t, "", "xterm", cmd.IsTerminal(w))
+	assertPlainAlignedTable(t, out)
+}
+
+func TestTableOutputPlainWithNoColorOnTerminal(t *testing.T) {
+	out := renderNodesTable(t, "1", "xterm", true)
+	assertPlainAlignedTable(t, out)
+}
+
+func TestTableOutputPlainOnDumbTerminal(t *testing.T) {
+	out := renderNodesTable(t, "", "dumb", true)
+	assertPlainAlignedTable(t, out)
+}
+
+func TestTableOutputStyledOnTerminal(t *testing.T) {
+	out := renderNodesTable(t, "", "xterm", true)
+	if !strings.Contains(out, "\x1b[") {
+		t.Fatalf("table on a terminal carries no ANSI styling, so the plain-output tests prove nothing:\n%q", out)
+	}
+}
+
+func assertPlainAlignedTable(t *testing.T, out string) {
+	t.Helper()
+	if strings.ContainsRune(out, '\x1b') {
+		t.Fatalf("output contains an escape byte:\n%q", out)
+	}
+
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	var rows []string
+	for _, l := range lines {
+		if strings.Contains(l, "node") || strings.HasPrefix(l, "NAME") {
+			rows = append(rows, l)
+		}
+	}
+	if len(rows) != 3 {
+		t.Fatalf("want header and two node rows, got %d:\n%s", len(rows), out)
+	}
+
+	// Columns line up only if the width calculation saw the same bytes it printed.
+	statusCol := strings.Index(rows[0], "STATUS")
+	ipCol := strings.Index(rows[0], "IP")
+	for _, row := range rows[1:] {
+		fields := strings.Fields(row)
+		var status string
+		for _, f := range fields {
+			if strings.Contains(f, "Ready") {
+				status = f
+			}
+		}
+		if got := strings.Index(row, status); got != statusCol {
+			t.Errorf("STATUS column at %d in %q, header at %d", got, row, statusCol)
+		}
+		if got := strings.Index(row, "10.0.0."); got != ipCol {
+			t.Errorf("IP column at %d in %q, header at %d", got, row, ipCol)
+		}
+	}
+
+	// grep -w Ready must match: the byte before the word is not a word character.
+	ready := strings.Index(rows[1], "Ready")
+	if ready <= 0 || rows[1][ready-1] != ' ' {
+		t.Errorf("Ready is not preceded by a space in %q", rows[1])
+	}
+}

cmd/spinifex/cmd/service.go

@@ -225,42 +225,6 @@ var predastoreStartCmd = &cobra.Command{
 	},
 }
 
-var predastoreStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the predastore service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping predastore service...")
-
-		// Only the pid directory matters here: Stop finds the running process
-		// through it, and start wrote the file there.
-		service, err := service.New("predastore", &predastore.Config{
-			BasePath: viper.GetString("predastore-base-path"),
-		})
-
-		if err != nil {
-			return fmt.Errorf("create predastore service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop predastore service: %w", err)
-		}
-
-		fmt.Println("Predastore service stopped")
-		return nil
-	},
-}
-
-var predastoreStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the predastore service",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Add your status logic here
-		fmt.Println("Predastore service status: ...")
-	},
-}
-
 // Repeat for viperblock.
 var viperblockStartCmd = &cobra.Command{
 	Use:           "start",
@@ -413,37 +377,6 @@ var viperblockStartCmd = &cobra.Command{
 	},
 }
 
-var viperblockStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the viperblock service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping viperblock service...")
-
-		service, err := service.New("viperblock", &viperblockd.Config{})
-
-		if err != nil {
-			return fmt.Errorf("create viperblock service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop viperblock service: %w", err)
-		}
-
-		fmt.Println("Viperblock service stopped")
-		return nil
-	},
-}
-
-var viperblockStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the viperblock service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("Viperblock service status: ...")
-	},
-}
-
 var qemunbdCmd = &cobra.Command{
 	Use:   "qemunbd",
 	Short: "Manage the qemunbd service",
@@ -518,37 +451,6 @@ var qemunbdStartCmd = &cobra.Command{
 	},
 }
 
-var qemunbdStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the qemunbd service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping qemunbd service...")
-
-		service, err := service.New("qemunbd", &qemunbdd.Config{})
-
-		if err != nil {
-			return fmt.Errorf("create qemunbd service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop qemunbd service: %w", err)
-		}
-
-		fmt.Println("qemunbd service stopped")
-		return nil
-	},
-}
-
-var qemunbdStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the qemunbd service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("qemunbd service status: ...")
-	},
-}
-
 // Repeat for nats.
 var natsStartCmd = &cobra.Command{
 	Use:           "start",
@@ -589,37 +491,6 @@ var natsStartCmd = &cobra.Command{
 	},
 }
 
-var natsStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the nats service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping nats service...")
-
-		service, err := service.New("nats", &nats.Config{})
-
-		if err != nil {
-			return fmt.Errorf("create nats service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop nats service: %w", err)
-		}
-
-		fmt.Println("Nats service stopped")
-		return nil
-	},
-}
-
-var natsStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the nats service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("Nats service status: ...")
-	},
-}
-
 // Repeat for spinifex.
 var spinifexStartCmd = &cobra.Command{
 	Use:           "start",
@@ -682,37 +553,6 @@ var spinifexStartCmd = &cobra.Command{
 	},
 }
 
-var spinifexStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the spinifex service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping spinifex service...")
-
-		service, err := service.New("spinifex", &config.ClusterConfig{})
-
-		if err != nil {
-			return fmt.Errorf("create spinifex service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop spinifex service: %w", err)
-		}
-
-		fmt.Println("Spinifex service stopped")
-		return nil
-	},
-}
-
-var spinifexStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the spinifex service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("Spinifex service status: ...")
-	},
-}
-
 // AWS GW
 
 var awsgwStartCmd = &cobra.Command{
@@ -785,37 +625,6 @@ var awsgwStartCmd = &cobra.Command{
 	},
 }
 
-var awsgwStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the awsgw service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping awsgw service...")
-
-		service, err := service.New("awsgw", &config.ClusterConfig{})
-
-		if err != nil {
-			return fmt.Errorf("create awsgw service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop awsgw service: %w", err)
-		}
-
-		fmt.Println("AWSGW service stopped")
-		return nil
-	},
-}
-
-var awsgwStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the awsgw service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("AWSGW service status: ...")
-	},
-}
-
 // consoleRegion is the region the console signs with. awsgw pins signature
 // verification to this same node field, so reading it from anywhere else —
 // including the cluster-wide [aws] region — would let the two drift and fail
@@ -876,50 +685,6 @@ var spinifexUIStartCmd = &cobra.Command{
 	},
 }
 
-var spinifexUIStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the spinifex-ui service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping spinifex-ui service...")
-
-		svc, err := service.New("spinifex-ui", &spinifexui.Config{})
-
-		if err != nil {
-			return fmt.Errorf("create spinifex-ui service: %w", err)
-		}
-
-		if err = svc.Stop(); err != nil {
-			return fmt.Errorf("stop spinifex-ui service: %w", err)
-		}
-		fmt.Println("spinifex-ui service stopped")
-		return nil
-	},
-}
-
-var spinifexUIStatusCmd = &cobra.Command{
-	Use:           "status",
-	Short:         "Get status of the spinifex-ui service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		svc, err := service.New("spinifex-ui", &spinifexui.Config{})
-
-		if err != nil {
-			return fmt.Errorf("create spinifex-ui service: %w", err)
-		}
-
-		status, err := svc.Status()
-		if err != nil {
-			return fmt.Errorf("get spinifex-ui service status: %w", err)
-		}
-
-		fmt.Println("spinifex-ui service status:", status)
-		return nil
-	},
-}
-
 // checkLegacyWanBridgeKey fails vpcd startup if the deprecated `wan_bridge`
 // TOML key or `SPINIFEX_VPCD_WAN_BRIDGE` env-var is present. There is no silent
 // alias or auto-rewrite; the operator must remove the key before vpcd will start.
@@ -1025,12 +790,15 @@ var vpcdStartCmd = &cobra.Command{
 			AZ:                      nodeConfig.AZ,
 			NorthstarBaseDomain:     handlers_dns.ResolveBaseDomain(&nodeConfig),
 			NorthstarInternalDomain: handlers_dns.ResolveInternalDomain(&nodeConfig),
+			ServicesDomain:          clusterConfig.AWS.ServicesDomain,
 			ResolverNameservers:     handlers_dns.ResolverNameserverIPs(clusterConfig),
 			NATExemptCIDRs:          clusterConfig.Network.NATExemptCIDRs,
 			IPSecEnabled:            clusterConfig.Network.IPSecEnabled,
 			UnderlayMTU:             clusterConfig.Network.UnderlayMTU,
 			BlockedWANPorts:         clusterConfig.Network.ResolvedBlockedWANPorts(),
 			EgressBlockExemptVPCs:   clusterConfig.Network.EgressBlockExemptVPCs,
+			IMDSHostMetaIP:          clusterConfig.Network.IMDSHostMetaIP,
+			IMDSHostDNSIP:           clusterConfig.Network.IMDSHostDNSIP,
 		})
 		if err != nil {
 			return fmt.Errorf("create vpcd service: %w", err)
@@ -1044,35 +812,6 @@ var vpcdStartCmd = &cobra.Command{
 	},
 }
 
-var vpcdStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the vpcd service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping vpcd service...")
-
-		svc, err := service.New("vpcd", &vpcd.Config{})
-		if err != nil {
-			return fmt.Errorf("create vpcd service: %w", err)
-		}
-
-		if err = svc.Stop(); err != nil {
-			return fmt.Errorf("stop vpcd service: %w", err)
-		}
-		fmt.Println("vpcd service stopped")
-		return nil
-	},
-}
-
-var vpcdStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the vpcd service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("vpcd service status: ...")
-	},
-}
-
 var northstarCmd = &cobra.Command{
 	Use:   "northstar",
 	Short: "Manage the northstar (DNS) service",
@@ -1195,35 +934,6 @@ func resolveNorthstarConfigPath(nodePath, override string) string {
 	return nodePath
 }
 
-var northstarStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the northstar service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping northstar service...")
-
-		svc, err := service.New("northstar", &northstar.Config{})
-		if err != nil {
-			return fmt.Errorf("create northstar service: %w", err)
-		}
-
-		if err = svc.Stop(); err != nil {
-			return fmt.Errorf("stop northstar service: %w", err)
-		}
-		fmt.Println("northstar service stopped")
-		return nil
-	},
-}
-
-var northstarStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the northstar service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("northstar service status: ...")
-	},
-}
-
 var qmpCollectorCmd = &cobra.Command{
 	Use:   "qmp-collector",
 	Short: "Manage the qmp-collector (guest metrics) service",
@@ -1267,34 +977,6 @@ var qmpCollectorStartCmd = &cobra.Command{
 	},
 }
 
-var qmpCollectorStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the qmp-collector service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping qmp-collector service...")
-
-		svc, err := service.New("qmp-collector", &qmpcollector.Config{})
-		if err != nil {
-			return fmt.Errorf("create qmp-collector service: %w", err)
-		}
-		if err = svc.Stop(); err != nil {
-			return fmt.Errorf("stop qmp-collector service: %w", err)
-		}
-		fmt.Println("qmp-collector service stopped")
-		return nil
-	},
-}
-
-var qmpCollectorStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the qmp-collector service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("qmp-collector service status: ...")
-	},
-}
-
 // registerPredastoreNamespacedFlags defines predastore's host, base-path and
 // config-path flags and binds them to viper. It must only run once (from
 // init()); pflag panics on a redefined flag.
@@ -1424,8 +1106,6 @@ func init() {
 	bindPredastoreCollisionEnv()
 
 	predastoreCmd.AddCommand(predastoreStartCmd)
-	predastoreCmd.AddCommand(predastoreStopCmd)
-	predastoreCmd.AddCommand(predastoreStatusCmd)
 
 	serviceCmd.AddCommand(viperblockCmd)
 
@@ -1451,8 +1131,6 @@ func init() {
 	viper.BindPFlag("viperblock-debug", viperblockCmd.PersistentFlags().Lookup("debug"))
 
 	viperblockCmd.AddCommand(viperblockStartCmd)
-	viperblockCmd.AddCommand(viperblockStopCmd)
-	viperblockCmd.AddCommand(viperblockStatusCmd)
 
 	serviceCmd.AddCommand(qemunbdCmd)
 
@@ -1461,15 +1139,11 @@ func init() {
 	viper.BindPFlag("qemunbd-debug", qemunbdCmd.PersistentFlags().Lookup("debug"))
 
 	qemunbdCmd.AddCommand(qemunbdStartCmd)
-	qemunbdCmd.AddCommand(qemunbdStopCmd)
-	qemunbdCmd.AddCommand(qemunbdStatusCmd)
 
 	// Nats
 	serviceCmd.AddCommand(natsCmd)
 
 	natsCmd.AddCommand(natsStartCmd)
-	natsCmd.AddCommand(natsStopCmd)
-	natsCmd.AddCommand(natsStatusCmd)
 
 	// Add NATS flags
 	natsCmd.PersistentFlags().Int("port", 4222, "NATS server port")
@@ -1491,8 +1165,6 @@ func init() {
 	serviceCmd.AddCommand(spinifexCmd)
 
 	spinifexCmd.AddCommand(spinifexStartCmd)
-	spinifexCmd.AddCommand(spinifexStopCmd)
-	spinifexCmd.AddCommand(spinifexStatusCmd)
 
 	spinifexCmd.PersistentFlags().String("wal-dir", "", "Write-ahead log (WAL) directory. Place on high-speed NVMe disk, or tmpfs for development.")
 	viper.BindEnv("wal-dir", "SPINIFEX_WAL_DIR")
@@ -1515,8 +1187,6 @@ func init() {
 	bindAwsgwCollisionEnv()
 
 	awsgwCmd.AddCommand(awsgwStartCmd)
-	awsgwCmd.AddCommand(awsgwStopCmd)
-	awsgwCmd.AddCommand(awsgwStatusCmd)
 
 	// spinifex-ui
 	serviceCmd.AddCommand(spinifexUICmd)
@@ -1546,15 +1216,11 @@ func init() {
 	viper.BindPFlag("spinifex-ui-ochre-enabled", spinifexUICmd.PersistentFlags().Lookup("ochre-enabled"))
 
 	spinifexUICmd.AddCommand(spinifexUIStartCmd)
-	spinifexUICmd.AddCommand(spinifexUIStopCmd)
-	spinifexUICmd.AddCommand(spinifexUIStatusCmd)
 
 	// vpcd
 	serviceCmd.AddCommand(vpcdCmd)
 
 	vpcdCmd.AddCommand(vpcdStartCmd)
-	vpcdCmd.AddCommand(vpcdStopCmd)
-	vpcdCmd.AddCommand(vpcdStatusCmd)
 
 	// northstar
 	serviceCmd.AddCommand(northstarCmd)
@@ -1564,13 +1230,9 @@ func init() {
 	viper.BindPFlag("northstar-config", northstarCmd.PersistentFlags().Lookup("northstar-config"))
 
 	northstarCmd.AddCommand(northstarStartCmd)
-	northstarCmd.AddCommand(northstarStopCmd)
-	northstarCmd.AddCommand(northstarStatusCmd)
 
 	// qmp-collector
 	serviceCmd.AddCommand(qmpCollectorCmd)
 
 	qmpCollectorCmd.AddCommand(qmpCollectorStartCmd)
-	qmpCollectorCmd.AddCommand(qmpCollectorStopCmd)
-	qmpCollectorCmd.AddCommand(qmpCollectorStatusCmd)
 }

cmd/spinifex/cmd/service_exit_test.go

@@ -62,16 +62,8 @@ func writeMalformedToml(t *testing.T) string {
 	return writeSpinifexToml(t, "[section\nkey = value without quotes\n")
 }
 
-// isolateRuntimeDir redirects the pid-file fallback (utils.pidPath, via
-// XDG_RUNTIME_DIR) to an empty temp dir, so a stop command with no explicit
-// directory override can't resolve against a real, shared, ambient pid dir.
-func isolateRuntimeDir(t *testing.T) {
-	t.Helper()
-	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
-}
-
 // TestServiceStartCmdsReturnErrorOnMissingConfig exercises each service
-// start/stop command's RunE directly and asserts a fatal precondition
+// start command's RunE directly and asserts a fatal precondition
 // failure surfaces as a returned error, not a silent zero-exit return.
 // Cases with a setup func drive further into RunE toward a specific branch.
 func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
@@ -339,99 +331,6 @@ func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
 			},
 			wantErr: "start qmp-collector service",
 		},
-		// The stop commands below resolve their pid directory to an isolated,
-		// always-empty temp dir, so Stop() safely fails to find a pid file
-		// rather than risking a real running process via an ambient path.
-		{
-			name: "predastore stop (no running process)",
-			cmd:  predastoreStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				viper.Set("predastore-base-path", t.TempDir())
-			},
-			wantErr: "stop predastore service",
-		},
-		{
-			name: "viperblock stop (no running process)",
-			cmd:  viperblockStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop viperblock service",
-		},
-		{
-			name: "qemunbd stop (no running process)",
-			cmd:  qemunbdStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop qemunbd service",
-		},
-		{
-			name: "nats stop (no running process)",
-			cmd:  natsStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop nats service",
-		},
-		{
-			name: "spinifex stop (no running process)",
-			cmd:  spinifexStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop spinifex service",
-		},
-		{
-			name: "awsgw stop (no running process)",
-			cmd:  awsgwStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop awsgw service",
-		},
-		{
-			name: "spinifex-ui stop (no running process)",
-			cmd:  spinifexUIStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop spinifex-ui service",
-		},
-		{
-			name: "vpcd stop (no running process)",
-			cmd:  vpcdStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop vpcd service",
-		},
-		{
-			name: "northstar stop (no running process)",
-			cmd:  northstarStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop northstar service",
-		},
-		{
-			name: "qmp-collector stop (no running process)",
-			cmd:  qmpCollectorStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop qmp-collector service",
-		},
 	}
 
 	for _, tt := range tests {
@@ -451,36 +350,6 @@ func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
 	}
 }
 
-// TestSpinifexUIStatusCmdReportsStoppedWithNoRunningProcess drives the
-// status command's success path: with no pid file at an isolated
-// XDG_RUNTIME_DIR, Status reports "stopped" and RunE must return nil.
-func TestSpinifexUIStatusCmdReportsStoppedWithNoRunningProcess(t *testing.T) {
-	isolateRuntimeDir(t)
-
-	require.NotNil(t, spinifexUIStatusCmd.RunE)
-	err := spinifexUIStatusCmd.RunE(spinifexUIStatusCmd, nil)
-	require.NoError(t, err)
-}
-
-// TestSpinifexUIStatusCmdReturnsErrorOnUnreadablePidDir drives the status
-// command's other branch: a genuine read failure (permission denied, not
-// ENOENT) must surface as an error rather than the benign "stopped" case.
-func TestSpinifexUIStatusCmdReturnsErrorOnUnreadablePidDir(t *testing.T) {
-	if os.Geteuid() == 0 {
-		t.Skip("root bypasses directory permission checks")
-	}
-
-	dir := t.TempDir()
-	require.NoError(t, os.WriteFile(filepath.Join(dir, "spinifex-ui.pid"), []byte("123"), 0o600))
-	require.NoError(t, os.Chmod(dir, 0o000))
-	t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
-	t.Setenv("XDG_RUNTIME_DIR", dir)
-
-	require.NotNil(t, spinifexUIStatusCmd.RunE)
-	err := spinifexUIStatusCmd.RunE(spinifexUIStatusCmd, nil)
-	require.ErrorContains(t, err, "get spinifex-ui service status")
-}
-
 // TestServiceStartExitsNonZeroOnFailure proves a fatal service-start error
 // terminates the process non-zero (systemd's Restart=on-failure depends on
 // it). os.Exit can't be observed in-process, so this re-execs the test

cmd/spinifex/cmd/templates/spinifex.toml

@@ -10,7 +10,7 @@ email = "{{.OperatorEmail}}"
 
 [aws]
 region = "{{.Region}}"
-internal_suffix = "spinifex.internal"
+services_domain = "services.internal"
 
 [network]
 ipsec_enabled = {{.IPSecEnabled}}

docs/DESIGN.md

@@ -289,31 +289,25 @@ Configs are generated by `spx admin init` (leader) or `spx admin join` (follower
 
 ```
 spinifex/
+├── cmd/
+│   ├── spinifex/            # spx CLI: service entry points and `spx admin`
+│   ├── installer/           # Platform installer
+│   └── ...                  # In-guest agents (lb, ecs, rds, eks helpers) and dev tools
 ├── spinifex/
-│   ├── services/                # Long-running services (awsgw, spinifex, viperblockd,
-│   │                            #   predastore, nats, spinifexui)
-│   ├── gateway/                 # AWS API surface
-│   │   ├── gateway.go           # chi router, SigV4 auth, throttling
-│   │   ├── auth.go              # SigV4 verification + IAM lookup
-│   │   ├── ec2.go               # EC2 action dispatcher
-│   │   ├── ec2/                 # Per-resource handlers (instance, volume, vpc, ...)
-│   │   ├── elbv2/, iam/, policy/
-│   ├── handlers/                # NATS-side service implementations (ec2, elbv2, iam, ...)
-│   ├── daemon/
-│   │   ├── daemon.go            # Daemon entry, NATS subscription table
-│   │   └── daemon_handlers_*.go # One file per resource family
-│   ├── admin/                   # Cluster bootstrap (init/join, master.key, CA, tokens)
-│   ├── config/                  # Configuration structures
-│   ├── instancetypes/           # t3.*, m5.*, sys.* definitions
-│   ├── lbagent/                 # In-VM agent for ELB target health
-│   ├── vm/                      # VM instance representation
-│   ├── vpcd/                    # VPC daemon (OVN/OVS network programming)
-│   └── qmp/                     # QEMU Machine Protocol client
-└── cmd/
-    ├── spinifex/                # spx CLI
-    ├── lb-agent/                # ELB target agent binary
-    ├── installer/               # Platform installer
-    └── ...                      # ECS agent, ECR credential provider, EKS helper binaries
+│   ├── services/            # Long-running services (awsgw, spinifex, nats, predastore, ...)
+│   ├── gateway/             # AWS API surface: routing, SigV4 auth, per-service dispatch
+│   ├── handlers/            # NATS-side service implementations (ec2, iam, eks, rds, ...)
+│   ├── daemon/              # Per-node daemon and its NATS subscription table
+│   ├── vm/, qmp/            # VM lifecycle and the QEMU Machine Protocol client
+│   ├── vpcd/, network/      # VPC daemon and the layered OVN/OVS network stack
+│   ├── admin/               # Cluster bootstrap (init/join, master.key, CA, tokens)
+│   └── config/              # Configuration structures
+├── internal/                # In-guest SigV4 clients and the AWS Smithy models
+├── build/                   # systemd units, node helper scripts, microVM build
+├── scripts/                 # Install, dev-environment, image-build scripts
+└── tests/
+    ├── integration/         # Gateway router against embedded NATS JetStream
+    └── e2e/                 # Live-cluster suites (see docs/testing/e2e-coverage.md)
 ```
 
 ## Development

docs/README.md

@@ -2,7 +2,11 @@
 
 This directory contains the Spinifex documentation. This is available in individual markdown files, and a generated website to aid navigation and browsing documentation at: [https://docs.mulgadc.com](https://docs.mulgadc.com/)
 
-The [AWS model operation coverage](aws-model-operation-coverage.md) report is generated from Spinifex's dispatch tables and the pinned AWS API models. Run `make aws-model-coverage` to regenerate and print it.
+The [AWS API coverage](coverage/README.md) pages are generated from Spinifex's dispatch tables and the pinned AWS API models, one page per service. Run `make aws-model-coverage` to regenerate them and print a summary. Only `pages.json` and the `intro.md` files under `coverage/` are hand-written; every `README.md` there is generated and will be overwritten.
+
+The same run writes [`coverage/coverage.json`](coverage/coverage.json), a versioned machine-readable model-versus-dispatch inventory. It is generated and must not be edited. Its operation states are structural: `implemented` means a registered action has neither a stub nor an explicit refusal. It is not a behavioural compatibility claim; conformance evidence and capability profiles establish that separately.
+
+A published page names the operations Spinifex serves, and nothing else. It carries no percentage, no unimplemented operation and no operation the platform will never offer: the same run writes those to [`coverage/internal-report.md`](coverage/internal-report.md), which no slug in the docs site's configuration points at, so it stays internal. A `notApplicable` declaration in `pages.json` therefore publishes nothing; it keeps a permanent absence out of the report's candidate work.
 
 ## Frontmatter
 

docs/admin/host-firewall/README.md

@@ -0,0 +1,233 @@
+---
+title: "Host Firewall"
+seoTitle: "Host Firewall and Cluster Membership — Spinifex Docs"
+description: "Arm the Spinifex host firewall, narrow SSH to your management networks, and learn how the peer-scoped cluster plane tracks membership changes on its own."
+category: "Admin"
+tags:
+  - security
+  - firewall
+  - nftables
+  - hardening
+resources:
+  - title: "Network Connections"
+    url: "/docs/network-connections"
+  - title: "Multi-Node Install"
+    url: "/docs/install-multi-node"
+  - title: "Setting Up Your Cluster"
+    url: "/docs/setting-up-your-cluster"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+---
+
+# Host Firewall
+
+> Arm the host firewall, narrow SSH, and keep the cluster plane off the public internet.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [What the policy allows](#what-the-policy-allows)
+- [Cluster membership](#cluster-membership)
+- [Is it on?](#is-it-on)
+- [Narrowing SSH access](#narrowing-ssh-access)
+- [Turning it off and on around cluster changes](#turning-it-off-and-on-around-cluster-changes)
+- [The files](#the-files)
+- [Checking it](#checking-it)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Spinifex ships an optional host firewall — an nftables input policy that defaults to drop and divides the node's ports into two groups:
+
+| Group | Ports | Who can reach them |
+|---|---|---|
+| **Public** | SSH, 443, 3000 (console), 8443 (S3), 9999 (AWS gateway), 53 (DNS) | anyone |
+| **Internal** | OVN, NATS, formation, Geneve and the rest of the cluster plane | **cluster members only** |
+
+The internal group is the point. Before this existed, OVN and NATS were reachable from the public internet on a WAN-facing node.
+
+**"Cluster members" is not a list you maintain.** Each node works it out from the cluster it belongs to and rewrites its own rules whenever membership changes — you never edit the peer list by hand.
+
+The policy governs inbound traffic only. Outbound is untouched, because the metadata reply path egresses under per-tap policy routing and a filter there would break it.
+
+## Instructions
+
+## What the Policy Allows
+
+The public plane is open to any source, IPv4 only — an accept with no source scope would otherwise also serve on any global IPv6 address the node picks up from an upstream router advertisement:
+
+| Ports | Serving |
+|---|---|
+| TCP 443 | Held for the TLS edge; no listener yet, so the rollout needs no firewall change |
+| TCP 3000 | Web console |
+| TCP 8443 | Predastore S3 gate |
+| TCP 9999 | AWS gateway — also how every guest agent (load balancer, EKS, ECS, RDS) reaches its control plane, since they speak SigV4 rather than NATS |
+| TCP and UDP 53 | Northstar authoritative DNS. Open on every node deliberately: a node serving no public zone still answers here, which is the accepted cost of not templating the policy per node |
+
+SSH is public too, but scoped by a variable you own — see [Narrowing SSH access](#narrowing-ssh-access).
+
+The cluster plane is peer-scoped. On a single-NIC node the planes collapse onto one address, which is why these rules match peer addresses rather than an interface or a CIDR:
+
+| Ports | Serving |
+|---|---|
+| TCP 4222, 4248 | NATS client and cluster routes |
+| TCP 4432 | Formation, then the daemon cluster manager |
+| TCP and UDP 5300 | Internal DNS |
+| TCP 6641, 6642 | OVN northbound and southbound |
+| TCP 6643, 6644 | OVN database clustering |
+| TCP 8660 | Predastore's admin listener |
+| UDP 6660, 7660 | Object shards and metadata consensus |
+| UDP 6081, 500, 4500, and ESP | Geneve, IKE, NAT-T and IPsec |
+
+Two of those are peer-scoped for reasons worth knowing. Outside a formation window, 4432 is the daemon cluster manager, whose routes report node topology, service inventory and running instances to anyone who asks. And 8660 is Predastore's unauthenticated `/healthz` and `/readyz`, which name the peers a gate cannot reach.
+
+**The formation port needs nothing from you.** `spx admin init` opens 4432 to any source for the length of the formation window and closes it again afterwards, because a node dialling in to join is not a peer yet. It is one slot rather than a list, so it cannot be grown into a general-purpose hole. The handshake behind it is TLS 1.3 with a bearer token.
+
+Beyond those groups the policy accepts what the platform cannot work without: established and related connections, loopback, the ICMP types PMTUD needs under a Geneve overlay, broadcast DHCP replies for the uplink and external-pool leases, and guest traffic to the metadata and VPC DNS endpoints on `169.254.169.254` and `169.254.169.253`. The guest rules are scoped to those two addresses and their ports rather than accepting everything arriving on a guest port, which would expose every wildcard-bound host service to any tenant.
+
+## Cluster Membership
+
+The daemon resolves who the node's peers are from the cluster it belongs to, writes them into `peers.nft`, and reloads the policy. A node that has not resolved membership yet is left reachable rather than cut off from a cluster it cannot name: the apply helper fails without touching the ruleset when the peer file is missing.
+
+One consequence matters in practice. **A node only recognises the members of the cluster it currently belongs to.** During formation the nodes do not yet know each other, so internal traffic between them is dropped — which is why forming or expanding a cluster starts by turning the policy off.
+
+## Is It On?
+
+| How the node was installed | Firewall |
+|---|---|
+| From the ISO | **on** |
+| Binary installer (`curl \| bash`) or `setup.sh` | **off** |
+| `setup.sh --firewall=on` | **on** |
+
+The binary installer defaults to off deliberately: it runs on servers that already have an operating system and services on them, and switching on a default-deny policy uninvited could cut off something Spinifex knows nothing about. The ISO is an appliance on hardware we define, so there the policy is the product.
+
+**For production, turn it on** — either at install time:
+
+```bash
+curl -fsSL https://install.mulgadc.com | bash -s -- --firewall=on
+```
+
+or afterwards, by setting it in `/etc/spinifex/spinifex.toml` and restarting the daemon:
+
+```toml
+[network]
+firewall_enabled = true
+```
+
+Before you do, check what else the machine is serving. Anything listening on a port outside the public group above stops accepting new connections.
+
+## Narrowing SSH Access
+
+SSH is accepted from **every source** by default. That is deliberate — arming the policy should not lock an operator out of the node they are arming it from — but it is the first thing to change on a production cluster.
+
+The sources live in `/etc/spinifex/firewall/custom.nft`, which the installer creates once and never rewrites, so an edit here survives upgrades, re-installs and node resets:
+
+```
+define trusted_ssh_peers = { 10.0.0.0/8, 192.168.1.0/24 }
+```
+
+Check the change before applying it, and check the policy file rather than this one — on its own `custom.nft` parses as a bare define and proves nothing, because the rule that uses the variable lives elsewhere:
+
+```bash
+sudo nft -c -f /etc/spinifex/firewall/spinifex.nft
+sudo nft -f /etc/spinifex/firewall/spinifex.nft
+```
+
+Never leave the variable empty. An undefined or empty set fails the whole ruleset, not just its own rule.
+
+The ports come from sshd's own configuration, re-read on every `setup.sh` run, so moving sshd to a non-standard port is picked up by an upgrade rather than needing an edit here. IPv4 only: sshd reached over IPv6 is not matched by this rule.
+
+## Turning It Off and On Around Cluster Changes
+
+Turn the firewall off while you form or expand a cluster, and on again once it is up:
+
+```bash
+# Off — before forming or expanding a cluster. Run on every node.
+sudo /usr/local/lib/spinifex/spinifex-firewall-apply disable
+
+# On — once the cluster is formed and verified. Run on every node.
+sudo systemctl restart spinifex-daemon
+```
+
+Restarting the daemon is what re-arms it: the node rebuilds its peer list from the cluster it is now part of, reloads the rules, and re-enables the boot-time unit so the policy survives a reboot. It also happens on its own within five minutes if you would rather wait.
+
+`disable` removes Spinifex's own table and the peer file and leaves every other nftables table alone. It is idempotent, and it is the recovery path — it runs even on a node with no policy installed, so it cannot leave you with a node you can neither arm nor disarm.
+
+> [!WARNING]
+> Do not flush the whole nftables ruleset to clear the policy. The VPC daemon writes SNAT and per-elastic-IP forward rules into the `ip nat` and `ip filter` tables and reinstalls them only when it starts, so `nft flush ruleset` breaks elastic IPs silently until the next `systemctl restart spinifex-vpcd`. Use `disable`, which only ever touches `inet spinifex_filter`.
+
+## The Files
+
+Everything lives under `/etc/spinifex/firewall/`. Which files are safe to edit follows from who writes them:
+
+| File | Written by | Survives an upgrade |
+|---|---|---|
+| `spinifex.nft` | `setup.sh` — the policy itself | No. Edits are overwritten |
+| `local.nft` | `setup.sh` — the sshd ports it detected | No. Refreshed on every run |
+| `peers.nft` | The daemon — cluster and encap peer addresses | No. Rewritten on every membership change |
+| `open-ports.nft` | The apply helper — the formation window only | No. Reset closed on install and upgrade |
+| `custom.nft` | **You** — `trusted_ssh_peers` and anything else you add | **Yes.** Created once, never rewritten |
+| `mode` | `setup.sh` — whether the install path armed the policy | No |
+
+The daemon cannot write `/etc` itself, and a blanket `nft` grant would be root-equivalent, so changes go through `/usr/local/lib/spinifex/spinifex-firewall-apply` — a root helper that accepts a fixed set of verbs and re-validates every address handed to it.
+
+## Checking It
+
+```bash
+sudo nft list table inet spinifex_filter
+```
+
+The peer list is an nft variable, expanded when the rules load, so it does not appear under a name of its own. Look instead at the `ip saddr { ... }` addresses on the cluster-plane rules — the ones accepting 4222, 6641, 6642 and the rest.
+
+Every node's addresses should be there. On a multi-NIC node that means its WAN, LAN and VPC addresses, so expect several entries per node. A missing node means its cluster traffic is being dropped.
+
+Dropped packets are logged, rate-limited to five a minute, so this tells you whether a connection problem is the firewall or something else:
+
+```bash
+sudo journalctl -k | grep 'spinifex-fw drop'
+```
+
+An empty log with a connection still failing means the policy is not the cause. A node with no table loaded at all is not armed:
+
+```bash
+systemctl is-enabled spinifex-firewall.service
+```
+
+## Troubleshooting
+
+### A Join Hangs Instead of Failing
+
+A hang rather than a quick refusal means packets are being dropped. Check the drop log on the node running init, and confirm the formation port opened — `spx admin init` prints `⚠️ Could not open port 4432 in the host firewall` when it could not:
+
+```bash
+sudo journalctl -k | grep 'spinifex-fw drop'
+```
+
+On ISO-installed nodes the usual cause is that each node's policy still trusts only itself, because that is the whole cluster as far as it knows. Turn the firewall off on **every** node, retry, and re-arm once the cluster is verified.
+
+### A Node Is Missing from the Peer List
+
+The daemon rewrites `peers.nft` from the cluster it belongs to, so a missing node means that node is not in the membership this one resolved. Confirm what the cluster thinks first, then restart the daemon to force a rebuild:
+
+```bash
+spx get nodes
+sudo systemctl restart spinifex-daemon
+```
+
+### One of My Own Services Stopped Answering
+
+Arming the policy closes every port outside the public group. Add what you need to `custom.nft`, which is the only file an upgrade leaves alone, and check the policy before applying it:
+
+```bash
+sudo nft -c -f /etc/spinifex/firewall/spinifex.nft
+```
+
+### Elastic IPs Stopped Working
+
+Something flushed the whole nftables ruleset, taking the VPC daemon's SNAT and forward rules with it. They are reinstalled on start:
+
+```bash
+sudo systemctl restart spinifex-vpcd
+```

docs/admin/predastore-distributed-storage/README.md

@@ -0,0 +1,250 @@
+---
+title: "Predastore Distributed Storage"
+seoTitle: "How Predastore Distributed Storage Works — Spinifex Docs"
+description: "How Spinifex stores objects across a cluster: gate, blob and meta nodes, Reed-Solomon erasure coding, placement, degraded writes, repair and encryption at rest."
+category: "Admin"
+tags:
+  - storage
+  - s3
+  - predastore
+  - erasure coding
+  - durability
+resources:
+  - title: "Predastore Repository"
+    url: "https://github.com/mulgadc/predastore"
+  - title: "Viperblock (EBS)"
+    url: "https://github.com/mulgadc/viperblock"
+  - title: "Multi-Node Install"
+    url: "/docs/install-multi-node"
+  - title: "Network Connections"
+    url: "/docs/network-connections"
+---
+
+# Predastore Distributed Storage
+
+> How Spinifex stores objects across a cluster, and what a server failure costs.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Hosts and nodes](#hosts-and-nodes)
+- [The configuration file](#the-configuration-file)
+- [Erasure coding and durability](#erasure-coding-and-durability)
+- [Placement](#placement)
+- [The write path](#the-write-path)
+- [The read path](#the-read-path)
+- [Repair](#repair)
+- [Encryption at rest](#encryption-at-rest)
+- [What a server failure costs](#what-a-server-failure-costs)
+- [Checking it](#checking-it)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Predastore is the S3-compatible object store underneath a Spinifex cluster, and almost everything durable ends up in it:
+
+- **Machine images** — the AMIs instances launch from
+- **Block storage** — EBS volume data and snapshots, written through [Viperblock](https://github.com/mulgadc/viperblock), which uses Predastore as its S3 backend
+- **User data** — cloud-init configuration and system artifacts
+- **DNS zone files** — the zones Northstar serves
+- **Anything your own workloads put in a bucket** through the S3 API
+
+This is background reading. You do not configure any of it by hand: `spx admin init` and `spx admin join` build the topology from the servers that actually form the cluster, and every machine gets the same file with its own host ID recorded in `spinifex.toml`.
+
+## Instructions
+
+## Hosts and Nodes
+
+A cluster is two levels.
+
+A **host** is one machine: one Predastore process owning that machine's data directory, TLS identity and encryption key. A **node** is a role pinned to a host, running inside that process on its own port. Every server in a Spinifex cluster is one host carrying three nodes:
+
+| Role | Port | Purpose |
+|---|---|---|
+| `gate` | TCP 8443 | Serves the S3 API — SigV4, IAM, erasure coding, placement. Every server runs one, so any of them answers an S3 request. |
+| `blob` | UDP 6660 | Holds erasure-coded object shards. One per machine. |
+| `meta` | UDP 7660 | Member of the Raft quorum over global state — buckets and the object index. |
+
+Ports have to be unique within a host but not across the cluster, so every machine uses the same three. That is why the port surface does not widen as the cluster grows — see [Network requirements](/docs/install-multi-node#network-requirements).
+
+Which transport a pair of nodes uses follows from where they sit, not from a mode flag. Nodes on the same machine talk over an in-process pipe and open no socket at all; nodes on different machines talk over QUIC, TLS 1.3, authenticated by the cluster CA. A single-server install is therefore one process listening on 8443 and nothing else.
+
+Nothing dials a gate, so a gate binds no cluster-plane listener. Its port is the S3 port and nothing more.
+
+## The Configuration File
+
+The whole cluster's storage topology lives in `/etc/spinifex/predastore/predastore.toml`, identical on every server apart from which host the local process is told to be.
+
+Each server is one `[[host]]` block carrying three `[[host.node]]` blocks:
+
+```toml
+[[host]]
+id = 1
+bind_addr = "10.2.0.2"                                   # cluster plane: raft and blob
+addr = "10.2.0.2"                                        # what the other hosts dial
+data_dir = "/var/lib/spinifex/predastore"
+tls_cert = "/etc/spinifex/server.pem"
+tls_key = "/etc/spinifex/server.key"
+encryption_key = "/etc/spinifex/predastore/encryption.key"
+admin_port = 8660                                        # /healthz, /readyz, /repair
+
+[[host.node]]
+id = 1
+role = "gate"
+port = 8443
+bind_addr = "0.0.0.0"                                    # public plane; only a gate may set this
+```
+
+The two bind addresses are the point. `bind_addr` on the host is the cluster plane — Raft consensus and shard traffic listen there and nowhere else, at the address peers dial rather than a wildcard, so on a multi-homed machine replication never reaches the public interface. The gate binds the public plane separately, because S3 is meant to be reachable.
+
+## Erasure Coding and Durability
+
+The erasure code is cluster configuration, not a per-request choice, and it is chosen at formation from the number of servers — each machine contributes exactly one blob node:
+
+| Servers | Code | What it survives |
+|---|---|---|
+| 1 | `RS(1,0)` | Nothing at this layer. One shard on one node; redundancy is the pool's job — ZFS raidz or a mirror under the data directory. |
+| 2 | `RS(1,1)` | One server's shards, but the metadata quorum has no majority to lose. |
+| 3 or more | `RS(2,1)` | Any one server's shards, with the metadata quorum intact. |
+
+An object is encoded end to end into `K` data and `M` parity shards with no intermediate chunking. Each shard is roughly `⌈size / K⌉` bytes, and any `K` of the `K+M` shards rebuild the object.
+
+`data + parity` must not exceed the number of blob nodes, because a stripe is spread over distinct nodes and a narrower cluster would fail every write. `RS(2,1)` surviving the loss of any one server is the reason [three servers is the recommended minimum](/docs/install-multi-node).
+
+## Placement
+
+Which nodes hold an object's shards comes from a consistent hash ring over the blob nodes, keyed by the object hash. `Nodes(hash, K+M)` returns the placement in shard order — shard `i` goes to the node at position `i` — so every gate in the cluster derives the same placement from the same object without consulting anything.
+
+The gate then **records** that placement in the metadata plane rather than trusting the ring at read time. A ring that has since changed shape does not lose objects placed under the old one.
+
+## The Write Path
+
+The gate authenticates the request, resolves the bucket, hashes the object name, places it on the ring, encodes the body into data and parity shards, streams each shard to its blob node in parallel, and records the placement.
+
+Three properties matter operationally:
+
+- **The write is two-phase.** Each shard put makes the bytes durable and invisible. The placement record landing in Raft is the commit point, and the shards are published only afterwards — so an interrupted writer costs a round trip rather than the object. Shards are always durable before metadata: a crash between the two leaves shards nothing points at, rather than metadata pointing at data that was never written.
+- **Every write mints an epoch**, stamped on every shard of the stripe and into the record. The record names one generation, and a shard either belongs to it or does not. That is what makes a half-completed overwrite a reconstruction rather than an object spliced from two generations.
+- **One server down does not refuse writes.** The write is acknowledged once `K` shards are durable, and the response carries `X-Spx-Degraded-Write: <n>`. `K` is the floor rather than `K+1` because any `K` of the `K+M` reconstruct — at `RS(2,1)` one more buys no redundancy and would refuse three quarters of writes with a single server down.
+
+A shard its owner will not take is written to the next node along the ring instead of being given up on — **hinted handoff** — and the response carries `X-Spx-Handoff: <n>`. The stripe is then complete and the object is as redundant as a full-width write; what is outstanding is only that the shards are not all where the record says. The holder is derived rather than recorded, so there is no hint to store, replicate or lose. Handoff is skipped when the cluster has no node to spare, since a second shard of one object on a node that already holds one is not redundancy.
+
+If any node reported storage pressure the response also carries `X-Predastore-Pool-Pressure: nearfull`; a node that refused outright surfaces as `507 InsufficientStorage`.
+
+## The Read Path
+
+The gate reads the recorded placement, fetches the data shards, and joins them. Parity is never touched on the happy path. Only if the join fails does it refetch including parity and reconstruct, and a read that had to reconstruct says so in `X-Spx-Degraded: <n>` — a cost report, not an error.
+
+Every shard read names the record's epoch, and a node holding another generation answers `epoch-mismatch`. **The gate counts that exactly as a missing shard**, so the read either rebuilds the object the record names or fails. It never returns plausible nonsense.
+
+A shard its owner cannot serve is looked for on the handoff node before it is counted as failed, since a write that could not reach the owner will have put it there.
+
+A `Range` request that lands entirely inside one data shard takes a fast path — a single ranged shard read — because Reed-Solomon splits data sequentially. Anything wider reconstructs the object and slices it.
+
+## Repair
+
+Degraded writes trade redundancy for availability, and repair is what pays that back. It restores shards a blob node owns but does not hold at the generation its object's record names, and it runs on every server with a local blob node.
+
+Two things are worth knowing about it:
+
+- **It carries no correctness weight.** A read compares each shard's epoch against the record it has already loaded and discards a stale one whether or not repair has ever run. What repair restores is redundancy — the number of further losses an object survives — and never the object.
+- **It rebuilds onto the owner**, so it closes the gap only once the missing server returns. At `RS(2,1)` an object written while a server is down holds no parity for the duration of the outage. That is the accepted position: refusing writes meanwhile is the worse failure.
+
+Every pass scans the placement records from the beginning, asks each holder which generation it has, and rebuilds the ones that disagree. A shard that was handed off is returned rather than recomputed — cheaper by a decode — and the holder's copy is released only after the owner's is published. If fewer than `K` peers hold the record's generation the rebuild fails and says so, rather than inventing a shard from a mixture of generations.
+
+## Encryption at Rest
+
+Every fragment is sealed independently under AES-256-GCM, with a deterministic nonce and additional data reconstructed at read time from the on-disk header plus per-data-directory state. There is no unencrypted code path: the process refuses to start without an encryption key.
+
+| Property | What it means |
+|---|---|
+| **Confidentiality** | Disk-level access yields ciphertext and tags. Plaintext is never written. |
+| **Authenticated integrity** | GCM is the sole integrity authority; there is no separate checksum. Corruption, tampering and a wrong key all surface as one integrity error. |
+| **Position binding** | The additional data binds each fragment to its slot, so swapping fragments between objects fails the tag. |
+| **Cross-directory defence** | A per-data-directory store ID enters the nonce, so splicing a fragment from one data directory into another fails the tag. |
+
+The key is one 32-byte AES-256 key per cluster, and **every server must be given the same file** — fragments sealed under one key cannot be opened under another. `spx admin init` generates it and `spx admin join` distributes it, which is also why joining a server that has already been in service discards its own key and orphans everything sealed under it. The loader is fail-closed on permissions: a group- or other-readable key file is rejected outright, with no override.
+
+## What a Server Failure Costs
+
+| What fails | Reads | Writes |
+|---|---|---|
+| One gate (of three or more) | Unaffected — any gate answers for the cluster | Unaffected |
+| One blob node, `RS(2,1)` | Served by reconstruction, reported in `X-Spx-Degraded` | Accepted at the `K` floor, reported in `X-Spx-Degraded-Write`, or placed complete via handoff |
+| More than `M` blob nodes | Objects placed on them are lost | Refused |
+| Metadata quorum | Serve whatever the surviving replicas last applied | Stopped entirely |
+
+Reads try the cached Raft leader first, then every replica, and report not-found only once every replica has answered not-found — a replica that has not applied a key yet must not be mistaken for absence. Writes go through the leader, and a replica that cannot commit redirects the client to the one that can.
+
+Replacing a server restores capacity, not the shards that were on the old one: repair rebuilds onto the owner it expects, so plan a replacement as a return of that node rather than as re-replication onto a new one.
+
+## Checking It
+
+Each host runs an unauthenticated admin listener on port 8660, bound to the cluster plane rather than the S3 address. It serves counters and readiness, never addresses or keys:
+
+```bash
+curl -s http://$(hostname -I | awk '{print $1}'):8660/readyz
+curl -s http://$(hostname -I | awk '{print $1}'):8660/repair
+```
+
+`/readyz` reports the most recent probe cycle for the nodes in that process, and answers `503` until one has completed — reporting ready on no information would send traffic to a process that has never answered the question. `/healthz` answers for the process itself: reaching the handler is the whole check.
+
+That the gate is serving is an S3 request like any other:
+
+```bash
+export AWS_PROFILE=spinifex
+aws s3 ls
+```
+
+To see whether a write or read was degraded, ask for the response headers:
+
+```bash
+aws s3api put-object --bucket predastore --key probe --body /etc/hostname --debug 2>&1 | grep -i 'x-spx-\|x-predastore-'
+```
+
+And `spx get nodes` should list `predastore` in `SERVICES` on every node — a short service list on one server means something failed to start there.
+
+## Troubleshooting
+
+### S3 Answers but Every Write Fails
+
+The gate is up and the metadata quorum is not. Check that a majority of servers are running `spinifex-predastore`:
+
+```bash
+spx get nodes
+sudo systemctl is-active spinifex-predastore
+```
+
+Reads keep serving what the surviving replicas last applied, which is why this presents as a write-only outage.
+
+### 507 InsufficientStorage, or `nearfull` on Every Response
+
+A blob node is out of space, or close enough that compaction has been kicked. Free space in the data directory on each server:
+
+```bash
+df -h /var/lib/spinifex/predastore
+```
+
+Capacity here is per server, not pooled: a stripe needs its shard on a specific node, so one full server refuses writes for the objects the ring places on it.
+
+### Reads Fail with an Integrity Error
+
+GCM is the only integrity authority, so corruption, tampering and a wrong encryption key are indistinguishable. Confirm the key first, since it is by far the most likely of the three — every server must hold the identical file:
+
+```bash
+sudo sha256sum /etc/spinifex/predastore/encryption.key
+```
+
+Compare the digest across servers. A mismatch means that server was initialized separately rather than joined, and the fragments it sealed cannot be opened by the cluster.
+
+### Degraded Reads That Never Clear
+
+`X-Spx-Degraded` on reads long after an outage means repair has not restored the shards. Repair rebuilds onto the node that owns the shard, so it cannot finish while that node is absent:
+
+```bash
+curl -s http://$(hostname -I | awk '{print $1}'):8660/repair
+```
+
+`"enabled": false` means the sweep is off on that host. Otherwise bring the missing server back and let a pass complete — the sweep runs every five minutes by default.

docs/admin/setting-up-your-cluster/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Setting Up Your Cluster"
 seoTitle: "Set Up Your First Spinifex Cluster — Spinifex Docs"
-description: "Import an AMI, create an SSH key pair and a VPC with a public subnet, then launch your first EC2 instance and connect to it on a fresh Spinifex cluster."
+description: "Import an AMI and the RDS, ECS and EKS base images, create an SSH key pair and a VPC with a public subnet, then launch your first EC2 instance on a fresh Spinifex cluster."
 category: "Admin"
 tags:
   - setup
@@ -17,24 +17,26 @@ resources:
 
 # Setting Up Your Cluster
 
-> Import an AMI, create a VPC with a public subnet, and launch your first instance.
+> Import an AMI and the system base images, create a VPC with a public subnet, and launch your first instance.
 
 ## Table of Contents
 
 - [1. Import an AMI](#1-import-an-ami)
+  - [Option C: Import the system base images](#option-c-import-the-system-base-images)
 - [2. Create an SSH Key](#2-create-an-ssh-key)
 - [3. Create a VPC and Public Subnet](#3-create-a-vpc-and-public-subnet)
 - [4. Launch an Instance](#4-launch-an-instance)
 - [5. Connect via SSH](#5-connect-via-ssh)
 - [6. Managing Instances](#6-managing-instances)
 - [7. Launching the Web UI](#7-launching-the-web-ui)
+- [8. Secure the Install](#8-secure-the-install)
 - [Troubleshooting](#troubleshooting)
 
 ---
 
 ## Overview
 
-This guide walks through the first steps on a freshly installed cluster: importing an AMI, creating an SSH key and a VPC with a public subnet, launching an instance, and connecting to it.
+This guide walks through the first steps on a freshly installed cluster: importing an AMI and the system base images RDS, ECS and EKS need, creating an SSH key and a VPC with a public subnet, launching an instance, and connecting to it.
 
 ## Instructions
 
@@ -69,10 +71,16 @@ ubuntu-26.04-x86_64     | ubuntu | 26.04   | x86_64 | uefi
 spx admin images import --name ubuntu-26.04-x86_64
 ```
 
+Verify the import and note the AMI ID:
+
+```bash
+AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
+```
+
 ### Option B: Import a local image file
 
 ```bash
-spx admin images import --file ~/images/ubuntu-26.04.img --distro ubuntu --version 26.04 --arch x86_64 --boot-mode uefi
+spx admin images import --file ~/images/ubuntu-26.04-server-cloudimg-amd64.img --checksum ~/images/SHA256SUMS --distro ubuntu --version 26.04 --arch x86_64 --boot-mode uefi
 ```
 
 Verify the import and note the AMI ID:
@@ -81,6 +89,41 @@ Verify the import and note the AMI ID:
 AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
 ```
 
+### Option C: Import the system base images
+
+**RDS, ECS and EKS each launch from an image of their own, and none of them is imported for you.** Until these are in the catalog, `CreateDBInstance`, `CreateCluster` and `CreateNodegroup` all fail at the point they go looking for a node image — so a cluster without them has those three services present in the API and unusable in practice. Import them once, now, and the install is functional from the start.
+
+Run these one at a time rather than in parallel, so predastore is not taking four concurrent uploads:
+
+```bash
+for img in spinifex-rds-postgres spinifex-rds-mariadb spinifex-ecs-node spinifex-eks-node; do
+    sudo spx admin images import --name "$img" --config /etc/spinifex/spinifex.toml
+done
+```
+
+| Image                   | Serves                           | Contents                                        |
+| ----------------------- | -------------------------------- | ----------------------------------------------- |
+| `spinifex-rds-postgres` | RDS, `--engine postgres`         | Alpine 3.24.1 + PostgreSQL 18 + `rds-init`      |
+| `spinifex-rds-mariadb`  | RDS, `--engine mariadb`          | Alpine 3.24.1 + MariaDB 11.8 + `rds-init`       |
+| `spinifex-ecs-node`     | ECS container instances          | Alpine 3.21.7 + containerd + ecs-agent          |
+| `spinifex-eks-node`     | EKS control plane and nodegroups | Alpine 3.21.7 + K3s v1.32.5 + eks-token-webhook |
+
+All four boot **BIOS**, unlike the distro images, which boot UEFI. That is a property of how they are built and nothing you need to pass.
+
+If you run GPU workloads, add `spinifex-ecs-node-gpu` and `spinifex-eks-node-gpu` as well — Ubuntu 26.04 with the NVIDIA driver, selected automatically for a GPU instance type. They boot UEFI and they are large, so import them only if you have GPUs.
+
+> [!IMPORTANT]
+> **These register under generic names, and that is correct.** `describe-images` shows `spinifex-eks-node` and `spinifex-ecs-node` both as `ami-alpine-3.21.7-x86_64`, and the two RDS images both as `ami-alpine-3.24.1-x86_64`. Nothing resolves them by name: ECS and EKS find theirs by the `spinifex:managed-by=ecs|eks` tag, and RDS by `spinifex:managed-by=rds` plus the `engine` and `engine-version` tags. **Verify the tags, not the names** — four rows with duplicate-looking names is what success looks like:
+>
+> ```bash
+> aws ec2 describe-images \
+>   --filters 'Name=tag-key,Values=spinifex:managed-by' \
+>   --query 'Images[].[ImageId,State,Tags[?Key==`spinifex:managed-by`].Value|[0],Tags[?Key==`engine`].Value|[0]]' \
+>   --output text
+> ```
+
+`spx admin images list --config /etc/spinifex/spinifex.toml` on a node shows the whole catalog, including the images not imported here.
+
 ## 2. Create an SSH Key
 
 ### Option A: Import an existing key
@@ -131,7 +174,7 @@ aws ec2 attach-internet-gateway \
 
 ### Create a Subnet
 
-Create the subnet your instances will launch into. The routing and public-IP steps below are what make it a *public* subnet.
+Create the subnet your instances will launch into. The routing and public-IP steps below are what make it a _public_ subnet.
 
 ```bash
 SUBNET_ID=$(aws ec2 create-subnet \
@@ -312,17 +355,18 @@ https://YOUR_NODE_IP:3000
 
 On first load, your browser will show a TLS warning — Spinifex generates a self-signed certificate at install time. This is expected.
 
-1. Accept the warning to reach the login page (exact wording varies by browser — e.g. Chrome: *Advanced → Proceed to ...*, Firefox: *Advanced → Accept the Risk and Continue*).
+1. Accept the warning to reach the login page (exact wording varies by browser — e.g. Chrome: _Advanced → Proceed to ..._, Firefox: _Advanced → Accept the Risk and Continue_).
 2. On the login page, click **Download Certificate** and save `spinifex-ca.pem` to your machine.
-3. Install the certificate as a **trusted root** on your workstation:
-
-   - **macOS:** open `spinifex-ca.pem` in Keychain Access → *System* keychain → set *Trust* to **Always Trust**.
-   - **Linux:** `sudo cp spinifex-ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates`
-   - **Windows:** double-click the file → *Install Certificate* → *Local Machine* → *Trusted Root Certification Authorities*.
-   - **Browser-only (Firefox):** *Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import* and tick *Trust this CA to identify websites*.
-
+3. Install the certificate as a **trusted root** on your workstation, following the steps for your platform below.
 4. Restart your browser and reload `https://YOUR_NODE_IP:3000`. The padlock should now show a valid certificate.
 
+Step 3, by platform:
+
+- **macOS:** open `spinifex-ca.pem` in Keychain Access → _System_ keychain → set _Trust_ to **Always Trust**.
+- **Linux:** `sudo cp spinifex-ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates`
+- **Windows:** double-click the file → _Install Certificate_ → _Local Machine_ → _Trusted Root Certification Authorities_.
+- **Browser-only (Firefox):** _Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import_ and tick _Trust this CA to identify websites_.
+
 > **Why this is required:** the UI logs in by reading your AWS credentials through a trusted TLS channel. Browsers refuse to send credentials over an untrusted connection, so the certificate must be installed as trusted — temporary "Proceed anyway" exceptions won't work for login.
 
 ### Log In with AWS Credentials
@@ -339,6 +383,22 @@ At the Spinifex login screen, paste the **Access Key ID** and **Secret Access Ke
 
 Once logged in, you have browser-based access to every Spinifex feature: launch and manage instances, attach EBS volumes, browse S3 buckets, configure VPCs and security groups, and manage IAM users and keys — all backed by the same AWS-compatible control plane the CLI uses.
 
+## 8. Secure the Install
+
+Your cluster works. Before it carries anything real, close the host down.
+
+**Arm the host firewall.** A binary install leaves it off, because it lands on a machine that was already doing something. Armed, it keeps OVN, NATS and the rest of the cluster plane reachable by cluster members only, and everything outside the public group closed:
+
+```toml
+# /etc/spinifex/spinifex.toml
+[network]
+firewall_enabled = true
+```
+
+**Narrow SSH.** It is accepted from every source by default, so that arming the policy cannot lock you out of the node you are arming it from. Replace `0.0.0.0/0` with your management networks in `/etc/spinifex/firewall/custom.nft` — the one file an upgrade never overwrites.
+
+[Host Firewall](/docs/host-firewall) covers both, what the policy allows, and how each node tracks cluster membership without a peer list you maintain.
+
 ## Additional Options
 
 ### Private Subnets (No Public IP)

docs/admin/spinifex-admin-cli/README.md

@@ -109,7 +109,7 @@ spx admin images list
 spx admin images import --name debian-13-arm64
 ```
 
-Catalog imports verify the image against the catalog-declared SHA-256/SHA-512 digest before extraction. Use `--file` to import operator-supplied media (verification skipped — operator is responsible for integrity), or `--force` to re-download after a checksum mismatch.
+Catalog imports verify the image against the catalog-declared SHA-256/SHA-512 digest before extraction. Use `--file` with `--checksum <sums-file>` to import operator-supplied media verified against the sums file it was published with, or `--force` to re-download after a checksum mismatch. A `--file` import without `--checksum` proceeds unverified and prints a notice. Every import records the source file's digest on the AMI; `spx admin images describe --image-id <ami>` shows it.
 
 ### EKS node image
 

docs/admin/update/README.md

@@ -37,20 +37,6 @@ For operators who want to review migrations before they are applied, a manual up
 > [!WARNING]
 > **Swapping the `spx` binary alone is not an upgrade.** Systemd unit files (`KillMode`, `TimeoutStopSec`, drain ordering, and similar) are written once at install time and never re-asserted just because a new binary is in place — a node "upgraded" by replacing `/usr/local/bin/spx` directly keeps running whatever units it was first installed with, which can silently disagree with the new binary's behaviour. Re-running the installer always reinstalls units unconditionally, so it is unaffected. `spx admin upgrade` now reconciles units too, so it closes this gap for operators who update the binary by hand. See [Checking for Unit Drift](#checking-for-unit-drift).
 
-> [!WARNING]
-> **Object storage is exempt when upgrading from v1.15.0 or earlier to v1.16.0.** Predastore's configuration schema and on-disk layout changed with the object storage cutover in v1.16.0, and no migration converts an installation from before it. `spx admin upgrade` will not report anything pending for `predastore.toml`, and updating the binary over such an installation leaves Predastore unable to start. Those clusters have to be re-initialised from scratch, which discards their stored objects — export anything you need first. Upgrades between v1.16.0 and later releases are unaffected.
-
-> [!WARNING]
-> **AMI metadata is exempt when upgrading from v1.16.0 or earlier to the release carrying the EBS-provider decoupling.** AMI metadata moved to `ebsmetadata` documents, and the legacy path that read it from `ami-<id>/config.json` was removed rather than migrated. There is no prefix scan, no fallback and no backfill at daemon start, so an AMI imported before the change becomes invisible to the control plane afterwards: `describe-images --image-ids` answers `InvalidAMIID.NotFound` and launches fail with `AMI has no snapshot ID, cannot perform zero-copy clone`. As with object storage above, `spx admin upgrade` reports nothing pending, because the gap is in stored data rather than in a config file.
->
-> **Re-import affected AMIs after upgrading**, using [`spx admin images import`](/docs/spinifex-admin-cli), which writes metadata in the new location. If the source images are no longer available, the installation has to be re-initialised. Verify before you rely on it — an AMI is only healthy if it resolves by ID, not merely if it appears in the list:
->
-> ```bash
-> aws ec2 describe-images --image-ids <ami-id>
-> ```
->
-> This warning covers AMI metadata specifically, which is what has been observed. Check any other imported state you depend on before upgrading a cluster you cannot rebuild.
-
 ## Instructions
 
 ## Step 1. Re-run the Installer

docs/aws-model-operation-coverage.md

@@ -1,577 +0,0 @@
-# AWS model operation coverage
-
-Generated from the cached `aws-sdk-go v1.55.8` `api-2.json` models and Spinifex's authoritative gateway dispatch tables.
-
-Implemented means a modelled operation is registered to a real handler. Stub and deliberately unsupported handlers are reported separately. Shape conformance does not imply behavioural conformance.
-
-| Service | API version | Modelled | Registered | Implemented | Stubbed | Unsupported | Missing | Extra |
-|---|---:|---:|---:|---:|---:|---:|---:|---:|
-| acm | 2015-12-08 | 15 | 9 | 9 | 0 | 0 | 6 | 0 |
-| ec2 | 2016-11-15 | 625 | 121 | 121 | 0 | 0 | 504 | 0 |
-| ecr | 2015-09-21 | 47 | 39 | 19 | 13 | 7 | 11 | 3 |
-| ecs | 2014-11-13 | 56 | 39 | 31 | 5 | 0 | 20 | 3 |
-| elasticloadbalancingv2 | 2015-12-01 | 46 | 37 | 33 | 0 | 0 | 13 | 4 |
-| iam | 2010-05-08 | 159 | 75 | 75 | 0 | 0 | 84 | 0 |
-| rds | 2014-10-31 | 162 | 43 | 26 | 0 | 12 | 124 | 5 |
-| s3 | 2006-03-01 | 99 | — | — | — | — | — | — |
-| sts | 2011-06-15 | 8 | 4 | 4 | 0 | 0 | 4 | 0 |
-
-## acm
-
-Implements **9 of 15** modelled operations (60.0%).
-
-<details><summary>Implemented (9)</summary>
-
-`AddTagsToCertificate`, `DeleteCertificate`, `DescribeCertificate`, `GetCertificate`, `ImportCertificate`, `ListCertificates`, `ListTagsForCertificate`, `RemoveTagsFromCertificate`
-
-`RequestCertificate`
-
-
-</details>
-
-<details><summary>Missing from dispatch (6)</summary>
-
-`ExportCertificate`, `GetAccountConfiguration`, `PutAccountConfiguration`, `RenewCertificate`, `ResendValidationEmail`, `UpdateCertificateOptions`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>
-
-## ec2
-
-Implements **121 of 625** modelled operations (19.4%).
-
-<details><summary>Implemented (121)</summary>
-
-`AllocateAddress`, `AssociateAddress`, `AssociateIamInstanceProfile`, `AssociateRouteTable`, `AttachInternetGateway`, `AttachNetworkInterface`, `AttachVolume`, `AuthorizeSecurityGroupEgress`
-
-`AuthorizeSecurityGroupIngress`, `CancelCapacityReservation`, `CancelSpotInstanceRequests`, `CopyImage`, `CopySnapshot`, `CreateCapacityReservation`, `CreateEgressOnlyInternetGateway`, `CreateImage`
-
-`CreateInternetGateway`, `CreateKeyPair`, `CreateLaunchTemplate`, `CreateLaunchTemplateVersion`, `CreateNatGateway`, `CreateNetworkInterface`, `CreatePlacementGroup`, `CreateRoute`
-
-`CreateRouteTable`, `CreateSecurityGroup`, `CreateSnapshot`, `CreateSubnet`, `CreateTags`, `CreateVolume`, `CreateVpc`, `DeleteEgressOnlyInternetGateway`
-
-`DeleteInternetGateway`, `DeleteKeyPair`, `DeleteLaunchTemplate`, `DeleteLaunchTemplateVersions`, `DeleteNatGateway`, `DeleteNetworkInterface`, `DeletePlacementGroup`, `DeleteRoute`
-
-`DeleteRouteTable`, `DeleteSecurityGroup`, `DeleteSnapshot`, `DeleteSubnet`, `DeleteTags`, `DeleteVolume`, `DeleteVpc`, `DeregisterImage`
-
-`DescribeAccountAttributes`, `DescribeAddresses`, `DescribeAddressesAttribute`, `DescribeAvailabilityZones`, `DescribeCapacityReservations`, `DescribeEgressOnlyInternetGateways`, `DescribeIamInstanceProfileAssociations`, `DescribeImageAttribute`
-
-`DescribeImages`, `DescribeInstanceAttribute`, `DescribeInstanceCreditSpecifications`, `DescribeInstanceStatus`, `DescribeInstanceTypes`, `DescribeInstances`, `DescribeInternetGateways`, `DescribeKeyPairs`
-
-`DescribeLaunchTemplateVersions`, `DescribeLaunchTemplates`, `DescribeNatGateways`, `DescribeNetworkInterfaces`, `DescribePlacementGroups`, `DescribeRegions`, `DescribeRouteTables`, `DescribeSecurityGroupRules`
-
-`DescribeSecurityGroups`, `DescribeSnapshots`, `DescribeSpotInstanceRequests`, `DescribeSubnets`, `DescribeTags`, `DescribeVolumeStatus`, `DescribeVolumes`, `DescribeVolumesModifications`
-
-`DescribeVpcAttribute`, `DescribeVpcs`, `DetachInternetGateway`, `DetachNetworkInterface`, `DetachVolume`, `DisableEbsEncryptionByDefault`, `DisableSerialConsoleAccess`, `DisassociateAddress`
-
-`DisassociateIamInstanceProfile`, `DisassociateRouteTable`, `EnableEbsEncryptionByDefault`, `EnableSerialConsoleAccess`, `GetConsoleOutput`, `GetEbsEncryptionByDefault`, `GetPasswordData`, `GetSerialConsoleAccessStatus`
-
-`ImportKeyPair`, `ModifyImageAttribute`, `ModifyInstanceAttribute`, `ModifyInstanceMetadataOptions`, `ModifyLaunchTemplate`, `ModifyNetworkInterfaceAttribute`, `ModifySubnetAttribute`, `ModifyVolume`
-
-`ModifyVpcAttribute`, `MonitorInstances`, `RebootInstances`, `RegisterImage`, `ReleaseAddress`, `ReplaceIamInstanceProfileAssociation`, `ReplaceRoute`, `ReplaceRouteTableAssociation`
-
-`RequestSpotInstances`, `ResetImageAttribute`, `RevokeSecurityGroupEgress`, `RevokeSecurityGroupIngress`, `RunInstances`, `StartInstances`, `StopInstances`, `TerminateInstances`
-
-`UnmonitorInstances`
-
-
-</details>
-
-<details><summary>Missing from dispatch (504)</summary>
-
-`AcceptAddressTransfer`, `AcceptReservedInstancesExchangeQuote`, `AcceptTransitGatewayMulticastDomainAssociations`, `AcceptTransitGatewayPeeringAttachment`, `AcceptTransitGatewayVpcAttachment`, `AcceptVpcEndpointConnections`, `AcceptVpcPeeringConnection`, `AdvertiseByoipCidr`
-
-`AllocateHosts`, `AllocateIpamPoolCidr`, `ApplySecurityGroupsToClientVpnTargetNetwork`, `AssignIpv6Addresses`, `AssignPrivateIpAddresses`, `AssignPrivateNatGatewayAddress`, `AssociateClientVpnTargetNetwork`, `AssociateDhcpOptions`
-
-`AssociateEnclaveCertificateIamRole`, `AssociateInstanceEventWindow`, `AssociateIpamByoasn`, `AssociateIpamResourceDiscovery`, `AssociateNatGatewayAddress`, `AssociateSubnetCidrBlock`, `AssociateTransitGatewayMulticastDomain`, `AssociateTransitGatewayPolicyTable`
-
-`AssociateTransitGatewayRouteTable`, `AssociateTrunkInterface`, `AssociateVpcCidrBlock`, `AttachClassicLinkVpc`, `AttachVerifiedAccessTrustProvider`, `AttachVpnGateway`, `AuthorizeClientVpnIngress`, `BundleInstance`
-
-`CancelBundleTask`, `CancelCapacityReservationFleets`, `CancelConversionTask`, `CancelExportTask`, `CancelImageLaunchPermission`, `CancelImportTask`, `CancelReservedInstancesListing`, `CancelSpotFleetRequests`
-
-`ConfirmProductInstance`, `CopyFpgaImage`, `CreateCapacityReservationFleet`, `CreateCarrierGateway`, `CreateClientVpnEndpoint`, `CreateClientVpnRoute`, `CreateCoipCidr`, `CreateCoipPool`
-
-`CreateCustomerGateway`, `CreateDefaultSubnet`, `CreateDefaultVpc`, `CreateDhcpOptions`, `CreateFleet`, `CreateFlowLogs`, `CreateFpgaImage`, `CreateInstanceConnectEndpoint`
-
-`CreateInstanceEventWindow`, `CreateInstanceExportTask`, `CreateIpam`, `CreateIpamExternalResourceVerificationToken`, `CreateIpamPool`, `CreateIpamResourceDiscovery`, `CreateIpamScope`, `CreateLocalGatewayRoute`
-
-`CreateLocalGatewayRouteTable`, `CreateLocalGatewayRouteTableVirtualInterfaceGroupAssociation`, `CreateLocalGatewayRouteTableVpcAssociation`, `CreateManagedPrefixList`, `CreateNetworkAcl`, `CreateNetworkAclEntry`, `CreateNetworkInsightsAccessScope`, `CreateNetworkInsightsPath`
-
-`CreateNetworkInterfacePermission`, `CreatePublicIpv4Pool`, `CreateReplaceRootVolumeTask`, `CreateReservedInstancesListing`, `CreateRestoreImageTask`, `CreateSnapshots`, `CreateSpotDatafeedSubscription`, `CreateStoreImageTask`
-
-`CreateSubnetCidrReservation`, `CreateTrafficMirrorFilter`, `CreateTrafficMirrorFilterRule`, `CreateTrafficMirrorSession`, `CreateTrafficMirrorTarget`, `CreateTransitGateway`, `CreateTransitGatewayConnect`, `CreateTransitGatewayConnectPeer`
-
-`CreateTransitGatewayMulticastDomain`, `CreateTransitGatewayPeeringAttachment`, `CreateTransitGatewayPolicyTable`, `CreateTransitGatewayPrefixListReference`, `CreateTransitGatewayRoute`, `CreateTransitGatewayRouteTable`, `CreateTransitGatewayRouteTableAnnouncement`, `CreateTransitGatewayVpcAttachment`
-
-`CreateVerifiedAccessEndpoint`, `CreateVerifiedAccessGroup`, `CreateVerifiedAccessInstance`, `CreateVerifiedAccessTrustProvider`, `CreateVpcEndpoint`, `CreateVpcEndpointConnectionNotification`, `CreateVpcEndpointServiceConfiguration`, `CreateVpcPeeringConnection`
-
-`CreateVpnConnection`, `CreateVpnConnectionRoute`, `CreateVpnGateway`, `DeleteCarrierGateway`, `DeleteClientVpnEndpoint`, `DeleteClientVpnRoute`, `DeleteCoipCidr`, `DeleteCoipPool`
-
-`DeleteCustomerGateway`, `DeleteDhcpOptions`, `DeleteFleets`, `DeleteFlowLogs`, `DeleteFpgaImage`, `DeleteInstanceConnectEndpoint`, `DeleteInstanceEventWindow`, `DeleteIpam`
-
-`DeleteIpamExternalResourceVerificationToken`, `DeleteIpamPool`, `DeleteIpamResourceDiscovery`, `DeleteIpamScope`, `DeleteLocalGatewayRoute`, `DeleteLocalGatewayRouteTable`, `DeleteLocalGatewayRouteTableVirtualInterfaceGroupAssociation`, `DeleteLocalGatewayRouteTableVpcAssociation`
-
-`DeleteManagedPrefixList`, `DeleteNetworkAcl`, `DeleteNetworkAclEntry`, `DeleteNetworkInsightsAccessScope`, `DeleteNetworkInsightsAccessScopeAnalysis`, `DeleteNetworkInsightsAnalysis`, `DeleteNetworkInsightsPath`, `DeleteNetworkInterfacePermission`
-
-`DeletePublicIpv4Pool`, `DeleteQueuedReservedInstances`, `DeleteSpotDatafeedSubscription`, `DeleteSubnetCidrReservation`, `DeleteTrafficMirrorFilter`, `DeleteTrafficMirrorFilterRule`, `DeleteTrafficMirrorSession`, `DeleteTrafficMirrorTarget`
-
-`DeleteTransitGateway`, `DeleteTransitGatewayConnect`, `DeleteTransitGatewayConnectPeer`, `DeleteTransitGatewayMulticastDomain`, `DeleteTransitGatewayPeeringAttachment`, `DeleteTransitGatewayPolicyTable`, `DeleteTransitGatewayPrefixListReference`, `DeleteTransitGatewayRoute`
-
-`DeleteTransitGatewayRouteTable`, `DeleteTransitGatewayRouteTableAnnouncement`, `DeleteTransitGatewayVpcAttachment`, `DeleteVerifiedAccessEndpoint`, `DeleteVerifiedAccessGroup`, `DeleteVerifiedAccessInstance`, `DeleteVerifiedAccessTrustProvider`, `DeleteVpcEndpointConnectionNotifications`
-
-`DeleteVpcEndpointServiceConfigurations`, `DeleteVpcEndpoints`, `DeleteVpcPeeringConnection`, `DeleteVpnConnection`, `DeleteVpnConnectionRoute`, `DeleteVpnGateway`, `DeprovisionByoipCidr`, `DeprovisionIpamByoasn`
-
-`DeprovisionIpamPoolCidr`, `DeprovisionPublicIpv4PoolCidr`, `DeregisterInstanceEventNotificationAttributes`, `DeregisterTransitGatewayMulticastGroupMembers`, `DeregisterTransitGatewayMulticastGroupSources`, `DescribeAddressTransfers`, `DescribeAggregateIdFormat`, `DescribeAwsNetworkPerformanceMetricSubscriptions`
-
-`DescribeBundleTasks`, `DescribeByoipCidrs`, `DescribeCapacityBlockOfferings`, `DescribeCapacityReservationFleets`, `DescribeCarrierGateways`, `DescribeClassicLinkInstances`, `DescribeClientVpnAuthorizationRules`, `DescribeClientVpnConnections`
-
-`DescribeClientVpnEndpoints`, `DescribeClientVpnRoutes`, `DescribeClientVpnTargetNetworks`, `DescribeCoipPools`, `DescribeConversionTasks`, `DescribeCustomerGateways`, `DescribeDhcpOptions`, `DescribeElasticGpus`
-
-`DescribeExportImageTasks`, `DescribeExportTasks`, `DescribeFastLaunchImages`, `DescribeFastSnapshotRestores`, `DescribeFleetHistory`, `DescribeFleetInstances`, `DescribeFleets`, `DescribeFlowLogs`
-
-`DescribeFpgaImageAttribute`, `DescribeFpgaImages`, `DescribeHostReservationOfferings`, `DescribeHostReservations`, `DescribeHosts`, `DescribeIdFormat`, `DescribeIdentityIdFormat`, `DescribeImportImageTasks`
-
-`DescribeImportSnapshotTasks`, `DescribeInstanceConnectEndpoints`, `DescribeInstanceEventNotificationAttributes`, `DescribeInstanceEventWindows`, `DescribeInstanceTopology`, `DescribeInstanceTypeOfferings`, `DescribeIpamByoasn`, `DescribeIpamExternalResourceVerificationTokens`
-
-`DescribeIpamPools`, `DescribeIpamResourceDiscoveries`, `DescribeIpamResourceDiscoveryAssociations`, `DescribeIpamScopes`, `DescribeIpams`, `DescribeIpv6Pools`, `DescribeLocalGatewayRouteTableVirtualInterfaceGroupAssociations`, `DescribeLocalGatewayRouteTableVpcAssociations`
-
-`DescribeLocalGatewayRouteTables`, `DescribeLocalGatewayVirtualInterfaceGroups`, `DescribeLocalGatewayVirtualInterfaces`, `DescribeLocalGateways`, `DescribeLockedSnapshots`, `DescribeMacHosts`, `DescribeManagedPrefixLists`, `DescribeMovingAddresses`
-
-`DescribeNetworkAcls`, `DescribeNetworkInsightsAccessScopeAnalyses`, `DescribeNetworkInsightsAccessScopes`, `DescribeNetworkInsightsAnalyses`, `DescribeNetworkInsightsPaths`, `DescribeNetworkInterfaceAttribute`, `DescribeNetworkInterfacePermissions`, `DescribePrefixLists`
-
-`DescribePrincipalIdFormat`, `DescribePublicIpv4Pools`, `DescribeReplaceRootVolumeTasks`, `DescribeReservedInstances`, `DescribeReservedInstancesListings`, `DescribeReservedInstancesModifications`, `DescribeReservedInstancesOfferings`, `DescribeScheduledInstanceAvailability`
-
-`DescribeScheduledInstances`, `DescribeSecurityGroupReferences`, `DescribeSnapshotAttribute`, `DescribeSnapshotTierStatus`, `DescribeSpotDatafeedSubscription`, `DescribeSpotFleetInstances`, `DescribeSpotFleetRequestHistory`, `DescribeSpotFleetRequests`
-
-`DescribeSpotPriceHistory`, `DescribeStaleSecurityGroups`, `DescribeStoreImageTasks`, `DescribeTrafficMirrorFilterRules`, `DescribeTrafficMirrorFilters`, `DescribeTrafficMirrorSessions`, `DescribeTrafficMirrorTargets`, `DescribeTransitGatewayAttachments`
-
-`DescribeTransitGatewayConnectPeers`, `DescribeTransitGatewayConnects`, `DescribeTransitGatewayMulticastDomains`, `DescribeTransitGatewayPeeringAttachments`, `DescribeTransitGatewayPolicyTables`, `DescribeTransitGatewayRouteTableAnnouncements`, `DescribeTransitGatewayRouteTables`, `DescribeTransitGatewayVpcAttachments`
-
-`DescribeTransitGateways`, `DescribeTrunkInterfaceAssociations`, `DescribeVerifiedAccessEndpoints`, `DescribeVerifiedAccessGroups`, `DescribeVerifiedAccessInstanceLoggingConfigurations`, `DescribeVerifiedAccessInstances`, `DescribeVerifiedAccessTrustProviders`, `DescribeVolumeAttribute`
-
-`DescribeVpcClassicLink`, `DescribeVpcClassicLinkDnsSupport`, `DescribeVpcEndpointConnectionNotifications`, `DescribeVpcEndpointConnections`, `DescribeVpcEndpointServiceConfigurations`, `DescribeVpcEndpointServicePermissions`, `DescribeVpcEndpointServices`, `DescribeVpcEndpoints`
-
-`DescribeVpcPeeringConnections`, `DescribeVpnConnections`, `DescribeVpnGateways`, `DetachClassicLinkVpc`, `DetachVerifiedAccessTrustProvider`, `DetachVpnGateway`, `DisableAddressTransfer`, `DisableAwsNetworkPerformanceMetricSubscription`
-
-`DisableFastLaunch`, `DisableFastSnapshotRestores`, `DisableImage`, `DisableImageBlockPublicAccess`, `DisableImageDeprecation`, `DisableImageDeregistrationProtection`, `DisableIpamOrganizationAdminAccount`, `DisableSnapshotBlockPublicAccess`
-
-`DisableTransitGatewayRouteTablePropagation`, `DisableVgwRoutePropagation`, `DisableVpcClassicLink`, `DisableVpcClassicLinkDnsSupport`, `DisassociateClientVpnTargetNetwork`, `DisassociateEnclaveCertificateIamRole`, `DisassociateInstanceEventWindow`, `DisassociateIpamByoasn`
-
-`DisassociateIpamResourceDiscovery`, `DisassociateNatGatewayAddress`, `DisassociateSubnetCidrBlock`, `DisassociateTransitGatewayMulticastDomain`, `DisassociateTransitGatewayPolicyTable`, `DisassociateTransitGatewayRouteTable`, `DisassociateTrunkInterface`, `DisassociateVpcCidrBlock`
-
-`EnableAddressTransfer`, `EnableAwsNetworkPerformanceMetricSubscription`, `EnableFastLaunch`, `EnableFastSnapshotRestores`, `EnableImage`, `EnableImageBlockPublicAccess`, `EnableImageDeprecation`, `EnableImageDeregistrationProtection`
-
-`EnableIpamOrganizationAdminAccount`, `EnableReachabilityAnalyzerOrganizationSharing`, `EnableSnapshotBlockPublicAccess`, `EnableTransitGatewayRouteTablePropagation`, `EnableVgwRoutePropagation`, `EnableVolumeIO`, `EnableVpcClassicLink`, `EnableVpcClassicLinkDnsSupport`
-
-`ExportClientVpnClientCertificateRevocationList`, `ExportClientVpnClientConfiguration`, `ExportImage`, `ExportTransitGatewayRoutes`, `GetAssociatedEnclaveCertificateIamRoles`, `GetAssociatedIpv6PoolCidrs`, `GetAwsNetworkPerformanceData`, `GetCapacityReservationUsage`
-
-`GetCoipPoolUsage`, `GetConsoleScreenshot`, `GetDefaultCreditSpecification`, `GetEbsDefaultKmsKeyId`, `GetFlowLogsIntegrationTemplate`, `GetGroupsForCapacityReservation`, `GetHostReservationPurchasePreview`, `GetImageBlockPublicAccessState`
-
-`GetInstanceMetadataDefaults`, `GetInstanceTpmEkPub`, `GetInstanceTypesFromInstanceRequirements`, `GetInstanceUefiData`, `GetIpamAddressHistory`, `GetIpamDiscoveredAccounts`, `GetIpamDiscoveredPublicAddresses`, `GetIpamDiscoveredResourceCidrs`
-
-`GetIpamPoolAllocations`, `GetIpamPoolCidrs`, `GetIpamResourceCidrs`, `GetLaunchTemplateData`, `GetManagedPrefixListAssociations`, `GetManagedPrefixListEntries`, `GetNetworkInsightsAccessScopeAnalysisFindings`, `GetNetworkInsightsAccessScopeContent`
-
-`GetReservedInstancesExchangeQuote`, `GetSecurityGroupsForVpc`, `GetSnapshotBlockPublicAccessState`, `GetSpotPlacementScores`, `GetSubnetCidrReservations`, `GetTransitGatewayAttachmentPropagations`, `GetTransitGatewayMulticastDomainAssociations`, `GetTransitGatewayPolicyTableAssociations`
-
-`GetTransitGatewayPolicyTableEntries`, `GetTransitGatewayPrefixListReferences`, `GetTransitGatewayRouteTableAssociations`, `GetTransitGatewayRouteTablePropagations`, `GetVerifiedAccessEndpointPolicy`, `GetVerifiedAccessGroupPolicy`, `GetVpnConnectionDeviceSampleConfiguration`, `GetVpnConnectionDeviceTypes`
-
-`GetVpnTunnelReplacementStatus`, `ImportClientVpnClientCertificateRevocationList`, `ImportImage`, `ImportInstance`, `ImportSnapshot`, `ImportVolume`, `ListImagesInRecycleBin`, `ListSnapshotsInRecycleBin`
-
-`LockSnapshot`, `ModifyAddressAttribute`, `ModifyAvailabilityZoneGroup`, `ModifyCapacityReservation`, `ModifyCapacityReservationFleet`, `ModifyClientVpnEndpoint`, `ModifyDefaultCreditSpecification`, `ModifyEbsDefaultKmsKeyId`
-
-`ModifyFleet`, `ModifyFpgaImageAttribute`, `ModifyHosts`, `ModifyIdFormat`, `ModifyIdentityIdFormat`, `ModifyInstanceCapacityReservationAttributes`, `ModifyInstanceCreditSpecification`, `ModifyInstanceEventStartTime`
-
-`ModifyInstanceEventWindow`, `ModifyInstanceMaintenanceOptions`, `ModifyInstanceMetadataDefaults`, `ModifyInstancePlacement`, `ModifyIpam`, `ModifyIpamPool`, `ModifyIpamResourceCidr`, `ModifyIpamResourceDiscovery`
-
-`ModifyIpamScope`, `ModifyLocalGatewayRoute`, `ModifyManagedPrefixList`, `ModifyPrivateDnsNameOptions`, `ModifyReservedInstances`, `ModifySecurityGroupRules`, `ModifySnapshotAttribute`, `ModifySnapshotTier`
-
-`ModifySpotFleetRequest`, `ModifyTrafficMirrorFilterNetworkServices`, `ModifyTrafficMirrorFilterRule`, `ModifyTrafficMirrorSession`, `ModifyTransitGateway`, `ModifyTransitGatewayPrefixListReference`, `ModifyTransitGatewayVpcAttachment`, `ModifyVerifiedAccessEndpoint`
-
-`ModifyVerifiedAccessEndpointPolicy`, `ModifyVerifiedAccessGroup`, `ModifyVerifiedAccessGroupPolicy`, `ModifyVerifiedAccessInstance`, `ModifyVerifiedAccessInstanceLoggingConfiguration`, `ModifyVerifiedAccessTrustProvider`, `ModifyVolumeAttribute`, `ModifyVpcEndpoint`
-
-`ModifyVpcEndpointConnectionNotification`, `ModifyVpcEndpointServiceConfiguration`, `ModifyVpcEndpointServicePayerResponsibility`, `ModifyVpcEndpointServicePermissions`, `ModifyVpcPeeringConnectionOptions`, `ModifyVpcTenancy`, `ModifyVpnConnection`, `ModifyVpnConnectionOptions`
-
-`ModifyVpnTunnelCertificate`, `ModifyVpnTunnelOptions`, `MoveAddressToVpc`, `MoveByoipCidrToIpam`, `ProvisionByoipCidr`, `ProvisionIpamByoasn`, `ProvisionIpamPoolCidr`, `ProvisionPublicIpv4PoolCidr`
-
-`PurchaseCapacityBlock`, `PurchaseHostReservation`, `PurchaseReservedInstancesOffering`, `PurchaseScheduledInstances`, `RegisterInstanceEventNotificationAttributes`, `RegisterTransitGatewayMulticastGroupMembers`, `RegisterTransitGatewayMulticastGroupSources`, `RejectTransitGatewayMulticastDomainAssociations`
-
-`RejectTransitGatewayPeeringAttachment`, `RejectTransitGatewayVpcAttachment`, `RejectVpcEndpointConnections`, `RejectVpcPeeringConnection`, `ReleaseHosts`, `ReleaseIpamPoolAllocation`, `ReplaceNetworkAclAssociation`, `ReplaceNetworkAclEntry`
-
-`ReplaceTransitGatewayRoute`, `ReplaceVpnTunnel`, `ReportInstanceStatus`, `RequestSpotFleet`, `ResetAddressAttribute`, `ResetEbsDefaultKmsKeyId`, `ResetFpgaImageAttribute`, `ResetInstanceAttribute`
-
-`ResetNetworkInterfaceAttribute`, `ResetSnapshotAttribute`, `RestoreAddressToClassic`, `RestoreImageFromRecycleBin`, `RestoreManagedPrefixListVersion`, `RestoreSnapshotFromRecycleBin`, `RestoreSnapshotTier`, `RevokeClientVpnIngress`
-
-`RunScheduledInstances`, `SearchLocalGatewayRoutes`, `SearchTransitGatewayMulticastGroups`, `SearchTransitGatewayRoutes`, `SendDiagnosticInterrupt`, `StartNetworkInsightsAccessScopeAnalysis`, `StartNetworkInsightsAnalysis`, `StartVpcEndpointServicePrivateDnsVerification`
-
-`TerminateClientVpnConnections`, `UnassignIpv6Addresses`, `UnassignPrivateIpAddresses`, `UnassignPrivateNatGatewayAddress`, `UnlockSnapshot`, `UpdateSecurityGroupRuleDescriptionsEgress`, `UpdateSecurityGroupRuleDescriptionsIngress`, `WithdrawByoipCidr`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>
-
-## ecr
-
-Implements **19 of 47** modelled operations (40.4%).
-
-<details><summary>Implemented (19)</summary>
-
-`BatchDeleteImage`, `BatchGetImage`, `CreateRepository`, `DeleteLifecyclePolicy`, `DeleteRepository`, `DeleteRepositoryPolicy`, `DescribeImages`, `DescribeRepositories`
-
-`GetAuthorizationToken`, `GetLifecyclePolicy`, `GetLifecyclePolicyPreview`, `GetRepositoryPolicy`, `ListImages`, `ListTagsForResource`, `PutImage`, `PutImageTagMutability`
-
-`PutLifecyclePolicy`, `SetRepositoryPolicy`, `StartLifecyclePolicyPreview`
-
-
-</details>
-
-<details><summary>Missing from dispatch (11)</summary>
-
-`CreatePullThroughCacheRule`, `CreateRepositoryCreationTemplate`, `DeletePullThroughCacheRule`, `DeleteRegistryPolicy`, `DeleteRepositoryCreationTemplate`, `DescribeImageReplicationStatus`, `DescribePullThroughCacheRules`, `DescribeRepositoryCreationTemplates`
-
-`UpdatePullThroughCacheRule`, `UpdateRepositoryCreationTemplate`, `ValidatePullThroughCacheRule`
-
-
-</details>
-
-<details><summary>Registered stubs (13)</summary>
-
-`BatchCheckLayerAvailability`, `CompleteLayerUpload`, `DescribeRegistry`, `GetDownloadUrlForLayer`, `GetRegistryPolicy`, `InitiateLayerUpload`, `ListRepositories`, `PutRegistryPolicy`
-
-`PutReplicationConfiguration`, `ReplicateImage`, `TagResource`, `UntagResource`, `UploadLayerPart`
-
-
-</details>
-
-<details><summary>Deliberately unsupported (7)</summary>
-
-`BatchGetRepositoryScanningConfiguration`, `DescribeImageScanFindings`, `GetImageScanningConfiguration`, `GetRegistryScanningConfiguration`, `PutImageScanningConfiguration`, `PutRegistryScanningConfiguration`, `StartImageScan`
-
-
-</details>
-
-<details><summary>Registered outside the pinned model (3)</summary>
-
-`GetImageScanningConfiguration`, `ListRepositories`, `ReplicateImage`
-
-
-</details>
-
-## ecs
-
-Implements **31 of 56** modelled operations (55.4%).
-
-<details><summary>Implemented (31)</summary>
-
-`CreateCapacityProvider`, `CreateCluster`, `CreateService`, `DeleteCapacityProvider`, `DeleteCluster`, `DeleteService`, `DeregisterContainerInstance`, `DeregisterTaskDefinition`
-
-`DescribeCapacityProviders`, `DescribeClusters`, `DescribeContainerInstances`, `DescribeServices`, `DescribeTaskDefinition`, `DescribeTasks`, `ListClusters`, `ListContainerInstances`
-
-`ListServices`, `ListTagsForResource`, `ListTaskDefinitions`, `ListTasks`, `PutClusterCapacityProviders`, `RegisterContainerInstance`, `RegisterTaskDefinition`, `RunTask`
-
-`StartTask`, `StopTask`, `SubmitTaskStateChange`, `TagResource`, `UntagResource`, `UpdateContainerInstancesState`, `UpdateService`
-
-
-</details>
-
-<details><summary>Missing from dispatch (20)</summary>
-
-`CreateTaskSet`, `DeleteAccountSetting`, `DeleteAttributes`, `DeleteTaskDefinitions`, `DeleteTaskSet`, `DescribeTaskSets`, `DiscoverPollEndpoint`, `ExecuteCommand`
-
-`GetTaskProtection`, `ListAttributes`, `PutAccountSettingDefault`, `PutAttributes`, `SubmitAttachmentStateChanges`, `SubmitContainerStateChange`, `UpdateCapacityProvider`, `UpdateClusterSettings`
-
-`UpdateContainerAgent`, `UpdateServicePrimaryTaskSet`, `UpdateTaskProtection`, `UpdateTaskSet`
-
-
-</details>
-
-<details><summary>Registered stubs (5)</summary>
-
-`ListAccountSettings`, `ListServicesByNamespace`, `ListTaskDefinitionFamilies`, `PutAccountSetting`, `UpdateCluster`
-
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (3)</summary>
-
-`PollAssignments`, `ProvisionCapacity`, `ReportTaskGPU`
-
-
-</details>
-
-## elasticloadbalancingv2
-
-Implements **33 of 46** modelled operations (71.7%).
-
-<details><summary>Implemented (33)</summary>
-
-`AddListenerCertificates`, `AddTags`, `CreateListener`, `CreateLoadBalancer`, `CreateRule`, `CreateTargetGroup`, `DeleteListener`, `DeleteLoadBalancer`
-
-`DeleteRule`, `DeleteTargetGroup`, `DeregisterTargets`, `DescribeListenerCertificates`, `DescribeListeners`, `DescribeLoadBalancerAttributes`, `DescribeLoadBalancers`, `DescribeRules`
-
-`DescribeSSLPolicies`, `DescribeTags`, `DescribeTargetGroupAttributes`, `DescribeTargetGroups`, `DescribeTargetHealth`, `ModifyListener`, `ModifyLoadBalancerAttributes`, `ModifyRule`
-
-`ModifyTargetGroup`, `ModifyTargetGroupAttributes`, `RegisterTargets`, `RemoveListenerCertificates`, `RemoveTags`, `SetIpAddressType`, `SetRulePriorities`, `SetSecurityGroups`
-
-`SetSubnets`
-
-
-</details>
-
-<details><summary>Missing from dispatch (13)</summary>
-
-`AddTrustStoreRevocations`, `CreateTrustStore`, `DeleteSharedTrustStoreAssociation`, `DeleteTrustStore`, `DescribeAccountLimits`, `DescribeTrustStoreAssociations`, `DescribeTrustStoreRevocations`, `DescribeTrustStores`
-
-`GetResourcePolicy`, `GetTrustStoreCaCertificatesBundle`, `GetTrustStoreRevocationContent`, `ModifyTrustStore`, `RemoveTrustStoreRevocations`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (4)</summary>
-
-`DescribeListenerAttributes`, `GetLBConfig`, `LBAgentHeartbeat`, `ModifyListenerAttributes`
-
-
-</details>
-
-## iam
-
-Implements **75 of 159** modelled operations (47.2%).
-
-<details><summary>Implemented (75)</summary>
-
-`AddRoleToInstanceProfile`, `AddUserToGroup`, `AttachGroupPolicy`, `AttachRolePolicy`, `AttachUserPolicy`, `CreateAccessKey`, `CreateGroup`, `CreateInstanceProfile`
-
-`CreateOpenIDConnectProvider`, `CreatePolicy`, `CreateRole`, `CreateUser`, `DeleteAccessKey`, `DeleteGroup`, `DeleteGroupPolicy`, `DeleteInstanceProfile`
-
-`DeleteOpenIDConnectProvider`, `DeletePolicy`, `DeleteRole`, `DeleteRolePolicy`, `DeleteUser`, `DeleteUserPolicy`, `DetachGroupPolicy`, `DetachRolePolicy`
-
-`DetachUserPolicy`, `GetAccountSummary`, `GetGroup`, `GetGroupPolicy`, `GetInstanceProfile`, `GetOpenIDConnectProvider`, `GetPolicy`, `GetPolicyVersion`
-
-`GetRole`, `GetRolePolicy`, `GetUser`, `GetUserPolicy`, `ListAccessKeys`, `ListAttachedGroupPolicies`, `ListAttachedRolePolicies`, `ListAttachedUserPolicies`
-
-`ListGroupPolicies`, `ListGroups`, `ListGroupsForUser`, `ListInstanceProfileTags`, `ListInstanceProfiles`, `ListInstanceProfilesForRole`, `ListOpenIDConnectProviderTags`, `ListOpenIDConnectProviders`
-
-`ListPolicies`, `ListPolicyTags`, `ListPolicyVersions`, `ListRolePolicies`, `ListRoleTags`, `ListRoles`, `ListUserPolicies`, `ListUserTags`
-
-`ListUsers`, `PutGroupPolicy`, `PutRolePolicy`, `PutUserPolicy`, `RemoveRoleFromInstanceProfile`, `RemoveUserFromGroup`, `TagInstanceProfile`, `TagOpenIDConnectProvider`
-
-`TagPolicy`, `TagRole`, `TagUser`, `UntagInstanceProfile`, `UntagOpenIDConnectProvider`, `UntagPolicy`, `UntagRole`, `UntagUser`
-
-`UpdateAccessKey`, `UpdateAssumeRolePolicy`, `UpdateRole`
-
-
-</details>
-
-<details><summary>Missing from dispatch (84)</summary>
-
-`AddClientIDToOpenIDConnectProvider`, `ChangePassword`, `CreateAccountAlias`, `CreateLoginProfile`, `CreatePolicyVersion`, `CreateSAMLProvider`, `CreateServiceLinkedRole`, `CreateServiceSpecificCredential`
-
-`CreateVirtualMFADevice`, `DeactivateMFADevice`, `DeleteAccountAlias`, `DeleteAccountPasswordPolicy`, `DeleteLoginProfile`, `DeletePolicyVersion`, `DeleteRolePermissionsBoundary`, `DeleteSAMLProvider`
-
-`DeleteSSHPublicKey`, `DeleteServerCertificate`, `DeleteServiceLinkedRole`, `DeleteServiceSpecificCredential`, `DeleteSigningCertificate`, `DeleteUserPermissionsBoundary`, `DeleteVirtualMFADevice`, `EnableMFADevice`
-
-`GenerateCredentialReport`, `GenerateOrganizationsAccessReport`, `GenerateServiceLastAccessedDetails`, `GetAccessKeyLastUsed`, `GetAccountAuthorizationDetails`, `GetAccountPasswordPolicy`, `GetContextKeysForCustomPolicy`, `GetContextKeysForPrincipalPolicy`
-
-`GetCredentialReport`, `GetLoginProfile`, `GetMFADevice`, `GetOrganizationsAccessReport`, `GetSAMLProvider`, `GetSSHPublicKey`, `GetServerCertificate`, `GetServiceLastAccessedDetails`
-
-`GetServiceLastAccessedDetailsWithEntities`, `GetServiceLinkedRoleDeletionStatus`, `ListAccountAliases`, `ListEntitiesForPolicy`, `ListMFADeviceTags`, `ListMFADevices`, `ListPoliciesGrantingServiceAccess`, `ListSAMLProviderTags`
-
-`ListSAMLProviders`, `ListSSHPublicKeys`, `ListServerCertificateTags`, `ListServerCertificates`, `ListServiceSpecificCredentials`, `ListSigningCertificates`, `ListVirtualMFADevices`, `PutRolePermissionsBoundary`
-
-`PutUserPermissionsBoundary`, `RemoveClientIDFromOpenIDConnectProvider`, `ResetServiceSpecificCredential`, `ResyncMFADevice`, `SetDefaultPolicyVersion`, `SetSecurityTokenServicePreferences`, `SimulateCustomPolicy`, `SimulatePrincipalPolicy`
-
-`TagMFADevice`, `TagSAMLProvider`, `TagServerCertificate`, `UntagMFADevice`, `UntagSAMLProvider`, `UntagServerCertificate`, `UpdateAccountPasswordPolicy`, `UpdateGroup`
-
-`UpdateLoginProfile`, `UpdateOpenIDConnectProviderThumbprint`, `UpdateRoleDescription`, `UpdateSAMLProvider`, `UpdateSSHPublicKey`, `UpdateServerCertificate`, `UpdateServiceSpecificCredential`, `UpdateSigningCertificate`
-
-`UpdateUser`, `UploadSSHPublicKey`, `UploadServerCertificate`, `UploadSigningCertificate`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>
-
-## rds
-
-Implements **26 of 162** modelled operations (16.0%).
-
-<details><summary>Implemented (26)</summary>
-
-`AddTagsToResource`, `CreateDBInstance`, `CreateDBParameterGroup`, `CreateDBSnapshot`, `CreateDBSubnetGroup`, `DeleteDBInstance`, `DeleteDBParameterGroup`, `DeleteDBSnapshot`
-
-`DeleteDBSubnetGroup`, `DescribeDBEngineVersions`, `DescribeDBInstanceAutomatedBackups`, `DescribeDBInstances`, `DescribeDBParameterGroups`, `DescribeDBParameters`, `DescribeDBSnapshots`, `DescribeDBSubnetGroups`
-
-`DescribeEvents`, `DescribeOrderableDBInstanceOptions`, `ListTagsForResource`, `ModifyDBInstance`, `ModifyDBParameterGroup`, `RebootDBInstance`, `RemoveTagsFromResource`, `RestoreDBInstanceFromDBSnapshot`
-
-`StartDBInstance`, `StopDBInstance`
-
-
-</details>
-
-<details><summary>Missing from dispatch (124)</summary>
-
-`AddRoleToDBCluster`, `AddRoleToDBInstance`, `AddSourceIdentifierToSubscription`, `ApplyPendingMaintenanceAction`, `AuthorizeDBSecurityGroupIngress`, `BacktrackDBCluster`, `CancelExportTask`, `CopyDBClusterParameterGroup`
-
-`CopyDBClusterSnapshot`, `CopyDBParameterGroup`, `CopyDBSnapshot`, `CopyOptionGroup`, `CreateBlueGreenDeployment`, `CreateCustomDBEngineVersion`, `CreateDBClusterEndpoint`, `CreateDBClusterParameterGroup`
-
-`CreateDBClusterSnapshot`, `CreateDBProxy`, `CreateDBProxyEndpoint`, `CreateDBSecurityGroup`, `CreateDBShardGroup`, `CreateEventSubscription`, `CreateGlobalCluster`, `CreateIntegration`
-
-`CreateTenantDatabase`, `DeleteBlueGreenDeployment`, `DeleteCustomDBEngineVersion`, `DeleteDBClusterAutomatedBackup`, `DeleteDBClusterEndpoint`, `DeleteDBClusterParameterGroup`, `DeleteDBClusterSnapshot`, `DeleteDBInstanceAutomatedBackup`
-
-`DeleteDBProxy`, `DeleteDBProxyEndpoint`, `DeleteDBSecurityGroup`, `DeleteDBShardGroup`, `DeleteEventSubscription`, `DeleteGlobalCluster`, `DeleteIntegration`, `DeleteTenantDatabase`
-
-`DeregisterDBProxyTargets`, `DescribeAccountAttributes`, `DescribeBlueGreenDeployments`, `DescribeCertificates`, `DescribeDBClusterAutomatedBackups`, `DescribeDBClusterBacktracks`, `DescribeDBClusterEndpoints`, `DescribeDBClusterParameterGroups`
-
-`DescribeDBClusterParameters`, `DescribeDBClusterSnapshotAttributes`, `DescribeDBClusterSnapshots`, `DescribeDBLogFiles`, `DescribeDBProxies`, `DescribeDBProxyEndpoints`, `DescribeDBProxyTargetGroups`, `DescribeDBProxyTargets`
-
-`DescribeDBRecommendations`, `DescribeDBSecurityGroups`, `DescribeDBShardGroups`, `DescribeDBSnapshotAttributes`, `DescribeDBSnapshotTenantDatabases`, `DescribeEngineDefaultClusterParameters`, `DescribeEngineDefaultParameters`, `DescribeEventCategories`
-
-`DescribeEventSubscriptions`, `DescribeExportTasks`, `DescribeGlobalClusters`, `DescribeIntegrations`, `DescribeOptionGroupOptions`, `DescribePendingMaintenanceActions`, `DescribeReservedDBInstances`, `DescribeReservedDBInstancesOfferings`
-
-`DescribeSourceRegions`, `DescribeTenantDatabases`, `DescribeValidDBInstanceModifications`, `DisableHttpEndpoint`, `DownloadDBLogFilePortion`, `EnableHttpEndpoint`, `FailoverGlobalCluster`, `ModifyActivityStream`
-
-`ModifyCertificates`, `ModifyCurrentDBClusterCapacity`, `ModifyCustomDBEngineVersion`, `ModifyDBClusterEndpoint`, `ModifyDBClusterParameterGroup`, `ModifyDBClusterSnapshotAttribute`, `ModifyDBProxy`, `ModifyDBProxyEndpoint`
-
-`ModifyDBProxyTargetGroup`, `ModifyDBRecommendation`, `ModifyDBShardGroup`, `ModifyDBSnapshot`, `ModifyDBSnapshotAttribute`, `ModifyDBSubnetGroup`, `ModifyEventSubscription`, `ModifyGlobalCluster`
-
-`ModifyIntegration`, `ModifyTenantDatabase`, `PromoteReadReplicaDBCluster`, `PurchaseReservedDBInstancesOffering`, `RebootDBCluster`, `RebootDBShardGroup`, `RegisterDBProxyTargets`, `RemoveFromGlobalCluster`
-
-`RemoveRoleFromDBCluster`, `RemoveRoleFromDBInstance`, `RemoveSourceIdentifierFromSubscription`, `ResetDBClusterParameterGroup`, `ResetDBParameterGroup`, `RestoreDBClusterFromS3`, `RestoreDBClusterFromSnapshot`, `RestoreDBClusterToPointInTime`
-
-`RestoreDBInstanceFromS3`, `RevokeDBSecurityGroupIngress`, `StartActivityStream`, `StartDBCluster`, `StartDBInstanceAutomatedBackupsReplication`, `StartExportTask`, `StopActivityStream`, `StopDBCluster`
-
-`StopDBInstanceAutomatedBackupsReplication`, `SwitchoverBlueGreenDeployment`, `SwitchoverGlobalCluster`, `SwitchoverReadReplica`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (12)</summary>
-
-`CreateDBCluster`, `CreateDBInstanceReadReplica`, `CreateOptionGroup`, `DeleteDBCluster`, `DeleteOptionGroup`, `DescribeDBClusters`, `DescribeOptionGroups`, `FailoverDBCluster`
-
-`ModifyDBCluster`, `ModifyOptionGroup`, `PromoteReadReplica`, `RestoreDBInstanceToPointInTime`
-
-
-</details>
-
-<details><summary>Registered outside the pinned model (5)</summary>
-
-`AcknowledgeDBBootstrap`, `GetDBBootstrapConfig`, `PollDBCommands`, `RegisterDBInstance`, `SubmitDBStateChange`
-
-
-</details>
-
-## s3
-
-Operation coverage is not enumerable: Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.
-
-## sts
-
-Implements **4 of 8** modelled operations (50.0%).
-
-<details><summary>Implemented (4)</summary>
-
-`AssumeRole`, `AssumeRoleWithWebIdentity`, `GetCallerIdentity`, `GetSessionToken`
-
-
-</details>
-
-<details><summary>Missing from dispatch (4)</summary>
-
-`AssumeRoleWithSAML`, `DecodeAuthorizationMessage`, `GetAccessKeyInfo`, `GetFederationToken`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>

docs/compute/gpu-passthrough/README.md

@@ -136,6 +136,21 @@ aws ec2 run-instances \
   --key-name spinifex-key
 ```
 
+### Multi-GPU Instances
+
+The GPU count comes from the instance type, exactly as it does in EC2. AWS multi-GPU types pair eight GPUs with a very large host — `p4d.24xlarge` is 96 vCPU and 1152 GiB, — so an edge rig with eight cards and 256 GiB cannot admit them even though it has the GPUs. For those hosts Spinifex offers its own `gpu.*` family, which is not an EC2 family:
+
+```bash
+aws ec2 run-instances \
+  --image-id $GPU_AMI \
+  --instance-type gpu.8x4c \
+  --key-name spinifex-key
+```
+
+`gpu.<count>x<vcpu>c` is that many GPUs at that many vCPU each, so `gpu.8x4c` is eight GPUs on 32 vCPU and 128 GiB. Counts run in three tiers — `2c`, `4c` and `8c` — all at 4 GiB of memory per vCPU. The GPU model and VRAM reported in `DescribeInstanceTypes` are whatever the node discovered, so a `gpu.8x4c` on an 8×RTX 3090 box advertises eight RTX 3090s at 24 GiB each.
+
+A node holding more than one distinct GPU model offers single-GPU types only, and logs each withheld type by name. Two devices count as the same model when their reported name and VRAM match, which keeps a mixed-SKU H100 node fully usable.
+
 To verify the GPU is visible from inside the instance, SSH in and run:
 
 ```bash

docs/compute/vpc-networking/README.md

@@ -161,6 +161,12 @@ Both support the same AWS features: public subnets, Elastic IPs, security groups
 
 ## `nat` — Shared SNAT (Simple)
 
+**`nat` is a supported mode, not a fallback.** It is covered by the nightly e2e suite (cells 19 and 30, single- and multi-node) and it is the only mode a cloud-hosted node can run, so the OCI deployment depends on it. Read "specialised" below as "fewer deployments need it", never as "less finished".
+
+**Start with `pool` unless one of the reasons below applies.** Pool mode is the default, is what most of this document describes, and is what the AWS feature set was built against. `nat` is the specialised mode: it exists for uplinks that cannot be bridged, and it reaches AWS parity through host-side plumbing rather than by putting VM addresses on the wire.
+
+Reach for `nat` when the uplink refuses to carry anything but its own MAC and its own addresses — WiFi and WWAN, and **any cloud-hosted node**, where the VNIC drops a frame whose source address is not registered to it. On such a host `nat` is not a preference, it is the only mode that works at all; pool mode's localnet needs to announce VM addresses by ARP and a cloud VNIC will not carry them. See [oci-integration](../../oci-integration/README.md).
+
 All VMs share a single external IP for outbound SNAT. By default there are no public IPs, no Elastic IPs, and no inbound from WAN — all subnets behave as private subnets with internet access. On routed-NAT nodes, adding a public pool restores full public IP parity (see below).
 
 > **Limitation (routed-NAT v1):** System instances (ECS/EKS/load-balancer
@@ -244,6 +250,24 @@ gateway     = "192.168.1.1"
 prefix_len  = 24
 ```
 
+### NAT gateways in NAT mode
+
+`CreateNatGateway` works here too, and draws its public address from the same public pool an EIP does. On the OVN side it is an `snat` row on the VPC router rewriting the private subnet's CIDR to that address — not the `dnat_and_snat` an EIP gets — so it is centralised on the VPC's gateway chassis by construction and there is no port or MAC to bind it to.
+
+**That address needs the same host plumbing as an EIP**, and on a routed-NAT node it is the host that delivers it: the `/32` route steering it into OVN, the proxy-ARP entry answering for it on the uplink, and the FORWARD accepts. Without them an egressing private instance leaves with a source address the host holds no route back to, while the masquerade rule — which matches the transit `100.127.0.0/24` alone — never sees the packet. Spinifex binds it when the gateway is created and re-asserts it on the host EIP pass, so no operator step is involved; `ip route show dev spx-nat-host` should list the gateway's public address alongside any EIPs.
+
+### NAT mode on more than one node
+
+`spx admin init --external-mode=nat --nodes N` works for N > 1, and each public IP is delivered by the node running its instance rather than by one gateway node. Three things make that true, and all three are automatic:
+
+- The `dnat_and_snat` row carries `external_mac` and `logical_port`, so OVN processes it on the instance's own chassis — the same mechanism pool mode has always used. It is safe here because `ext-shared` is a localnet, so the transit veth is node-local and only the node running the instance can answer for the address on it.
+- Each node installs a host `/32` for its own instances' addresses and leaves the rest alone, asking local OVS which ENIs are plugged in here. Run `ip route show dev spx-nat-host` on each node: exactly one node should hold each public address.
+- Every node's `spx-nat-host` carries the same MAC, `02:00:64:7f:00:01`. OVN keeps one MAC binding for the transit next hop across the whole cluster, so a per-node address would send every chassis's return traffic to whichever node wrote the binding last. `setup-ovn.sh` sets it and vpcd re-asserts it on every start.
+
+Default egress — an instance with no public IP — is still centralised on the VPC's gateway chassis, which is unchanged and is what the gateway LRP's transit address is for.
+
+**On a cloud host the address must also be registered on that node's VNIC.** The allocator runs per node and registers on the local VNIC, which is the right one exactly because the traffic now leaves there. `oci_vnic_iface` is the setting; `oci_vnic_id` is only needed to pin a pool to a specific VNIC and should not be set for this.
+
 **Caveat — reaching an EIP from the spinifex host itself.** Host-sourced traffic enters OVN from the transit net, which is exempt from NAT (that is what makes the jumpbox pattern work) — so a host connection to an EIP that carries the transit source IP would skip DNAT. Spinifex stamps the EIP route with the uplink's LAN IP as source to avoid this, but if no uplink address can be determined, connect to the instance's **private IP** from the host instead. Other machines on the LAN are unaffected.
 
 ## Disabled (Empty/Omitted)
@@ -263,6 +287,9 @@ VPC networking is overlay-only. No external connectivity. Instances can only com
 | DescribeInstances shows public IP | Yes             | Yes           | With public pool  | No       |
 | Admin must reserve IP range       | Yes             | No            | Only static pool  | No       |
 | Needs router DHCP                 | No              | Yes           | Optional          | No       |
+| Works on a non-bridgeable uplink  | No              | No            | Yes               | n/a      |
+| NAT gateways                      | Yes             | Yes           | With public pool  | No       |
+| More than one node                | Yes             | Yes           | Yes               | Yes      |
 
 If you start with `nat` and later need public subnets: on a bridgeable uplink switch to `pool` and define a range (or use `source = "dhcp"`); on a routed-NAT node just add a public pool alongside `nat-transit` — no data migration needed.
 
@@ -460,8 +487,8 @@ external_mode = "pool"    # "pool", "nat", or "" (disabled)
 
 | Value          | Behavior                                                          |
 | -------------- | ----------------------------------------------------------------- |
-| `"pool"`       | Full public networking — public subnets, auto-assign, Elastic IPs |
-| `"nat"`        | Outbound-only SNAT — all VMs share one external IP                |
+| `"pool"`       | **Default.** Full public networking — public subnets, auto-assign, Elastic IPs |
+| `"nat"`        | Outbound-only SNAT — all VMs share one external IP. Add a public pool for full parity. For uplinks that cannot be bridged, including every cloud host |
 | `""` / omitted | Overlay-only — no external connectivity                           |
 
 ## IP Pools: network.external_pools
@@ -842,18 +869,8 @@ one entry covers every security group in that VPC.
 own `spinifex.toml`, and there is no live distribution of it. Two properties of
 the current implementation make the operator responsible for consistency:
 
-- **One node writes the ACLs.** SG reconcile runs on a single CAS-elected vpcd
-  leader, which programs the shared OVN northbound DB. Whichever node holds the
-  lease is the one whose `blocked_ports_wan` / `egress_block_exempt_vpcs` is in
-  force. Leadership moves on restart or crash, so if the node configs disagree,
-  the effective policy **changes when the leader changes** and the drift pass
-  flaps the ACLs between the two states. Edit the value **identically on every
-  node**.
-- **It is read at vpcd startup, not hot-reloaded.** The policy is built once when
-  vpcd starts; editing the TOML does nothing until vpcd restarts. Deploy the
-  config change to all nodes and restart vpcd cluster-wide (take the target down
-  and confirm no `spx` process survives — a selective single-service restart is
-  not reliable on the shared binary).
+- **One node writes the ACLs.** SG reconcile runs on a single CAS-elected vpcd leader, which programs the shared OVN northbound DB. Whichever node holds the lease is the one whose `blocked_ports_wan` / `egress_block_exempt_vpcs` is in force. Leadership moves on restart or crash, so if the node configs disagree, the effective policy **changes when the leader changes** and the drift pass flaps the ACLs between the two states. Edit the value **identically on every node**.
+- **It is read at vpcd startup, not hot-reloaded.** The policy is built once when vpcd starts; editing the TOML does nothing until vpcd restarts. Deploy the config change to all nodes and restart vpcd cluster-wide (take the target down and confirm no `spx` process survives — a selective single-service restart is not reliable on the shared binary).
 
 This is the current implementation and is deliberately minimal. A future revision
 will move the exemption into shared cluster state (so a single edit propagates
@@ -933,7 +950,7 @@ firewall_enabled = true
 
 Before enabling it, check what else the machine is serving — anything listening
 outside the public port group stops accepting new connections. See
-[Firewall and Cluster Membership](/docs/install-multi-node#firewall-and-cluster-membership)
+[Host Firewall](/docs/host-firewall)
 for the full port policy and the cluster-formation steps.
 
 ### Verifying it

docs/containers/ecr/README.md

@@ -146,12 +146,11 @@ Repositories can be managed as code with the standard `aws_ecr_repository` resou
 
 ```hcl
 provider "aws" {
-  region                      = "ap-southeast-2"
-  skip_credentials_validation = true
-  skip_requesting_account_id  = true
+  region = "ap-southeast-2"
 
   endpoints {
     ecr = "https://<your-spinifex-gateway>"
+    sts = "https://<your-spinifex-gateway>"
   }
 }
 

docs/containers/eks/README.md

@@ -55,7 +55,7 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 
 **Spinifex specifics**
 
-- **Auth mode is API-only.** `authentication_mode` must be `API`; the legacy `CONFIG_MAP` and `API_AND_CONFIG_MAP` modes are rejected. Grant access with access entries, not an `aws-auth` ConfigMap.
+- **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
 - **Security groups are auto-managed.** Spinifex creates the cluster and node-group security groups deterministically (e.g. `eks-cluster-<name>-nodegroup-sg`); `vpc_config.security_group_ids` is ignored. To expose a workload you add an ingress rule to the auto-managed node-group SG.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
 - **Default Kubernetes version is `1.32`.**
@@ -82,12 +82,11 @@ Before creating a cluster you need the supporting infrastructure in place. The T
 - **Spinifex running**, with the AWS CLI configured for the `spinifex` profile (see [Installing Spinifex](/docs/install)) and `kubectl` installed.
 - **The `eks-node` image** registered on the cluster. The console blocks cluster creation until it is present.
 - **A VPC with at least one subnet.** The control plane launches in your first subnet; workers run in the subnets you pass to the node group.
-- **Egress for the workers.** Workers must reach the internet to pull container images:
-  - **Public workers** — an **Internet Gateway** with a default route (`0.0.0.0/0 → IGW`) and public IPs.
-  - **Private workers** — a **NAT Gateway** in a public subnet with a default route (`0.0.0.0/0 → NAT`) from the private subnets.
-- **Two IAM roles:**
-  - A **cluster role** trusted by `eks.amazonaws.com` with `AmazonEKSClusterPolicy` attached.
-  - A **node role** trusted by `ec2.amazonaws.com` with `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly` attached (the last lets workers pull images from [ECR](/docs/ecr)).
+- **Egress for the workers.** Workers must reach the internet to pull container images.
+- **Public workers** need an **Internet Gateway** with a default route (`0.0.0.0/0 → IGW`) and public IPs.
+- **Private workers** need a **NAT Gateway** in a public subnet with a default route (`0.0.0.0/0 → NAT`) from the private subnets.
+- **A cluster role** trusted by `eks.amazonaws.com` with `AmazonEKSClusterPolicy` attached.
+- **A node role** trusted by `ec2.amazonaws.com` with `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly` attached (the last lets workers pull images from [ECR](/docs/ecr)).
 
 ## Instructions
 
@@ -263,7 +262,7 @@ The workbooks form a ladder. **eks-quickstart** is a NodePort demo; **eks-https-
 
 **Workload unreachable from outside the cluster.** The node-group security group only allows intra-cluster traffic by default. Add an ingress rule for your NodePort to `eks-cluster-<name>-nodegroup-sg`, as shown in the CLI tab.
 
-**`create-cluster` rejected with an authentication-mode error.** `authentication_mode` must be `API`. Remove any `CONFIG_MAP` / `API_AND_CONFIG_MAP` setting and grant access with access entries instead.
+**`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
 
 **Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, leave `addon_version` unset so Spinifex selects the catalog default.
 
@@ -273,4 +272,4 @@ The workbooks form a ladder. **eks-quickstart** is a NodePort demo; **eks-https-
 
 A complete end-to-end example of Spinifex EKS in production: a GPU-accelerated AI inference platform running on a Supermicro X14 2U with two NVIDIA RTX Pro 6000 Blackwell GPUs. The guide provisions an EKS cluster with two GPU worker nodes via VFIO PCIe passthrough, deploys an OpenAI-compatible LLM API (llama.cpp / Llama 3.2 3B) and a real-time YOLO object-detection stream on separate GPU nodes, and wires them together behind an ALB using IAM, ECR, ACM, and EBS — all with standard AWS tooling and a single `AWS_PROFILE` swap.
 
-→ [EKS AI Platform Reference Architecture](/docs/RTX-Pro-6000-EKS)
+→ [EKS AI Platform Reference Architecture](/hardware/supermicro/rtx-pro-6000)

docs/coverage/README.md

@@ -0,0 +1,34 @@
+---
+title: "AWS API Coverage"
+seoTitle: "Which AWS API Operations Spinifex Implements — Spinifex Docs"
+description: "The AWS API operations Spinifex serves, generated from its gateway dispatch tables and the pinned AWS SDK service models on each build, service by service."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - api
+  - operations
+---
+
+# AWS API Coverage
+
+## Overview
+
+The platform serves **392 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
+
+| Service | Operations |
+|---|---:|
+| [ACM](/coverage/acm) | 9 |
+| [EC2](/coverage/ec2) | 127 |
+| [ECR](/coverage/ecr) | 23 |
+| [ECS](/coverage/ecs) | 31 |
+| [EKS](/coverage/eks) | 27 |
+| [ELBv2](/coverage/elbv2) | 36 |
+| [IAM](/coverage/iam) | 81 |
+| [RDS](/coverage/rds) | 27 |
+| [S3](/coverage/s3) | 26 |
+| [STS](/coverage/sts) | 5 |
+| **Total** | **392** |

docs/coverage/acm/README.md

@@ -0,0 +1,43 @@
+---
+title: "ACM API Coverage"
+seoTitle: "AWS Certificate Manager API Coverage — Spinifex Docs"
+description: "The AWS Certificate Manager API operations Spinifex implements, generated from the gateway dispatch tables and the pinned AWS service model on each build."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - acm
+  - certificates
+  - tls
+---
+
+# ACM API Coverage
+
+## Overview
+
+Spinifex implements **9 operations** in the ACM `2015-12-08` API model.
+
+### Import and issuance
+
+Spinifex both stores externally-issued certificates and issues its own for load balancer listener references. Certificates are account-scoped, and a delete is refused while any listener still references the ARN — there is no force flag, matching AWS.
+
+`RequestCertificate` mints an ARN immediately and returns `PENDING_VALIDATION`; it never issues inline except against a tenant private CA, which has no domain to validate. The validation mode is derived from deployment state rather than configured: the DNS provider API where a credential exists, a manual TXT record where the platform hosts the zone, and a private CA otherwise — the only option for a deployment with no publicly delegated domain.
+
+Terraform's canonical certificate, DNS record and validation flow works unmodified in every mode. Where Spinifex owns the record write it emits no `ResourceRecord`, so iterating the validation options yields zero records and the validation resource still blocks correctly by polling until the certificate is issued.
+
+### Operations
+
+| Operation |
+|---|
+| `AddTagsToCertificate` |
+| `DeleteCertificate` |
+| `DescribeCertificate` |
+| `GetCertificate` |
+| `ImportCertificate` |
+| `ListCertificates` |
+| `ListTagsForCertificate` |
+| `RemoveTagsFromCertificate` |
+| `RequestCertificate` |

docs/coverage/acm/intro.md

@@ -0,0 +1,7 @@
+### Import and issuance
+
+Spinifex both stores externally-issued certificates and issues its own for load balancer listener references. Certificates are account-scoped, and a delete is refused while any listener still references the ARN — there is no force flag, matching AWS.
+
+`RequestCertificate` mints an ARN immediately and returns `PENDING_VALIDATION`; it never issues inline except against a tenant private CA, which has no domain to validate. The validation mode is derived from deployment state rather than configured: the DNS provider API where a credential exists, a manual TXT record where the platform hosts the zone, and a private CA otherwise — the only option for a deployment with no publicly delegated domain.
+
+Terraform's canonical certificate, DNS record and validation flow works unmodified in every mode. Where Spinifex owns the record write it emits no `ResourceRecord`, so iterating the validation options yields zero records and the validation resource still blocks correctly by polling until the certificate is issued.