release: update go.mod for v1.21.0

fix: fail the capabilities benchmark on a nil response instead of panicking
benchCapabilities read resp.Capabilities after checking only err, so a provider returning (nil, nil) panicked the benchmark rather than failing it.
2026-10-24 13:14:23 +00:00 · 2026-09-29 10:10:36 +10:00 · 2026-09-29 09:50:54 +10:00 · 2026-09-28 18:48:27 -05:00 · 2026-09-29 09:35:11 +10:00 · 2026-09-28 18:34:09 -05:00
1401 changed files with 74530 additions and 13948 deletions
@@ -134,21 +134,22 @@ var cascadeMarkers = []string{
 	"Expected value not to be nil",
 }

-// teardownPatterns match the content a test logs from t.Cleanup or from the
-// harness's on-failure diagnostics — i.e. after it has already failed. They are
-// the LAST file:line lines in the failure body, so extractErrorLine's
-// keep-the-last rule picks them over the assertion unless they are skipped.
-// Anchored deliberately: a fragment loose enough to match an assertion would
-// blank out the one line the report exists to show.
-var teardownPatterns = []*regexp.Regexp{
+// nonFailurePatterns match content a test emits AFTER the assertion that failed
+// — t.Cleanup, harness diagnostics, or a t.Logf on a passing branch — which
+// extractErrorLine's keep-the-last rule would otherwise pick over the assertion.
+// Go emits t.Logf and t.Errorf identically, so anchor every entry: a fragment
+// loose enough to match an assertion blanks out the line the report exists to
+// show. `\bas expected\b` unanchored also matches "did not fail as expected".
+var nonFailurePatterns = []*regexp.Regexp{
 	regexp.MustCompile(`^DB instance \S+ is gone$`),
 	regexp.MustCompile(`^db diagnostics \S+:`),
+	regexp.MustCompile(`^\S+: management bridge \S+:\d+ refused as expected$`),
 }

-// isTeardownLine reports whether a candidate error line was emitted after the
-// failure it would otherwise be mistaken for.
-func isTeardownLine(content string) bool {
-	for _, re := range teardownPatterns {
+// isNonFailureLine reports whether a candidate error line was emitted after the
+// failure it would otherwise be mistaken for, or reports an expected outcome.
+func isNonFailureLine(content string) bool {
+	for _, re := range nonFailurePatterns {
 		if re.MatchString(content) {
 			return true
 		}
@@ -245,12 +246,12 @@ func extractErrorLine(body string) string {
 			// progress lines from the same file:line pattern before the
 			// failing assertion finally fires. Cleanup runs after it, though,
 			// so its lines are last of all and have to be skipped.
-			if content := strings.TrimSpace(m[1]); !isTeardownLine(content) {
+			if content := strings.TrimSpace(m[1]); !isNonFailureLine(content) {
 				goFileLine = content
 			}
 			continue
 		}
-		if fileHintRe.MatchString(l) && !isTeardownLine(l) {
+		if fileHintRe.MatchString(l) && !isNonFailureLine(l) {
 			lastTestLine = l
 		}
 	}
@@ -296,15 +297,23 @@ func extractFileHint(body string) string {
 	}
 	// Scanned line by line rather than across the body so a cleanup line's own
 	// file:line does not become the site the report points a reader at.
-	hint := ""
+	hint, filtered := "", ""
 	for _, raw := range strings.Split(body, "\n") {
 		l := strings.TrimSpace(raw)
-		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isTeardownLine(strings.TrimSpace(m[1])) {
+		all := fileHintRe.FindAllString(l, -1)
+		if len(all) == 0 {
 			continue
 		}
-		if all := fileHintRe.FindAllString(l, -1); len(all) > 0 {
-			hint = all[len(all)-1]
+		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isNonFailureLine(strings.TrimSpace(m[1])) {
+			filtered = all[len(all)-1]
+			continue
 		}
+		hint = all[len(all)-1]
+	}
+	// A body whose only file:line was allowlisted still gets a pointer: coarse
+	// beats none, and extractErrorLine has the same last-resort shape.
+	if hint == "" {
+		return filtered
 	}
 	return hint
 }
@@ -141,6 +141,46 @@ func TestExtractErrorLine_SkipsCleanupLoggedAfterTheFailure(t *testing.T) {
 	}
 }

+// A subtest looping over two subjects calls t.Errorf for the first and t.Logf
+// for the second, so the success line is last in the body. Nightly run
+// 34383800404 reported that log line as the failure.
+func TestExtractErrorLine_SkipsSuccessLoggedAfterTheFailure(t *testing.T) {
+	body := `    isolation_test.go:80: rds-e2e-iso-a-1788981456: management bridge 10.15.8.10:5432 could not be tested: want connection refused, got dial tcp 10.15.8.10:5432: i/o timeout
+    isolation_test.go:84: rds-e2e-iso-b-1788981456: management bridge 10.15.8.11:5432 refused as expected`
+	got := extractErrorLine(body)
+	if !strings.Contains(got, "could not be tested") {
+		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
+	}
+	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
+	}
+}
+
+// "as expected" reads the same in an affirmation and in its negation, so an
+// unanchored pattern suppresses the assertion it was meant to outrank and
+// promotes a harness progress line in its place.
+func TestExtractErrorLine_KeepsNegatedExpectationAssertions(t *testing.T) {
+	body := `    isolation_test.go:60: waiting for the db instance to become available
+    isolation_test.go:80: rds-e2e-iso-a: management bridge 10.15.8.10:5432 was not refused as expected: got a connection`
+	got := extractErrorLine(body)
+	if !strings.Contains(got, "was not refused as expected") {
+		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
+	}
+	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
+	}
+}
+
+// Every candidate allowlisted leaves extractFileHint with nothing to return,
+// and unlike extractErrorLine it has no last-resort branch. A coarse pointer
+// beats sending the reader to no file at all.
+func TestExtractFileHint_FallsBackWhenEveryCandidateFiltered(t *testing.T) {
+	body := `    isolation_test.go:84: rds-e2e-iso-b: management bridge 10.15.8.11:5432 refused as expected`
+	if hint := extractFileHint(body); hint != "isolation_test.go:84" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:84", hint)
+	}
+}
+
 func TestExtractFileHint_PicksLastMatch(t *testing.T) {
 	body := "ec2helpers.go:50: setup\n    vpc_test.go:227: Eventually: condition not met"
 	got := extractFileHint(body)
@@ -0,0 +1,80 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="836" viewBox="0 0 1020 836" role="img" aria-label="Spinifex on a single OCI instance: the instance sits inside the VCN public subnet 10.200.0.0/23; its primary VNIC carries the default route and the cluster mesh, its secondary VNIC is the br-wan bridge holding registered secondary private IPs paired with reserved public IPs; guests reach them through the spx-nat transit veth, and an iSCSI block volume backs /var/lib/spinifex.">
+<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
+<rect x="0" y="0" width="1020" height="836" rx="20" fill="#14181D" stroke="#2A2F36" stroke-width="1"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON OCI — SINGLE NODE</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one instance · routed NAT · real public addresses</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">One OCI instance, single Spinifex node</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">AWS SDKs, SSH and guest egress all arrive here</text>
+<rect x="46" y="126" width="272.7" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="182.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="327.7" y="126" width="177.2" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="416.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest public IP</text>
+<rect x="513.92" y="126" width="218.1" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="623.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510.0" y1="170" x2="510.0" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="582" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">OCI VCN 10.200.0.0/16  ·  PUBLIC SUBNET 10.200.0.0/23</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">virtual router 10.200.0.1 · internet gateway</text>
+<rect x="48" y="232" width="924" height="520" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="68" y="266" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">BM.Standard.E2.64</text>
+<text x="260" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">spinifex-node-01  ·  FAULT-DOMAIN-1</text>
+<text x="952" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one host = one Spinifex region</text>
+<line x1="68" y1="282" x2="952" y2="282" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="68" y="298" width="434.0" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">VNIC 0 — PRIMARY</text>
+<text x="482.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">enp0s9</text>
+<text x="88" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">10.200.0.252/23  ·  holds the default route</text>
+<text x="88" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">advertise IP · ovn-encap-ip · NATS · API :9999</text>
+<rect x="518.0" y="298" width="434.0" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="538.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">VNIC 1 — SECONDARY</text>
+<text x="932.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">br-wan (Linux bridge)</text>
+<text x="538.0" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">10.200.1.233/32  +  ip rule → table 200</text>
+<text x="538.0" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">secondary private IPs ⇄ reserved public IPs</text>
+<line x1="735.0" y1="406" x2="735.0" y2="422" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="424" width="884" height="40" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="88" y="449" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="600" text-anchor="start">spx-nat-host ⇄ spx-nat-ovs</text>
+<text x="296" y="449" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">transit 100.127.0.0/24  ·  nft chain spinifex-nat-egress masquerades out br-wan</text>
+<line x1="510.0" y1="464" x2="510.0" y2="480" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="482" width="884" height="100" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">SPINIFEX CONTROL PLANE</text>
+<text x="932" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">every service is one binary: spx service &lt;name&gt;</text>
+<rect x="86" y="530" width="109.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="140.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw :9999</text>
+<rect x="204.01999999999998" y="530" width="74.9" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="241.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="287.94" y="530" width="61.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="318.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="358.22" y="530" width="61.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="388.9" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">NATS</text>
+<rect x="428.5" y="530" width="102.2" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="479.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="539.7" y="530" width="102.2" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="590.8" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="650.9000000000001" y="530" width="95.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="698.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">northstar</text>
+<rect x="755.2800000000001" y="530" width="88.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="799.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">ui :3000</text>
+<text x="86" y="568" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">SigV4 in, NATS in the middle, QEMU/KVM · OVN/OVS · Predastore · Viperblock underneath</text>
+<line x1="248" y1="582" x2="248" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="772" y1="582" x2="772" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<rect x="68" y="600" width="532" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">TENANT VPCs — OVN br-int</text>
+<text x="580" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">geneve-local</text>
+<rect x="86" y="644" width="496" height="34" rx="9" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-a  10.0.0.0/16</text>
+<text x="310" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  10.0.1.4  →  150.230.13.131</text>
+<rect x="86" y="686" width="496" height="34" rx="9" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-b  172.31.0.0/16</text>
+<text x="310" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  172.31.4.9  ·  private, egress only</text>
+<rect x="616" y="600" width="336" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="636" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">BLOCK VOLUME — iSCSI</text>
+<text x="932" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">256 GB+</text>
+<text x="636" y="658" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">169.254.2.2:3260  →  /dev/sdb</text>
+<text x="636" y="680" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">/var/lib/spinifex</text>
+<text x="636" y="700" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">mounted by UUID, defaults,_netdev,nofail</text>
+<text x="636" y="718" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">viperblock · predastore · JetStream</text>
+<text x="28" y="796" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">A guest's public address is a real OCI reserved public IP, attached to a secondary private IP on br-wan.</text>
+<text x="28" y="814" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Inbound and outbound are therefore the same address, and OCI's per-VNIC source check passes.</text>
+</svg>
@@ -0,0 +1,116 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="796" viewBox="0 0 1020 796" role="img" aria-label="Spinifex across three OCI instances: all three sit inside the VCN public subnet 10.200.0.0/23, one per fault domain. Each carries a primary VNIC for the cluster mesh and a br-wan secondary VNIC holding its reserved public addresses, its own iSCSI block volume, and its share of the guests. Geneve, NATS JetStream, predastore erasure coding and the OVN raft join them east-west.">
+<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
+<rect x="0" y="0" width="1020" height="796" rx="20" fill="#14181D" stroke="#2A2F36" stroke-width="1"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON OCI — THREE NODES</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">three fault domains · one region · one API endpoint</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">Three OCI instances, redundant multi-node install</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one endpoint, whichever node answers</text>
+<rect x="46" y="126" width="272.7" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="182.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="327.7" y="126" width="177.2" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="416.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest public IP</text>
+<rect x="513.92" y="126" width="218.1" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="623.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510.0" y1="170" x2="510.0" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="464" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">OCI VCN 10.200.0.0/16  ·  PUBLIC SUBNET 10.200.0.0/23</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">virtual router 10.200.0.1 · internet gateway</text>
+<rect x="48.0" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="66.0" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-01</text>
+<text x="327.3333333333333" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-1</text>
+<text x="66.0" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
+<line x1="66.0" y1="292" x2="327.3333333333333" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<text x="66.0" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0  10.200.0.252</text>
+<text x="66.0" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
+<text x="66.0" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1  br-wan  10.200.1.233/32</text>
+<text x="66.0" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
+<rect x="66.0" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="91.5" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
+<rect x="123.0" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="151.6" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="186.2" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="208.6" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="66.0" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="88.4" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
+<rect x="116.8" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="157.8" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="204.8" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="245.8" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="66.0" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76.0" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-23d2…  10.0.1.4 → 150.230.13.131</text>
+<rect x="66.0" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76.0" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
+<line x1="345.3333333333333" y1="364" x2="361.3333333333333" y2="364" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.5" stroke-dasharray="2 4"/>
+<rect x="361.3333333333333" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="379.3333333333333" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-02</text>
+<text x="640.6666666666666" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-2</text>
+<text x="379.3333333333333" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
+<line x1="379.3333333333333" y1="292" x2="640.6666666666666" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<text x="379.3333333333333" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0  10.200.0.118</text>
+<text x="379.3333333333333" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
+<text x="379.3333333333333" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1  br-wan  10.200.1.96/32</text>
+<text x="379.3333333333333" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
+<rect x="379.3333333333333" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="404.8" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
+<rect x="436.3333333333333" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="464.9" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="499.5333333333333" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="521.9" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="379.3333333333333" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="401.7" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
+<rect x="430.1333333333333" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="471.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="518.1333333333333" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="559.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="379.3333333333333" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="389.3333333333333" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-6b91…  10.0.1.9 → 137.23.9.125</text>
+<rect x="379.3333333333333" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="389.3333333333333" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
+<line x1="658.6666666666666" y1="364" x2="674.6666666666666" y2="364" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.5" stroke-dasharray="2 4"/>
+<rect x="674.6666666666666" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="692.6666666666666" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-03</text>
+<text x="954.0" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-3</text>
+<text x="692.6666666666666" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
+<line x1="692.6666666666666" y1="292" x2="954.0" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<text x="692.6666666666666" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0  10.200.0.41</text>
+<text x="692.6666666666666" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
+<text x="692.6666666666666" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1  br-wan  10.200.1.12/32</text>
+<text x="692.6666666666666" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
+<rect x="692.6666666666666" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="718.2" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
+<rect x="749.6666666666666" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="778.3" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="812.8666666666667" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="835.3" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="692.6666666666666" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="715.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
+<rect x="743.4666666666666" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="784.5" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="831.4666666666666" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="872.5" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="692.6666666666666" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702.6666666666666" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-0ae4…  10.0.2.6 → 150.230.4.88</text>
+<rect x="692.6666666666666" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702.6666666666666" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
+<rect x="48" y="524" width="924" height="104" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="68" y="554" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">EAST–WEST, INSIDE THE VCN</text>
+<text x="952" y="554" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">no public address is involved</text>
+<rect x="66" y="574" width="136.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="134.2" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">Geneve UDP 6081</text>
+<rect x="211.3" y="574" width="149.9" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="286.3" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">NATS JetStream R3</text>
+<rect x="370.24" y="574" width="156.8" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="448.6" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore RS(2,1)</text>
+<rect x="536.0" y="574" width="163.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="617.8" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">OVN Northbound raft</text>
+<rect x="708.58" y="574" width="68.1" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="742.6" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">OVSDB</text>
+<text x="66" y="612" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Guest-to-guest traffic rides Geneve between the nodes' primary VNICs — it never leaves the subnet and never touches br-wan.</text>
+<rect x="28" y="668" width="964" height="46" rx="12" fill="#EE4B12" fill-opacity="0.07" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.55"/>
+<text x="48" y="688" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13" fill="#F5F3EE" font-weight="700" text-anchor="start">A guest that moves takes its address with it.</text>
+<text x="48" y="705" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">The reserved public IP follows the secondary private IP to whichever node's br-wan the guest lands on — the tenant sees no change.</text>
+<text x="28" y="756" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Three nodes is the smallest cluster that tolerates losing one: NATS JetStream keeps quorum, predastore RS(2,1)</text>
+<text x="28" y="774" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">keeps the data readable, and the OVN Northbound raft keeps a leader. One node per fault domain is what makes that real.</text>
+</svg>
@@ -0,0 +1,44 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
+<rect width="760" height="470" fill="#ffffff"/>
+<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Guest block storage throughput (Viperblock)</text>
+<text x="74" y="49" font-size="12.5" fill="#6b7280">fio, 4 jobs, iodepth 32, direct I/O, per-guest sequential runs — Spinifex v1.20.0</text>
+<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">0</text>
+<line x1="74" y1="305.0" x2="734" y2="305.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="309.0" font-size="12" text-anchor="end" fill="#6b7280">100</text>
+<line x1="74" y1="224.0" x2="734" y2="224.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="228.0" font-size="12" text-anchor="end" fill="#6b7280">200</text>
+<line x1="74" y1="143.0" x2="734" y2="143.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="147.0" font-size="12" text-anchor="end" fill="#6b7280">300</text>
+<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">400</text>
+<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">MiB/s</text>
+<rect x="111.4" y="81.4" width="45.4" height="304.6" rx="2.5" fill="#2563eb"/>
+<text x="134.1" y="75.4" font-size="11" text-anchor="middle" fill="#1f2937">376</text>
+<rect x="159.8" y="198.1" width="45.4" height="187.9" rx="2.5" fill="#0d9488"/>
+<text x="182.5" y="192.1" font-size="11" text-anchor="middle" fill="#1f2937">232</text>
+<rect x="208.2" y="299.3" width="45.4" height="86.7" rx="2.5" fill="#f59e0b"/>
+<text x="230.9" y="293.3" font-size="11" text-anchor="middle" fill="#1f2937">107</text>
+<text x="184.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">GPU</text>
+<rect x="331.4" y="85.5" width="45.4" height="300.5" rx="2.5" fill="#2563eb"/>
+<text x="354.1" y="79.5" font-size="11" text-anchor="middle" fill="#1f2937">371</text>
+<rect x="379.8" y="202.9" width="45.4" height="183.1" rx="2.5" fill="#0d9488"/>
+<text x="402.5" y="196.9" font-size="11" text-anchor="middle" fill="#1f2937">226</text>
+<rect x="428.2" y="321.2" width="45.4" height="64.8" rx="2.5" fill="#f59e0b"/>
+<text x="450.9" y="315.2" font-size="11" text-anchor="middle" fill="#1f2937">80</text>
+<text x="404.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU1</text>
+<rect x="551.4" y="84.7" width="45.4" height="301.3" rx="2.5" fill="#2563eb"/>
+<text x="574.1" y="78.7" font-size="11" text-anchor="middle" fill="#1f2937">372</text>
+<rect x="599.8" y="201.3" width="45.4" height="184.7" rx="2.5" fill="#0d9488"/>
+<text x="622.5" y="195.3" font-size="11" text-anchor="middle" fill="#1f2937">228</text>
+<rect x="648.2" y="321.2" width="45.4" height="64.8" rx="2.5" fill="#f59e0b"/>
+<text x="670.9" y="315.2" font-size="11" text-anchor="middle" fill="#1f2937">80</text>
+<text x="624.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU2</text>
+<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
+<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
+<text x="93" y="455" font-size="12.5" fill="#1f2937">128K random read</text>
+<rect x="216.8" y="444" width="13" height="13" rx="2.5" fill="#0d9488"/>
+<text x="235.8" y="455" font-size="12.5" fill="#1f2937">128K mixed read</text>
+<rect x="352.3" y="444" width="13" height="13" rx="2.5" fill="#f59e0b"/>
+<text x="371.3" y="455" font-size="12.5" fill="#1f2937">16K random read</text>
+</svg>
@@ -0,0 +1,35 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
+<rect width="760" height="470" fill="#ffffff"/>
+<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Network throughput: overlay vs physical fabric</text>
+<text x="74" y="49" font-size="12.5" fill="#6b7280">iperf3 TCP receiver goodput, median across node pairs — Spinifex v1.20.0</text>
+<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">20</text>
+<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">25</text>
+<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">Gbit/s</text>
+<text x="140.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">1</text>
+<text x="272.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">2</text>
+<text x="404.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">4</text>
+<text x="536.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">8</text>
+<text x="668.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">16</text>
+<path d="M140.0,164.4 L272.0,160.5 L404.0,160.5 L536.0,160.5 L668.0,160.5" fill="none" stroke="#2563eb" stroke-width="3"/>
+<circle cx="140.0" cy="164.4" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="272.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="404.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="536.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="668.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<text x="664.0" y="149.5" font-size="11.5" text-anchor="end" font-weight="600" fill="#2563eb">23.48</text>
+<path d="M140.0,255.1 L272.0,222.1 L404.0,222.1 L536.0,222.1 L668.0,228.5" fill="none" stroke="#0d9488" stroke-width="3"/>
+<circle cx="140.0" cy="255.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="272.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="404.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="536.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="668.0" cy="228.5" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<text x="664.0" y="217.5" font-size="11.5" text-anchor="end" font-weight="600" fill="#0d9488">22.43</text>
+<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
+<text x="404.0" y="430.0" font-size="12.5" text-anchor="middle" fill="#6b7280">parallel TCP streams</text>
+<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
+<text x="93" y="455" font-size="12.5" fill="#1f2937">Host physical fabric (25GbE)</text>
+<rect x="304.4" y="444" width="13" height="13" rx="2.5" fill="#0d9488"/>
+<text x="323.4" y="455" font-size="12.5" fill="#1f2937">Guest overlay (OVN)</text>
+</svg>
@@ -0,0 +1,39 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
+<rect width="760" height="470" fill="#ffffff"/>
+<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Predastore S3 read throughput by client</text>
+<text x="74" y="49" font-size="12.5" fill="#6b7280">1 GiB objects, same presigned-curl client on every path — Spinifex v1.20.0</text>
+<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">0</text>
+<line x1="74" y1="278.0" x2="734" y2="278.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="282.0" font-size="12" text-anchor="end" fill="#6b7280">200</text>
+<line x1="74" y1="170.0" x2="734" y2="170.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="174.0" font-size="12" text-anchor="end" fill="#6b7280">400</text>
+<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">600</text>
+<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">MiB/s</text>
+<rect x="102.0" y="202.4" width="51.5" height="183.6" rx="2.5" fill="#2563eb"/>
+<text x="127.8" y="196.4" font-size="11" text-anchor="middle" fill="#1f2937">340</text>
+<rect x="156.5" y="136.5" width="51.5" height="249.5" rx="2.5" fill="#f59e0b"/>
+<text x="182.2" y="130.5" font-size="11" text-anchor="middle" fill="#1f2937">462</text>
+<text x="156.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">Host</text>
+<rect x="267.1" y="170.5" width="51.5" height="215.5" rx="2.5" fill="#2563eb"/>
+<text x="292.8" y="164.5" font-size="11" text-anchor="middle" fill="#1f2937">399</text>
+<rect x="321.5" y="174.9" width="51.5" height="211.1" rx="2.5" fill="#f59e0b"/>
+<text x="347.2" y="168.9" font-size="11" text-anchor="middle" fill="#1f2937">391</text>
+<text x="321.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">GPU guest</text>
+<rect x="432.1" y="102.5" width="51.5" height="283.5" rx="2.5" fill="#2563eb"/>
+<text x="457.8" y="96.5" font-size="11" text-anchor="middle" fill="#1f2937">525</text>
+<rect x="486.5" y="197.0" width="51.5" height="189.0" rx="2.5" fill="#f59e0b"/>
+<text x="512.2" y="191.0" font-size="11" text-anchor="middle" fill="#1f2937">350</text>
+<text x="486.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU1 guest</text>
+<rect x="597.0" y="106.8" width="51.5" height="279.2" rx="2.5" fill="#2563eb"/>
+<text x="622.8" y="100.8" font-size="11" text-anchor="middle" fill="#1f2937">517</text>
+<rect x="651.5" y="196.5" width="51.5" height="189.5" rx="2.5" fill="#f59e0b"/>
+<text x="677.2" y="190.5" font-size="11" text-anchor="middle" fill="#1f2937">351</text>
+<text x="651.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU2 guest</text>
+<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
+<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
+<text x="93" y="455" font-size="12.5" fill="#1f2937">Single-stream GET</text>
+<rect x="224.1" y="444" width="13" height="13" rx="2.5" fill="#f59e0b"/>
+<text x="243.1" y="455" font-size="12.5" fill="#1f2937">Chunked/concurrent GET</text>
+</svg>
@@ -130,7 +130,7 @@
    "runner": "ci-single",
    "extra": "rds",
    "suite": "rds",
-    "timeout_minutes": 100
+    "timeout_minutes": 120
  },
  {
    "cell": 29,
@@ -143,5 +143,52 @@
    "extra": "",
    "suite": "storagefault",
    "timeout_minutes": 100
+  },
+  {
+    "cell": 30,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "nat",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "nat-multi",
+    "suite": ""
+  },
+  {
+    "cell": 31,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "instancerecovery",
+    "timeout_minutes": 105
+  },
+  {
+    "cell": 32,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "lbrecovery",
+    "timeout_minutes": 90
+  },
+  {
+    "cell": 33,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "kvquorum",
+    "timeout_minutes": 55
  }
 ]
@@ -40,7 +40,24 @@
 E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"

 # Suites runnable against a multi-node environment.
-E2E_SUITES_MULTI="multinode lb cert quota storagefault"
+#
+# instancerecovery is multi-only and needs three: recovery is inert below that,
+# because there is no survivor to move a guest to that is not the node that
+# failed. Like storagefault it takes a node away and freezes a cluster-wide
+# service, so it runs last and owns its environment.
+#
+# lbrecovery is multi-only and needs three for two reasons rather than one: a
+# survivor to move the load balancer to, and a node for its backends that is not
+# the load balancer's, so taking that node away does not take the backends with
+# it. It runs beside instancerecovery at the end because it also takes a node
+# away and owns its environment while it does.
+#
+# kvquorum is multi-only and needs three, for the same reason as those two: a
+# surviving majority. It is what proves the control plane's own state is
+# replicated rather than sitting on one node, so it runs before the pair that
+# take a node away — a cluster whose buckets are already wrong should report
+# that, not report a recovery failing.
+E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault"

 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
@@ -64,6 +81,10 @@ E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
 # after a failed seal do not exist on one node, and they are the assertions
 # worth a nightly.
 E2E_SUITES_NIGHTLY_SINGLE="single cert iam"
+#
+# The multi-node nightly subset carries kvquorum's read-only half through
+# `multinode`, which sweeps every KV bucket rather than the three the daemon
+# owns. The node-loss half is too slow for a 35-minute cell and has its own.
 E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"

 # How long a suite gets, for the same reason the lists above live here: the
@@ -77,8 +98,9 @@ e2e_suite_timeout() {
    # what the queue costs, not what the tests do: single tests logged 11m, 22m
    # and 26m waiting for budget. 17 tests at that concurrency is over an hour
    # before anything goes wrong, and the first pass to run deep into the suite
-    # still had three left at 60m.
-    rds) echo "90m" ;;
+    # still had three left at 60m. The loaded graceful-reboot case adds an
+    # instance that holds its slot through a pgbench load as well as a create.
+    rds) echo "105m" ;;
    # diskperf runs two 16 GiB fio profiles per repetition, each on a volume it
    # creates first. One pass is ~20 minutes on the bare-metal cell and a
    # baseline capture at SPINIFEX_DISKPERF_REPS=3 is three times that.
@@ -87,6 +109,23 @@ e2e_suite_timeout() {
    # SIGSTOPped for seven minutes to prove a guest survives it. A full pass on
    # dev-prod is ~63 minutes, so the 30m default truncated it mid-suite.
    storagefault) echo "90m" ;;
+    # instancerecovery is almost entirely waiting, and none of it is avoidable:
+    # a dead owner has to be seen stale twice, a guest has to boot, and the
+    # storage case waits for a pause measured at 4m47s and then holds four
+    # minutes to prove nothing moved. Three tests, each taking a node away and
+    # bringing it back, and the partition case adds a lease validity, a lease
+    # TTL and a boot on top of everything the other two wait for.
+    instancerecovery) echo "90m" ;;
+    # lbrecovery pays for a load balancer twice: once to stand one up with two
+    # backends placed around it, and once for the recovery itself — a stale
+    # owner, a lease, a boot, HAProxy's own start, and its health checks against
+    # the backends before it forwards anything. Then the node comes back and has
+    # to be shown not to answer for an address it no longer holds.
+    lbrecovery) echo "75m" ;;
+    # kvquorum audits every bucket in seconds and then spends the rest of its
+    # budget on a node stopping and starting: seven units down, a raft election,
+    # and the whole service stack back up with every stream caught up.
+    kvquorum) echo "40m" ;;
    *) echo "30m" ;;
  esac
 }
@@ -33,8 +33,6 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        # The suites the nightly E2E matrix actually installs on. This matches
-        # what detect_os accepts: Debian 13+ and Ubuntu 24.04+.
        image:
          - ubuntu:24.04
          - ubuntu:26.04
@@ -55,15 +55,12 @@ permissions:

 env:
  GIT_BRANCH: ${{ github.ref_name }}
-  # Same central ES/APM sink as the other four e2e workflows. Bare-metal had
-  # no default here before — this cell's telemetry never reached the sink at
-  # all (not just missing ci.cell), see bm-bootstrap.sh's telemetry.env step.
  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}

 jobs:
  e2e_baremetal:
    name: Bare-Metal Single-Node E2E
-    runs-on: [ self-hosted, ci-baremetal ]
+    runs-on: [self-hosted, ci-baremetal]
    timeout-minutes: 240
    concurrency:
      group: e2e-baremetal-cluster
@@ -72,9 +69,6 @@ jobs:
      BM_ISO_CACHE: /var/cache/spinifex-iso-builder
      BM_CELL: ${{ inputs.cell || '20' }}
      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
-      # Caller-supplied on workflow_call. A dispatch run has no such input, and
-      # leaving it empty cost this cell its ci.run_id: its telemetry reached the
-      # sink but could not be found by run, only by guessing a time window.
      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
    steps:
      - name: Clean Workspace
@@ -190,8 +184,6 @@ jobs:
        run: |
          mkdir -p "${ARTIFACT_DIR}"
          set -o pipefail
-          # Per-suite budgets are declared once in suite-sets.sh. The script
-          # applies one timeout to every suite in the list, so the widest wins.
          . "${GITHUB_WORKSPACE}/.github/scripts/suite-sets.sh"
          BM_TEST_TIMEOUT=""
          for suite in ${BM_SUITES}; do
@@ -1,7 +1,5 @@
 name: E2E DDIL

-# This workflow is manually triggered while the DDIL scenarios are being
-# validated.
 on:
  workflow_dispatch:
    inputs:
@@ -10,37 +8,31 @@ on:
        required: false
        default: ""

-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
+# Shared with e2e-nightly.yml and e2e-suites.yml: every one of them provisions
+# many cells on the same runner pools and the same hypervisors, so two in
 # flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
+# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
+# is outside the group: one cluster per run is cheap enough to overlap.
 concurrency:
  group: e2e-hypervisors
  cancel-in-progress: false

 permissions:
  contents: read
-  # actions: read lets the otel-job-id action look up this job's numeric id via
-  # the Actions API — github.job is only the YAML key, no context exposes it.
  actions: read

 env:
  GIT_BRANCH: ${{ github.ref_name }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  ddil:
    name: DDIL Real Multi-Node
-    runs-on: [ self-hosted, ci-multi ]
+    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 60
    env:
      TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
-      # Job-level env wins over the workflow-level default. ci.cell=ddil marks
-      # this workflow's telemetry so a DDIL run's partition-induced errors are
-      # never mistaken for a healthy multi-node run's in the sink.
      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=ddil
    steps:
      - name: Clean Workspace
@@ -1,19 +1,20 @@
 name: E2E Nightly

+# Triggered nightly by a Cloudflare Worker that dispatches this workflow once on
+# `dev` and once on `main`, rather than by a `schedule:` here. GitHub registers a
+# schedule only from the default branch, so a cron in this file can never run
+# `main` and `dev` as separate nightlies, and it would always execute `main`'s
+# copy of this file — leaving a fix to the workflow itself untestable until it
+# had already shipped. A dispatch takes both the file and the source from its ref.
 on:
-  schedule:
-    - cron: "0 14 * * 0-4" # 00:00 Mon-Fri AEST = 14:00 Sun-Thu UTC
  workflow_dispatch:
    inputs:
      cells:
-        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection"
+        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
        required: false

 permissions:
  contents: read
-  # actions: read lets the "Resolve CI job id" step below look up this job's
-  # numeric id via the Actions API — github.job is only the YAML key, no
-  # context exposes the id itself.
  actions: read

 # Every cell builds its cluster on the same five hypervisors, so two runs in
@@ -21,30 +22,21 @@ permissions:
 # second rather than cancelling it: a dispatch is usually a verification run
 # and discarding it loses the result someone is waiting on.
 #
-# The group is shared with e2e.yml, e2e-suites.yml and e2e-ddil.yml, which draw
-# on the same runner pools. A per-workflow group let a push-triggered E2E land
-# on top of a nightly cell and take libvirt down under both.
+# The group is shared with e2e-suites.yml and e2e-ddil.yml, which draw on the
+# same runner pools. e2e.yml is deliberately outside it: a single-cluster push
+# run is cheap enough to overlap, and grouping it stalled every push behind a
+# multi-hour nightly.
 concurrency:
  group: e2e-hypervisors
  cancel-in-progress: false

-# Scheduled runs are loaded from the default branch (main), so the schedule
-# trigger must stay in sync on main for cron to fire — and so must the
-# concurrency block above. main currently has none, which is why the nightly
-# cron runs ungrouped and a dispatch on any branch can land on top of it.
 env:
-  GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+  GIT_BRANCH: ${{ github.ref_name }}
  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
-  # The matrix is built here rather than filtered inside each cell. A job whose
-  # steps all skip still reports success, so gating steps made a one-cell
-  # dispatch look like a green nineteen-job suite. A cell that was not selected
-  # now produces no job at all.
  select:
    runs-on: ubuntu-26.04
    outputs:
@@ -60,9 +52,6 @@ jobs:
      - id: pick
        env:
          CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
-          # Provision, install and teardown around the suite. Measured at ~6
-          # minutes single-node and ~10 multi-node, so this is the floor the
-          # job budget must carry on top of the suite's own.
          SETUP_ALLOWANCE_MIN: 10
        run: |
          if [ -z "$CELLS" ]; then
@@ -125,15 +114,11 @@ jobs:
      max-parallel: 3
      matrix:
        include: ${{ fromJSON(needs.select.outputs.cells) }}
-    runs-on: [ self-hosted, "${{ matrix.runner }}" ]
+    runs-on: [self-hosted, "${{ matrix.runner }}"]
    timeout-minutes: ${{ matrix.timeout_minutes || 50 }}
    env:
      TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
-      # Job-level env wins over the workflow-level default above, so telemetry
-      # from this cell carries ci.cell/ci.matrix. Twelve cells run under one
-      # run id and every cell names its hosts node1/node2/node3, so without
-      # this every cell's telemetry collapses into one unattributable bucket.
      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}

    steps:
@@ -293,8 +278,31 @@ jobs:
             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
             if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"

+      # natuplink proves a guest with a public IP reaches the internet. The two
+      # surfaces it never touches are a private subnet's isolation and its NAT
+      # gateway, and TestVPCEgressPaths is the authority on both — it skipped in
+      # nat mode until the fixture learned that routed NAT carries EIPs too.
+      - name: Run VPC Egress Paths in NAT mode
+        if: matrix.extra == 'nat-single'
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             echo '::group::suite single/TestVPCEgressPaths'; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-vpcegress-nat.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./single.test -test.v -test.timeout=30m -test.run '^TestVPCEgressPaths\$' \
+               2>&1 | tee '${ARTIFACT_DIR}/test-vpcegress-nat.log'; \
+             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
+             if [ \$rc -ne 0 ]; then echo 'vpcegress-nat: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
+
      - name: Run Multi Node E2E
-        if: matrix.topo == 'real-multi' && matrix.suite == ''
+        if: matrix.topo == 'real-multi' && matrix.suite == '' && matrix.extra == ''
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
@@ -329,6 +337,40 @@ jobs:
               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
             done; exit \$FAIL"

+      # The same suite as the nat-single cell, run against three nodes: there it
+      # proves routed NAT works at all, here that each node delivers its own
+      # guests' public IPs and no others — which one node cannot show.
+      - name: Run NAT Uplink Multi E2E
+        if: matrix.extra == 'nat-multi'
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+
+          NODES_CSV=""
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
+          done
+
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
+             echo '::group::suite natuplink'; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+               SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
+               SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./natuplink.test -test.v -test.timeout=30m \
+               2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
+             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
+             if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
+
      # The dedicated suite cells only. Resolved once here rather than in each
      # step below, which would repeat the same three lines five times.
      - name: Resolve VM SSH details
@@ -690,6 +732,66 @@ jobs:
            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"

+      # Whether br-mgmt is a per-host island or one L2 segment spanning the
+      # cluster decides whether a system instance can reach the gateway address
+      # it was baked with. Nothing in a failing run records which one was built,
+      # and the environment is destroyed before it can be asked.
+      - name: Collect management-fabric topology
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          OUT="${ARTIFACT_DIR}/mgmt-fabric.txt"
+          INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")
+
+          declare -a WAN=() MGMT=()
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            WAN[$i]=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+          done
+
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            idx=$((i + 1))
+            {
+              echo "=== node${idx} (${WAN[$i]}) ==="
+              ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+                'sudo ovs-vsctl list-ports br-mgmt 2>&1 | sed "s/^/ports: /"; \
+                 ip -4 addr show br-mgmt 2>&1 | sed "s/^/addr:  /"; \
+                 sudo ovs-vsctl get Bridge br-mgmt fail_mode 2>&1 | sed "s/^/fail:  /"'
+              echo
+            } >> "$OUT" 2>&1
+            MGMT[$i]=$(ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+              "ip -4 -o addr show br-mgmt 2>/dev/null | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null)
+          done
+
+          # An island cluster reaches only its own address and that is correct;
+          # a spanned one must reach every peer. Either result is a finding, and
+          # a mixed result is the failure this matrix exists to catch.
+          {
+            echo "=== br-mgmt reachability (row = prober, column = target) ==="
+            printf '%-10s' "from\\to"
+            for i in $(seq 0 $((NODE_COUNT - 1))); do printf '%-18s' "node$((i+1)):${MGMT[$i]:-none}"; done
+            echo
+            for i in $(seq 0 $((NODE_COUNT - 1))); do
+              printf '%-10s' "node$((i+1))"
+              for j in $(seq 0 $((NODE_COUNT - 1))); do
+                if [ -z "${MGMT[$j]}" ]; then printf '%-18s' "-"; continue; fi
+                if ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+                     "ping -c1 -W1 ${MGMT[$j]} >/dev/null 2>&1"; then
+                  printf '%-18s' ok
+                else
+                  printf '%-18s' FAIL
+                fi
+              done
+              echo
+            done
+          } >> "$OUT" 2>&1
+
+          cat "$OUT"
+
      - name: Install go-junit-report
        if: always()
        run: |
@@ -775,12 +877,11 @@ jobs:

  # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
  # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
-  # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
-  # (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
+  # matrix cell ("cell 20"): blank cells is the whole nightly, or name 20 to run
+  # it alone. A reusable call needs its own ref, since it cannot read GIT_BRANCH.
  baremetal:
    name: cell-20 (baremetal/pxe/real-hw)
    if: >-
-      github.event_name == 'schedule' ||
      inputs.cells == '' ||
      contains(format(',{0},', inputs.cells), ',20,')
    permissions:
@@ -788,7 +889,7 @@ jobs:
      statuses: write
    uses: ./.github/workflows/e2e-baremetal.yml
    with:
-      ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+      ref: ${{ github.ref_name }}
      # A reusable-workflow call gets none of this workflow's context (env:
      # above, matrix.cell) for free — cell-20 needs its own attrs string.
      # There's no matrix leg here, just the one fixed baremetal cell.
@@ -808,7 +909,6 @@ jobs:
    needs: baremetal
    if: >-
      !cancelled() && (
-        github.event_name == 'schedule' ||
        inputs.cells == '' ||
        contains(format(',{0},', inputs.cells), ',28,')
      )
@@ -817,7 +917,7 @@ jobs:
      statuses: write
    uses: ./.github/workflows/e2e-baremetal.yml
    with:
-      ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+      ref: ${{ github.ref_name }}
      cell: "28"
      suites: diskperf
      otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=28,ci.matrix=baremetal/pxe/diskperf
@@ -834,7 +934,6 @@ jobs:
  iso-single:
    name: cell-21 (iso/dhcp/single)
    if: >-
-      github.event_name == 'schedule' ||
      inputs.cells == '' ||
      contains(format(',{0},', inputs.cells), ',21,')
    runs-on: [self-hosted, ci-single]
@@ -1089,7 +1188,6 @@ jobs:
    needs: [iso-single]
    if: >-
      !cancelled() && (
-      github.event_name == 'schedule' ||
      inputs.cells == '' ||
      contains(format(',{0},', inputs.cells), ',22,') )
    runs-on: [self-hosted, ci-multi]
@@ -1251,7 +1349,6 @@ jobs:
    needs: [matrix, baremetal, iso-single, iso-multi]
    if: >-
      !cancelled() && (
-      github.event_name == 'schedule' ||
      inputs.cells == '' ||
      contains(format(',{0},', inputs.cells), ',23,') )
    runs-on: [self-hosted, ci-single]
@@ -1282,7 +1379,7 @@ jobs:
      - name: Resolve sibling refs
        id: siblings
        env:
-          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+          BRANCH: ${{ github.ref_name }}
        run: bash .github/resolve-sibling-refs.sh
        working-directory: spinifex

@@ -1363,7 +1460,6 @@ jobs:
    needs: [matrix, baremetal, iso-single, iso-multi, performance]
    if: >-
      !cancelled() && (
-      github.event_name == 'schedule' ||
      inputs.cells == '' ||
      contains(format(',{0},', inputs.cells), ',24,') )
    runs-on: [self-hosted, ci-single]
@@ -1407,7 +1503,7 @@ jobs:
      - name: Resolve sibling refs
        id: siblings
        env:
-          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+          BRANCH: ${{ github.ref_name }}
        run: bash .github/resolve-sibling-refs.sh
        working-directory: spinifex

@@ -1446,8 +1542,16 @@ jobs:

      # Leftovers from a run killed before its cleanup trap: a surviving node
      # holds its ports and fails this run somewhere unrelated.
+      #
+      # Under the host benchmark lock, because this pattern matches every
+      # predastore node on the runner and the predastore repository runs the
+      # same harnesses on the same machine. Holding the lock is what makes
+      # "still running" mean "stale" rather than "somebody else's benchmark":
+      # on 2026-09-16 this step killed a predastore run 32s into its workload.
      - name: Clear stale nodes
-        run: pkill -KILL -f '/bin/s3d -config' || true
+        run: |
+          flock -w 3600 "${PREDA_BENCH_LOCK:-/var/lock/predastore-bench.lock}" \
+            pkill -KILL -f '/bin/s3d -config' || true

      - name: Run fault injection
        working-directory: predastore
@@ -1467,10 +1571,14 @@ jobs:
          if-no-files-found: warn
          retention-days: 90

+      # Under the lock for the same reason as "Clear stale nodes": by the time
+      # this runs the harness has released it, so another repository's benchmark
+      # can already have a cluster of its own up.
      - name: Clean up
        if: always()
        run: |
-          pkill -KILL -f '/bin/s3d -config' || true
+          flock -w 3600 "${PREDA_BENCH_LOCK:-/var/lock/predastore-bench.lock}" \
+            pkill -KILL -f '/bin/s3d -config' || true
          rm -rf "$STRESS_RESULTS_ROOT"

  # Reading a nightly meant opening every cell job, and the Actions UI does not
@@ -1478,7 +1586,17 @@ jobs:
  # every cell so this runs last; always() so a red run is the one it describes.
  summary:
    name: run summary
-    needs: [select, matrix, baremetal, diskperf, iso-single, iso-multi, performance, stress]
+    needs:
+      [
+        select,
+        matrix,
+        baremetal,
+        diskperf,
+        iso-single,
+        iso-multi,
+        performance,
+        stress,
+      ]
    if: always()
    runs-on: ubuntu-26.04
    steps:
@@ -1488,7 +1606,7 @@ jobs:
          sparse-checkout-cone-mode: false

      - name: Collect junit
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
        with:
          pattern: junit-cell-*-${{ github.run_id }}
          path: junit
@@ -43,26 +43,22 @@ on:
        required: false
        default: ""

-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
+# Shared with e2e-nightly.yml and e2e-ddil.yml: every one of them provisions
+# many cells on the same runner pools and the same hypervisors, so two in
 # flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
+# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
+# is outside the group: one cluster per run is cheap enough to overlap.
 concurrency:
  group: e2e-hypervisors
  cancel-in-progress: false

 permissions:
  contents: read
-  # actions: read lets the otel-job-id action look up a provision job's numeric
-  # id via the Actions API — github.job is only the YAML key, no context
-  # exposes it.
  actions: read

 env:
  GIT_BRANCH: ${{ github.ref_name }}
  TEST_RUN: ${{ inputs.test_run || '' }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

@@ -81,12 +77,6 @@ jobs:
      matrix: ${{ steps.suites.outputs.matrix }}
    env:
      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
-      # This job (not single_suite) is where "Bootstrap via Binary Install"
-      # bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env, before the
-      # suite matrix even exists — single_suite's legs share this one
-      # already-booted VM, so daemon telemetry can't be scoped any finer
-      # than "single env" per suite leg. ci.cell=single distinguishes it
-      # from multi_provision's VMs within the same run id.
      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
    steps:
      - name: Reap Orphan Procs
@@ -24,27 +24,15 @@ on:
        required: false
        default: ""

-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
-# flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
-concurrency:
-  group: e2e-hypervisors
-  cancel-in-progress: false
-
 permissions:
  contents: read
  statuses: write
-  # actions: read lets the otel-job-id action look up a job's numeric id via
-  # the Actions API — github.job is only the YAML key, no context exposes it.
  actions: read

 env:
  GIT_BRANCH: ${{ github.ref_name }}
  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam e2e-partialblock' }}
  TEST_RUN: ${{ inputs.test_run || '' }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

@@ -56,15 +56,7 @@ concurrency:
  cancel-in-progress: false

 env:
-  # OTLP export to the central Elastic sink (wattle is the sink itself — see
-  # Part A.2). Repo variable overrides; carried for correlation even though
-  # the GPU suite's own daemon-side spans are emitted by wattle's already-
-  # configured obs-agent, not by this job.
  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
-  # ci.cell=gpu keeps this workflow's telemetry separable from the tofu-env
-  # workflows'. Nothing in this job bakes these attrs into a daemon (see the
-  # note above), so the label only takes effect once wattle's obs-agent is
-  # given them; there is deliberately no ci.job_id lookup for the same reason.
  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=gpu
  WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
  WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
@@ -78,8 +70,6 @@ jobs:
    env:
      ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
    steps:
-      # Manual-work mutex gate — see setup note 4 above. Checked BEFORE any
-      # SSH to wattle so a paused run touches the box zero times.
      - name: Manual-work mutex gate
        id: gate
        env:
@@ -122,8 +112,6 @@ jobs:
          echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
          echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"

-      # Healthcheck: confirm wattle's persistent AWS-compat control plane is
-      # actually up before spending time building/shipping the test binary.
      - name: Healthcheck — describe-regions
        if: steps.gate.outputs.paused != 'true'
        run: |
@@ -131,18 +119,6 @@ jobs:
          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
            'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'

-      # Ensure the base EC2 GPU AMI is present — the raw-EC2 GPU passthrough
-      # test is the always-on baseline, so CI self-provisions it rather than
-      # depending on a manual import. `--name` selects the catalog entry (pulls
-      # the prebuilt image from iso.mulgadc.com); `--ami-name` is REQUIRED to
-      # name the AMI ubuntu-26.04-nvidia-gpu-x86_64 — without it the import
-      # falls back to the generic ami-<distro>-<version>-<arch>, which the
-      # tests/e2e/gpu filter would never match. Idempotent — skipped once
-      # wattle's persistent loop-fs already has the named AMI (the 4.4G source
-      # qcow2 is cached in the images dir, so a re-import wouldn't re-download).
-      # `spx admin images import` writes local spinifex storage, so it needs
-      # the `spinifex` group active (sg); aws-CLI calls hit the gateway over
-      # HTTPS and don't.
      - name: Ensure base GPU AMI present (import-if-absent)
        if: steps.gate.outputs.paused != 'true'
        run: |
@@ -171,11 +147,6 @@ jobs:
          '
          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"

-      # Preflight — the base GPU AMI is REQUIRED: the setup step above just
-      # ensured it, so a miss here means the import silently failed. Hard-fail
-      # rather than run a false-green all-skipped suite. ECS/EKS node AMIs are
-      # opt-in tiers (heavier — node AMIs + cluster spin-up); their tests
-      # self-skip when absent, so only warn.
      - name: Preflight — verify GPU AMIs
        if: steps.gate.outputs.paused != 'true'
        run: |
@@ -205,10 +176,6 @@ jobs:
          '
          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"

-      # Same build command CI's other e2e workflows use to cross-compile a
-      # suite into a shippable binary (tests/e2e/Makefile e2e-build target,
-      # SUITES_BUILD scoped to just gpu). Building on the runner keeps go
-      # toolchain / module cache off wattle entirely.
      - name: Build GPU e2e test binary on runner
        if: steps.gate.outputs.paused != 'true'
        working-directory: tests/e2e
@@ -231,12 +198,6 @@ jobs:
          ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
          scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test

-      # Same invocation shape e2e.yml uses to run a pre-built suite binary
-      # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
-      # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
-      # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
-      # the advertised gateway, so the override is required here. The harness
-      # should prefer spinifex.toml's advertised bridge.
      - name: Run GPU e2e suite on wattle
        id: e2e
        if: steps.gate.outputs.paused != 'true'
@@ -287,8 +248,6 @@ jobs:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          title: "GPU E2E results"

-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
      - name: Ship test results to Elasticsearch
        if: always() && steps.gate.outputs.paused != 'true'
        continue-on-error: true
@@ -313,14 +272,6 @@ jobs:
          if-no-files-found: warn
          retention-days: 14

-      # Cleanup is NEVER a host teardown: it never runs setup-ovn.sh,
-      # dev-reset, or cluster-reset. It only reconciles EC2-API-level state —
-      # terminate any GPU-typed instance still pending/running (the Go
-      # suite's own t.Cleanup already does this on normal pass/fail exit;
-      # this is the backstop for a hard `-test.timeout` kill or a workflow
-      # cancellation, which skip deferred Go cleanups). Termination triggers
-      # gpuManager.Release() on wattle, returning the A1000 to the pool for
-      # the next queued run or manual test.
      - name: Cleanup — terminate leftover GPU instances, release GPU
        if: always() && steps.gate.outputs.paused != 'true'
        run: |
@@ -1,9 +1,5 @@
 name: Image Scripts

-# Separate from spinifex.yml on purpose: scripts/images/ and images/ are shell
-# and mkosi assets baked into system images, not Go, so they get their own
-# path-filtered gate instead of running on every Go contributor's push (see
-# ui.yml for the same pattern applied to the frontend).
 on:
  push:
    branches:
@@ -38,15 +34,6 @@ jobs:
      - name: Run image script tests + shellcheck
        run: make test-images

-  # The mkosi EKS image is what customers run, and until now nothing built it:
-  # the composition existed in mkosi-build.sh but had no caller, so a change
-  # that broke it was only found by hand. Cell-25 still boots the Alpine image,
-  # so this is the only gate the Ubuntu one has.
-  #
-  # Self-hosted because the build needs Docker and several GB of disk, matching
-  # publish-system-ami.yml. Its own concurrency group, not e2e-hypervisors: the
-  # build provisions no libvirt guest, and sharing that group would queue image
-  # builds behind whole nightlies.
  eks_mkosi_image:
    name: Build mkosi EKS image
    runs-on: [self-hosted, ci-single]
@@ -58,9 +45,6 @@ jobs:
      - name: Checkout Code
        uses: actions/checkout@v7

-      # The profile binaries (eks-token-webhook, eks-gateway-publish, ...) are
-      # built on the host, not in the builder container, so Go is needed here
-      # even though the toolchain image carries mkosi itself.
      - name: Set up Go
        uses: actions/setup-go@v7
        with:
@@ -69,7 +53,5 @@ jobs:
          # disk, and cache-save would re-upload hundreds of MB for nothing.
          cache: false

-      # stage_profile_binaries() sets GOWORK=off, so the sub-repo resolves its
-      # own go.mod and the sibling monorepo checkouts are not needed here.
      - name: Build the image
        run: make build-eks-node-mkosi-image
@@ -0,0 +1,415 @@
+name: Predastore 3-Node Performance
+
+on:
+  workflow_dispatch:
+    inputs:
+      perf_preset:
+        description: "Warp workload preset: smoke (30s) or compare (2m)"
+        required: false
+        default: compare
+        type: choice
+        options:
+          - smoke
+          - compare
+      cluster_ref:
+        description: "Git ref to build the cluster from, e.g. v1.18.0. Empty uses this branch."
+        required: false
+        default: ""
+        type: string
+
+# Shared with e2e.yml and e2e-nightly.yml: this job provisions on the same
+# hypervisors they do, so a run in flight there and here would compete for
+# disks and libvirt guests. Queued, not cancelled, same as those workflows.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  GIT_BRANCH: ${{ github.ref_name }}
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+
+jobs:
+  perf_3node:
+    name: Predastore Throughput (3-node)
+    # ci-multi is a systemd runner on the LAN, not the ephemeral pool. A pool
+    # VM sits behind the geneve overlay at MTU 1408, which would put the
+    # runner's own network path inside the thing being measured.
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 90
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-perf3node-${{ github.run_id }}.log
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=perf-3node
+    steps:
+      - name: Reap Orphan Procs
+        run: |
+          set +e
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM
+          sleep 3
+          reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
+      - name: Clean Workspace
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      # ci-multi and ci-single resolve to the same physical host, so the
+      # nvme path cell-23 uses for its throughput run is already there.
+      # Pointing TMPDIR at it keeps warp's payloads off a small /tmp tmpfs.
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-3node-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Re-enable Go toolchain auto-download
+        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
+
+      # Same-named branch when one exists, dev otherwise. Matches how the
+      # e2e path picks siblings, so this measures the same predastore
+      # commit a cross-repo change would be built and tested against.
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ env.GIT_BRANCH }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - uses: actions/checkout@v7
+        id: predastore
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # The cluster's ref is a knob and the harness's is not. Measuring an old
+      # release with a new harness is the only way its number can be set beside
+      # a current one; rebuilding both would change the ruler and the thing.
+      - name: Build Install Artifacts
+        working-directory: mulga/scripts/tofu-cluster
+        env:
+          CLUSTER_REF: ${{ inputs.cluster_ref }}
+        run: |
+          ./build-install-artifacts.sh "${CLUSTER_REF:-$GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
+
+      # The libvirt provider dials the hypervisors by bare hostname. A pool VM
+      # sits inside a VPC and cannot resolve them; a LAN runner already can, so
+      # this is self-gating on the lookup rather than on the runner's labels.
+      - name: Map hypervisor names
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          if getent hosts banksia >/dev/null; then
+            echo "hypervisor names already resolve"
+          else
+            sudo tee -a /etc/hosts < network/hosts >/dev/null
+            echo "added $(grep -c '^[0-9]' network/hosts) entries to /etc/hosts"
+          fi
+          # Pinned rather than StrictHostKeyChecking=no: these hops reach the
+          # hypervisors themselves, so an unverified key is the one place in
+          # this job where accepting anything would matter.
+          if ssh-keygen -F banksia >/dev/null 2>&1; then
+            echo "hypervisor host keys already known"
+          else
+            install -d -m 700 ~/.ssh
+            cat network/known_hosts >> ~/.ssh/known_hosts
+            chmod 600 ~/.ssh/known_hosts
+            echo "pinned $(grep -c '^[a-z]' network/known_hosts) host keys"
+          fi
+
+      - name: Provision VMs
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
+          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
+          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ok "3-node cluster provisioned"
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
+          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Resolve cluster SSH details
+        id: cluster
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
+          REGION=$(echo "$INVENTORY" | jq -r '.region')
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          NODES_CSV=""
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
+          done
+          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
+          echo "ssh_user=${SSH_USER}" >> "$GITHUB_OUTPUT"
+          echo "region=${REGION}" >> "$GITHUB_OUTPUT"
+          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
+
+      # PERF_EXTERNAL_HOSTS wants host:port, and the gate listens on 8443 on
+      # every node. run-info.txt from this run is then comparable with the
+      # nightly's loopback one, since the harness itself never changes.
+      - name: Resolve PERF_EXTERNAL_HOSTS
+        id: perf_hosts
+        run: |
+          set -euo pipefail
+          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
+          HOSTS=""
+          for ip in "${NODE_LIST[@]}"; do
+            [ -z "$ip" ] && continue
+            if [ -z "$HOSTS" ]; then HOSTS="$ip:8443"; else HOSTS="$HOSTS,$ip:8443"; fi
+          done
+          echo "hosts=${HOSTS}" >> "$GITHUB_OUTPUT"
+
+      # A slow bootstrap should fail here, legibly, rather than inside warp
+      # with a wall of connection-refused errors. Bounded at 5 minutes per
+      # gate so a genuinely dead node still fails the job in finite time.
+      - name: Wait for gates to answer on :8443
+        run: |
+          set -euo pipefail
+          IFS=',' read -ra GATES <<< "${{ steps.perf_hosts.outputs.hosts }}"
+          for gate in "${GATES[@]}"; do
+            echo "waiting for https://$gate"
+            up=0
+            for i in $(seq 1 60); do
+              if curl -k -sS --max-time 5 -o /dev/null "https://$gate/"; then
+                up=1
+                break
+              fi
+              sleep 5
+            done
+            [ "$up" -eq 1 ] || { echo "::error::gate $gate never answered HTTPS on :8443"; exit 1; }
+            echo "gate $gate is up"
+          done
+
+      # Warp reads whole objects, so no workload here touches a ranged read and
+      # the throughput figures stay healthy while ranged GETs are broken. This
+      # probe is the only thing in CI that looks at them.
+      - name: Ranged GET probe
+        env:
+          GATE: ${{ steps.cluster.outputs.node1_ip }}
+          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
+          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
+          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
+          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
+        run: |
+          set -euo pipefail
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
+          AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          echo "::add-mask::$SK"
+          B="ranged-probe-${GITHUB_RUN_ID}"
+          AWSX=(--region "$CLUSTER_REGION" --endpoint-url "https://${GATE}:8443" --no-verify-ssl)
+          export AWS_ACCESS_KEY_ID="$AK" AWS_SECRET_ACCESS_KEY="$SK"
+          aws s3 mb "s3://$B" "${AWSX[@]}"
+          trap 'aws s3 rb "s3://$B" --force "${AWSX[@]}" >/dev/null 2>&1 || true' EXIT
+          # Two sizes: read amplification makes a boundary-crossing range cost
+          # the whole object, so the fault shows as time tracking object size.
+          for mb in 64 256; do
+            head -c $((mb*1024*1024)) /dev/urandom > "${TMPDIR}/probe-${mb}.bin"
+            aws s3 cp "${TMPDIR}/probe-${mb}.bin" "s3://$B/probe-${mb}.bin" --only-show-errors "${AWSX[@]}"
+            rm -f "${TMPDIR}/probe-${mb}.bin"
+          done
+          t() {
+            curl -sk --aws-sigv4 "aws:amz:${CLUSTER_REGION}:s3" -u "$AK:$SK" \
+              -H "Range: bytes=0-$1" -o /dev/null -w '%{time_total}' \
+              --url "https://${GATE}:8443/${B}/probe-${2}.bin"
+          }
+          # 4194304 is the last byte inside the first 4 MiB block; one more
+          # crosses into the second and leaves the single-shard fast path.
+          IN64=$(t 4194303 64);  OUT64=$(t 4194304 64)
+          IN256=$(t 4194303 256); OUT256=$(t 4194304 256)
+          printf '%-10s in-block=%ss  crossing=%ss\n' "64MiB" "$IN64" "$OUT64"
+          printf '%-10s in-block=%ss  crossing=%ss\n' "256MiB" "$IN256" "$OUT256"
+          # The 4x object-size ratio is the signal, and it is compared against
+          # the crossing reads alone so a slow runner cancels out of both sides.
+          awk -v a="$OUT64" -v b="$OUT256" 'BEGIN{
+            r = (a > 0) ? b/a : 0
+            printf "crossing-range cost ratio 256MiB/64MiB = %.2f (object size ratio is 4.00)\n", r
+            if (r > 2.0) {
+              print "::error::a boundary-crossing ranged GET scales with object size, so it is reading the whole object"
+              exit 1
+            }
+          }'
+          echo "ranged GET does not scale with object size"
+
+      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
+      # free-space watermark, so this asserts the same headroom cell-23 does.
+      # tmpfs is rejected outright: that would measure RAM, not the gates.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          case "$fstype" in
+            tmpfs|ramfs)
+              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
+              exit 1 ;;
+          esac
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
+            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
+            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      - name: Install warp
+        working-directory: predastore
+        run: make warp-install
+
+      # PERF_EXTERNAL_HOSTS skips profile selection, port offsetting, and
+      # cluster start/stop, and measures these three gates directly. Same
+      # workloads and analysis as the nightly loopback run, so the numbers
+      # from the two are directly comparable.
+      #
+      # The cluster's credentials are read and used inside this one step, the
+      # way e2e-ddil.yml does it: a step-level env: block is echoed verbatim
+      # into the run log, and a shell variable is not.
+      - name: Run 3-node throughput benchmark
+        working-directory: predastore
+        env:
+          PERF_EXTERNAL_HOSTS: ${{ steps.perf_hosts.outputs.hosts }}
+          PERF_CONFIGS: spinifex-3node
+          PERF_PRESET: ${{ inputs.perf_preset || 'compare' }}
+          # What is deployed, not what the harness was checked out at. With a
+          # cluster_ref those differ, and run-info.txt records the cluster.
+          PERF_EXTERNAL_SHA: ${{ inputs.cluster_ref || steps.predastore.outputs.commit }}
+          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
+          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
+          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
+          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
+        run: |
+          set -euo pipefail
+          # spx admin init mints a random pair at bootstrap and writes it to
+          # node1's ~/.aws/credentials. The benchmark's built-in key is the one
+          # the loopback configs trust, which this cluster has never heard of.
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
+          PERF_ACCESS_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          PERF_SECRET_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          if [ -z "$PERF_ACCESS_KEY" ] || [ -z "$PERF_SECRET_KEY" ]; then
+            echo "::error::could not read spinifex AWS credentials from ${NODE1_IP}"
+            exit 1
+          fi
+          echo "::add-mask::$PERF_SECRET_KEY"
+          mkdir -p "$PERF_RESULTS_ROOT"
+          PERF_ACCESS_KEY="$PERF_ACCESS_KEY" \
+          PERF_SECRET_KEY="$PERF_SECRET_KEY" \
+          PERF_REGION="$CLUSTER_REGION" \
+            make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"
+
+      # Report-only, matching cell-23: the baseline comparison and any
+      # regression gate are future work once run-to-run spread is known from
+      # real bare-metal runs.
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-perf-3node-${{ github.run_id }}
+          path: ${{ env.PERF_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up results
+        if: always()
+        run: rm -rf "${PERF_RESULTS_ROOT}"
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM
+          sleep 2
+          reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
+          ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-perf3node-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
@@ -60,7 +60,7 @@ concurrency:
 jobs:
  matrix:
    name: Resolve image matrix
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
    outputs:
      images: ${{ steps.gen.outputs.images }}
    steps:
@@ -84,7 +84,7 @@ jobs:
  publish:
    name: Build + publish ${{ matrix.name }}
    needs: matrix
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
    strategy:
      fail-fast: false
      # ci-single is one runner and each GPU build needs ~16G of scratch disk;
@@ -131,9 +131,7 @@ jobs:
        with:
          go-version-file: "spinifex/go.mod"
          # Self-hosted runner: the Go module + build cache already persists on
-          # local disk between runs. setup-go's cache-save ("Post Setup Go")
-          # otherwise re-uploads hundreds of MB to the Actions cache backend,
-          # taking longer than the build itself.
+          # local disk between runs.
          cache: false

      - name: Setup Go Workspace
@@ -160,7 +158,6 @@ jobs:
      - name: Build + publish ${{ matrix.name }} to R2
        working-directory: spinifex
        env:
-          # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
@@ -56,7 +56,6 @@ linters:
    - unconvert # removes unnecessary type conversions
    - usestdlibvars # detects hardcoded values that should use stdlib constants
    - usetesting # catches os.TempDir() etc. in tests — should use t.TempDir()
-    - wastedassign # finds assignments to variables that are never subsequently read
    - whitespace # detects leading and trailing whitespace in functions

  settings:
@@ -151,7 +150,7 @@ linters:
        - -ST1020 # godoc on exported types — not enforcing yet
        - -ST1021 # godoc on exported type groups — not enforcing yet
        - -ST1022 # godoc on exported vars/consts — not enforcing yet
-        - -SA4006 # value overwritten — ineffassign + wastedassign cover this better
+        - -SA4006 # value overwritten — ineffassign catches strictly more of these
        - -SA9005 # marshaling structs with no exported fields — intended {} for empty API-response types

    testifylint:
@@ -0,0 +1,96 @@
+# How to Contribute
+
+We welcome contributions to Spinifex!
+
+Report security vulnerabilities privately through [SECURITY.md](SECURITY.md), not as a public issue.
+
+**AWS parity is the design rule.** When Spinifex's behaviour is in question, the answer is what AWS does. If you believe Spinifex diverges from AWS, state what AWS does and show it with a test or a reproduction against a running cluster.
+
+## Understanding the Codebase
+
+Every AWS call enters through a SigV4-authenticated gateway, is published to NATS, and is answered by the daemon that claims it. Start with these:
+
+| Doc | Covers |
+|---|---|
+| [`docs/DESIGN.md`](docs/DESIGN.md) | Architecture, request flow, daemons, storage integration, file layout. |
+| [`docs/COMMANDS.md`](docs/COMMANDS.md) | The `spx admin` CLI and the systemd units it manages. |
+| [`docs/coverage/`](docs/coverage/README.md) | Which AWS API operations each service implements. |
+| [`scripts/README.md`](scripts/README.md) | Install, dev-environment, and image-build scripts. |
+
+User-facing docs are published at [docs.mulgadc.com](https://docs.mulgadc.com).
+
+## Development Setup
+
+These steps build Spinifex from source and run it as a single-node cluster, following the [Source Install](https://docs.mulgadc.com/docs/install-source) guide.
+
+### Prerequisites
+
+- **Ubuntu 26.04** or **Debian 13**, on x86_64 or aarch64
+- A host with hardware virtualisation (KVM) and `sudo` access
+- The host's WAN interface enslaved to a Linux bridge named `br-wan`, with the host IP, default route and DHCP on the bridge rather than the bare NIC (see [VPC Networking → Bridge Setup](https://docs.mulgadc.com/docs/vpc-networking#bridge-setup-physical-network-wiring))
+
+Check the bridge before continuing:
+
+```bash
+ip -br link show br-wan   # bridge exists and is UP
+ip route                  # default route's dev is br-wan
+```
+
+### Install and Run
+
+```bash
+mkdir -p ~/Development/mulga && cd ~/Development/mulga
+git clone https://github.com/mulgadc/spinifex.git
+sudo make -C spinifex quickinstall        # system packages, Go 1.27, AWS CLI v2
+export PATH=$PATH:/usr/local/go/bin
+
+cd spinifex
+./scripts/clone-deps.sh                   # sibling repos: viperblock, predastore, northstar, bluebottle
+./scripts/dev-install.sh                  # build, install, init a one-node cluster, start services
+
+AWS_PROFILE=spinifex aws ec2 describe-instance-types
+```
+
+A list of instance types means the cluster is up.
+
+`make preflight` also needs `shellcheck` and `golangci-lint` (CI pins v2.13.0):
+
+```bash
+sudo apt-get install -y shellcheck
+curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v2.13.0
+```
+
+The web console additionally needs Node.js 24 and `pnpm`.
+
+## Development Workflow
+
+| Command | Use it to |
+|---|---|
+| `make deploy` | Rebuild, install `spx`, and restart `spinifex.target` after a code change. |
+| `make reinstall` | Re-run `dev-install.sh` after changing systemd units, helper scripts, or logrotate config. |
+| `./scripts/reset-dev-env.sh` | Wipe all cluster state and reinstall from scratch, keeping your network topology. |
+| `make test` | Run unit tests (`make test-race` for the race detector). |
+| `make test-integration` | Run the real gateway router against embedded NATS JetStream. |
+| `make preflight` | Run the pre-commit gate: lint, `govulncheck`, coverage, and manifest and script checks. Must pass. |
+
+Follow the services with `journalctl -u 'spinifex-*' -f`.
+
+The web console lives in `spinifex/services/spinifexui/frontend/` (`pnpm install`, `pnpm dev`, `pnpm lint`, `pnpm test`). `spx` embeds its checked-in `dist/`, so run `make build-ui` and commit the rebuilt `dist/` with your change.
+
+## Creating a Pull Request
+
+1. Fork [mulgadc/spinifex](https://github.com/mulgadc/spinifex) and clone your fork into the same parent directory as the sibling repositories, branching off `dev`:
+
+   ```bash
+   cd ~/Development/mulga
+   git clone git@github.com:YOUR_USERNAME/spinifex.git && cd spinifex
+   git remote add upstream git@github.com:mulgadc/spinifex.git
+   git fetch upstream && git checkout -b feat/my-feature upstream/dev
+   ```
+
+2. Make your change, then verify it:
+   - `make preflight` passes.
+   - `go mod tidy` leaves only intended changes in `go.mod` and `go.sum`, with none of the `clone-deps.sh` `replace` lines.
+   - Runtime changes are exercised on your dev cluster with `make deploy`.
+
+3. Commit using [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `refactor:`, `docs:`, `chore:`), push, and open a pull request against `dev`, filling in the template.
@@ -149,21 +149,38 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share

 # the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
 # integration and race tests skipped to keep quick, they run in CI.
+#
+# -j2 overlaps the only two heavy gates, lint and test-cover: alone each leaves
+# most of the machine idle, so pairing them cuts a cold run by about a quarter.
+# -Otarget keeps each gate's output in one block instead of interleaving them.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
+	@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
 	@echo -e "\n ✅ Preflight passed — safe to commit."

 # Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
 # ship on every node (unlike scripts/images/, kept in preflight: a wrong
 # restart decision here is an outage, not asset churn).
+#
+# NODE_SCRIPTS are the four that setup.sh installs into
+# /usr/local/share/spinifex on every node. They are covered here for the same
+# reason: between them they install the software, wire OVN, erase the state and
+# remove the install, so a shellcheck warning in one is an outage or a wrong
+# deletion, not a style note.
+NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/node-reset.sh scripts/uninstall-spx.sh
+
 test-build-scripts:
 	@echo -e "\n....Running build/scripts/**/*_test.sh...."
 	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
 		echo "-- $$t"; \
 		sh "$$t" || exit 1; \
 	done
-	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh...."
-	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort)
+	@echo -e "\n....Running scripts/*_test.sh...."
+	@for t in $$(find scripts -maxdepth 1 -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		sh "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
+	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
 	@echo "  test-build-scripts ok"

 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
@@ -181,24 +198,25 @@ test-harness:
 # JetStream, with only the daemon-side NATS subjects stubbed.
 AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
 AWS_MODEL_CONFORMANCE_MODE ?= fail
-AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
+AWS_MODEL_OPERATION_COVERAGE_DIR ?= $(CURDIR)/docs/coverage
 # -count=1 is load-bearing: the conformance report is written by the test binary,
 # so a cached pass skips the run, leaves no report, and the cat below fails the
 # target. It also keeps the conformance gate honest — a cached result would mean
-# the check never ran against this commit.
+# the check never ran against this commit. The generated request sweep adds
+# about 15s, most of it EC2 fan-outs waiting out their window for instances
+# no daemon-lite owns.
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
-	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 120s ./tests/integration/... $(_RACEQ)
 	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
 	@$(MAKE) --no-print-directory generate-aws-model-coverage
-	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
+	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_DIR)"

 generate-aws-model-coverage:
-	@mkdir -p $(dir $(AWS_MODEL_OPERATION_COVERAGE_REPORT))
-	@go run ./cmd/aws-model-coverage > $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+	@go run ./cmd/aws-model-coverage -out $(AWS_MODEL_OPERATION_COVERAGE_DIR) -json $(AWS_MODEL_OPERATION_COVERAGE_DIR)/coverage.json

 aws-model-coverage: generate-aws-model-coverage
-	@cat $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+	@go run ./cmd/aws-model-coverage

 # Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
 # checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
@@ -6,6 +6,7 @@
  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
  <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
  <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
+  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
 </p>

 <p align="center">
@@ -79,7 +80,7 @@ Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all wo
 | **ECR** | Container Registry | Available |
 | **ECS** | Container Service | Available |
 | **RDS** | Databases | Available |
-| **Bedrock** | AI Deployment | Q3 2026 |
+| **Bedrock** | AI Deployment | Q4 2026 |

 Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).

@@ -1,9 +1,11 @@
 package main

 import (
+	"flag"
 	"fmt"
 	"os"

+	"github.com/mulgadc/predastore/s3api"
 	"github.com/mulgadc/spinifex/internal/awsmodel"
 	"github.com/mulgadc/spinifex/spinifex/gateway"

@@ -11,6 +13,36 @@ import (
 )

 func main() {
+	outputDir := flag.String("out", "", "write one publishable page per service into this directory")
+	jsonOutput := flag.String("json", "", "write a machine-readable coverage inventory to this file")
+	flag.Parse()
+
+	coverages, err := compareAll()
+	if err != nil {
+		fail(err)
+	}
+	if *outputDir == "" && *jsonOutput == "" {
+		fmt.Print(awsmodel.RenderCoverageSummary(coverages))
+		return
+	}
+	if *outputDir != "" {
+		if err := awsmodel.WritePages(*outputDir, coverages); err != nil {
+			fail(err)
+		}
+	}
+	if *jsonOutput != "" {
+		if err := awsmodel.WriteCoverageJSON(*jsonOutput, coverages); err != nil {
+			fail(err)
+		}
+	}
+}
+
+func fail(err error) {
+	fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
+	os.Exit(1)
+}
+
+func compareAll() ([]awsmodel.OperationCoverage, error) {
 	dispatch := gateway.AWSOperationInventory()
 	coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
 	for _, service := range awsmodel.Services() {
@@ -23,23 +55,19 @@ func main() {
 				Stubbed:     inventory.Stubbed,
 				Unsupported: inventory.Unsupported,
 			}
+		// Predastore serves the S3 REST surface, and names the operations each
+		// of its routes answers, so the comparison is the same one.
 		case service == awsmodel.S3:
-			modelInventory = awsmodel.DispatchInventory{
-				Opaque: true,
-				Note:   "Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.",
-			}
+			modelInventory = awsmodel.DispatchInventory{Registered: s3api.Operations()}
 		default:
-			fmt.Fprintf(os.Stderr, "aws-model-coverage: no gateway inventory for %s\n", service)
-			os.Exit(1)
+			return nil, fmt.Errorf("no gateway inventory for %s", service)
 		}

 		coverage, err := awsmodel.CompareOperations(service, modelInventory)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
-			os.Exit(1)
+			return nil, err
 		}
 		coverages = append(coverages, coverage)
 	}
-
-	fmt.Print(awsmodel.RenderCoverageMarkdown(coverages))
+	return coverages, nil
 }
@@ -30,10 +30,13 @@ type Agent struct {
 	cfg      config
 	id       identity
 	cp       controlPlane
-	puller   ctrruntime.ImagePuller
-	runner   ctrruntime.Runner
 	resolver ctrruntime.Resolver

+	// rt holds the container runtime, which is resolved by a loop rather than
+	// once at boot. Empty until the containerd socket answers, and emptied again
+	// if it stops answering.
+	rt *runtimeGate
+
 	reg *registrar
 	hb  *heartbeater

@@ -48,26 +51,34 @@ type Agent struct {
 	// nvidia-smi discovery.
 	gpu *gpuLedger

+	// dialRuntime connects to the container runtime. nil in unit tests, which
+	// drive the gate directly.
+	dialRuntime func() (ctrruntime.Runtime, error)
+
 	closers []func() error
 }

 // newAgent assembles an Agent from already-built seams. Tests use this directly
-// with fakes; New builds the production seams and delegates here. runner may be
-// nil when containerd is unavailable; the assign path then reports the task
-// STOPPED rather than crashing.
-func newAgent(cfg config, id identity, cp controlPlane, puller ctrruntime.ImagePuller, runner ctrruntime.Runner, resolver ctrruntime.Resolver) *Agent {
-	return &Agent{
+// with fakes; New builds the production seams and delegates here. rt may be nil
+// when containerd is unavailable; the assign path then reports the task STOPPED
+// rather than crashing, and the watch loop keeps dialling for it.
+func newAgent(cfg config, id identity, cp controlPlane, rt ctrruntime.Runtime, resolver ctrruntime.Resolver) *Agent {
+	gate := newRuntimeGate(rt)
+	a := &Agent{
 		cfg:      cfg,
 		id:       id,
 		cp:       cp,
-		puller:   puller,
-		runner:   runner,
 		resolver: resolver,
-		reg:      newRegistrar(cp, id),
-		hb:       newHeartbeater(cp, id, cfg.Heartbeat),
+		rt:       gate,
 		netns:    newTaskNetns(execNetRunner{}),
 		gpu:      newGPULedger(id.Capacity.GPUIDs),
 	}
+	// Registration reports the host's capacity only while a runtime is there to
+	// use it, so an agent that cannot run a container advertises none and takes
+	// no placement.
+	a.reg = newRegistrar(cp, id, gate.available)
+	a.hb = newHeartbeater(cp, id, cfg.Heartbeat, gate.available)
+	return a
 }

 // New builds an Agent from config: it resolves the host identity from IMDS,
@@ -89,6 +100,7 @@ func New(cfg config) (*Agent, error) {
 		ClusterName:  cfg.ClusterName,
 		InstanceID:   meta.InstanceID,
 		AZ:           meta.AZ,
+		InstanceType: meta.InstanceType,
 		Hostname:     host,
 		Capacity:     detectCapacity(discoverNvidiaGPUs(execCommandRunner)),
 		AgentVersion: version,
@@ -97,13 +109,12 @@ func New(cfg config) (*Agent, error) {
 	creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
 	resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)

-	var puller ctrruntime.ImagePuller
-	var runner ctrruntime.Runner
-	if rt, perr := ctrruntime.New(cfg.ContainerdSocket); perr != nil {
-		slog.Warn("ecs-agent: containerd unavailable at boot, image pulls disabled", "err", perr)
-	} else {
-		puller = rt
-		runner = rt
+	// A dial that fails here is not fatal and not final: the node starts the
+	// container daemon and the agent back to back, so the socket is often not
+	// accepting yet. Run's watch loop keeps dialling.
+	rt, perr := ctrruntime.New(cfg.ContainerdSocket)
+	if perr != nil {
+		slog.Warn("ecs-agent: containerd unavailable at boot, will keep retrying", "err", perr)
 	}

 	cp, err := newGatewayControlPlane(cfg, creds)
@@ -111,12 +122,10 @@ func New(cfg config) (*Agent, error) {
 		return nil, fmt.Errorf("build gateway control-plane: %w", err)
 	}

-	a := newAgent(cfg, id, cp, puller, runner, resolver)
+	a := newAgent(cfg, id, cp, rt, resolver)
 	a.cred = newCredEndpoint(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA,
 		cfg.CredEndpointIP, cfg.CredEndpointPort, execNetRunner{})
-	if puller != nil {
-		a.closers = append(a.closers, puller.Close)
-	}
+	a.dialRuntime = func() (ctrruntime.Runtime, error) { return ctrruntime.New(cfg.ContainerdSocket) }
 	return a, nil
 }

@@ -137,6 +146,12 @@ func (a *Agent) Run(ctx context.Context) error {

 	go a.hb.Run(ctx)

+	// Keeps the runtime current for the life of the process: dials while there
+	// is none, and drops one that stops answering.
+	if a.dialRuntime != nil {
+		go a.rt.watch(ctx, a.dialRuntime, a.cfg.RuntimeRetry)
+	}
+
 	// Re-adopt containers still running from before this restart, then poll with
 	// their tasks pre-seeded so re-delivered assignments are acked, not re-run.
 	go a.pollAssignments(ctx, a.reconcile(ctx))
@@ -156,6 +171,7 @@ func (a *Agent) Stop() error {
 			firstErr = err
 		}
 	}
+	a.rt.clear()
 	return firstErr
 }

@@ -12,7 +12,7 @@ func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
 	cp := &fakeCP{}
 	cfg := config{Heartbeat: 5 * time.Millisecond, PollInterval: 5 * time.Millisecond}
 	puller := &ctrruntime.FakePuller{}
-	a := newAgent(cfg, testIdentity(), cp, puller, puller, nil)
+	a := newAgent(cfg, testIdentity(), cp, puller, nil)
 	a.closers = append(a.closers, puller.Close)

 	ctx, cancel := context.WithCancel(context.Background())
@@ -33,6 +33,7 @@ type config struct {
 	ContainerdSocket string
 	Heartbeat        time.Duration
 	PollInterval     time.Duration
+	RuntimeRetry     time.Duration
 	CredEndpointIP   string
 	CredEndpointPort int
 }
@@ -75,6 +76,12 @@ func loadConfig(envFile string) config {
 			cfg.PollInterval = d
 		}
 	}
+	cfg.RuntimeRetry = defaultRuntimeRetry
+	if v := get("ECS_RUNTIME_RETRY"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.RuntimeRetry = d
+		}
+	}
 	cfg.CredEndpointIP = get("ECS_CRED_ENDPOINT_IP")
 	if cfg.CredEndpointIP == "" {
 		cfg.CredEndpointIP = defaultCredEndpointIP
@@ -76,6 +76,10 @@ func (g *gatewayControlPlane) Register(id identity) error {
 			{Name: aws.String("MEMORY"), Type: aws.String("INTEGER"), IntegerValue: aws.Int64(int64(id.Capacity.MemoryMiB))},
 		},
 		VersionInfo: &ecs.VersionInfo{AgentVersion: aws.String(id.AgentVersion)},
+		// The built-in container-instance attributes AWS's own agent reports.
+		// The scheduler sizes the host's awsvpc ENI capacity from the instance
+		// type, so a registration without one is not ENI-gated.
+		Attributes: instanceAttributes(id),
 	}
 	if len(id.Capacity.GPUIDs) > 0 {
 		in.TotalResources = append(in.TotalResources, &ecs.Resource{
@@ -14,18 +14,23 @@ type heartbeater struct {
 	cp       controlPlane
 	id       identity
 	interval time.Duration
+	// runtimeReady reports whether a container runtime is resolved, so a beat
+	// advertises capacity only while there is something to run it with. nil
+	// means always ready.
+	runtimeReady func() bool
 }

-func newHeartbeater(cp controlPlane, id identity, interval time.Duration) *heartbeater {
+func newHeartbeater(cp controlPlane, id identity, interval time.Duration, runtimeReady func() bool) *heartbeater {
 	if interval <= 0 {
 		interval = defaultHeartbeat
 	}
-	return &heartbeater{cp: cp, id: id, interval: interval}
+	return &heartbeater{cp: cp, id: id, interval: interval, runtimeReady: runtimeReady}
 }

 // beat re-registers the instance once, refreshing its LastSeen.
 func (h *heartbeater) beat() error {
-	return h.cp.Register(h.id)
+	ready := h.runtimeReady == nil || h.runtimeReady()
+	return h.cp.Register(advertisedIdentity(h.id, ready))
 }

 // Run beats every interval until ctx is cancelled. A failed beat is logged and
@@ -13,9 +13,10 @@ import (

 // instanceMetadata is the IMDS-derived identity of the host the agent runs on.
 type instanceMetadata struct {
-	AccountID  string
-	InstanceID string
-	AZ         string
+	AccountID    string
+	InstanceID   string
+	AZ           string
+	InstanceType string
 }

 // fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2
@@ -39,10 +40,14 @@ func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata,
 		return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
 	}

+	// The instance type comes off the identity document already in hand rather
+	// than a further meta-data read; the scheduler sizes the host's awsvpc ENI
+	// capacity from it.
 	return instanceMetadata{
-		AccountID:  doc.AccountID,
-		InstanceID: strings.TrimSpace(instanceID),
-		AZ:         strings.TrimSpace(az),
+		AccountID:    doc.AccountID,
+		InstanceID:   strings.TrimSpace(instanceID),
+		AZ:           strings.TrimSpace(az),
+		InstanceType: strings.TrimSpace(doc.InstanceType),
 	}, nil
 }

@@ -24,10 +24,11 @@ import (
 // agent falls back to its pre-reconcile behaviour (poll + run).
 func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 	adopted := map[string]bool{}
-	if a.runner == nil {
+	runner := a.rt.get()
+	if runner == nil {
 		return adopted
 	}
-	containers, err := a.runner.List(ctx)
+	containers, err := runner.List(ctx)
 	if err != nil {
 		slog.Warn("ecs-agent: reconcile list failed", "err", err)
 		return adopted
@@ -96,9 +97,13 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 // is STOPPED, so the scheduler stops counting a task this instance is not
 // running and its service can place a replacement.
 func (a *Agent) reportStoppedTask(ctx context.Context, as *bus.Assign, dead []ctrruntime.Container) {
+	runner := a.rt.get()
 	statuses := make([]bus.ContainerStatus, 0, len(dead))
 	for _, c := range dead {
-		if err := a.runner.Remove(ctx, c.ID); err != nil {
+		if runner == nil {
+			break
+		}
+		if err := runner.Remove(ctx, c.ID); err != nil {
 			slog.Warn("ecs-agent: reconcile remove failed", "task", as.TaskID, "container", c.ID, "err", err)
 		}
 		statuses = append(statuses, bus.ContainerStatus{
@@ -55,7 +55,7 @@ func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
 			adoptedContainer("t-001", "web", "default", "t-001", testTaskRole, "52:54:00:de:ad:01", true),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, nil)
 	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)

 	adopted := a.reconcile(context.Background())
@@ -84,7 +84,7 @@ func TestReconcile_ReportsTaskWhoseContainerDied(t *testing.T) {
 			adoptedContainer("t-dead", "web", "default", "t-dead", testTaskRole, "52:54:00:de:ad:02", false),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, nil)
 	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)

 	adopted := a.reconcile(context.Background())
@@ -116,7 +116,7 @@ func TestReconcile_PartiallyDeadTaskStaysRunning(t *testing.T) {
 			adoptedContainer("t-mixed", "sidecar", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", false),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, nil)
 	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)

 	a.reconcile(context.Background())
@@ -145,7 +145,7 @@ func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
 		nil,
 	}}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)

 	ctx, cancel := context.WithCancel(context.Background())
 	go a.pollAssignments(ctx, map[string]bool{"t-001": true})
@@ -197,7 +197,7 @@ func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
 			adoptedContainer("t-ok", "web", "default", "", "", "", true),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, nil)

 	adopted := a.reconcile(context.Background())

@@ -213,13 +213,13 @@ func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {

 // A nil runner or a List error degrades to an empty set without panicking.
 func TestReconcile_DegradesGracefully(t *testing.T) {
-	noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil, nil)
+	noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil)
 	if got := noRunner.reconcile(context.Background()); len(got) != 0 {
 		t.Errorf("nil runner: adopted = %v, want empty", got)
 	}

 	rt := &ctrruntime.FakePuller{ListErr: errors.New("boom")}
-	listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+	listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, nil)
 	if got := listErr.reconcile(context.Background()); len(got) != 0 {
 		t.Errorf("list error: adopted = %v, want empty", got)
 	}
@@ -3,6 +3,9 @@ package main
 import (
 	"fmt"

+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ecs"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )

@@ -14,25 +17,61 @@ type identity struct {
 	ClusterName  string
 	InstanceID   string
 	AZ           string
+	InstanceType string
 	Hostname     string
 	Capacity     bus.InstanceCapacity
 	AgentVersion string
 }

+// instanceAttributes builds the built-in container-instance attributes from the
+// host's identity. Only attributes whose value was actually discovered are
+// reported, so an empty one is absent rather than present and blank.
+func instanceAttributes(id identity) []*ecs.Attribute {
+	var attrs []*ecs.Attribute
+	add := func(name, value string) {
+		if value == "" {
+			return
+		}
+		attrs = append(attrs, &ecs.Attribute{Name: aws.String(name), Value: aws.String(value)})
+	}
+	add(handlers_ecs.AttrInstanceType, id.InstanceType)
+	add(handlers_ecs.AttrAvailabilityZone, id.AZ)
+	return attrs
+}
+
+// advertisedIdentity is the identity to register with, given whether a
+// container runtime is available. A host with no runtime can run nothing, so it
+// advertises no capacity: the scheduler places nothing on it, and the control
+// plane reads the zero as an involuntary drain. The identity is otherwise
+// unchanged, so the instance stays visible rather than disappearing.
+func advertisedIdentity(id identity, runtimeReady bool) identity {
+	if runtimeReady {
+		return id
+	}
+	id.Capacity = bus.InstanceCapacity{}
+	return id
+}
+
 // registrar registers the host as a container instance through the gateway.
 type registrar struct {
 	cp controlPlane
 	id identity
+	// runtimeReady reports whether a container runtime is resolved. nil means
+	// always ready, which is what a test with no gate wants.
+	runtimeReady func() bool
 }

-func newRegistrar(cp controlPlane, id identity) *registrar {
-	return &registrar{cp: cp, id: id}
+func newRegistrar(cp controlPlane, id identity, runtimeReady func() bool) *registrar {
+	return &registrar{cp: cp, id: id, runtimeReady: runtimeReady}
 }

+// ready reports the runtime state, defaulting to ready when no gate is wired.
+func (r *registrar) ready() bool { return r.runtimeReady == nil || r.runtimeReady() }
+
 // Register calls the gateway's RegisterContainerInstance. The scheduler records
 // (or refreshes) the container instance on receipt.
 func (r *registrar) Register() error {
-	if err := r.cp.Register(r.id); err != nil {
+	if err := r.cp.Register(advertisedIdentity(r.id, r.ready())); err != nil {
 		return fmt.Errorf("register instance %s: %w", r.id.InstanceID, err)
 	}
 	return nil
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"slices"
 	"sync"
 	"testing"
 	"time"
@@ -17,14 +18,16 @@ import (
 type fakeCP struct {
 	mu sync.Mutex

-	registers    int
-	states       []bus.TaskState
-	gpuReports   []gpuReport
-	pollAcks     [][]string
-	pollStopAcks [][]string
-	pollReplies  [][]bus.Assign
-	stopReplies  [][]bus.StopDirective
-	pollCalls    int
+	registers      int
+	lastRegistered identity
+	states         []bus.TaskState
+	gpuReports     []gpuReport
+	pollAcks       [][]string
+	pollStopAcks   [][]string
+	pollReplies    [][]bus.Assign
+	stopReplies    [][]bus.StopDirective
+	stopPending    []bus.StopDirective
+	pollCalls      int

 	registerErr bool
 	submitErr   bool
@@ -39,16 +42,24 @@ type gpuReport struct {

 var _ controlPlane = (*fakeCP)(nil)

-func (f *fakeCP) Register(_ identity) error {
+func (f *fakeCP) Register(id identity) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	if f.registerErr {
 		return errors.New("register boom")
 	}
 	f.registers++
+	f.lastRegistered = id
 	return nil
 }

+// registered returns the identity of the most recent Register call.
+func (f *fakeCP) registered() identity {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.lastRegistered
+}
+
 func (f *fakeCP) SubmitTaskState(st bus.TaskState) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -72,6 +83,12 @@ func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]
 	if f.pollCalls < len(f.stopReplies) {
 		stops = f.stopReplies[f.pollCalls]
 	}
+	// stopPending is the durable inbox the gateway keeps: an acked directive is
+	// deleted, and everything still in it is redelivered on every poll.
+	f.stopPending = slices.DeleteFunc(f.stopPending, func(sd bus.StopDirective) bool {
+		return slices.Contains(ackStops, sd.TaskID)
+	})
+	stops = slices.Concat(stops, f.stopPending)
 	f.pollCalls++
 	return out, stops, nil
 }
@@ -142,7 +159,7 @@ func testIdentityWithGPUs(uuids ...string) identity {

 func TestRegistrar_Register(t *testing.T) {
 	cp := &fakeCP{}
-	if err := newRegistrar(cp, testIdentity()).Register(); err != nil {
+	if err := newRegistrar(cp, testIdentity(), nil).Register(); err != nil {
 		t.Fatalf("Register: %v", err)
 	}
 	if cp.registerCount() != 1 {
@@ -152,14 +169,14 @@ func TestRegistrar_Register(t *testing.T) {

 func TestRegistrar_RegisterError(t *testing.T) {
 	cp := &fakeCP{registerErr: true}
-	if err := newRegistrar(cp, testIdentity()).Register(); err == nil {
+	if err := newRegistrar(cp, testIdentity(), nil).Register(); err == nil {
 		t.Fatal("expected register error")
 	}
 }

 func TestHeartbeater_Beat(t *testing.T) {
 	cp := &fakeCP{}
-	if err := newHeartbeater(cp, testIdentity(), time.Second).beat(); err != nil {
+	if err := newHeartbeater(cp, testIdentity(), time.Second, nil).beat(); err != nil {
 		t.Fatalf("beat: %v", err)
 	}
 	if cp.registerCount() != 1 {
@@ -168,7 +185,7 @@ func TestHeartbeater_Beat(t *testing.T) {
 }

 func TestHeartbeater_DefaultInterval(t *testing.T) {
-	h := newHeartbeater(&fakeCP{}, testIdentity(), 0)
+	h := newHeartbeater(&fakeCP{}, testIdentity(), 0, nil)
 	if h.interval != defaultHeartbeat {
 		t.Errorf("interval = %v, want default", h.interval)
 	}
@@ -176,7 +193,7 @@ func TestHeartbeater_DefaultInterval(t *testing.T) {

 func TestHeartbeater_RunStopsOnContext(t *testing.T) {
 	cp := &fakeCP{}
-	h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond)
+	h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond, nil)
 	ctx, cancel := context.WithCancel(context.Background())

 	done := make(chan struct{})
@@ -193,3 +210,48 @@ func TestHeartbeater_RunStopsOnContext(t *testing.T) {
 		t.Error("expected at least one re-register before cancel")
 	}
 }
+
+// TestRegistrar_NoRuntimeAdvertisesNoCapacity is the placement half of the boot
+// bug: an agent with no container runtime kept registering its full CPU and
+// memory, so the scheduler placed tasks it could only fail to start. Capacity is
+// advertised only while a runtime is there to use it.
+func TestRegistrar_NoRuntimeAdvertisesNoCapacity(t *testing.T) {
+	cp := &fakeCP{}
+	ready := false
+	r := newRegistrar(cp, testIdentity(), func() bool { return ready })
+
+	if err := r.Register(); err != nil {
+		t.Fatalf("register: %v", err)
+	}
+	if got := cp.registered().Capacity; got.CPU != 0 || got.MemoryMiB != 0 {
+		t.Fatalf("advertised capacity with no runtime: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
+	}
+
+	ready = true
+	if err := r.Register(); err != nil {
+		t.Fatalf("register: %v", err)
+	}
+	want := testIdentity().Capacity
+	if got := cp.registered().Capacity; got.CPU != want.CPU || got.MemoryMiB != want.MemoryMiB {
+		t.Fatalf("capacity not restored once the runtime resolved: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
+	}
+}
+
+// TestHeartbeater_NoRuntimeAdvertisesNoCapacity is the same rule on the beat,
+// which is how a runtime lost after boot reaches the control plane.
+func TestHeartbeater_NoRuntimeAdvertisesNoCapacity(t *testing.T) {
+	cp := &fakeCP{}
+	h := newHeartbeater(cp, testIdentity(), time.Second, func() bool { return false })
+
+	if err := h.beat(); err != nil {
+		t.Fatalf("beat: %v", err)
+	}
+	if got := cp.registered().Capacity; got.CPU != 0 || got.MemoryMiB != 0 {
+		t.Fatalf("beat advertised capacity with no runtime: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
+	}
+	// The instance still identifies itself, so it stays visible as drained rather
+	// than vanishing from the cluster.
+	if cp.registered().InstanceID != testIdentity().InstanceID {
+		t.Error("beat dropped the instance identity")
+	}
+}
@@ -3,6 +3,7 @@ package runtime
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"

 	containerd "github.com/containerd/containerd/v2/client"
@@ -22,6 +23,11 @@ const servingProbeTimeout = 3 * time.Second
 // containerdPuller drives a real containerd daemon over its unix socket.
 type containerdPuller struct {
 	client *containerd.Client
+
+	// stdinDone holds one channel per interactive container, closed when that
+	// container is stopped or removed so its stdin reader returns EOF.
+	mu        sync.Mutex
+	stdinDone map[string]chan struct{}
 }

 var _ ImagePuller = (*containerdPuller)(nil)
@@ -3,9 +3,14 @@ package runtime
 import (
 	"context"
 	"fmt"
+	"io"
 	"sort"
+	"strings"
+	"syscall"
+	"time"

 	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/core/containers"
 	"github.com/containerd/containerd/v2/pkg/cdi"
 	"github.com/containerd/containerd/v2/pkg/cio"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
@@ -41,6 +46,106 @@ func cdiSpecOpts(gpuIDs []string) []oci.SpecOpts {
 	return []oci.SpecOpts{cdi.WithCDIDevices(devices...)}
 }

+// ociCapName renders an ECS Linux capability (Docker's bare name, e.g.
+// "SYS_ADMIN") as the OCI runtime-spec name ("CAP_SYS_ADMIN") capabilities.Add/
+// Drop expect.
+func ociCapName(name string) string {
+	if strings.HasPrefix(name, "CAP_") {
+		return name
+	}
+	return "CAP_" + name
+}
+
+// ociCapNames maps a list of ECS capability names to their OCI form.
+func ociCapNames(names []string) []string {
+	if len(names) == 0 {
+		return nil
+	}
+	out := make([]string, len(names))
+	for i, n := range names {
+		out[i] = ociCapName(n)
+	}
+	return out
+}
+
+// withSysctls sets Linux.Sysctl on the OCI spec from systemControls namespace/
+// value pairs. containerd has no built-in SpecOpts for sysctls.
+func withSysctls(ctls []SystemControl) oci.SpecOpts {
+	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
+		if len(ctls) == 0 {
+			return nil
+		}
+		if s.Linux == nil {
+			s.Linux = &specs.Linux{}
+		}
+		if s.Linux.Sysctl == nil {
+			s.Linux.Sysctl = map[string]string{}
+		}
+		for _, c := range ctls {
+			s.Linux.Sysctl[c.Namespace] = c.Value
+		}
+		return nil
+	}
+}
+
+// heldStdin backs an interactive container's stdin: reads block instead of
+// hitting an immediate EOF, which is the "stdin held open" behaviour
+// interactive=true asks for, until done closes and the read reports EOF.
+type heldStdin struct{ done <-chan struct{} }
+
+func (h heldStdin) Read([]byte) (int, error) {
+	<-h.done
+	return 0, io.EOF
+}
+
+// ioStreamOpts describes the task IO for a container. A terminal container needs
+// a terminal FIFO set: oci.WithTTY alone puts Terminal in the OCI spec while the
+// shim creates no console socket, and runc refuses that pair outright. A
+// terminal carries no separate stderr, so that stream stays nil and the shim
+// merges it into the console.
+func ioStreamOpts(stdin io.Reader, terminal bool) []cio.Opt {
+	if terminal {
+		return []cio.Opt{cio.WithStreams(stdin, io.Discard, nil), cio.WithTerminal}
+	}
+	return []cio.Opt{cio.WithStreams(stdin, io.Discard, io.Discard)}
+}
+
+// containerIOCreator returns cio.NullIO for a container needing neither stdin
+// nor a terminal, else a creator carrying whichever it asked for. An interactive
+// container's stdin is held open until releaseStdin; containerd copies stdin on
+// a goroutine that exits only once the read returns, so a reader that never
+// returned would park it for the life of the agent. A terminal container that is
+// not interactive gets no stdin stream at all, which is what a nil reader means.
+func (p *containerdPuller) containerIOCreator(containerID string, interactive, terminal bool) cio.Creator {
+	if !interactive && !terminal {
+		return cio.NullIO
+	}
+	var stdin io.Reader
+	if interactive {
+		done := make(chan struct{})
+		p.mu.Lock()
+		if p.stdinDone == nil {
+			p.stdinDone = make(map[string]chan struct{})
+		}
+		p.stdinDone[containerID] = done
+		p.mu.Unlock()
+		stdin = heldStdin{done: done}
+	}
+	return cio.NewCreator(ioStreamOpts(stdin, terminal)...)
+}
+
+// releaseStdin closes a held stdin so containerd's copier goroutine exits. Safe
+// for a container that never held one, and safe to call more than once.
+func (p *containerdPuller) releaseStdin(containerID string) {
+	p.mu.Lock()
+	done, ok := p.stdinDone[containerID]
+	delete(p.stdinDone, containerID)
+	p.mu.Unlock()
+	if ok {
+		close(done)
+	}
+}
+
 var _ Runner = (*containerdPuller)(nil)

 // Run creates a container from an already-pulled image and starts its task. The
@@ -81,6 +186,26 @@ func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (st
 		specOpts = append(specOpts, oci.WithEnv(envSlice(spec.Env)))
 	}
 	specOpts = append(specOpts, cdiSpecOpts(spec.GPUIDs)...)
+	if spec.User != "" {
+		specOpts = append(specOpts, oci.WithUser(spec.User))
+	}
+	if spec.ReadonlyRootFilesystem != nil && *spec.ReadonlyRootFilesystem {
+		specOpts = append(specOpts, oci.WithRootFSReadonly())
+	}
+	if spec.Privileged != nil && *spec.Privileged {
+		specOpts = append(specOpts, oci.WithPrivileged)
+	}
+	terminal := spec.PseudoTerminal != nil && *spec.PseudoTerminal
+	if terminal {
+		specOpts = append(specOpts, oci.WithTTY)
+	}
+	specOpts = append(specOpts, withSysctls(spec.SystemControls))
+	if len(spec.CapAdd) > 0 {
+		specOpts = append(specOpts, oci.WithAddedCapabilities(ociCapNames(spec.CapAdd)))
+	}
+	if len(spec.CapDrop) > 0 {
+		specOpts = append(specOpts, oci.WithDroppedCapabilities(ociCapNames(spec.CapDrop)))
+	}

 	container, err := p.client.NewContainer(ctx, id,
 		containerd.WithNewSnapshot(id+"-snapshot", image),
@@ -91,12 +216,15 @@ func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (st
 		return "", fmt.Errorf("create container %s: %w", id, err)
 	}

-	task, err := container.NewTask(ctx, cio.NullIO)
+	interactive := spec.Interactive != nil && *spec.Interactive
+	task, err := container.NewTask(ctx, p.containerIOCreator(id, interactive, terminal))
 	if err != nil {
+		p.releaseStdin(id)
 		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
 		return "", fmt.Errorf("create task %s: %w", id, err)
 	}
 	if err := task.Start(ctx); err != nil {
+		p.releaseStdin(id)
 		_, _ = task.Delete(ctx)
 		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
 		return "", fmt.Errorf("start task %s: %w", id, err)
@@ -148,9 +276,35 @@ func (p *containerdPuller) List(ctx context.Context) ([]Container, error) {
 	return out, nil
 }

+// Stop enforces a container's stopTimeout: SIGTERM the task, wait up to
+// timeout for it to exit on its own, then fall through to Remove's forced
+// kill+delete regardless of whether it exited in time. A container with no
+// live task (already gone) or that fails to accept the signal goes straight to
+// Remove, matching Remove's own already-gone tolerance.
+func (p *containerdPuller) Stop(ctx context.Context, containerID string, timeout time.Duration) error {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	container, err := p.client.LoadContainer(ctx, containerID)
+	if err != nil {
+		return nil //nolint:nilerr // load failure means the container is already gone
+	}
+	task, terr := container.Task(ctx, nil)
+	if terr != nil {
+		return p.Remove(ctx, containerID)
+	}
+	statusC, werr := task.Wait(ctx)
+	if werr == nil && task.Kill(ctx, syscall.SIGTERM) == nil {
+		select {
+		case <-statusC:
+		case <-time.After(timeout):
+		}
+	}
+	return p.Remove(ctx, containerID)
+}
+
 // Remove kills and deletes the container's task, then the container + snapshot.
 func (p *containerdPuller) Remove(ctx context.Context, containerID string) error {
 	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	p.releaseStdin(containerID)
 	container, err := p.client.LoadContainer(ctx, containerID)
 	if err != nil {
 		return nil //nolint:nilerr // load failure means the container is already gone
@@ -1,8 +1,15 @@
 package runtime

 import (
+	"context"
+	"errors"
+	"io"
 	"reflect"
 	"testing"
+	"time"
+
+	"github.com/containerd/containerd/v2/pkg/cio"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 )

 // cdiDeviceNames maps a container's pinned GPU UUIDs to CDI device names
@@ -45,3 +52,166 @@ func TestCDISpecOpts_Wiring(t *testing.T) {
 		t.Error("GPU container: spec opt is nil")
 	}
 }
+
+// TestOCICapNames verifies ECS's bare capability names (Docker convention,
+// e.g. "SYS_PTRACE") are prefixed to the OCI runtime-spec form the process
+// capability sets expect, and a name already carrying the prefix passes
+// through unchanged.
+func TestOCICapNames(t *testing.T) {
+	cases := []struct {
+		name string
+		in   []string
+		want []string
+	}{
+		{name: "empty", in: nil, want: nil},
+		{name: "bare", in: []string{"SYS_PTRACE"}, want: []string{"CAP_SYS_PTRACE"}},
+		{name: "already prefixed", in: []string{"CAP_NET_ADMIN"}, want: []string{"CAP_NET_ADMIN"}},
+		{name: "mixed", in: []string{"SYS_PTRACE", "CAP_NET_ADMIN"}, want: []string{"CAP_SYS_PTRACE", "CAP_NET_ADMIN"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := ociCapNames(tc.in)
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("ociCapNames(%v) = %v, want %v", tc.in, got, tc.want)
+			}
+		})
+	}
+}
+
+// TestWithSysctls verifies systemControls namespace/value pairs land on
+// Linux.Sysctl, and an empty list leaves the spec's Linux section untouched
+// rather than allocating an empty map (the "requests nothing" case).
+func TestWithSysctls(t *testing.T) {
+	s := &specs.Spec{}
+	if err := withSysctls(nil)(context.Background(), nil, nil, s); err != nil {
+		t.Fatalf("empty ctls: %v", err)
+	}
+	if s.Linux != nil {
+		t.Errorf("empty ctls: want no Linux section, got %+v", s.Linux)
+	}
+
+	ctls := []SystemControl{{Namespace: "net.core.somaxconn", Value: "1024"}}
+	if err := withSysctls(ctls)(context.Background(), nil, nil, s); err != nil {
+		t.Fatalf("with ctls: %v", err)
+	}
+	if s.Linux == nil || s.Linux.Sysctl["net.core.somaxconn"] != "1024" {
+		t.Errorf("want sysctl net.core.somaxconn=1024, got %+v", s.Linux)
+	}
+}
+
+// An interactive container's stdin read blocks until the container is released,
+// then reports EOF so containerd's copier goroutine exits instead of parking
+// for the life of the agent.
+func TestHeldStdin_ReadsEOFOnceReleased(t *testing.T) {
+	p := &containerdPuller{}
+	p.containerIOCreator("c1", true, false)
+
+	p.mu.Lock()
+	done := p.stdinDone["c1"]
+	p.mu.Unlock()
+	if done == nil {
+		t.Fatal("interactive container registered no stdin channel")
+	}
+
+	read := make(chan error, 1)
+	go func() {
+		_, err := heldStdin{done: done}.Read(make([]byte, 1))
+		read <- err
+	}()
+
+	select {
+	case err := <-read:
+		t.Fatalf("read returned before release: %v", err)
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	p.releaseStdin("c1")
+
+	select {
+	case err := <-read:
+		if !errors.Is(err, io.EOF) {
+			t.Fatalf("want io.EOF after release, got %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("read did not return after release")
+	}
+}
+
+// A non-interactive container uses cio.NullIO and registers nothing to release.
+func TestContainerIOCreator_NonInteractiveRegistersNothing(t *testing.T) {
+	p := &containerdPuller{}
+	p.containerIOCreator("c1", false, false)
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if len(p.stdinDone) != 0 {
+		t.Fatalf("want no registered stdin channels, got %d", len(p.stdinDone))
+	}
+}
+
+// applyIOOpts resolves the cio options onto a Streams the way cio.NewCreator
+// does, which is the only way to observe what the creator was built with.
+func applyIOOpts(opts []cio.Opt) cio.Streams {
+	var s cio.Streams
+	for _, o := range opts {
+		o(&s)
+	}
+	return s
+}
+
+// A terminal container needs the terminal flag set on its FIFO set, else the
+// shim creates no console socket and runc refuses to start the container at all.
+// The terminal carries no separate stderr stream.
+func TestIOStreamOpts_TerminalRequestsAConsole(t *testing.T) {
+	s := applyIOOpts(ioStreamOpts(nil, true))
+	if !s.Terminal {
+		t.Error("terminal container: want Terminal set")
+	}
+	if s.Stderr != nil {
+		t.Errorf("terminal container: want no stderr stream, got %#v", s.Stderr)
+	}
+	if s.Stdout == nil {
+		t.Error("terminal container: want a stdout stream")
+	}
+}
+
+// A container with stdin but no terminal keeps both output streams and asks for
+// no console.
+func TestIOStreamOpts_NonTerminalKeepsStderr(t *testing.T) {
+	s := applyIOOpts(ioStreamOpts(heldStdin{}, false))
+	if s.Terminal {
+		t.Error("non-terminal container: want Terminal unset")
+	}
+	if s.Stderr == nil {
+		t.Error("non-terminal container: want a stderr stream")
+	}
+	if s.Stdin == nil {
+		t.Error("non-terminal interactive container: want a stdin stream")
+	}
+}
+
+// A terminal container that is not interactive still gets a creator rather than
+// cio.NullIO, and registers no stdin to release.
+func TestContainerIOCreator_TerminalWithoutInteractive(t *testing.T) {
+	p := &containerdPuller{}
+	if c := p.containerIOCreator("c1", false, true); c == nil {
+		t.Fatal("terminal container: want a creator")
+	}
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if len(p.stdinDone) != 0 {
+		t.Fatalf("want no registered stdin channels, got %d", len(p.stdinDone))
+	}
+}
+
+// Remove calls releaseStdin unconditionally and Stop falls through to Remove,
+// so an unknown container and a second release must both be no-ops.
+func TestReleaseStdin_UnknownAndRepeatedAreSafe(t *testing.T) {
+	p := &containerdPuller{}
+	p.releaseStdin("never-registered")
+
+	p.containerIOCreator("c1", true, false)
+	p.releaseStdin("c1")
+	p.releaseStdin("c1")
+}
@@ -4,7 +4,10 @@
 // scheduler-only build run against a fake.
 package runtime

-import "context"
+import (
+	"context"
+	"time"
+)

 // PullSpec describes one image to pull.
 type PullSpec struct {
@@ -35,6 +38,12 @@ type ImagePuller interface {
 	Close() error
 }

+// SystemControl is a sysctl namespace/value pair applied to the OCI spec.
+type SystemControl struct {
+	Namespace string
+	Value     string
+}
+
 // RunSpec describes a single container the agent must create and start. v1 runs
 // the container in the host network namespace (bridge/CNI task networking lands
 // in a later sprint); Labels carry the mulga.ecs.* task identity for the reboot
@@ -54,6 +63,22 @@ type RunSpec struct {
 	// per GPU requested. Empty for non-GPU containers or if pinning fell short.
 	// The runner injects these as CDI devices into the OCI spec.
 	GPUIDs []string
+	// User is the OCI process user (oci.WithUser). Empty runs as the image
+	// default.
+	User string
+	// ReadonlyRootFilesystem, Privileged, PseudoTerminal and Interactive are
+	// pointers so the runner can tell "not requested" (nil) from an explicit
+	// false, which still needs echoing but applies nothing.
+	ReadonlyRootFilesystem *bool
+	Privileged             *bool
+	PseudoTerminal         *bool
+	Interactive            *bool
+	// SystemControls are sysctls set on the OCI spec's Linux.Sysctl map.
+	SystemControls []SystemControl
+	// CapAdd / CapDrop are Linux capabilities added/dropped from the OCI
+	// spec's process capability sets (linuxParameters.capabilities).
+	CapAdd  []string
+	CapDrop []string
 }

 // RunStatus is a finished container's outcome.
@@ -71,12 +96,16 @@ type Container struct {
 }

 // Runner creates and starts containers from already-pulled images. id is a
-// caller-unique container ID; Wait blocks until the container exits; Remove
-// tears down the container + its task; List enumerates known containers so the
-// reboot reconciler can re-adopt the ones still running.
+// caller-unique container ID; Wait blocks until the container exits; Stop
+// requests a graceful exit (SIGTERM, then a forced kill+remove once timeout
+// elapses) — the agent's enforcement of a container's stopTimeout; Remove
+// tears down the container + its task immediately, with no grace period; List
+// enumerates known containers so the reboot reconciler can re-adopt the ones
+// still running.
 type Runner interface {
 	Run(ctx context.Context, id string, spec RunSpec) (containerID string, err error)
 	Wait(ctx context.Context, containerID string) (RunStatus, error)
+	Stop(ctx context.Context, containerID string, timeout time.Duration) error
 	Remove(ctx context.Context, containerID string) error
 	List(ctx context.Context) ([]Container, error)
 }
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 )

 // FakePuller is an in-memory ImagePuller for tests and the scheduler-only build.
@@ -26,11 +27,21 @@ type FakePuller struct {
 	Removed  []string
 	OnRun    func(string, RunSpec) (string, error)

-	// List bookkeeping: Containers is replayed by List; ListErr forces a failure;
-	// Listed records that List was called.
+	// Stop bookkeeping: StopTimeouts pairs the container ID with the timeout it
+	// was stopped with, in call order, so a test can assert both the target and
+	// the enforced stopTimeout without a fake containerd.
+	Stopped      []string
+	StopTimeouts []time.Duration
+	StopErr      error
+
+	// List bookkeeping: Containers is replayed by List; ListErr forces a failure,
+	// limited to the first ListErrFor calls when that is non-zero, so a test can
+	// model a runtime that recovers. Listed and ListCalls record the calls.
 	Containers []Container
 	ListErr    error
+	ListErrFor int
 	Listed     bool
+	ListCalls  int
 }

 var (
@@ -98,6 +109,15 @@ func (f *FakePuller) Wait(_ context.Context, containerID string) (RunStatus, err
 	return RunStatus{ExitCode: f.WaitCode}, f.WaitErr
 }

+// Stop records the container ID and the timeout it was stopped with.
+func (f *FakePuller) Stop(_ context.Context, containerID string, timeout time.Duration) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Stopped = append(f.Stopped, containerID)
+	f.StopTimeouts = append(f.StopTimeouts, timeout)
+	return f.StopErr
+}
+
 // Remove records the container ID.
 func (f *FakePuller) Remove(_ context.Context, containerID string) error {
 	f.mu.Lock()
@@ -113,13 +133,36 @@ func (f *FakePuller) Waits() []string {
 	return append([]string(nil), f.Waited...)
 }

+// RunCalls returns a copy of the specs passed to Run, for assertions made while
+// the agent's goroutines may still be live.
+func (f *FakePuller) RunCalls() []RunSpec {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]RunSpec(nil), f.Runs...)
+}
+
+// Reaped returns a copy of the container IDs passed to Stop.
+func (f *FakePuller) Reaped() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.Stopped...)
+}
+
 // List replays the programmed container set (or error).
 func (f *FakePuller) List(_ context.Context) ([]Container, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	f.Listed = true
-	if f.ListErr != nil {
+	f.ListCalls++
+	if f.ListErr != nil && (f.ListErrFor == 0 || f.ListCalls <= f.ListErrFor) {
 		return nil, f.ListErr
 	}
 	return f.Containers, nil
 }
+
+// Lists reports how many times List was called.
+func (f *FakePuller) Lists() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.ListCalls
+}
@@ -0,0 +1,116 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"sync"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+)
+
+// runtimeProbeTimeout bounds the liveness probe against a runtime the gate is
+// already holding, so a wedged socket cannot stall the watch loop.
+const runtimeProbeTimeout = 5 * time.Second
+
+// defaultRuntimeRetry is how often the gate dials a runtime it does not have,
+// and probes one it does. The node starts containerd and the agent back to
+// back, so the first few dials are expected to fail.
+const defaultRuntimeRetry = 10 * time.Second
+
+// runtimeGate holds the container runtime the agent has resolved, or nothing
+// when the socket has not answered. The runtime is resolved by a loop rather
+// than once at boot: the node's service manager reports the container daemon
+// started when the process spawned, not when its socket began accepting, so a
+// dial that fails in the first second of boot is a transient condition.
+//
+// Every read of the runtime goes through the gate, so a runtime that arrives
+// after boot is picked up on the next task without restarting the agent.
+type runtimeGate struct {
+	mu sync.RWMutex
+	rt ctrruntime.Runtime
+}
+
+// newRuntimeGate returns a gate holding rt, which may be nil.
+func newRuntimeGate(rt ctrruntime.Runtime) *runtimeGate {
+	return &runtimeGate{rt: rt}
+}
+
+// get returns the current runtime, or nil when none is resolved.
+func (g *runtimeGate) get() ctrruntime.Runtime {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+	return g.rt
+}
+
+// available reports whether a runtime is resolved. It is what registration
+// consults to decide whether the host has any capacity to advertise.
+func (g *runtimeGate) available() bool { return g.get() != nil }
+
+// set replaces the held runtime, closing the one it displaces.
+func (g *runtimeGate) set(rt ctrruntime.Runtime) {
+	g.mu.Lock()
+	old := g.rt
+	g.rt = rt
+	g.mu.Unlock()
+	if old != nil && old != rt {
+		_ = old.Close()
+	}
+}
+
+// clear drops and closes the held runtime, returning whether it held one.
+func (g *runtimeGate) clear() bool {
+	g.mu.Lock()
+	old := g.rt
+	g.rt = nil
+	g.mu.Unlock()
+	if old == nil {
+		return false
+	}
+	_ = old.Close()
+	return true
+}
+
+// watch keeps the gate current until ctx is cancelled: it dials while the gate
+// is empty, and probes while it is full, clearing the gate when the probe fails
+// so the next tick re-dials. A containerd stopped and restarted underneath a
+// running agent is therefore picked up the same way a slow boot is.
+func (g *runtimeGate) watch(ctx context.Context, dial func() (ctrruntime.Runtime, error), interval time.Duration) {
+	if interval <= 0 {
+		interval = defaultRuntimeRetry
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			g.step(ctx, dial)
+		}
+	}
+}
+
+// step performs one pass of the watch loop: dial when empty, probe when full.
+func (g *runtimeGate) step(ctx context.Context, dial func() (ctrruntime.Runtime, error)) {
+	if rt := g.get(); rt != nil {
+		probeCtx, cancel := context.WithTimeout(ctx, runtimeProbeTimeout)
+		// A container list is what reconcile already asks the runtime at boot and
+		// the cheapest thing it answers, so it doubles as the liveness probe.
+		_, err := rt.List(probeCtx)
+		cancel()
+		if err != nil && ctx.Err() == nil {
+			slog.Warn("ecs-agent: container runtime stopped answering, will re-dial", "err", err)
+			g.clear()
+		}
+		return
+	}
+
+	rt, err := dial()
+	if err != nil {
+		slog.Debug("ecs-agent: container runtime still unavailable", "err", err)
+		return
+	}
+	slog.Info("ecs-agent: container runtime available")
+	g.set(rt)
+}
@@ -0,0 +1,124 @@
+//test:in-package — package main cannot be imported by an external test package.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+)
+
+// TestRuntimeGate_DialsUntilTheSocketAnswers is the boot case the agent used to
+// lose: the node starts containerd and the agent together, so the first dials
+// fail. Resolving once left the agent runtime-less for the life of the process.
+func TestRuntimeGate_DialsUntilTheSocketAnswers(t *testing.T) {
+	g := newRuntimeGate(nil)
+	if g.available() {
+		t.Fatal("empty gate reported a runtime")
+	}
+
+	rt := &ctrruntime.FakePuller{}
+	dials := 0
+	dial := func() (ctrruntime.Runtime, error) {
+		dials++
+		if dials < 3 {
+			return nil, errors.New("dial containerd: no such file or directory")
+		}
+		return rt, nil
+	}
+
+	for range 2 {
+		g.step(t.Context(), dial)
+		if g.available() {
+			t.Fatalf("gate resolved a runtime after %d failed dials", dials)
+		}
+	}
+
+	g.step(t.Context(), dial)
+	if g.get() != rt {
+		t.Fatalf("gate did not pick up the runtime once the socket answered (dials=%d)", dials)
+	}
+}
+
+// TestRuntimeGate_DropsARuntimeThatStopsAnswering covers a containerd restarted
+// underneath a live agent: the held runtime stops answering, so the gate clears
+// it and the next pass re-dials rather than handing out a dead client forever.
+func TestRuntimeGate_DropsARuntimeThatStopsAnswering(t *testing.T) {
+	dead := &ctrruntime.FakePuller{ListErr: errors.New("connection refused")}
+	fresh := &ctrruntime.FakePuller{}
+	g := newRuntimeGate(dead)
+
+	dial := func() (ctrruntime.Runtime, error) { return fresh, nil }
+
+	g.step(t.Context(), dial)
+	if g.available() {
+		t.Fatal("gate kept a runtime whose probe failed")
+	}
+	if !dead.Closed {
+		t.Error("displaced runtime was not closed")
+	}
+
+	g.step(t.Context(), dial)
+	if g.get() != fresh {
+		t.Fatal("gate did not re-dial after clearing")
+	}
+}
+
+// TestRuntimeGate_KeepsAHealthyRuntime guards the other direction: a probe that
+// succeeds must not churn the runtime, or every pass would drop a working client.
+func TestRuntimeGate_KeepsAHealthyRuntime(t *testing.T) {
+	rt := &ctrruntime.FakePuller{}
+	g := newRuntimeGate(rt)
+
+	dial := func() (ctrruntime.Runtime, error) {
+		t.Error("dialled while holding a healthy runtime")
+		return nil, errors.New("unexpected dial")
+	}
+	g.step(t.Context(), dial)
+
+	if g.get() != rt {
+		t.Fatal("gate dropped a runtime whose probe succeeded")
+	}
+	if rt.Closed {
+		t.Error("healthy runtime was closed")
+	}
+}
+
+// TestRuntimeGate_ClearIsIdempotent keeps Stop safe to call on an agent that
+// never resolved a runtime.
+func TestRuntimeGate_ClearIsIdempotent(t *testing.T) {
+	g := newRuntimeGate(nil)
+	if g.clear() {
+		t.Error("clear reported dropping a runtime the gate never held")
+	}
+
+	rt := &ctrruntime.FakePuller{}
+	g.set(rt)
+	if !g.clear() {
+		t.Error("clear did not report dropping the held runtime")
+	}
+	if !rt.Closed {
+		t.Error("cleared runtime was not closed")
+	}
+	if g.clear() {
+		t.Error("second clear reported dropping a runtime")
+	}
+}
+
+// TestRuntimeGate_WatchStopsWithContext keeps the watch loop from outliving the
+// agent it belongs to.
+func TestRuntimeGate_WatchStopsWithContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(t.Context())
+	g := newRuntimeGate(nil)
+
+	done := make(chan struct{})
+	go func() {
+		g.watch(ctx, func() (ctrruntime.Runtime, error) { return nil, errors.New("no socket") }, 0)
+		close(done)
+	}()
+
+	cancel()
+	<-done
+}
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"log/slog"
 	"maps"
+	"strconv"
+	"sync"
 	"time"

 	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
@@ -12,6 +14,10 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )

+// defaultStopTimeout is applied when a container carries no explicit
+// stopTimeout, matching the EC2 launch-type default ECS itself uses.
+const defaultStopTimeout = 30 * time.Second
+
 // pollAssignments drains the instance's assignment inbox on a ticker until ctx is
 // cancelled. Each assign is dispatched to runTask exactly once; the taskIDs seen
 // on a poll are acked on the next poll so the gateway can drop them — a crash
@@ -23,7 +29,7 @@ func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool)
 	ticker := time.NewTicker(a.cfg.PollInterval)
 	defer ticker.Stop()

-	stopping := map[string]bool{}
+	reaps := newStopLedger()
 	var ackAssigns, ackStops []string
 	for {
 		select {
@@ -40,17 +46,18 @@ func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool)
 			ackStops = ackStops[:0]
 			for i := range stops {
 				sd := stops[i]
-				if !stopping[sd.TaskID] {
-					stopping[sd.TaskID] = true
+				switch {
+				case reaps.done(sd.TaskID):
+					ackStops = append(ackStops, sd.TaskID)
+				case reaps.begin(sd.TaskID):
 					slog.Info("ecs-agent: task stop requested", "task", sd.TaskID, "reason", sd.Reason)
-					go a.stopTask(ctx, sd)
+					go func() { reaps.end(sd.TaskID, a.stopTask(ctx, sd)) }()
 				}
-				ackStops = append(ackStops, sd.TaskID)
 			}
 			ackAssigns = ackAssigns[:0]
 			for i := range assigns {
 				as := assigns[i]
-				if !dispatched[as.TaskID] && !stopping[as.TaskID] {
+				if !dispatched[as.TaskID] && !reaps.seen(as.TaskID) {
 					dispatched[as.TaskID] = true
 					slog.Info("ecs-agent: task assigned", "task", as.TaskID, "containers", len(as.Containers))
 					go a.runTask(ctx, &as)
@@ -61,27 +68,83 @@ func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool)
 	}
 }

-// stopTask reaps a task's containers on a control-plane stop directive: it lists
-// the runtime's containers, removes the ones labeled with this task (kill +
-// delete, releasing host ports/netns), then reports the task STOPPED with the
-// directive's reason. Idempotent — a task with no live containers still reports
-// STOPPED so the scheduler releases its capacity.
-func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) {
-	if a.runner == nil {
-		return
+// stopLedger tracks reaps in flight and reaps that reported STOPPED. The two
+// are separate because only the second may be acked: acking the first deletes a
+// directive that was never performed, and nothing re-posts it.
+type stopLedger struct {
+	mu       sync.Mutex
+	inflight map[string]bool
+	stopped  map[string]bool
+}
+
+func newStopLedger() *stopLedger {
+	return &stopLedger{inflight: map[string]bool{}, stopped: map[string]bool{}}
+}
+
+// begin claims a task for a reap, reporting false when one is already running.
+func (l *stopLedger) begin(taskID string) bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.inflight[taskID] {
+		return false
 	}
-	containers, err := a.runner.List(ctx)
+	l.inflight[taskID] = true
+	return true
+}
+
+// end releases the claim, recording the task as stopped only if the reap got
+// that far. A failed reap leaves nothing behind, so the next poll redelivers
+// the directive and dispatches it again.
+func (l *stopLedger) end(taskID string, reported bool) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	delete(l.inflight, taskID)
+	if reported {
+		l.stopped[taskID] = true
+	}
+}
+
+// done reports whether the task's reap reported STOPPED, which is the only
+// condition under which its directive may be acked.
+func (l *stopLedger) done(taskID string) bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.stopped[taskID]
+}
+
+// seen reports whether a stop for the task is in flight or complete, so an
+// assignment arriving alongside its own stop is never started.
+func (l *stopLedger) seen(taskID string) bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.inflight[taskID] || l.stopped[taskID]
+}
+
+// stopTask reaps a task's containers on a control-plane stop directive: it
+// lists the runtime's containers, gracefully stops (SIGTERM, wait up to the
+// container's stopTimeout, then force kill + delete) the ones labeled with
+// this task, then reports the task STOPPED with the directive's reason.
+// Idempotent — a task with no live containers still reports STOPPED so the
+// scheduler releases its capacity. Reports whether it got as far as saying
+// STOPPED: a caller may only ack the directive when it did.
+func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) bool {
+	runner := a.rt.get()
+	if runner == nil {
+		slog.Warn("ecs-agent: stop deferred, no container runtime", "task", sd.TaskID)
+		return false
+	}
+	containers, err := runner.List(ctx)
 	if err != nil {
 		slog.Warn("ecs-agent: stop list failed", "task", sd.TaskID, "err", err)
-		return
+		return false
 	}
 	statuses := make([]bus.ContainerStatus, 0)
 	for _, c := range containers {
 		if c.Labels[labelTaskID] != sd.TaskID {
 			continue
 		}
-		if rerr := a.runner.Remove(ctx, c.ID); rerr != nil {
-			slog.Warn("ecs-agent: stop remove failed", "task", sd.TaskID, "container", c.ID, "err", rerr)
+		if rerr := runner.Stop(ctx, c.ID, containerStopTimeout(c.Labels)); rerr != nil {
+			slog.Warn("ecs-agent: stop failed", "task", sd.TaskID, "container", c.ID, "err", rerr)
 		}
 		statuses = append(statuses, bus.ContainerStatus{
 			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
@@ -93,13 +156,15 @@ func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) {
 	}
 	a.reportTaskState(&bus.Assign{TaskID: sd.TaskID}, bus.TaskStatusStopped, reason, statuses)
 	slog.Info("ecs-agent: task reaped", "task", sd.TaskID, "containers", len(statuses))
+	return true
 }

 // runTask pulls each container image, starts the containers, and reports task
 // state on the bus. A missing runtime or any per-container failure reports the
 // task STOPPED with a reason; success reports RUNNING and waits for exit.
 func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
-	if a.puller == nil || a.runner == nil {
+	rt := a.rt.get()
+	if rt == nil {
 		a.reportTaskState(as, bus.TaskStatusStopped, "containerd unavailable on agent", nil)
 		return
 	}
@@ -117,7 +182,12 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {

 	statuses := make([]bus.ContainerStatus, 0, len(as.Containers))
 	for _, c := range as.Containers {
-		if _, err := a.puller.Pull(ctx, ctrruntime.PullSpec{Ref: c.Image}, resolver); err != nil {
+		// startTimeout bounds pull-through-start as a unit: real ECS measures a
+		// container's time-to-RUNNING from the same starting point.
+		runCtx, cancel := containerStartCtx(ctx, c.StartTimeout)
+
+		if _, err := rt.Pull(runCtx, ctrruntime.PullSpec{Ref: c.Image}, resolver); err != nil {
+			cancel()
 			slog.Error("ecs-agent: pull failed", "task", as.TaskID, "image", c.Image, "err", err)
 			a.teardownTaskNetns(as)
 			a.reportTaskState(as, bus.TaskStatusStopped, "image pull failed: "+err.Error(), statuses)
@@ -126,16 +196,29 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {

 		cid := containerID(as.TaskID, c.Name)
 		gpuIDs := a.pinContainerGPUs(as.TaskID, c.Name, c.GPU)
-		spec := ctrruntime.RunSpec{
-			Image:     c.Image,
-			Command:   c.Command,
-			Env:       withCredEnv(c.Environment, credID),
-			Labels:    taskLabels(as, c.Name),
-			NetnsPath: netnsPath,
-			GPU:       c.GPU,
-			GPUIDs:    gpuIDs,
+		labels := taskLabels(as, c.Name)
+		if c.StopTimeout != nil {
+			labels[labelStopTimeout] = strconv.FormatInt(*c.StopTimeout, 10)
 		}
-		id, err := a.runner.Run(ctx, cid, spec)
+		spec := ctrruntime.RunSpec{
+			Image:                  c.Image,
+			Command:                c.Command,
+			Env:                    withCredEnv(c.Environment, credID),
+			Labels:                 labels,
+			NetnsPath:              netnsPath,
+			GPU:                    c.GPU,
+			GPUIDs:                 gpuIDs,
+			User:                   c.User,
+			ReadonlyRootFilesystem: c.ReadonlyRootFilesystem,
+			Privileged:             c.Privileged,
+			PseudoTerminal:         c.PseudoTerminal,
+			Interactive:            c.Interactive,
+			SystemControls:         toRuntimeSystemControls(c.SystemControls),
+			CapAdd:                 c.CapAdd,
+			CapDrop:                c.CapDrop,
+		}
+		id, err := rt.Run(runCtx, cid, spec)
+		cancel()
 		if err != nil {
 			slog.Error("ecs-agent: run failed", "task", as.TaskID, "container", c.Name, "err", err)
 			a.teardownTaskNetns(as)
@@ -155,7 +238,12 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 // waitContainer blocks until a container exits, then reports the task STOPPED
 // with the exit code. v1 stops the whole task when its first container exits.
 func (a *Agent) waitContainer(ctx context.Context, as *bus.Assign, name, containerID string) {
-	status, err := a.runner.Wait(ctx, containerID)
+	runner := a.rt.get()
+	if runner == nil {
+		slog.Warn("ecs-agent: cannot wait on container, no container runtime", "task", as.TaskID, "container", name)
+		return
+	}
+	status, err := runner.Wait(ctx, containerID)
 	if err != nil {
 		slog.Warn("ecs-agent: wait container failed", "task", as.TaskID, "container", name, "err", err)
 		return
@@ -240,6 +328,44 @@ func withCredEnv(env map[string]string, credID string) map[string]string {
 	return out
 }

+// containerStartCtx bounds pull+run by a container's startTimeout, if set. A
+// nil startTimeout (the common case) returns ctx unchanged with a no-op
+// cancel, matching today's unbounded behaviour.
+func containerStartCtx(ctx context.Context, startTimeout *int64) (context.Context, context.CancelFunc) {
+	if startTimeout == nil {
+		return ctx, func() {}
+	}
+	return context.WithTimeout(ctx, time.Duration(*startTimeout)*time.Second)
+}
+
+// containerStopTimeout reads the stopTimeout stamped on a container's labels
+// by runTask, falling back to defaultStopTimeout when absent or unparseable.
+func containerStopTimeout(labels map[string]string) time.Duration {
+	v, ok := labels[labelStopTimeout]
+	if !ok {
+		return defaultStopTimeout
+	}
+	secs, err := strconv.ParseInt(v, 10, 64)
+	if err != nil {
+		return defaultStopTimeout
+	}
+	return time.Duration(secs) * time.Second
+}
+
+// toRuntimeSystemControls maps the assign's sysctl pairs to the runtime
+// package's own type, keeping cmd/ecs-agent/runtime decoupled from the ecs
+// handlers' bus package.
+func toRuntimeSystemControls(in []bus.SystemControl) []ctrruntime.SystemControl {
+	if len(in) == 0 {
+		return nil
+	}
+	out := make([]ctrruntime.SystemControl, len(in))
+	for i, sc := range in {
+		out[i] = ctrruntime.SystemControl{Namespace: sc.Namespace, Value: sc.Value}
+	}
+	return out
+}
+
 // pinContainerGPUs reserves n device UUIDs for a container from the local
 // ledger, before the container is started so the runner can inject them as
 // CDI devices. A short ledger (fewer free devices than requested, e.g.
@@ -317,6 +443,9 @@ const (
 	labelCredID        = "mulga.ecs.credID"
 	labelTaskRoleARN   = "mulga.ecs.taskRoleArn"
 	labelENIMac        = "mulga.ecs.eniMac"
+	// labelStopTimeout carries a container's stopTimeout (seconds) so stopTask
+	// can read it back from List's labels; it has no assign to consult there.
+	labelStopTimeout = "mulga.ecs.stopTimeout"
 )

 // taskLabels are the mulga.ecs.* labels stamped on a container. The cred/role/MAC
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"slices"
 	"testing"
 	"time"

@@ -24,13 +25,13 @@ func testAssign() *bus.Assign {
 }

 func newRunAgent(cp controlPlane, rt *ctrruntime.FakePuller) *Agent {
-	return newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	return newAgent(config{}, testIdentity(), cp, rt, nil)
 }

 // runTask with no runtime reports the task STOPPED instead of crashing.
 func TestRunTask_NoRuntimeReportsStopped(t *testing.T) {
 	cp := &fakeCP{}
-	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
+	a := newAgent(config{}, testIdentity(), cp, nil, nil)
 	a.runTask(context.Background(), testAssign())

 	st := cp.taskStates()
@@ -70,6 +71,157 @@ func TestRunTask_PullRunReportsRunning(t *testing.T) {
 	}
 }

+// Each enforced field maps onto RunSpec exactly, asserted through the fake
+// runtime — the mapping this change exists to wire, without a real containerd.
+func TestRunTask_RuntimeFieldsCarriedToRunSpec(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	ro := true
+	priv := false
+	tty := true
+	interactive := true
+	as.Containers[0].User = "1000:1000"
+	as.Containers[0].ReadonlyRootFilesystem = &ro
+	as.Containers[0].Privileged = &priv
+	as.Containers[0].PseudoTerminal = &tty
+	as.Containers[0].Interactive = &interactive
+	as.Containers[0].SystemControls = []bus.SystemControl{{Namespace: "net.core.somaxconn", Value: "1024"}}
+	as.Containers[0].CapAdd = []string{"SYS_PTRACE"}
+	as.Containers[0].CapDrop = []string{"NET_RAW"}
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 1 {
+		t.Fatalf("expected one run, got %+v", rt.Runs)
+	}
+	got := rt.Runs[0]
+	if got.User != "1000:1000" {
+		t.Errorf("User = %q, want 1000:1000", got.User)
+	}
+	if got.ReadonlyRootFilesystem == nil || !*got.ReadonlyRootFilesystem {
+		t.Errorf("ReadonlyRootFilesystem = %v, want true", got.ReadonlyRootFilesystem)
+	}
+	if got.Privileged == nil || *got.Privileged {
+		t.Errorf("Privileged = %v, want false (present, not applied)", got.Privileged)
+	}
+	if got.PseudoTerminal == nil || !*got.PseudoTerminal {
+		t.Errorf("PseudoTerminal = %v, want true", got.PseudoTerminal)
+	}
+	if got.Interactive == nil || !*got.Interactive {
+		t.Errorf("Interactive = %v, want true", got.Interactive)
+	}
+	if len(got.SystemControls) != 1 || got.SystemControls[0].Namespace != "net.core.somaxconn" {
+		t.Errorf("SystemControls = %+v, want one net.core.somaxconn entry", got.SystemControls)
+	}
+	if len(got.CapAdd) != 1 || got.CapAdd[0] != "SYS_PTRACE" {
+		t.Errorf("CapAdd = %+v, want [SYS_PTRACE]", got.CapAdd)
+	}
+	if len(got.CapDrop) != 1 || got.CapDrop[0] != "NET_RAW" {
+		t.Errorf("CapDrop = %+v, want [NET_RAW]", got.CapDrop)
+	}
+}
+
+// An assign carrying none of the new fields leaves RunSpec's pointer fields
+// nil rather than defaulting to a false indistinguishable from "explicitly
+// not requested".
+func TestRunTask_NilRuntimeFieldsLeaveRunSpecUnset(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	if len(rt.Runs) != 1 {
+		t.Fatalf("expected one run, got %+v", rt.Runs)
+	}
+	got := rt.Runs[0]
+	if got.ReadonlyRootFilesystem != nil || got.Privileged != nil || got.PseudoTerminal != nil || got.Interactive != nil {
+		t.Errorf("want all runtime-field pointers nil, got %+v", got)
+	}
+	if len(got.SystemControls) != 0 || len(got.CapAdd) != 0 || len(got.CapDrop) != 0 {
+		t.Errorf("want no sysctls/capabilities, got %+v", got)
+	}
+}
+
+// TestRunTask_StampsStopTimeoutLabel verifies runTask stamps a container's
+// stopTimeout as a label — the only way stopTask (which only has List's
+// labels, not the original assign) can recover it later.
+func TestRunTask_StampsStopTimeoutLabel(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	stop := int64(45)
+	as.Containers[0].StopTimeout = &stop
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 1 {
+		t.Fatalf("expected one run, got %+v", rt.Runs)
+	}
+	if got := rt.Runs[0].Labels[labelStopTimeout]; got != "45" {
+		t.Errorf("stopTimeout label = %q, want 45", got)
+	}
+}
+
+// TestRunTask_NoStopTimeoutLabelWhenUnset verifies a container with no
+// stopTimeout carries no label, so stopTask falls back to the default rather
+// than reading a stray "0".
+func TestRunTask_NoStopTimeoutLabelWhenUnset(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	if len(rt.Runs) != 1 {
+		t.Fatalf("expected one run, got %+v", rt.Runs)
+	}
+	if _, ok := rt.Runs[0].Labels[labelStopTimeout]; ok {
+		t.Errorf("want no stopTimeout label, got %+v", rt.Runs[0].Labels)
+	}
+}
+
+// TestStopTask_UsesContainerStopTimeout verifies that stopTask reads a
+// container's stopTimeout label and calls the runner's graceful Stop (SIGTERM
+// + bounded wait) with it, instead of the hard kill+delete used before this
+// fix.
+func TestStopTask_UsesContainerStopTimeout(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{{
+		ID: "t-001-web",
+		Labels: map[string]string{
+			labelTaskID: "t-001", labelContainerName: "web", labelStopTimeout: "45",
+		},
+	}}}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
+		t.Fatalf("want t-001-web stopped, got %+v", rt.Stopped)
+	}
+	if len(rt.StopTimeouts) != 1 || rt.StopTimeouts[0] != 45*time.Second {
+		t.Fatalf("want stopTimeout 45s, got %+v", rt.StopTimeouts)
+	}
+}
+
+// TestStopTask_DefaultStopTimeoutWhenUnset verifies a container with no
+// stopTimeout label gets the EC2 launch-type default rather than an unbounded
+// or zero grace period.
+func TestStopTask_DefaultStopTimeoutWhenUnset(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{{
+		ID:     "t-001-web",
+		Labels: map[string]string{labelTaskID: "t-001", labelContainerName: "web"},
+	}}}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(rt.StopTimeouts) != 1 || rt.StopTimeouts[0] != defaultStopTimeout {
+		t.Fatalf("want default stopTimeout %v, got %+v", defaultStopTimeout, rt.StopTimeouts)
+	}
+}
+
 // TestRunTask_GPUCarriedToRunSpec verifies that the AssignContainer's GPU count
 // reaches the runtime RunSpec unchanged; a
 // container with no GPU request keeps RunSpec.GPU at zero (regression).
@@ -100,7 +252,7 @@ func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
 func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
 	cp := &fakeCP{}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, nil)
 	as := testAssign()
 	as.Containers = append(as.Containers, bus.AssignContainer{
 		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
@@ -144,7 +296,7 @@ func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
 func TestRunTask_GPUIDsReachRunSpec(t *testing.T) {
 	cp := &fakeCP{}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, nil)
 	as := testAssign()
 	as.Containers = append(as.Containers, bus.AssignContainer{
 		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
@@ -209,7 +361,7 @@ func TestStopTask_ReleasesGPUsBackToLedger(t *testing.T) {
 			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "trainer",
 		}},
 	}}
-	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, rt, nil)
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, nil)
 	if _, err := a.gpu.Pin(gpuKey("t-001", "trainer"), 1); err != nil {
 		t.Fatalf("seed pin: %v", err)
 	}
@@ -287,7 +439,7 @@ func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
 		nil,   // poll 3: gateway dropped it after the ack
 	}}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)

 	ctx, cancel := context.WithCancel(context.Background())
 	go a.pollAssignments(ctx, map[string]bool{})
@@ -342,8 +494,8 @@ func TestStopTask_ReapsLabeledContainersAndReportsStopped(t *testing.T) {

 	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})

-	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
-		t.Fatalf("want only t-001-web removed, got %+v", rt.Removed)
+	if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
+		t.Fatalf("want only t-001-web stopped, got %+v", rt.Stopped)
 	}
 	st := cp.taskStates()
 	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
@@ -381,12 +533,12 @@ func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
 	as := testAssign()
 	cp := &fakeCP{
 		pollReplies: [][]bus.Assign{{*as}, {*as}, nil},
-		stopReplies: [][]bus.StopDirective{{{TaskID: as.TaskID, Reason: "bye"}}},
+		stopPending: []bus.StopDirective{{TaskID: as.TaskID, Reason: "bye"}},
 	}
 	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
 		{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
 	}}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)

 	ctx, cancel := context.WithCancel(context.Background())
 	go a.pollAssignments(ctx, map[string]bool{})
@@ -425,10 +577,126 @@ func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
 	}
 }

+// countStopAcks reports how many polls carried taskID in their stop-ack list.
+func countStopAcks(acks [][]string, taskID string) int {
+	n := 0
+	for _, ack := range acks {
+		if slices.Contains(ack, taskID) {
+			n++
+		}
+	}
+	return n
+}
+
+// A reap that cannot run is never acked, so the directive stays in the gateway's
+// inbox and is redelivered and retried until the runtime comes back.
+func TestPollAssignments_UnperformableStopIsNotAckedAndRetried(t *testing.T) {
+	cp := &fakeCP{stopPending: []bus.StopDirective{{TaskID: "t-001", Reason: "bye"}}}
+	rt := &ctrruntime.FakePuller{ListErr: errors.New("containerd is down")}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	deadline := time.After(time.Second)
+	for rt.Lists() < 3 {
+		select {
+		case <-deadline:
+			t.Fatalf("failed reap was not retried; List calls=%d", rt.Lists())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	cancel()
+
+	if n := countStopAcks(cp.stopAcks(), "t-001"); n != 0 {
+		t.Errorf("a reap that never ran was acked %d times; stopAcks=%v", n, cp.stopAcks())
+	}
+	if stopped := countStatus(cp.taskStates(), "t-001", bus.TaskStatusStopped); stopped != 0 {
+		t.Errorf("a reap that never ran reported STOPPED %d times", stopped)
+	}
+}
+
+// Once the runtime recovers, the redelivered directive is reaped, reported
+// STOPPED and acked exactly once, which deletes it from the inbox.
+func TestPollAssignments_RecoveredStopIsAckedExactlyOnce(t *testing.T) {
+	cp := &fakeCP{stopPending: []bus.StopDirective{{TaskID: "t-001", Reason: "bye"}}}
+	rt := &ctrruntime.FakePuller{
+		ListErr:    errors.New("containerd is down"),
+		ListErrFor: 1,
+		Containers: []ctrruntime.Container{
+			{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
+		},
+	}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	deadline := time.After(2 * time.Second)
+	for countStopAcks(cp.stopAcks(), "t-001") == 0 {
+		select {
+		case <-deadline:
+			t.Fatalf("recovered reap was never acked; states=%+v stopAcks=%v", cp.taskStates(), cp.stopAcks())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(30 * time.Millisecond)
+	cancel()
+
+	if n := countStopAcks(cp.stopAcks(), "t-001"); n != 1 {
+		t.Errorf("stop acked %d times, want exactly 1; stopAcks=%v", n, cp.stopAcks())
+	}
+	if stopped := countStatus(cp.taskStates(), "t-001", bus.TaskStatusStopped); stopped != 1 {
+		t.Errorf("task reported STOPPED %d times, want exactly 1", stopped)
+	}
+	if reaped := rt.Reaped(); len(reaped) != 1 || reaped[0] != "t-001-web" {
+		t.Errorf("want t-001-web reaped once, got %+v", reaped)
+	}
+}
+
+// A task whose reap completed is never started again by an assign the gateway
+// is still redelivering.
+func TestPollAssignments_StoppedTaskIsNotRerunByRedeliveredAssign(t *testing.T) {
+	as := testAssign()
+	cp := &fakeCP{
+		pollReplies: [][]bus.Assign{{*as}, {*as}, {*as}, {*as}, {*as}, {*as}},
+		stopPending: []bus.StopDirective{{TaskID: as.TaskID, Reason: "bye"}},
+	}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
+	}}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	deadline := time.After(time.Second)
+	for countStopAcks(cp.stopAcks(), as.TaskID) == 0 {
+		select {
+		case <-deadline:
+			t.Fatalf("stop never completed; states=%+v", cp.taskStates())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	// Let the remaining scripted assigns arrive after the reap has finished.
+	time.Sleep(40 * time.Millisecond)
+	cancel()
+
+	if runs := rt.RunCalls(); len(runs) != 0 {
+		t.Errorf("stopped task was started by a redelivered assign: %+v", runs)
+	}
+	if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 0 {
+		t.Errorf("stopped task reported RUNNING %d times", running)
+	}
+}
+
 // stopTask with no runtime is a no-op: nothing to reap, no state reported.
 func TestStopTask_NilRunnerNoOp(t *testing.T) {
 	cp := &fakeCP{}
-	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
+	a := newAgent(config{}, testIdentity(), cp, nil, nil)
 	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
 	if len(cp.taskStates()) != 0 {
 		t.Fatalf("nil runner should report no state, got %+v", cp.taskStates())
@@ -459,8 +727,8 @@ func TestStopTask_EmptyReasonReapsWithDefault(t *testing.T) {
 	a := newRunAgent(cp, rt)
 	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001"})

-	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
-		t.Fatalf("want container t-001-web reaped, got %+v", rt.Removed)
+	if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
+		t.Fatalf("want container t-001-web reaped, got %+v", rt.Stopped)
 	}
 	st := cp.taskStates()
 	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
@@ -539,7 +807,7 @@ func (sentinelResolver) Authorize(context.Context, string) (string, string, stri
 // exec-role resolver is built.
 func TestPullResolver(t *testing.T) {
 	fake := sentinelResolver{}
-	a := newAgent(config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, nil, fake)
+	a := newAgent(config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, fake)

 	if got := a.pullResolver(&bus.Assign{TaskID: "t1"}); got != fake {
 		t.Fatalf("no execution role: want instance resolver fallback")
@@ -33,12 +33,12 @@ func serverTLSConfig(cert tls.Certificate) *tls.Config {
 // valid even if the webhook process is restarted. Returns the parsed cert and
 // the certificate PEM (for embedding as the kubeconfig CA bundle).
 func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error) {
-	if certPEM, keyPEM, ok := readCertPair(certPath, keyPath); ok {
-		cert, err := tls.X509KeyPair(certPEM, keyPEM)
-		if err == nil {
+	// Fall through to regenerate on a missing/corrupt/incompatible persisted
+	// pair. The cert PEM is re-read because tls.Certificate keeps only DER.
+	if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
+		if certPEM, err := os.ReadFile(certPath); err == nil {
 			return cert, certPEM, nil
 		}
-		// Fall through to regenerate on a corrupt/incompatible persisted pair.
 	}

 	certPEM, keyPEM, err := generateSelfSigned()
@@ -58,18 +58,6 @@ func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error
 	return cert, certPEM, nil
 }

-func readCertPair(certPath, keyPath string) (certPEM, keyPEM []byte, ok bool) {
-	c, err := os.ReadFile(certPath)
-	if err != nil {
-		return nil, nil, false
-	}
-	k, err := os.ReadFile(keyPath)
-	if err != nil {
-		return nil, nil, false
-	}
-	return c, k, true
-}
-
 func generateSelfSigned() (certPEM, keyPEM []byte, err error) {
 	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {
@@ -30,6 +30,7 @@ import (
 	"github.com/mulgadc/bluebottle/pkg/masterkey"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/mulgadc/spinifex/spinifex/clustersize"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
@@ -200,6 +201,15 @@ takes effect immediately. Prompts for confirmation unless --yes is passed.`,
 	Run: runimagesPromoteCmd,
 }

+var imagesDescribeCmd = &cobra.Command{
+	Use:   "describe",
+	Short: "Show an AMI's metadata and source digest",
+	Long: `Print an AMI's control-plane metadata, including the digest of the
+artifact it was imported from and who verified it. The digest describes the
+imported source file, not the volume's current contents.`,
+	Run: runimagesDescribeCmd,
+}
+
 var volumesCmd = &cobra.Command{
 	Use:   "volumes",
 	Short: "Inspect block storage volumes",
@@ -294,7 +304,7 @@ spx admin images import --name debian-13-x86_64
 spx admin images list

 // Manually import a path
-spx admin images import --file /path/to/image --distro debian --version 13 --arch x86_64
+spx admin images import --file /path/to/image --checksum /path/to/SHA512SUMS --distro debian --version 13 --arch x86_64

 -> x <-
 */
@@ -328,6 +338,7 @@ func init() {
 	imagesCmd.AddCommand(imagesListCmd)
 	imagesCmd.AddCommand(imagesRemoveCmd)
 	imagesCmd.AddCommand(imagesPromoteCmd)
+	imagesCmd.AddCommand(imagesDescribeCmd)

 	adminCmd.AddCommand(volumesCmd)
 	volumesCmd.AddCommand(volumesOrphansCmd)
@@ -358,10 +369,11 @@ func init() {

 	// Flags for admin init
 	adminInitCmd.Flags().Bool("force", false, "Force re-initialization (overwrites existing config)")
+	adminInitCmd.Flags().Bool("discard-jetstream", true, "With --nodes >= 2, remove this node's existing JetStream streams once every node has joined; =false refuses to form over a non-empty store instead. Ignored with --nodes 1")
 	adminInitCmd.Flags().String("region", "ap-southeast-2", "Mulga region to create")
 	adminInitCmd.Flags().String("az", "ap-southeast-2a", "Mulga AZ to create")
 	adminInitCmd.Flags().String("node", "node1", "Node name, increment for additional nodes (default, node1)")
-	adminInitCmd.Flags().Int("nodes", 3, "Number of nodes to expect for cluster")
+	adminInitCmd.Flags().Int("nodes", 3, "Number of nodes to expect for cluster (1, or 3 and up)")
 	adminInitCmd.Flags().String("host", "", "Leader node to join (if not specified, tries multicast discovery)")
 	adminInitCmd.Flags().Int("port", 4432, "Port to bind cluster services on")
 	adminInitCmd.Flags().String("bind", "0.0.0.0", "IP address to bind services to (e.g., 10.11.12.1 for multi-node). Default 0.0.0.0 listens on all interfaces.")
@@ -426,6 +438,7 @@ func init() {
 	imagesImportCmd.Flags().StringSlice("tag", nil, "Tag to apply to the imported AMI as key=value (repeatable; e.g. --tag spinifex:managed-by=elbv2)")
 	imagesImportCmd.Flags().Bool("force", false, "Force command execution (overwrites existing files)")
 	imagesImportCmd.Flags().Bool("skip-verify", false, "Skip catalog-image checksum verification (INSECURE; operator assumes integrity responsibility)")
+	imagesImportCmd.Flags().String("checksum", "", "Sums file to verify a --file image against (SHA256SUMS, SHA512SUMS, <image>.sha256/.sha512)")

 	imagesRemoveCmd.Flags().String("image-id", "", "AMI ID to remove (required)")
 	imagesRemoveCmd.Flags().Bool("force", false, "Bypass dependency, ownership and config-corrupt checks (salvage mode)")
@@ -435,6 +448,9 @@ func init() {
 	imagesPromoteCmd.Flags().String("image-id", "", "AMI ID to promote to system image (required)")
 	imagesPromoteCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
 	_ = imagesPromoteCmd.MarkFlagRequired("image-id")
+
+	imagesDescribeCmd.Flags().String("image-id", "", "AMI ID to describe (required)")
+	_ = imagesDescribeCmd.MarkFlagRequired("image-id")
 }

 const bytesPerGiB = 1024 * 1024 * 1024
@@ -463,6 +479,108 @@ func amiVolumeSizeGiB(sizeBytes int64) uint64 {
 	return safecast.Int64ToUint64((sizeBytes + bytesPerGiB - 1) / bytesPerGiB)
 }

+// validateChecksumFlags rejects --checksum combinations that would otherwise be silently ignored.
+// An explicitly empty --checksum (e.g. an unset shell variable) must not fall through to an unverified import.
+func validateChecksumFlags(checksumPath string, checksumSet bool, localFile string, skipVerify bool) error {
+	if !checksumSet {
+		return nil
+	}
+	if strings.TrimSpace(checksumPath) == "" {
+		return errors.New("--checksum requires a sums file path")
+	}
+	if localFile == "" {
+		return errors.New("--checksum requires --file")
+	}
+	if skipVerify {
+		return errors.New("--checksum and --skip-verify are mutually exclusive")
+	}
+	return nil
+}
+
+// catalogPinnedDigestSource records that the expected digest came from this
+// binary's catalog, so a recorded digest never reads as a fetched sums file.
+const catalogPinnedDigestSource = "catalog:pinned"
+
+// importSource is the file an import is about to extract, and what it is to be verified against.
+type importSource struct {
+	imageFile, imageName, localFile            string
+	image                                      utils.Images
+	checksumPath, checksumAlgo, expectedDigest string
+	skipVerify                                 bool
+}
+
+// resolveDigest verifies or hashes the source before extract and returns the digest to record.
+// Upstream digests are of the artifact as supplied. Failures are reported to errOut before
+// returning; the file is left in place, and a catalog download recovers with --force.
+func (src importSource) resolveDigest(out, errOut io.Writer) (ebsmetadata.ImageDigest, error) {
+	digest := ebsmetadata.ImageDigest{Filename: filepath.Base(src.imageFile)}
+	switch sourceDigestMode(src.localFile, src.checksumPath, src.skipVerify) {
+	case ebsmetadata.DigestCatalog:
+		if src.image.ChecksumType == "" || (src.image.Checksum == "" && src.image.ChecksumDigest == "") {
+			fmt.Fprintf(errOut, "Catalog entry %q is missing Checksum/ChecksumDigest/ChecksumType; refusing import.\n", src.imageName)
+			return digest, errors.New("catalog entry has no checksum")
+		}
+		// A pinned digest is checked against this file rather than a sums file
+		// fetched from the same host that served the image.
+		if src.image.ChecksumDigest != "" {
+			actual, err := utils.VerifyImageDigest(src.imageFile, src.image.ChecksumType, src.image.ChecksumDigest)
+			if err != nil {
+				printChecksumError(errOut, src.imageFile, src.imageName, src.image, err)
+				return digest, err
+			}
+			fmt.Fprintf(out, "✅ Verified image checksum (%s) against the pinned catalog digest\n", src.image.ChecksumType)
+			digest.Algorithm, digest.Value, digest.Source = src.image.ChecksumType, actual, catalogPinnedDigestSource
+			digest.Verification = ebsmetadata.DigestCatalog
+			return digest, nil
+		}
+		actual, err := utils.VerifyImageChecksum(src.imageFile, src.image.Checksum, src.image.ChecksumType)
+		if err != nil {
+			printChecksumError(errOut, src.imageFile, src.imageName, src.image, err)
+			return digest, err
+		}
+		fmt.Fprintf(out, "✅ Verified image checksum (%s)\n", src.image.ChecksumType)
+		digest.Algorithm, digest.Value, digest.Source = src.image.ChecksumType, actual, src.image.Checksum
+		digest.Verification = ebsmetadata.DigestCatalog
+	case ebsmetadata.DigestOperator:
+		actual, err := utils.VerifyImageDigest(src.localFile, src.checksumAlgo, src.expectedDigest)
+		if err != nil {
+			fmt.Fprintf(errOut, "Image integrity verification failed: %v\n", err)
+			fmt.Fprintf(errOut, "  file:     %s\n", src.localFile)
+			fmt.Fprintf(errOut, "  checksum: %s\n", src.checksumPath)
+			return digest, err
+		}
+		fmt.Fprintf(out, "✅ Verified image checksum (%s) against %s\n", src.checksumAlgo, src.checksumPath)
+		digest.Algorithm, digest.Value, digest.Source = src.checksumAlgo, actual, filepath.Base(src.checksumPath)
+		digest.Verification = ebsmetadata.DigestOperator
+	default:
+		if src.localFile != "" {
+			fmt.Fprintf(errOut, "Importing %s without checksum validation (pass --checksum <sums-file> to verify)\n", src.localFile)
+		} else {
+			fmt.Fprintf(errOut, "⚠️  --skip-verify set: checksum verification skipped for %s\n", src.imageName)
+		}
+		actual, err := utils.HashImageFile(src.imageFile, "sha256")
+		if err != nil {
+			fmt.Fprintf(errOut, "Could not hash image: %v\n", err)
+			return digest, err
+		}
+		digest.Algorithm, digest.Value = "sha256", actual
+		digest.Verification = ebsmetadata.DigestUnverified
+	}
+	return digest, nil
+}
+
+// sourceDigestMode picks how an import's source digest is obtained.
+func sourceDigestMode(localFile, checksumPath string, skipVerify bool) string {
+	switch {
+	case localFile != "" && checksumPath != "":
+		return ebsmetadata.DigestOperator
+	case localFile != "" || skipVerify:
+		return ebsmetadata.DigestUnverified
+	default:
+		return ebsmetadata.DigestCatalog
+	}
+}
+
 func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	var image utils.Images

@@ -502,6 +620,20 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}

+	checksumPath, _ := cmd.Flags().GetString("checksum")
+	if err := validateChecksumFlags(checksumPath, cmd.Flags().Changed("checksum"), localFile, skipVerify); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	var checksumAlgo, expectedDigest string
+	if checksumPath != "" {
+		checksumAlgo, expectedDigest, err = utils.ReadExpectedDigest(checksumPath, filepath.Base(localFile))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Could not read checksum from %s: %v\n", checksumPath, err)
+			os.Exit(1)
+		}
+	}
+
 	if imageName != "" {
 		var exists bool
 		image, exists = utils.AvailableImages[imageName]
@@ -588,8 +720,8 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}

-	// Next, if the file is selected to download, fetch it, extract disk image, and save to path
-	if imageName != "" && localFile == "" {
+	// Next, if the file is selected to download, fetch it and save to path
+	if localFile == "" {
 		// Download the file to the image path
 		filename := path.Base(image.URL)
 		imageFile = fmt.Sprintf("%s/%s", imagePath, filename)
@@ -605,24 +737,20 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 				os.Exit(1)
 			}
 		}
+	}

-		// Verify before extract: the catalog digest is of the artifact as it
-		// sits on the mirror (.tar.xz/.img/.raw). Also catches a poisoned
-		// cache on the re-run path — failure leaves the file on disk so the
-		// operator can inspect; recover with --force.
-		if skipVerify {
-			fmt.Fprintf(os.Stderr, "⚠️  --skip-verify set: checksum verification skipped for %s\n", imageName)
-		} else {
-			if image.Checksum == "" || image.ChecksumType == "" {
-				fmt.Fprintf(os.Stderr, "Catalog entry %q is missing Checksum/ChecksumType; refusing import.\n", imageName)
-				os.Exit(1)
-			}
-			if err := utils.VerifyImageChecksum(imageFile, image.Checksum, image.ChecksumType); err != nil {
-				printChecksumError(os.Stderr, imageFile, imageName, image, err)
-				os.Exit(1)
-			}
-			fmt.Printf("✅ Verified image checksum (%s)\n", image.ChecksumType)
-		}
+	sourceDigest, err := importSource{
+		imageFile:      imageFile,
+		imageName:      imageName,
+		localFile:      localFile,
+		image:          image,
+		checksumPath:   checksumPath,
+		checksumAlgo:   checksumAlgo,
+		expectedDigest: expectedDigest,
+		skipVerify:     skipVerify,
+	}.resolveDigest(os.Stdout, os.Stderr)
+	if err != nil {
+		os.Exit(1)
 	}

 	// Next, validate if the image is raw, tar, gz, xv, etc. We need to upload the raw image
@@ -659,7 +787,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		Description:     fmt.Sprintf("%s cloud image prepared for Spinifex", amiName),
 		Architecture:    image.Arch,
 		PlatformDetails: image.Platform,
-		CreationDate:    time.Now(),
+		CreationDate:    time.Now().UTC(),
 		RootDeviceType:  "ebs",
 		Virtualization:  "hvm",
 		ImageOwnerAlias: "system",
@@ -668,6 +796,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		BootMode:        image.BootMode,
 		Distro:          image.Distro,
 		DistroFamily:    utils.DistroFamily(image.Distro),
+		SourceDigest:    &sourceDigest,
 	}

 	// Copy catalog-provided tags (e.g. spinifex:managed-by for system AMIs)
@@ -738,6 +867,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	}

 	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
+	fmt.Printf("✅ Source digest: %s:%s (%s)\n", sourceDigest.Algorithm, sourceDigest.Value, sourceDigest.Verification)
 }

 // registerImportedAMISnapshot writes the EC2 control plane's snapshot document
@@ -1000,6 +1130,62 @@ func runimagesPromoteCmd(cmd *cobra.Command, args []string) {
 	fmt.Printf("✅ Promoted %s to system image (owner: %s).\n", imageID, admin.SystemOwnerAlias)
 }

+func runimagesDescribeCmd(cmd *cobra.Command, args []string) {
+	imageID, _ := cmd.Flags().GetString("image-id")
+
+	cfgFile, _ := cmd.Flags().GetString("config")
+	if cfgFile == "" {
+		cfgFile = DefaultConfigFile()
+	}
+
+	appConfig, err := config.LoadConfig(cfgFile)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Error loading config file:", err)
+		os.Exit(1)
+	}
+
+	node := appConfig.Nodes[appConfig.Node]
+	store := objectstore.NewS3ObjectStoreFromConfig(
+		node.Predastore.Host,
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+
+	meta, err := admin.GetAMIMetadata(store, node.Predastore.Bucket, imageID)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Failed to describe AMI:", err)
+		os.Exit(1)
+	}
+
+	printAMIDescription(os.Stdout, meta)
+}
+
+// printAMIDescription writes an AMI's metadata and source digest for an operator.
+func printAMIDescription(w io.Writer, meta ebsmetadata.AMI) {
+	fmt.Fprintf(w, "Image ID:       %s\n", meta.ImageID)
+	fmt.Fprintf(w, "Name:           %s\n", meta.Name)
+	fmt.Fprintf(w, "Owner:          %s\n", meta.ImageOwnerAlias)
+	if !meta.CreationDate.IsZero() {
+		fmt.Fprintf(w, "Created:        %s\n", meta.CreationDate.UTC().Format("2006-01-02T15:04:05Z"))
+	}
+	fmt.Fprintf(w, "Boot mode:      %s\n", meta.BootMode)
+	fmt.Fprintf(w, "Architecture:   %s\n", meta.Architecture)
+
+	d := meta.SourceDigest
+	if d == nil {
+		fmt.Fprintln(w, "Source digest:  not recorded (imported before digest recording)")
+		return
+	}
+	fmt.Fprintf(w, "Source digest:  %s:%s\n", d.Algorithm, d.Value)
+	if d.Source != "" {
+		fmt.Fprintf(w, "Verification:   %s (%s)\n", d.Verification, d.Source)
+	} else {
+		fmt.Fprintf(w, "Verification:   %s\n", d.Verification)
+	}
+	fmt.Fprintf(w, "Source file:    %s\n", d.Filename)
+}
+
 // List remote images available.
 func runimagesListCmd(cmd *cobra.Command, args []string) {
 	//fmt.Println(availableImages)
@@ -1027,6 +1213,26 @@ func runimagesListCmd(cmd *cobra.Command, args []string) {
 	pterm.Println("spx admin images import --name <image-name>")
 }

+// validateNodeCount accepts the cluster sizes Spinifex supports: one server, or
+// three and up.
+//
+// Two is rejected rather than allowed and warned about. On two servers OVN runs
+// standalone, the storage metadata quorum has no majority to lose, and the
+// control plane's own KV state has no replica count that works — one replica
+// disappears with its node, and two stop accepting writes when either goes. A
+// cluster that cannot survive the event it was built for is worth refusing at the
+// point someone asks for it.
+func validateNodeCount(nodes int) error {
+	if nodes < 1 {
+		return fmt.Errorf("a cluster needs at least one node, got %d", nodes)
+	}
+	if nodes > 1 && nodes < clustersize.MinHANodes {
+		return fmt.Errorf("%d nodes is not a supported cluster size: use 1 for a single server, or %d and up for a cluster that can survive losing one",
+			nodes, clustersize.MinHANodes)
+	}
+	return nil
+}
+
 // TODO: Move all logic to a module, use minimal application logic in viper commands.
 func runAdminInit(cmd *cobra.Command, args []string) {
 	if os.Getuid() != 0 {
@@ -1058,6 +1264,15 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	clusterName, _ := cmd.Flags().GetString("cluster-name")
 	services, _ := cmd.Flags().GetStringSlice("services")

+	// Rejected before anything is written, because a two-node cluster has no
+	// correct answer rather than a worse one: no layer of the stack has a
+	// majority to lose, and no replica count for its state both survives a node
+	// going away and still accepts writes.
+	if err := validateNodeCount(nodes); err != nil {
+		fmt.Fprintf(os.Stderr, "--nodes: %v\n", err)
+		os.Exit(1)
+	}
+
 	// Optional operator email — validated up-front so a bad address fails
 	// before we touch any config state. Empty is allowed here; reset / repeat
 	// inits on a box that already has an email in /etc/spinifex/spinifex.toml
@@ -1115,7 +1330,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	detected, err := admin.DetectNetwork()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "⚠️  Network auto-detection failed: %v\n", err)
-		fmt.Fprintf(os.Stderr, "   Use --external-mode=nat for outbound-only VMs on a non-bridgeable uplink, or specify --external-* flags manually.\n")
+		fmt.Fprintf(os.Stderr, "   Use --external-mode=nat on a non-bridgeable uplink, or specify --external-* flags manually.\n")
 	} else {
 		detectedNet = detected

@@ -1153,7 +1368,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			if externalMode == "" && !cmd.Flags().Changed("external-mode") {
 				if isNonBridgeableUplink(detected.WAN.Name) {
 					fmt.Fprintf(os.Stderr, "\n❌ Detected WAN interface %s cannot be bridged (WiFi/cellular/PPP).\n", detected.WAN.Name)
-					fmt.Fprintf(os.Stderr, "   Use routed NAT mode instead (outbound-only VM networking):\n")
+					fmt.Fprintf(os.Stderr, "   Use routed NAT mode instead:\n")
 					fmt.Fprintf(os.Stderr, "     ./scripts/setup-ovn.sh --management --nat-uplink\n")
 					fmt.Fprintf(os.Stderr, "     spx admin init --external-mode=nat\n")
 					os.Exit(1)
@@ -1175,9 +1390,14 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	// the transit segment claims externalGateway below.
 	natPublicGateway := externalGateway
 	if externalMode == "nat" {
+		// Verified on three OCI nodes 2026-09-26: each node delivers its own
+		// instances' public IPs, inbound and outbound. Still flagged because no
+		// nightly cell covers multi-node nat yet.
 		if nodes >= 2 {
-			fmt.Fprintf(os.Stderr, "❌ Error: --external-mode=nat is single-node only (v1); use --nodes=1\n")
-			os.Exit(1)
+			fmt.Fprintf(os.Stderr, "⚠️  --external-mode=nat on %d nodes is NEW.\n", nodes)
+			fmt.Fprintf(os.Stderr, "   Public IPs are delivered by the node running the instance; default egress stays on the VPC's gateway chassis.\n")
+			fmt.Fprintf(os.Stderr, "   Verified by hand on three nodes, but not yet covered by a nightly cell.\n")
+			fmt.Fprintf(os.Stderr, "   Check with: ip route show dev spx-nat-host — exactly one node should hold each public address.\n")
 		}
 		if !natPublicPool && (externalBindBridge != "" || gatewayIP != "") {
 			fmt.Fprintf(os.Stderr, "❌ Error: --external-bind-bridge/--gateway-ip require --external-pool or --external-source in --external-mode=nat\n")
@@ -1313,6 +1533,15 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(0)
 	}

+	if spxRoot == "" {
+		spxRoot = DefaultDataDir()
+	}
+	discardJetStream, _ := cmd.Flags().GetBool("discard-jetstream")
+	if err := checkInitJetStreamStore(jetStreamStoreDir(spxRoot), nodes, discardJetStream); err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+
 	// Preserve the previously-captured operator email across --force re-inits
 	// when --email is omitted (e.g. reset-dev-env.sh workflows). Without this
 	// a reset would silently blank the address.
@@ -1429,7 +1658,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}

 	// Generate SSL certificates (with bind IP in SANs for multi-node support)
-	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP, region, config.DefaultAWSInternalSuffix)
+	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP, region, config.DefaultAWSServicesDomain)

 	// Generate per-node IPsec peer cert when cluster-wide IPsec is enabled
 	// (default true). Reuses the cluster CA — no intermediate strongSwan PKI.
@@ -1498,14 +1727,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			bootstrapSubnetId := utils.GenerateResourceID("subnet")
 			bootstrapIgwId := utils.GenerateResourceID("igw")
 			networkConfig.ExternalMode = externalMode
-			networkConfig.PoolName = "wan"
-			networkConfig.PoolSource = externalSource
-			networkConfig.PoolBindBridge = externalBindBridge
-			networkConfig.PoolStart = poolStart
-			networkConfig.PoolEnd = poolEnd
-			networkConfig.PoolGateway = externalGateway
-			networkConfig.PoolGatewayIP = gatewayIP
-			networkConfig.PoolPrefixLen = externalPrefixLen
+			networkConfig.Pools = externalPools
 			networkConfig.PoolDNSServers = dnsServers
 			networkConfig.BootstrapAccountId = admin.DefaultAccountID()
 			networkConfig.BootstrapVpcId = bootstrapVpcId
@@ -1658,7 +1880,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 				fmt.Printf("  Public pool:   dhcp via %s\n", externalBindBridge)
 			}
 		} else {
-			fmt.Printf("\n📡 External networking: nat (routed, outbound-only — no public IPs/EIPs)\n")
+			fmt.Printf("\n📡 External networking: nat (routed; no public pool configured yet)\n")
 		}
 		fmt.Printf("  Transit:       %s via %s (host masquerades out any uplink)\n", host.NATTransitCIDR, host.NATTransitHostEnd)
 		fmt.Printf("  Host setup:    ./scripts/setup-ovn.sh --nat-uplink (run before starting services)\n")
@@ -1845,6 +2067,17 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,

 	fmt.Printf("✅ All %d nodes joined!\n", expectedNodes)

+	// The pre-start check refused a store with streams unless discard was asked
+	// for, so anything left here is removed now that the cluster is committed.
+	if discard, _ := cmd.Flags().GetBool("discard-jetstream"); discard {
+		storeDir := jetStreamStoreDir(spxRoot)
+		if err := discardJetStreamStore(storeDir); err != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("✅ Discarded this node's pre-formation JetStream store: %s\n", storeDir)
+	}
+
 	// Build cluster topology from formation data
 	allNodes := fs.Nodes()
 	clusterRoutes := formation.BuildClusterRoutes(allNodes)
@@ -2002,19 +2235,27 @@ func joinRetryable(err error, statusCode int) bool {
 }

 // checkJoinPreconditions rejects a join that would silently destroy this node's
-// existing cluster identity. Joining adopts the primary's CA and master key,
-// overwriting whatever is here: correct on a freshly installed node, and on one
-// that has been in service it orphans every fragment and volume sealed under the
-// old key. runAdminInit guards the same way before re-initializing.
-func checkJoinPreconditions(configDir string, force bool) error {
+// identity or JetStream data; --force accepts that loss. A running NATS service
+// is refused even with --force, because the store cannot be removed under it.
+func checkJoinPreconditions(configDir, storeDir string, force bool) error {
+	if err := checkStoreDiscardable(); err != nil {
+		return err
+	}
 	if force {
 		return nil
 	}
 	tomlPath := filepath.Join(configDir, "spinifex.toml")
-	if !admin.FileExists(tomlPath) {
-		return nil
+	if admin.FileExists(tomlPath) {
+		return fmt.Errorf("this node is already initialized: %s", tomlPath)
 	}
-	return fmt.Errorf("this node is already initialized: %s", tomlPath)
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return err
+	}
+	if len(streams) > 0 {
+		return fmt.Errorf("this node's JetStream store %s holds %d stream(s)", storeDir, len(streams))
+	}
+	return nil
 }

 // joinDiscardsIdentityMsg spells out what a forced join throws away. Kept out of
@@ -2023,6 +2264,8 @@ const joinDiscardsIdentityMsg = `Joining will discard this node's own cluster id
  - CA certificate and key
  - master key, and any data sealed under it
  - viperblock key, and any volumes encrypted under it
+  - JetStream store (<data-dir>/nats/jetstream): every stream and KV bucket
+    this node holds, so it takes the cluster's copy instead of diverging from it

 That is safe on a freshly installed node — an ISO install initializes a
 single-node cluster at first boot, and nothing has been sealed under these keys
@@ -2103,7 +2346,11 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	// Checked before any network call so a node that will not join says so
 	// immediately. Unlike init this exits non-zero: a node that did not join
 	// must not look like success to a provisioning script.
-	if err := checkJoinPreconditions(configDir, force); err != nil {
+	if dataDir == "" {
+		dataDir = DefaultDataDir()
+	}
+	storeDir := jetStreamStoreDir(dataDir)
+	if err := checkJoinPreconditions(configDir, storeDir, force); err != nil {
 		fmt.Fprintf(os.Stderr, "⚠️  %v\n\n%s\n", err, joinDiscardsIdentityMsg)
 		os.Exit(1)
 	}
@@ -2113,11 +2360,6 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		clusterBind = bindIP
 	}

-	// Set default data directory
-	if dataDir == "" {
-		dataDir = DefaultDataDir()
-	}
-
 	fmt.Println("🚀 Joining Spinifex cluster...")
 	fmt.Printf("Node: %s\n", node)
 	fmt.Printf("Leader: %s\n", leaderHost)
@@ -2298,6 +2540,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}

+	// Removed only once formation has succeeded, so a failed join leaves the
+	// node's single-node cluster intact.
+	if err := discardJetStreamStore(storeDir); err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("✅ Discarded this node's pre-formation JetStream store: %s\n", storeDir)
+
 	// Set up config directory
 	if configDir == "" {
 		configDir = DefaultConfigDir()
@@ -2385,7 +2635,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}

 	// Generate server cert signed by CA with this node's bind IP
-	if err := admin.GenerateServerCertOnly(configDir, bindIP, region, config.DefaultAWSInternalSuffix); err != nil {
+	if err := admin.GenerateServerCertOnly(configDir, bindIP, region, config.DefaultAWSServicesDomain); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
 		os.Exit(1)
 	}
@@ -2604,7 +2854,7 @@ func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterCo

 	// Background: this runs at CLI top level, where there is no request to
 	// inherit a deadline from and the process exits after the one command.
-	svc, err := handlers_iam.NewIAMServiceImpl(context.Background(), nc, masterKey, len(cfg.Nodes))
+	svc, err := handlers_iam.NewIAMServiceImpl(context.Background(), nc, masterKey)
 	if err != nil {
 		nc.Close()
 		return nil, nil, nil, nil, fmt.Errorf("init IAM service: %w", err)
@@ -2994,18 +3244,11 @@ func applyNetworkConfig(settings *admin.ConfigSettings, nc *formation.NetworkCon
 	settings.IPSecEnabled = nc.IPSecEnabled
 	settings.ExternalMode = nc.ExternalMode
 	settings.PoolDNSServers = nc.PoolDNSServers
+	// Derived from the mode rather than carried in NetworkConfig, so the leader
+	// and every joiner pin the same value from the same rule as single-node init.
+	settings.BridgeMode = bridgeModeFor(nc.ExternalMode)
 	if nc.ExternalMode != "" {
-		settings.Pools = []admin.PoolData{{
-			Name:       nc.PoolName,
-			Source:     nc.PoolSource,
-			BindBridge: nc.PoolBindBridge,
-			Start:      nc.PoolStart,
-			End:        nc.PoolEnd,
-			Gateway:    nc.PoolGateway,
-			GatewayIP:  nc.PoolGatewayIP,
-			PrefixLen:  nc.PoolPrefixLen,
-			DNSServers: nc.PoolDNSServers,
-		}}
+		settings.Pools = nc.Pools
 	}

 	settings.BootstrapAccountId = nc.BootstrapAccountId
@@ -3706,7 +3949,12 @@ func resolveIfaceIP(iface string) string {
 func printChecksumError(w io.Writer, imageFile, imageName string, image utils.Images, err error) {
 	fmt.Fprintf(w, "Image integrity verification failed: %v\n", err)
 	fmt.Fprintf(w, "  file:     %s\n", imageFile)
-	fmt.Fprintf(w, "  source:   %s\n", image.Checksum)
+	if image.ChecksumDigest != "" {
+		fmt.Fprintf(w, "  source:   %s (%s)\n", catalogPinnedDigestSource, image.ChecksumDigest)
+		fmt.Fprintf(w, "  url:      %s\n", image.URL)
+	} else {
+		fmt.Fprintf(w, "  source:   %s\n", image.Checksum)
+	}
 	fmt.Fprintln(w)
 	fmt.Fprintln(w, "The cached file was left in place. To re-download and retry:")
 	fmt.Fprintf(w, "  spx admin images import --name %s --force\n", imageName)
@@ -0,0 +1,451 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/highwayhash"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/spf13/cobra"
+)
+
+var kvCmd = &cobra.Command{
+	Use:   "kv",
+	Short: "Inspect JetStream KV state",
+}
+
+var kvDigestCmd = &cobra.Command{
+	Use:   "digest",
+	Short: "Print a digest of every stream held by this node's JetStream replica",
+	Long: `Read this node's own copy of every JetStream stream and print one line per
+stream: name, first and last sequence, message count, and a SHA-256 over every
+stored message (sequence, subject, timestamp, headers, data).
+
+Run it on every node and compare the output. Replicas of one stream must
+produce identical lines; any difference means the replicas have diverged, and
+--seqs prints a hash per sequence to find where. For a check that tells a
+diverged replica from one that is merely behind, run it with --json --seqs on
+every node and pass the results to "spx admin kv compare".
+
+It never touches the live store. A direct get over NATS may be answered by any
+replica, so instead the streams tree is copied to --work-dir and read by a
+private in-process server with no listener, which is removed afterwards. The
+copy's stream config is set to one replica with no expiry so it can be loaded.
+
+Writes landing during the copy, or applied but not yet flushed to disk, can
+make a replica's tail differ, so gate API writes and re-run before calling a
+tail difference divergence.`,
+	Run: runKVDigest,
+}
+
+func init() {
+	adminCmd.AddCommand(kvCmd)
+	kvCmd.AddCommand(kvDigestCmd)
+	kvDigestCmd.Flags().String("store-dir", "", "JetStream store to read (default <spinifex-dir>/nats/jetstream)")
+	kvDigestCmd.Flags().String("work-dir", os.TempDir(), "Directory to hold the temporary copy")
+	kvDigestCmd.Flags().StringSlice("stream", nil, "Only these streams (repeatable)")
+	kvDigestCmd.Flags().Bool("seqs", false, "Also print a hash per sequence")
+	kvDigestCmd.Flags().Bool("json", false, "Print one JSON document, the input to \"spx admin kv compare\"")
+}
+
+// nodeDigest is one node's digest report, the unit "kv compare" reads.
+type nodeDigest struct {
+	Host     string         `json:"host"`
+	Store    string         `json:"store"`
+	CopiedAt time.Time      `json:"copied_at"`
+	Skipped  []string       `json:"skipped,omitempty"`
+	Streams  []streamDigest `json:"streams"`
+}
+
+// streamDigest is one stream's content as held by one replica.
+type streamDigest struct {
+	streamMeta
+
+	Name     string      `json:"name"`
+	FirstSeq uint64      `json:"first_seq"`
+	LastSeq  uint64      `json:"last_seq"`
+	Msgs     uint64      `json:"msgs"`
+	Digest   string      `json:"sha256"`
+	Seqs     []seqDigest `json:"seqs,omitempty"`
+}
+
+// streamMeta is the part of the live stream config that decides which replica
+// differences are legitimate. It is read before the copy is rewritten.
+type streamMeta struct {
+	Replicas    int           `json:"replicas,omitempty"`
+	Retention   string        `json:"retention,omitempty"`
+	MaxAge      time.Duration `json:"max_age_ns,omitempty"`
+	AllowMsgTTL bool          `json:"allow_msg_ttl,omitempty"`
+	MaxMsgs     int64         `json:"max_msgs,omitempty"`
+	MaxBytes    int64         `json:"max_bytes,omitempty"`
+}
+
+// seqDigest is the hash of one stored sequence, or "deleted" when the replica
+// has no message at that sequence. Subject, time and TTL are empty then.
+type seqDigest struct {
+	Seq     uint64        `json:"seq"`
+	Hash    string        `json:"hash"`
+	Subject string        `json:"subject,omitempty"`
+	Time    time.Time     `json:"time,omitzero"`
+	TTL     time.Duration `json:"ttl_ns,omitempty"`
+}
+
+// deletedHash marks a sequence the replica holds no message for.
+const deletedHash = "deleted"
+
+func runKVDigest(cmd *cobra.Command, _ []string) {
+	storeDir, _ := cmd.Flags().GetString("store-dir")
+	if storeDir == "" {
+		spxRoot, _ := cmd.Flags().GetString("spinifex-dir")
+		if spxRoot == "" {
+			spxRoot = DefaultDataDir()
+		}
+		storeDir = jetStreamStoreDir(spxRoot)
+	}
+	workParent, _ := cmd.Flags().GetString("work-dir")
+	only, _ := cmd.Flags().GetStringSlice("stream")
+	withSeqs, _ := cmd.Flags().GetBool("seqs")
+	asJSON, _ := cmd.Flags().GetBool("json")
+
+	work, err := os.MkdirTemp(workParent, "spx-kv-digest-")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		os.Exit(1)
+	}
+	defer os.RemoveAll(work)
+
+	copiedAt := time.Now().UTC()
+	skipped, metas, err := copyStreamTree(storeDir, filepath.Join(work, "jetstream"))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		os.RemoveAll(work)
+		os.Exit(1)
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Minute)
+	defer cancel()
+	digests, err := digestStore(ctx, work, only, withSeqs, metas)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		cancel()
+		os.RemoveAll(work)
+		os.Exit(1)
+	}
+
+	host, _ := os.Hostname()
+	if asJSON {
+		enc := json.NewEncoder(cmd.OutOrStdout())
+		if err := enc.Encode(nodeDigest{Host: host, Store: storeDir, CopiedAt: copiedAt, Skipped: skipped, Streams: digests}); err != nil {
+			fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+			os.Exit(1)
+		}
+		return
+	}
+	writeDigests(cmd.OutOrStdout(), host, storeDir, copiedAt, skipped, digests)
+}
+
+// writeDigests prints the report. Only the "#" lines carry node-specific
+// values, so `grep -v '^#'` output from two replicas diffs cleanly.
+func writeDigests(w io.Writer, host, storeDir string, copiedAt time.Time, skipped []string, digests []streamDigest) {
+	fmt.Fprintf(w, "# host=%s store=%s copied_at=%s\n", host, storeDir, copiedAt.Format(time.RFC3339))
+	for _, s := range skipped {
+		fmt.Fprintf(w, "# skipped account stream (not read): %s\n", s)
+	}
+	for _, d := range digests {
+		fmt.Fprintf(w, "%s\tfirst=%d\tlast=%d\tmsgs=%d\tsha256=%s\n", d.Name, d.FirstSeq, d.LastSeq, d.Msgs, d.Digest)
+		for _, s := range d.Seqs {
+			fmt.Fprintf(w, "%s\tseq=%d\t%s\n", d.Name, s.Seq, s.Hash)
+		}
+	}
+}
+
+// globalAccountDir is the account Spinifex's token-authenticated clients use.
+// Streams under any other account are listed but not read.
+const globalAccountDir = "$G"
+
+// copyStreamTree copies <storeDir>/$G/streams to <dst>/$G/streams, leaving out
+// consumer state (obs). It returns streams found under other accounts and each
+// copied stream's config as the live server held it, keyed by stream name.
+func copyStreamTree(storeDir, dst string) (skipped []string, metas map[string]streamMeta, err error) {
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return nil, nil, err
+	}
+	if len(streams) == 0 {
+		return nil, nil, fmt.Errorf("no streams found under %s", storeDir)
+	}
+	for _, s := range streams {
+		if !strings.HasPrefix(s, globalAccountDir+"/") {
+			skipped = append(skipped, s)
+		}
+	}
+
+	src := filepath.Join(storeDir, globalAccountDir, "streams")
+	out := filepath.Join(dst, globalAccountDir, "streams")
+	err = filepath.WalkDir(src, func(path string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		rel, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		// <stream>/obs holds consumers, which are not stream content.
+		if parts := strings.Split(rel, string(filepath.Separator)); d.IsDir() && len(parts) == 2 && parts[1] == "obs" {
+			return filepath.SkipDir
+		}
+		target := filepath.Join(out, rel)
+		if d.IsDir() {
+			return os.MkdirAll(target, 0o700)
+		}
+		if !d.Type().IsRegular() {
+			return nil
+		}
+		return copyFile(path, target)
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("copy streams from %s: %w", src, err)
+	}
+	entries, err := os.ReadDir(out)
+	if err != nil {
+		return nil, nil, err
+	}
+	metas = make(map[string]streamMeta, len(entries))
+	for _, e := range entries {
+		if e.IsDir() {
+			meta, err := rewriteStreamMetaStandalone(filepath.Join(out, e.Name()))
+			if err != nil {
+				return nil, nil, err
+			}
+			metas[e.Name()] = meta
+		}
+	}
+	return skipped, metas, nil
+}
+
+// rewriteStreamMetaStandalone makes a copied stream loadable by a standalone
+// server, which refuses replicas > 1 and would expire aged messages on
+// recovery. meta.sum is highwayhash64 keyed by sha256 of the directory name.
+// It returns the settings as they were before the rewrite.
+func rewriteStreamMetaStandalone(streamDir string) (streamMeta, error) {
+	metaPath := filepath.Join(streamDir, "meta.inf")
+	raw, err := os.ReadFile(metaPath)
+	if errors.Is(err, os.ErrNotExist) {
+		return streamMeta{}, nil
+	}
+	if err != nil {
+		return streamMeta{}, err
+	}
+	var orig struct {
+		Replicas    int           `json:"num_replicas"`
+		Retention   string        `json:"retention"`
+		MaxAge      time.Duration `json:"max_age"`
+		AllowMsgTTL bool          `json:"allow_msg_ttl"`
+		MaxMsgs     int64         `json:"max_msgs"`
+		MaxBytes    int64         `json:"max_bytes"`
+	}
+	if err := json.Unmarshal(raw, &orig); err != nil {
+		return streamMeta{}, fmt.Errorf("parse %s: %w", metaPath, err)
+	}
+	kept := streamMeta{
+		Replicas: orig.Replicas, Retention: orig.Retention, MaxAge: orig.MaxAge,
+		AllowMsgTTL: orig.AllowMsgTTL, MaxMsgs: orig.MaxMsgs, MaxBytes: orig.MaxBytes,
+	}
+	dec := json.NewDecoder(bytes.NewReader(raw))
+	dec.UseNumber()
+	var meta map[string]any
+	if err := dec.Decode(&meta); err != nil {
+		return streamMeta{}, fmt.Errorf("parse %s: %w", metaPath, err)
+	}
+	meta["num_replicas"] = 1
+	meta["max_age"] = 0
+	delete(meta, "allow_msg_ttl")
+	delete(meta, "subject_delete_marker_ttl")
+	delete(meta, "placement")
+	out, err := json.Marshal(meta)
+	if err != nil {
+		return streamMeta{}, err
+	}
+	key := sha256.Sum256([]byte(filepath.Base(streamDir)))
+	hh, err := highwayhash.New64(key[:])
+	if err != nil {
+		return streamMeta{}, err
+	}
+	hh.Write(out)
+	if err := os.WriteFile(metaPath, out, 0o600); err != nil {
+		return streamMeta{}, err
+	}
+	return kept, os.WriteFile(filepath.Join(streamDir, "meta.sum"), []byte(hex.EncodeToString(hh.Sum(nil))), 0o600)
+}
+
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(out, in); err != nil {
+		out.Close()
+		return err
+	}
+	return out.Close()
+}
+
+// digestStore opens storeParent (holding jetstream/) in a standalone server
+// that accepts in-process connections only, and digests each stream.
+func digestStore(ctx context.Context, storeParent string, only []string, withSeqs bool, metas map[string]streamMeta) ([]streamDigest, error) {
+	ns, err := server.NewServer(&server.Options{
+		ServerName: "spx-kv-digest",
+		JetStream:  true,
+		StoreDir:   storeParent,
+		DontListen: true,
+		NoLog:      true,
+		NoSigs:     true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("create reader server: %w", err)
+	}
+	ns.Start()
+	defer func() {
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}()
+	if !ns.ReadyForConnections(30 * time.Second) {
+		return nil, errors.New("reader server did not become ready")
+	}
+
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	if err != nil {
+		return nil, fmt.Errorf("connect to reader server: %w", err)
+	}
+	defer nc.Close()
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return nil, err
+	}
+
+	var names []string
+	lister := js.StreamNames(ctx)
+	for name := range lister.Name() {
+		if len(only) == 0 || slices.Contains(only, name) {
+			names = append(names, name)
+		}
+	}
+	if err := lister.Err(); err != nil {
+		return nil, fmt.Errorf("list streams: %w", err)
+	}
+	sort.Strings(names)
+
+	digests := make([]streamDigest, 0, len(names))
+	for _, name := range names {
+		d, err := digestStream(ctx, js, name, withSeqs)
+		if err != nil {
+			return nil, err
+		}
+		d.streamMeta = metas[name]
+		digests = append(digests, d)
+	}
+	return digests, nil
+}
+
+func digestStream(ctx context.Context, js jetstream.JetStream, name string, withSeqs bool) (streamDigest, error) {
+	s, err := js.Stream(ctx, name)
+	if err != nil {
+		return streamDigest{}, fmt.Errorf("open stream %s: %w", name, err)
+	}
+	info, err := s.Info(ctx)
+	if err != nil {
+		return streamDigest{}, fmt.Errorf("stream info %s: %w", name, err)
+	}
+	st := info.State
+	d := streamDigest{Name: name, FirstSeq: st.FirstSeq, LastSeq: st.LastSeq, Msgs: st.Msgs}
+
+	whole := sha256.New()
+	for seq := st.FirstSeq; st.Msgs > 0 && seq <= st.LastSeq; seq++ {
+		msg, err := s.GetMsg(ctx, seq)
+		sd := seqDigest{Seq: seq}
+		switch {
+		case errors.Is(err, jetstream.ErrMsgNotFound):
+			sd.Hash = deletedHash
+		case err != nil:
+			return streamDigest{}, fmt.Errorf("read %s seq %d: %w", name, seq, err)
+		default:
+			sd.Hash = hashStoredMsg(msg)
+			sd.Subject = msg.Subject
+			sd.Time = msg.Time.UTC()
+			sd.TTL = msgTTL(msg.Header.Get(jetstream.MsgTTLHeader))
+		}
+		fmt.Fprintf(whole, "%d:%s\n", seq, sd.Hash)
+		if withSeqs {
+			d.Seqs = append(d.Seqs, sd)
+		}
+	}
+	d.Digest = hex.EncodeToString(whole.Sum(nil))
+	return d, nil
+}
+
+// hashStoredMsg hashes everything a replica stores for one sequence. Header
+// keys are sorted so the hash does not depend on map iteration order.
+func hashStoredMsg(m *jetstream.RawStreamMsg) string {
+	h := sha256.New()
+	var n [8]byte
+	field := func(b []byte) {
+		binary.BigEndian.PutUint64(n[:], uint64(len(b)))
+		h.Write(n[:])
+		h.Write(b)
+	}
+	binary.BigEndian.PutUint64(n[:], m.Sequence)
+	h.Write(n[:])
+	field([]byte(m.Subject))
+	binary.BigEndian.PutUint64(n[:], uint64(m.Time.UnixNano()))
+	h.Write(n[:])
+	keys := make([]string, 0, len(m.Header))
+	for k := range m.Header {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		field([]byte(k))
+		for _, v := range m.Header[k] {
+			field([]byte(v))
+		}
+	}
+	field(m.Data)
+	return hex.EncodeToString(h.Sum(nil))[:16]
+}
+
+// msgTTL parses a Nats-TTL header: a Go duration or whole seconds. "never", an
+// absent header or an unparseable one yield zero, meaning no expiry.
+func msgTTL(v string) time.Duration {
+	if v == "" {
+		return 0
+	}
+	if d, err := time.ParseDuration(v); err == nil && d > 0 {
+		return d
+	}
+	if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
+		return time.Duration(n) * time.Second
+	}
+	return 0
+}
@@ -0,0 +1,360 @@
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+var kvCompareCmd = &cobra.Command{
+	Use:   "compare <digest.json>...",
+	Short: "Compare the kv digests of every node and report replicas that have diverged",
+	Long: `Read the "spx admin kv digest --json --seqs" output of every node in the
+cluster, one file per node ("-" reads any number of them from stdin), and
+check that every stream's replicas hold the same content.
+
+A replica that is merely behind is not divergent, so only these fail:
+
+  - the same sequence holds different content on two replicas;
+  - a message held by one replica is missing from another, and nothing on
+    that replica explains it: no later write to the same subject, no expiry
+    under max_age or a message TTL, and no stream limit dropping its head;
+  - a stream is held by fewer nodes than its replica count, or by more;
+  - replicas disagree on the replica count;
+  - no sequence range is common to every replica, so nothing can be checked.
+
+Sequences past the slowest replica's last one are not compared. Digests are
+taken from live stores a few seconds apart, so re-run before trusting a
+failure that could be a write landing between two copies.
+
+Exits 0 when every stream is consistent, 1 on divergence, 2 on bad input.`,
+	Args: cobra.MinimumNArgs(1),
+	Run:  runKVCompare,
+}
+
+func init() {
+	kvCmd.AddCommand(kvCompareCmd)
+}
+
+func runKVCompare(cmd *cobra.Command, args []string) {
+	var nodes []nodeDigest
+	for _, arg := range args {
+		var r io.Reader
+		if arg == "-" {
+			r = cmd.InOrStdin()
+		} else {
+			f, err := os.Open(arg)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+				os.Exit(2)
+			}
+			defer f.Close()
+			r = f
+		}
+		got, err := loadNodeDigests(r, arg)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+			os.Exit(2)
+		}
+		nodes = append(nodes, got...)
+	}
+	res, err := compareNodeDigests(nodes)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+		os.Exit(2)
+	}
+	writeCompareResult(cmd.OutOrStdout(), nodes, res)
+	if res.divergent() {
+		os.Exit(1)
+	}
+}
+
+// loadNodeDigests reads every JSON digest document in r.
+func loadNodeDigests(r io.Reader, source string) ([]nodeDigest, error) {
+	dec := json.NewDecoder(r)
+	var out []nodeDigest
+	for {
+		var d nodeDigest
+		err := dec.Decode(&d)
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("%s: document %d: %w", source, len(out)+1, err)
+		}
+		if d.Host == "" {
+			return nil, fmt.Errorf("%s: document %d has no host; is it \"kv digest --json\" output?", source, len(out)+1)
+		}
+		out = append(out, d)
+	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("%s: no digest documents", source)
+	}
+	return out, nil
+}
+
+// maxFindingsPerStream caps what one stream prints; the rest are counted.
+const maxFindingsPerStream = 5
+
+type streamVerdict struct {
+	Name     string
+	Holders  []string
+	Problems []string
+	Notes    []string
+}
+
+type kvCompareResult struct {
+	Streams []streamVerdict
+}
+
+func (r kvCompareResult) divergent() bool {
+	return slices.ContainsFunc(r.Streams, func(v streamVerdict) bool { return len(v.Problems) > 0 })
+}
+
+// replicaView is one node's copy of one stream, indexed for the per-sequence rules.
+type replicaView struct {
+	host          string
+	copiedAt      time.Time
+	d             *streamDigest
+	bySeq         map[uint64]seqDigest
+	lastBySubject map[string]uint64
+}
+
+func compareNodeDigests(nodes []nodeDigest) (kvCompareResult, error) {
+	if len(nodes) < 2 {
+		return kvCompareResult{}, fmt.Errorf("need the digests of at least two nodes, got %d", len(nodes))
+	}
+	hosts := make([]string, 0, len(nodes))
+	byName := map[string][]*replicaView{}
+	for i := range nodes {
+		n := &nodes[i]
+		if slices.Contains(hosts, n.Host) {
+			return kvCompareResult{}, fmt.Errorf("two digests are from host %s", n.Host)
+		}
+		hosts = append(hosts, n.Host)
+		for j := range n.Streams {
+			d := &n.Streams[j]
+			v := &replicaView{host: n.Host, copiedAt: n.CopiedAt, d: d,
+				bySeq: make(map[uint64]seqDigest, len(d.Seqs)), lastBySubject: map[string]uint64{}}
+			for _, s := range d.Seqs {
+				v.bySeq[s.Seq] = s
+				if s.Hash != deletedHash && s.Seq > v.lastBySubject[s.Subject] {
+					v.lastBySubject[s.Subject] = s.Seq
+				}
+			}
+			byName[d.Name] = append(byName[d.Name], v)
+		}
+	}
+	names := make([]string, 0, len(byName))
+	for name := range byName {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	var res kvCompareResult
+	for _, name := range names {
+		res.Streams = append(res.Streams, compareStream(name, hosts, byName[name]))
+	}
+	return res, nil
+}
+
+func compareStream(name string, hosts []string, views []*replicaView) streamVerdict {
+	v := streamVerdict{Name: name}
+	for _, r := range views {
+		v.Holders = append(v.Holders, r.host)
+	}
+
+	counts := map[int][]string{}
+	replicas := 0
+	for _, r := range views {
+		if n := r.d.Replicas; n > 0 {
+			counts[n] = append(counts[n], r.host)
+			replicas = max(replicas, n)
+		}
+	}
+	if len(counts) > 1 {
+		var parts []string
+		for n, hs := range counts {
+			parts = append(parts, fmt.Sprintf("%d on %s", n, strings.Join(hs, ",")))
+		}
+		sort.Strings(parts)
+		v.Problems = append(v.Problems, "replicas disagree on the replica count: "+strings.Join(parts, "; "))
+	}
+	switch {
+	case replicas > 0 && len(views) < replicas:
+		missing := slices.DeleteFunc(slices.Clone(hosts), func(h string) bool { return slices.Contains(v.Holders, h) })
+		msg := fmt.Sprintf("held by %d of %d replicas; missing from %s", len(views), replicas, strings.Join(missing, ","))
+		if len(hosts) < replicas {
+			msg += fmt.Sprintf(" (only %d digests given)", len(hosts))
+		}
+		v.Problems = append(v.Problems, msg)
+	case replicas > 0 && len(views) > replicas:
+		v.Problems = append(v.Problems, fmt.Sprintf("held by %d nodes but configured for %d replicas; a node outside the stream's group still holds a copy", len(views), replicas))
+	}
+	if len(views) < 2 {
+		v.Notes = append(v.Notes, "single copy, nothing to compare")
+		return v
+	}
+
+	identical := true
+	for _, r := range views[1:] {
+		if r.d.Digest != views[0].d.Digest || r.d.FirstSeq != views[0].d.FirstSeq || r.d.LastSeq != views[0].d.LastSeq {
+			identical = false
+			break
+		}
+	}
+	if identical {
+		v.Notes = append(v.Notes, fmt.Sprintf("identical, seqs %d-%d", views[0].d.FirstSeq, views[0].d.LastSeq))
+		return v
+	}
+	for _, r := range views {
+		if r.d.Msgs > 0 && len(r.d.Seqs) == 0 {
+			v.Problems = append(v.Problems, "contents differ and "+r.host+"'s digest has no per-sequence detail; re-run digest with --seqs")
+			return v
+		}
+	}
+
+	lo, hi := views[0].d.FirstSeq, views[0].d.LastSeq
+	seqSet := map[uint64]struct{}{}
+	for _, r := range views {
+		lo, hi = max(lo, r.d.FirstSeq), min(hi, r.d.LastSeq)
+		for s := range r.bySeq {
+			seqSet[s] = struct{}{}
+		}
+	}
+	if lo > hi {
+		v.Problems = append(v.Problems, "no sequence range is common to every replica, so nothing could be compared")
+		return v
+	}
+	seqs := make([]uint64, 0, len(seqSet))
+	for s := range seqSet {
+		if s <= hi {
+			seqs = append(seqs, s)
+		}
+	}
+	slices.Sort(seqs)
+
+	var problems []string
+	explained := 0
+	for _, seq := range seqs {
+		var present, absent []*replicaView
+		for _, r := range views {
+			if e, ok := r.bySeq[seq]; ok && e.Hash != deletedHash {
+				present = append(present, r)
+			} else {
+				absent = append(absent, r)
+			}
+		}
+		if len(present) == 0 {
+			continue
+		}
+		byHash := map[string][]string{}
+		for _, r := range present {
+			h := r.bySeq[seq].Hash
+			byHash[h] = append(byHash[h], r.host)
+		}
+		if len(byHash) > 1 {
+			var parts []string
+			for h, hs := range byHash {
+				parts = append(parts, strings.Join(hs, ",")+"="+h)
+			}
+			sort.Strings(parts)
+			problems = append(problems, fmt.Sprintf("seq %d %s holds different content: %s", seq, present[0].bySeq[seq].Subject, strings.Join(parts, " ")))
+			continue
+		}
+		msg := present[0].bySeq[seq]
+		for _, r := range absent {
+			if absenceExplained(r, msg) {
+				explained++
+				continue
+			}
+			problems = append(problems, fmt.Sprintf("seq %d %s is on %s but missing from %s, with no later write to that subject, expiry or limit to explain it",
+				seq, msg.Subject, hostsOf(present), r.host))
+		}
+	}
+	if len(problems) > maxFindingsPerStream {
+		problems = append(problems[:maxFindingsPerStream], fmt.Sprintf("... and %d more", len(problems)-maxFindingsPerStream))
+	}
+	v.Problems = append(v.Problems, problems...)
+
+	note := fmt.Sprintf("seqs %d-%d compared", lo, hi)
+	if explained > 0 {
+		note += fmt.Sprintf(", %d superseded or expired messages missing from some replicas", explained)
+	}
+	var ahead []string
+	for _, r := range views {
+		if r.d.LastSeq > hi {
+			ahead = append(ahead, fmt.Sprintf("%s +%d", r.host, r.d.LastSeq-hi))
+		}
+	}
+	if len(ahead) > 0 {
+		note += ", tail not compared: " + strings.Join(ahead, " ")
+	}
+	v.Notes = append(v.Notes, note)
+	return v
+}
+
+// absenceExplained reports whether a replica lacking msg removed it legitimately:
+// the subject was written again later, the message aged out before that replica
+// was copied, or a stream limit dropped it from a head the replica has moved past.
+func absenceExplained(r *replicaView, msg seqDigest) bool {
+	if msg.Subject == "" {
+		return false
+	}
+	if r.lastBySubject[msg.Subject] > msg.Seq {
+		return true
+	}
+	meta := r.d.streamMeta
+	if meta.Retention != "" && meta.Retention != "limits" {
+		return true
+	}
+	if !msg.Time.IsZero() {
+		if meta.MaxAge > 0 && msg.Time.Add(meta.MaxAge).Before(r.copiedAt) {
+			return true
+		}
+		if msg.TTL > 0 && msg.Time.Add(msg.TTL).Before(r.copiedAt) {
+			return true
+		}
+	}
+	return msg.Seq < r.d.FirstSeq && (meta.MaxMsgs > 0 || meta.MaxBytes > 0)
+}
+
+func hostsOf(views []*replicaView) string {
+	hs := make([]string, len(views))
+	for i, r := range views {
+		hs[i] = r.host
+	}
+	return strings.Join(hs, ",")
+}
+
+func writeCompareResult(w io.Writer, nodes []nodeDigest, res kvCompareResult) {
+	fmt.Fprintf(w, "compared %d digests:", len(nodes))
+	for _, n := range nodes {
+		fmt.Fprintf(w, " %s@%s", n.Host, n.CopiedAt.Format(time.TimeOnly))
+	}
+	fmt.Fprintln(w)
+	bad := 0
+	for _, v := range res.Streams {
+		status := "ok"
+		if len(v.Problems) > 0 {
+			status = "DIVERGED"
+			bad++
+		}
+		fmt.Fprintf(w, "%-9s %s  [%s]\n", status, v.Name, strings.Join(v.Holders, ","))
+		for _, p := range v.Problems {
+			fmt.Fprintf(w, "    ! %s\n", p)
+		}
+		for _, n := range v.Notes {
+			fmt.Fprintf(w, "      %s\n", n)
+		}
+	}
+	fmt.Fprintf(w, "%d streams: %d consistent, %d diverged\n", len(res.Streams), len(res.Streams)-bad, bad)
+}
@@ -0,0 +1,233 @@
+package cmd_test
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var t0 = time.Date(2026, 9, 15, 10, 0, 0, 0, time.UTC)
+
+type msg struct {
+	seq     uint64
+	subject string
+	hash    string // "deleted" for a gap
+	at      time.Time
+}
+
+type stream struct {
+	name     string
+	replicas int
+	maxAge   time.Duration
+	maxMsgs  int64
+	msgs     []msg
+	noSeqs   bool
+}
+
+// doc renders one node's digest the way "kv digest --json --seqs" prints it.
+func doc(t *testing.T, host string, copiedAt time.Time, streams ...stream) string {
+	t.Helper()
+	var out []map[string]any
+	for _, s := range streams {
+		var seqs []map[string]any
+		var first, last, count uint64
+		digest := host + ":" // only equality matters to compare
+		for _, m := range s.msgs {
+			if first == 0 {
+				first = m.seq
+			}
+			last = m.seq
+			e := map[string]any{"seq": m.seq, "hash": m.hash}
+			if m.hash != "deleted" {
+				count++
+				e["subject"] = m.subject
+				at := m.at
+				if at.IsZero() {
+					at = t0
+				}
+				e["time"] = at
+			}
+			digest += m.hash + ","
+			seqs = append(seqs, e)
+		}
+		sd := map[string]any{"name": s.name, "first_seq": first, "last_seq": last, "msgs": count, "sha256": digest}
+		if s.replicas > 0 {
+			sd["replicas"] = s.replicas
+		}
+		if s.maxAge > 0 {
+			sd["max_age_ns"] = s.maxAge
+		}
+		if s.maxMsgs > 0 {
+			sd["max_msgs"] = s.maxMsgs
+		}
+		if !s.noSeqs {
+			sd["seqs"] = seqs
+		}
+		out = append(out, sd)
+	}
+	raw, err := json.Marshal(map[string]any{"host": host, "store": "/s", "copied_at": copiedAt, "streams": out})
+	require.NoError(t, err)
+	return string(raw)
+}
+
+func kvs(msgs ...msg) []msg { return msgs }
+
+func TestKVCompareIdentical(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 3, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0})}
+	// The digest field embeds the host, so this exercises the per-seq path too.
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, s), doc(t, "n3", t0, s))
+	require.NoError(t, err)
+	assert.False(t, divergent, report)
+	assert.Contains(t, report, "1 streams: 1 consistent, 0 diverged")
+}
+
+// The prod failure: each node wrote its own copy before formation, so the same
+// sequence holds different content on different replicas.
+func TestKVCompareSameSeqDifferentContent(t *testing.T) {
+	a := stream{name: "KV_acct", replicas: 2, msgs: kvs(msg{1, "$KV.acct.000000000002", "aaaa", t0})}
+	b := stream{name: "KV_acct", replicas: 2, msgs: kvs(msg{1, "$KV.acct.000000000002", "bbbb", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "node1", t0, a), doc(t, "node2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "DIVERGED  KV_acct")
+	assert.Contains(t, report, "seq 1 $KV.acct.000000000002 holds different content: node1=aaaa node2=bbbb")
+}
+
+func TestKVCompareMissingMessage(t *testing.T) {
+	full := kvs(msg{1, "k", "h1", t0}, msg{2, "j", "h2", t0}, msg{3, "k", "h3", t0})
+
+	t.Run("superseded by a later write to the subject", func(t *testing.T) {
+		lagging := stream{name: "KV_a", replicas: 2, msgs: full}
+		purged := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "", "deleted", time.Time{}}, full[1], full[2])}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, lagging), doc(t, "n2", t0, purged))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+		assert.Contains(t, report, "1 superseded or expired")
+	})
+
+	t.Run("nothing explains it", func(t *testing.T) {
+		have := stream{name: "KV_a", replicas: 2, msgs: full}
+		lost := stream{name: "KV_a", replicas: 2, msgs: kvs(full[0], msg{2, "", "deleted", time.Time{}}, full[2])}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, lost))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "seq 2 j is on n1 but missing from n2")
+	})
+
+	t.Run("aged out under max_age before the replica was copied", func(t *testing.T) {
+		old := t0.Add(-time.Hour)
+		have := stream{name: "KV_lease", replicas: 2, maxAge: time.Minute, msgs: kvs(msg{1, "l", "h1", old}, msg{2, "m", "h2", t0})}
+		gone := stream{name: "KV_lease", replicas: 2, maxAge: time.Minute, msgs: kvs(msg{2, "m", "h2", t0})}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, gone))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+
+		have.maxAge, gone.maxAge = 0, 0
+		divergent, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, gone))
+		require.NoError(t, err)
+		assert.True(t, divergent, "without max_age the same gap is unexplained")
+	})
+
+	t.Run("head dropped by a stream limit", func(t *testing.T) {
+		behind := stream{name: "S", replicas: 2, maxMsgs: 2, msgs: kvs(msg{1, "a", "h1", t0}, msg{2, "b", "h2", t0})}
+		ahead := stream{name: "S", replicas: 2, maxMsgs: 2, msgs: kvs(msg{2, "b", "h2", t0}, msg{3, "c", "h3", t0})}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, behind), doc(t, "n2", t0, ahead))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+	})
+}
+
+func TestKVCompareTailLag(t *testing.T) {
+	behind := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0})}
+	ahead := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0}, msg{3, "k", "h3", t0}, msg{4, "k", "h4", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, behind), doc(t, "n2", t0, ahead))
+	require.NoError(t, err)
+	assert.False(t, divergent, report)
+	assert.Contains(t, report, "tail not compared: n2 +2")
+}
+
+func TestKVCompareNoCommonRange(t *testing.T) {
+	a := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0})}
+	b := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{5, "k", "h5", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, a), doc(t, "n2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "no sequence range is common")
+}
+
+func TestKVCompareReplicaPlacement(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 3, msgs: kvs(msg{1, "k", "h1", t0})}
+	other := stream{name: "KV_b", replicas: 1, msgs: kvs(msg{1, "k", "h1", t0})}
+
+	t.Run("fewer holders than replicas", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, s), doc(t, "n3", t0, other))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "held by 2 of 3 replicas; missing from n3")
+	})
+
+	t.Run("more holders than replicas", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, other), doc(t, "n2", t0, other))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "held by 2 nodes but configured for 1 replicas")
+	})
+
+	t.Run("replica counts disagree", func(t *testing.T) {
+		r1 := s
+		r1.replicas = 1
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, r1))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "replicas disagree on the replica count")
+	})
+
+	t.Run("single replica stream on one node", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, other), doc(t, "n2", t0))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+	})
+}
+
+func TestKVCompareNeedsSeqs(t *testing.T) {
+	a := stream{name: "KV_a", replicas: 2, noSeqs: true, msgs: kvs(msg{1, "k", "h1", t0})}
+	b := stream{name: "KV_a", replicas: 2, noSeqs: true, msgs: kvs(msg{1, "k", "h2", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, a), doc(t, "n2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "re-run digest with --seqs")
+}
+
+func TestKVCompareInput(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0})}
+
+	// Several documents in one stream, as "-" reads them from stdin.
+	divergent, _, err := cmd.CompareKVDigests(doc(t, "n1", t0, s) + "\n" + doc(t, "n2", t0, s))
+	require.NoError(t, err)
+	assert.False(t, divergent)
+
+	_, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, s))
+	assert.ErrorContains(t, err, "at least two nodes")
+
+	_, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n1", t0, s))
+	assert.ErrorContains(t, err, "two digests are from host n1")
+
+	_, _, err = cmd.CompareKVDigests(`KV_a	first=1	last=1`)
+	assert.Error(t, err, "text digest output is rejected")
+
+	_, _, err = cmd.CompareKVDigests(`{"streams":[]}`)
+	assert.ErrorContains(t, err, "no host")
+
+	_, _, err = cmd.CompareKVDigests("  ")
+	assert.ErrorContains(t, err, "no digest documents")
+}
+
+func TestAdminInitDiscardsJetStreamByDefault(t *testing.T) {
+	assert.Equal(t, "true", cmd.AdminInitFlagDefault("discard-jetstream"))
+	assert.True(t, strings.HasPrefix(cmd.AdminInitFlagDefault("ipsec"), "true"))
+}
@@ -0,0 +1,201 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/mulgadc/spinifex/spinifex/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/spf13/cobra"
+)
+
+var kvReplicasCmd = &cobra.Command{
+	Use:   "replicas",
+	Short: "Report how many nodes each KV bucket is replicated across",
+	Long: `Print one line per JetStream KV bucket: how many replicas it has, how many the
+cluster's size entitles it to, and which nodes currently hold it.
+
+A bucket with fewer replicas than the cluster has nodes lives on fewer nodes
+than exist, and every node depends on those. Losing one of them makes the
+bucket unreadable and unwritable from every node at once, which is a
+cluster-wide outage caused by a single-node event.
+
+A bucket also has to be able to serve, which is a different question from how
+many replicas it was configured for. ONLINE counts the replicas that are caught
+up and reachable; a bucket whose ONLINE is not a majority of REPLICAS cannot
+accept a write no matter what its replica count says.
+
+--repair raises any under-replicated bucket to the cluster's replica count,
+capped at JetStream's maximum of five. It changes nothing else about a bucket
+and never lowers a replica count, so it is safe to run on a healthy cluster and
+safe to run twice.
+
+Exit status separates the two problems, so this can gate a deployment without
+parsing the output:
+
+  0  every bucket is at the cluster's replica count and can serve
+  1  at least one bucket is under-replicated — --repair is the answer
+  2  at least one bucket has no quorum, or the report could not be produced —
+     a node needs looking at, and --repair will not help`,
+	Run: runKVReplicas,
+}
+
+func init() {
+	kvCmd.AddCommand(kvReplicasCmd)
+	kvReplicasCmd.Flags().Bool("repair", false, "raise every under-replicated bucket to the cluster's node count")
+	kvReplicasCmd.Flags().Bool("json", false, "emit the report as JSON")
+}
+
+func runKVReplicas(cmd *cobra.Command, _ []string) {
+	repair, _ := cmd.Flags().GetBool("repair")
+	asJSON, _ := cmd.Flags().GetBool("json")
+
+	// Anything that stops the report being produced exits unreachable rather
+	// than under-replicated: an empty answer is not a cluster with no buckets,
+	// and --repair is not the response to it.
+	_, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "connect to cluster: %v\n", err)
+		os.Exit(exitUnreachable)
+	}
+	defer nc.Close()
+
+	js, err := jetstream.New(nc)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "get JetStream context: %v\n", err)
+		os.Exit(exitUnreachable)
+	}
+
+	// Background: this runs at CLI top level, where there is no request to
+	// inherit a deadline from and the process exits after the one command.
+	ctx := context.Background()
+	reports, err := kvutil.AuditBucketReplicas(ctx, js)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "audit KV bucket replicas: %v\n", err)
+		os.Exit(exitUnreachable)
+	}
+
+	if repair {
+		repairReplicaReports(ctx, js, reports, os.Stderr)
+	}
+
+	if err := writeReplicaReport(os.Stdout, reports, asJSON); err != nil {
+		fmt.Fprintf(os.Stderr, "write report: %v\n", err)
+		os.Exit(exitUnreachable)
+	}
+
+	os.Exit(replicaReportExit(reports))
+}
+
+// Exit statuses, documented in the command's own help because a deployment gate
+// reads them rather than the output.
+const (
+	exitHealthy = 0
+	// exitUnderReplicated is the repairable problem: the buckets serve, but on
+	// fewer nodes than the cluster has.
+	exitUnderReplicated = 1
+	// exitUnreachable is the problem repair cannot touch — a bucket that cannot
+	// accept a write, or a report that could not be produced at all.
+	exitUnreachable = 2
+)
+
+// replicaReportExit is the exit status for a set of reports.
+//
+// No quorum outranks under-replicated because they need different responses: one
+// is fixed by raising a count, the other by finding out why a node is not
+// answering. A bucket that is both should send the operator to the node.
+func replicaReportExit(reports []kvutil.BucketReport) int {
+	status := exitHealthy
+	for _, r := range reports {
+		if !r.HasQuorum() {
+			return exitUnreachable
+		}
+		if r.UnderReplicated() {
+			status = exitUnderReplicated
+		}
+	}
+	return status
+}
+
+// repairReplicaReports raises every under-replicated bucket and updates reports
+// in place for the ones that were raised.
+//
+// One bucket that cannot be placed must not stop the others being repaired, so a
+// failure is named on errOut and the sweep continues. The bucket it failed on
+// stays UNDER-REPLICATED in the report, which is already what makes the exit
+// status non-zero, so nothing is lost by not returning an error here.
+func repairReplicaReports(ctx context.Context, js jetstream.JetStream, reports []kvutil.BucketReport, errOut io.Writer) {
+	for i, r := range reports {
+		if !r.UnderReplicated() {
+			continue
+		}
+		if err := kvutil.RaiseBucketReplicas(ctx, js, r.Bucket, r.Want); err != nil {
+			fmt.Fprintf(errOut, "repair %s: %v\n", r.Bucket, err)
+			continue
+		}
+		reports[i].Replicas = r.Want
+	}
+}
+
+// underReplicatedCount is how many buckets are below the cluster's replica count.
+func underReplicatedCount(reports []kvutil.BucketReport) int {
+	under := 0
+	for _, r := range reports {
+		if r.UnderReplicated() {
+			under++
+		}
+	}
+	return under
+}
+
+// noQuorumCount is how many buckets cannot currently accept a write.
+func noQuorumCount(reports []kvutil.BucketReport) int {
+	stuck := 0
+	for _, r := range reports {
+		if !r.HasQuorum() {
+			stuck++
+		}
+	}
+	return stuck
+}
+
+// replicaStatus is the one word for a bucket's condition.
+//
+// No quorum wins over under-replicated when both are true, matching the exit
+// status: a bucket that cannot serve is the more urgent of the two and is not
+// what --repair is for.
+func replicaStatus(r kvutil.BucketReport) string {
+	switch {
+	case !r.HasQuorum():
+		return "NO-QUORUM"
+	case r.UnderReplicated():
+		return "UNDER-REPLICATED"
+	default:
+		return "ok"
+	}
+}
+
+// writeReplicaReport renders reports as a table, or as JSON when asJSON.
+func writeReplicaReport(out io.Writer, reports []kvutil.BucketReport, asJSON bool) error {
+	if asJSON {
+		return json.NewEncoder(out).Encode(reports)
+	}
+
+	w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "BUCKET\tREPLICAS\tWANT\tONLINE\tSTATUS\tLEADER\tHELD BY")
+	for _, r := range reports {
+		fmt.Fprintf(w, "%s\t%d\t%d\t%d\t%s\t%s\t%s\n",
+			r.Bucket, r.Replicas, r.Want, r.Online, replicaStatus(r), r.Leader, strings.Join(r.Peers, ","))
+	}
+	if err := w.Flush(); err != nil {
+		return err
+	}
+	_, err := fmt.Fprintf(out, "\n%d buckets, %d under-replicated, %d without quorum\n",
+		len(reports), underReplicatedCount(reports), noQuorumCount(reports))
+	return err
+}
@@ -0,0 +1,133 @@
+//test:in-package — it tests this command's unexported rendering and repair
+// helpers, which exist only to be exercised without a live cluster.
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/clustersize"
+	"github.com/mulgadc/spinifex/spinifex/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/testutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWriteReplicaReport_NamesEveryUnderReplicatedBucket(t *testing.T) {
+	reports := []kvutil.BucketReport{
+		{Bucket: "healthy", Replicas: 3, Want: 3, Online: 3, Leader: "a", Peers: []string{"a", "b", "c"}},
+		{Bucket: "short", Replicas: 1, Want: 3, Online: 1, Leader: "a", Peers: []string{"a"}},
+		{Bucket: "stuck", Replicas: 3, Want: 3, Online: 1, Peers: []string{"a", "b", "c"}},
+	}
+
+	var out bytes.Buffer
+	require.NoError(t, writeReplicaReport(&out, reports, false))
+	got := out.String()
+
+	assert.Contains(t, got, "BUCKET")
+	assert.Contains(t, got, "HELD BY")
+	assert.Regexp(t, `healthy\s+3\s+3\s+3\s+ok\s+a\s+a,b,c`, got)
+	assert.Regexp(t, `short\s+1\s+3\s+1\s+UNDER-REPLICATED\s+a\s+a`, got)
+	// A bucket that cannot serve reads as NO-QUORUM even though its configured
+	// count is right, because raising a count is not the response to it.
+	assert.Regexp(t, `stuck\s+3\s+3\s+1\s+NO-QUORUM`, got)
+	assert.Contains(t, got, "3 buckets, 1 under-replicated, 1 without quorum")
+}
+
+// TestReplicaReportExit pins the three exit classes a deployment gate reads. They
+// are separate because the responses are: a raise fixes an under-replicated
+// bucket, and nothing the command can do fixes one that will not answer.
+func TestReplicaReportExit(t *testing.T) {
+	healthy := kvutil.BucketReport{Bucket: "a", Replicas: 3, Want: 3, Online: 3}
+	short := kvutil.BucketReport{Bucket: "b", Replicas: 1, Want: 3, Online: 1}
+	stuck := kvutil.BucketReport{Bucket: "c", Replicas: 3, Want: 3, Online: 1}
+
+	assert.Equal(t, exitHealthy, replicaReportExit(nil))
+	assert.Equal(t, exitHealthy, replicaReportExit([]kvutil.BucketReport{healthy}))
+	assert.Equal(t, exitUnderReplicated, replicaReportExit([]kvutil.BucketReport{healthy, short}))
+	assert.Equal(t, exitUnreachable, replicaReportExit([]kvutil.BucketReport{healthy, stuck}))
+	// Both problems at once sends the operator to the node rather than to --repair.
+	assert.Equal(t, exitUnreachable, replicaReportExit([]kvutil.BucketReport{short, stuck}))
+}
+
+// TestWriteReplicaReport_JSONIsWhatTheE2ESuiteDecodes keeps the machine-readable
+// form honest: the e2e suite and anything gating a deployment parse these field
+// names, so a rename here breaks them silently.
+func TestWriteReplicaReport_JSONIsWhatTheE2ESuiteDecodes(t *testing.T) {
+	var out bytes.Buffer
+	require.NoError(t, writeReplicaReport(&out, []kvutil.BucketReport{
+		{Bucket: "short", Replicas: 1, Want: 3, Cluster: "spinifex", Online: 1, Leader: "a", Peers: []string{"a"}},
+	}, true))
+
+	assert.Contains(t, out.String(), `"bucket":"short"`)
+
+	var decoded []kvutil.BucketReport
+	require.NoError(t, json.Unmarshal(out.Bytes(), &decoded))
+	require.Len(t, decoded, 1)
+	assert.Equal(t, "short", decoded[0].Bucket)
+	assert.Equal(t, 1, decoded[0].Replicas)
+	assert.Equal(t, 3, decoded[0].Want)
+	assert.Equal(t, 1, decoded[0].Online)
+	assert.Equal(t, "a", decoded[0].Leader)
+	assert.True(t, decoded[0].UnderReplicated())
+	assert.True(t, decoded[0].HasQuorum())
+}
+
+func TestUnderReplicatedCount(t *testing.T) {
+	assert.Equal(t, 0, underReplicatedCount(nil))
+	assert.Equal(t, 2, underReplicatedCount([]kvutil.BucketReport{
+		{Bucket: "a", Replicas: 1, Want: 3},
+		{Bucket: "b", Replicas: 3, Want: 3},
+		{Bucket: "c", Replicas: 2, Want: 3},
+	}))
+}
+
+// TestRepairReplicaReports_KeepsGoingPastABucketItCannotPlace is the behaviour
+// that makes --repair usable mid-growth: the buckets it can raise are raised,
+// and the one it cannot is named rather than aborting the run.
+func TestRepairReplicaReports_KeepsGoingPastABucketItCannotPlace(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	defer nc.Close()
+
+	for _, b := range []string{"repair-alpha", "repair-beta"} {
+		_, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: b, History: 1})
+		require.NoError(t, err)
+	}
+
+	// One embedded server cannot hold two replicas, so every raise fails — and
+	// every bucket must still have been attempted and named.
+	reports := []kvutil.BucketReport{
+		{Bucket: "repair-alpha", Replicas: 1, Want: 2},
+		{Bucket: "repair-beta", Replicas: 1, Want: 2},
+	}
+	var errOut bytes.Buffer
+	repairReplicaReports(t.Context(), js, reports, &errOut)
+
+	assert.Contains(t, errOut.String(), "repair repair-alpha")
+	assert.Contains(t, errOut.String(), "repair repair-beta")
+	assert.Equal(t, 2, underReplicatedCount(reports), "a failed raise must not be reported as repaired")
+}
+
+// TestRepairReplicaReports_RaisesWhatItCan covers the success side against a real
+// server, so the in-place report update is proved rather than assumed.
+func TestRepairReplicaReports_RaisesWhatItCan(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	defer nc.Close()
+	clustersize.DeclareForTest(t, 1)
+
+	_, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "repair-ok", History: 1})
+	require.NoError(t, err)
+
+	// Want 1 against a bucket already at 1: the raise is a no-op that must still
+	// leave the report reading as healthy rather than as repaired-and-wrong.
+	reports := []kvutil.BucketReport{{Bucket: "repair-ok", Replicas: 1, Want: 1}}
+	var errOut bytes.Buffer
+	repairReplicaReports(t.Context(), js, reports, &errOut)
+
+	assert.Empty(t, strings.TrimSpace(errOut.String()))
+	assert.Equal(t, 0, underReplicatedCount(reports))
+}
@@ -0,0 +1,319 @@
+//test:in-package: exercises the unexported digest helpers against a real clustered
+// store, which the cobra command only reaches through a live node.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// kvServer runs a standalone JetStream server on storeParent and returns a KV
+// bucket on it plus a stop function.
+func kvServer(t *testing.T, storeParent, bucket string) (jetstream.KeyValue, func()) {
+	t.Helper()
+	ns, err := server.NewServer(&server.Options{
+		ServerName: "kv-test", JetStream: true, StoreDir: storeParent,
+		DontListen: true, NoLog: true, NoSigs: true,
+	})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: bucket, History: 5})
+	require.NoError(t, err)
+	return kv, func() {
+		nc.Close()
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}
+}
+
+func digestOf(t *testing.T, storeDir string, withSeqs bool) []streamDigest {
+	t.Helper()
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(storeDir, filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	d, err := digestStore(context.Background(), work, nil, withSeqs, metas)
+	require.NoError(t, err)
+	return d
+}
+
+func TestKVDigest(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	kv, stop := kvServer(t, parent, "test")
+	for _, kvp := range [][2]string{{"a", "1"}, {"b", "2"}, {"a", "3"}} {
+		_, err := kv.PutString(ctx, kvp[0], kvp[1])
+		require.NoError(t, err)
+	}
+	require.NoError(t, kv.Delete(ctx, "b"))
+	store := filepath.Join(parent, "jetstream")
+
+	// Copied while the server is live, exactly as it runs on a node.
+	live := digestOf(t, store, true)
+	stop()
+
+	require.Len(t, live, 1)
+	assert.Equal(t, "KV_test", live[0].Name)
+	assert.Equal(t, uint64(1), live[0].FirstSeq)
+	assert.Equal(t, uint64(4), live[0].LastSeq)
+	assert.Equal(t, uint64(4), live[0].Msgs)
+	require.Len(t, live[0].Seqs, 4)
+
+	t.Run("stable across reads and a stopped server", func(t *testing.T) {
+		again := digestOf(t, store, false)
+		assert.Equal(t, live[0].Digest, again[0].Digest)
+	})
+
+	t.Run("source store is left untouched", func(t *testing.T) {
+		_, err := os.Stat(filepath.Join(store, "$G", "streams", "KV_test"))
+		assert.NoError(t, err)
+	})
+
+	// Same keys and values written separately differ in timestamps, which is
+	// exactly what separates two nodes' pre-formation writes.
+	t.Run("independently written store differs", func(t *testing.T) {
+		other := t.TempDir()
+		kv2, stop2 := kvServer(t, other, "test")
+		for _, kvp := range [][2]string{{"a", "1"}, {"b", "2"}, {"a", "3"}} {
+			_, err := kv2.PutString(ctx, kvp[0], kvp[1])
+			require.NoError(t, err)
+		}
+		require.NoError(t, kv2.Delete(ctx, "b"))
+		stop2()
+		d := digestOf(t, filepath.Join(other, "jetstream"), true)
+		assert.NotEqual(t, live[0].Digest, d[0].Digest)
+
+		// Two stores adopted as replicas of one stream: same subjects and
+		// sequences, different content, which compare must call divergent.
+		res, err := compareNodeDigests([]nodeDigest{
+			{Host: "node1", CopiedAt: time.Now(), Streams: live},
+			{Host: "node2", CopiedAt: time.Now(), Streams: d},
+		})
+		require.NoError(t, err)
+		require.True(t, res.divergent())
+		require.Len(t, res.Streams, 1)
+		assert.Contains(t, strings.Join(res.Streams[0].Problems, "\n"), "seq 1 $KV.test.a holds different content")
+	})
+
+	t.Run("report lines", func(t *testing.T) {
+		var buf bytes.Buffer
+		writeDigests(&buf, "node1", store, time.Unix(0, 0), []string{"ACC/S"}, live)
+		out := buf.String()
+		assert.Contains(t, out, "# host=node1 ")
+		assert.Contains(t, out, "# skipped account stream (not read): ACC/S")
+		assert.Contains(t, out, "KV_test\tfirst=1\tlast=4\tmsgs=4\tsha256="+live[0].Digest)
+		assert.Equal(t, 4, strings.Count(out, "KV_test\tseq="))
+	})
+}
+
+func TestKVDigestStreamFilterAndDeletedSeq(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	// The KV bucket is only there to be excluded by the stream filter.
+	_, stop := kvServer(t, parent, "other")
+	stop()
+
+	// KV streams deny message deletes, so an interior gap needs a plain stream.
+	ns, err := server.NewServer(&server.Options{JetStream: true, StoreDir: parent, DontListen: true, NoLog: true, NoSigs: true})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	s, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ONE", Subjects: []string{"one.>"}})
+	require.NoError(t, err)
+	for _, v := range []string{"v1", "v2", "v3"} {
+		_, err = js.Publish(ctx, "one.k", []byte(v))
+		require.NoError(t, err)
+	}
+	require.NoError(t, s.DeleteMsg(ctx, 2))
+	nc.Close()
+	ns.Shutdown()
+	ns.WaitForShutdown()
+
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(filepath.Join(parent, "jetstream"), filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	d, err := digestStore(ctx, work, []string{"ONE"}, true, metas)
+	require.NoError(t, err)
+	require.Len(t, d, 1)
+	assert.Equal(t, "ONE", d[0].Name)
+	assert.Equal(t, uint64(2), d[0].Msgs)
+	require.Len(t, d[0].Seqs, 3)
+	assert.Equal(t, seqDigest{Seq: 2, Hash: "deleted"}, d[0].Seqs[1])
+	for _, i := range []int{0, 2} {
+		assert.Equal(t, "one.k", d[0].Seqs[i].Subject)
+		assert.False(t, d[0].Seqs[i].Time.IsZero())
+	}
+	assert.Equal(t, "limits", d[0].Retention)
+	assert.Equal(t, 1, d[0].Replicas)
+}
+
+func freePort(t *testing.T) int {
+	t.Helper()
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	defer l.Close()
+	return l.Addr().(*net.TCPAddr).Port
+}
+
+// A replicated stream's meta.inf carries num_replicas > 1, which a standalone
+// server refuses to load; this is the case every production node presents.
+func TestKVDigestClusteredReplicas(t *testing.T) {
+	ctx := context.Background()
+	const n = 3
+	ports := make([]int, n)
+	routes := make([]string, n)
+	for i := range ports {
+		ports[i] = freePort(t)
+		routes[i] = fmt.Sprintf("nats://127.0.0.1:%d", ports[i])
+	}
+	stores := make([]string, n)
+	servers := make([]*server.Server, n)
+	for i := range servers {
+		stores[i] = t.TempDir()
+		ns, err := server.NewServer(&server.Options{
+			ServerName: fmt.Sprintf("s%d", i+1), JetStream: true, StoreDir: stores[i],
+			Host: "127.0.0.1", Port: -1, NoLog: true, NoSigs: true,
+			Cluster: server.ClusterOpts{Name: "digest", Host: "127.0.0.1", Port: ports[i]},
+			Routes:  server.RoutesFromStr(strings.Join(routes, ",")),
+		})
+		require.NoError(t, err)
+		ns.Start()
+		servers[i] = ns
+	}
+	t.Cleanup(func() {
+		for _, ns := range servers {
+			ns.Shutdown()
+			ns.WaitForShutdown()
+		}
+	})
+
+	nc, err := nats.Connect("", nats.InProcessServer(servers[0]))
+	require.NoError(t, err)
+	defer nc.Close()
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+
+	var kv jetstream.KeyValue
+	require.Eventually(t, func() bool {
+		kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "rep", History: 3, Replicas: n, TTL: time.Hour})
+		return err == nil
+	}, 30*time.Second, 250*time.Millisecond, "clustered KV create: %v", err)
+	for i := range 5 {
+		_, err := kv.PutString(ctx, fmt.Sprintf("k%d", i%2), fmt.Sprintf("v%d", i))
+		require.NoError(t, err)
+	}
+	s, err := js.Stream(ctx, "KV_rep")
+	require.NoError(t, err)
+	require.Eventually(t, func() bool {
+		info, err := s.Info(ctx)
+		if err != nil || info.Cluster == nil || len(info.Cluster.Replicas) != n-1 {
+			return false
+		}
+		for _, r := range info.Cluster.Replicas {
+			if !r.Current || r.Lag != 0 {
+				return false
+			}
+		}
+		return true
+	}, 30*time.Second, 250*time.Millisecond)
+
+	// Current means applied, not flushed; shutdown flushes every block to disk.
+	nc.Close()
+	for _, ns := range servers {
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}
+
+	var first string
+	var nodes []nodeDigest
+	for i, store := range stores {
+		d := digestOf(t, filepath.Join(store, "jetstream"), true)
+		require.Len(t, d, 1, "replica %d", i+1)
+		assert.Equal(t, uint64(5), d[0].Msgs, "replica %d", i+1)
+		assert.Equal(t, n, d[0].Replicas, "live replica count survives the standalone rewrite")
+		assert.Equal(t, time.Hour, d[0].MaxAge)
+		nodes = append(nodes, nodeDigest{Host: fmt.Sprintf("s%d", i+1), CopiedAt: time.Now(), Streams: d})
+		if i == 0 {
+			first = d[0].Digest
+			continue
+		}
+		assert.Equal(t, first, d[0].Digest, "replica %d matches replica 1", i+1)
+	}
+	res, err := compareNodeDigests(nodes)
+	require.NoError(t, err)
+	assert.False(t, res.divergent(), "%+v", res)
+}
+
+// Recovery applies MaxAge, so without the rewrite an expired bucket copy
+// would read as empty and mask whatever the replica holds.
+func TestKVDigestIgnoresExpiry(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	ns, err := server.NewServer(&server.Options{JetStream: true, StoreDir: parent, DontListen: true, NoLog: true, NoSigs: true})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "ttl", TTL: time.Second})
+	require.NoError(t, err)
+	_, err = kv.PutString(ctx, "k", "v")
+	require.NoError(t, err)
+
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(filepath.Join(parent, "jetstream"), filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	nc.Close()
+	ns.Shutdown()
+	ns.WaitForShutdown()
+
+	time.Sleep(1500 * time.Millisecond)
+	d, err := digestStore(ctx, work, nil, false, metas)
+	require.NoError(t, err)
+	require.Len(t, d, 1)
+	assert.Equal(t, uint64(1), d[0].Msgs)
+}
+
+func TestCopyStreamTree(t *testing.T) {
+	store := seedStore(t, "KV_a")
+	require.NoError(t, os.MkdirAll(filepath.Join(store, "$G", "streams", "KV_a", "obs", "c1"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(store, "$G", "streams", "KV_a", "obs", "c1", "o.dat"), []byte("x"), 0o600))
+	require.NoError(t, os.MkdirAll(filepath.Join(store, "ACME", "streams", "ORDERS"), 0o755))
+
+	dst := filepath.Join(t.TempDir(), "jetstream")
+	skipped, _, err := copyStreamTree(store, dst)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"ACME/ORDERS"}, skipped)
+
+	_, err = os.Stat(filepath.Join(dst, "$G", "streams", "KV_a", "msgs", "1.blk"))
+	assert.NoError(t, err, "message blocks are copied")
+	_, err = os.Stat(filepath.Join(dst, "$G", "streams", "KV_a", "obs"))
+	assert.True(t, os.IsNotExist(err), "consumer state is left out")
+
+	_, _, err = copyStreamTree(filepath.Join(t.TempDir(), "absent"), dst)
+	assert.Error(t, err)
+}
@@ -0,0 +1,39 @@
+//test:in-package — validateNodeCount is an unexported guard on this command's
+// flags, and exporting it would widen the package's API for a test's convenience.
+
+package cmd
+
+import (
+	"strings"
+	"testing"
+)
+
+// TestValidateNodeCount pins the supported cluster sizes. Two is rejected rather
+// than warned about: on two servers no layer of the stack has a majority to lose,
+// and the control plane's own state has no replica count that both survives a node
+// going away and still accepts writes.
+func TestValidateNodeCount(t *testing.T) {
+	for _, nodes := range []int{1, 3, 4, 5, 9} {
+		if err := validateNodeCount(nodes); err != nil {
+			t.Errorf("validateNodeCount(%d) = %v, want nil", nodes, err)
+		}
+	}
+
+	for _, nodes := range []int{0, -1} {
+		if err := validateNodeCount(nodes); err == nil {
+			t.Errorf("validateNodeCount(%d) returned no error", nodes)
+		}
+	}
+
+	err := validateNodeCount(2)
+	if err == nil {
+		t.Fatal("validateNodeCount(2) returned no error")
+	}
+	// The message has to say what to do instead, because the operator typed a
+	// number and needs to know which one to type next.
+	for _, want := range []string{"2", "1", "3"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("error %q does not mention %s", err, want)
+		}
+	}
+}
@@ -854,7 +854,7 @@ func runOchreWeightsStage(cmd *cobra.Command, _ []string) {
 		ochreExit(1)
 		return
 	}
-	weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
+	weightsStore := gateway_bedrock.NewWeightsStore(js)
 	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)

 	provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout)
@@ -930,7 +930,7 @@ func runOchreWeightsList(_ *cobra.Command, _ []string) {
 		ochreExit(1)
 		return
 	}
-	weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
+	weightsStore := gateway_bedrock.NewWeightsStore(js)
 	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
 	checkSnapshotLive := newWeightsSnapshotChecker(store, node.Predastore.Bucket)

@@ -967,7 +967,7 @@ func removeWeights(ctx context.Context, weightsStore *gateway_bedrock.WeightsSto
 func runOchreWeightsRemove(cmd *cobra.Command, _ []string) {
 	modelID, _ := cmd.Flags().GetString("model-id")

-	appConfig, nc, err := loadConfigAndConnectFn()
+	_, nc, err := loadConfigAndConnectFn()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 		ochreExit(1)
@@ -981,7 +981,7 @@ func runOchreWeightsRemove(cmd *cobra.Command, _ []string) {
 		ochreExit(1)
 		return
 	}
-	weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
+	weightsStore := gateway_bedrock.NewWeightsStore(js)

 	entry, err := removeWeights(context.Background(), weightsStore, modelID)
 	if err != nil {
@@ -1014,7 +1014,7 @@ func resolveHFToken(ctx context.Context, cmd *cobra.Command, cfg *config.Cluster
 	if err != nil {
 		return ""
 	}
-	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil)
+	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, nil)
 	token, ok, err := credStore.Resolve(ctx, utils.GlobalAccountID, vendorHuggingFace)
 	if err != nil || !ok {
 		return ""
@@ -1088,7 +1088,7 @@ func ochreCredentialsStore() (*gateway_bedrock.CredentialStore, func(), error) {
 		nc.Close()
 		return nil, nil, fmt.Errorf("jetstream context: %w", err)
 	}
-	return gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil), func() { nc.Close() }, nil
+	return gateway_bedrock.NewCredentialStore(js, masterKey, nil), func() { nc.Close() }, nil
 }

 func runOchreCredentialsSet(cmd *cobra.Command, _ []string) {
@@ -1186,7 +1186,7 @@ func init() {
 // ochreAccessStore connects to the cluster and returns the grant store along
 // with a cleanup that closes the connection.
 func ochreAccessStore() (*gateway_bedrock.ModelAccessStore, func(), error) {
-	cfg, nc, err := loadConfigAndConnectFn()
+	_, nc, err := loadConfigAndConnectFn()
 	if err != nil {
 		return nil, nil, fmt.Errorf("connect to cluster: %w", err)
 	}
@@ -1195,7 +1195,7 @@ func ochreAccessStore() (*gateway_bedrock.ModelAccessStore, func(), error) {
 		nc.Close()
 		return nil, nil, fmt.Errorf("jetstream context: %w", err)
 	}
-	return gateway_bedrock.NewModelAccessStore(js, len(cfg.Nodes)), func() { nc.Close() }, nil
+	return gateway_bedrock.NewModelAccessStore(js), func() { nc.Close() }, nil
 }

 // ochreTargetModels resolves the --model-id / --all-models pair into the model
@@ -470,7 +470,7 @@ func TestResolveHFToken_FallsBackToStoredPlatformCredential(t *testing.T) {
 	require.NoError(t, os.MkdirAll(filepath.Join(baseDir, "config"), 0750))
 	require.NoError(t, handlers_iam.SaveMasterKey(filepath.Join(baseDir, "config", "master.key"), masterKey))

-	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, 1, nil)
+	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, nil)
 	require.NoError(t, credStore.PutCredential(context.Background(), utils.GlobalAccountID, vendorHuggingFace, "hf_stored_token"))

 	cmd := newOchreWeightsPullTestCmd(t, selfHostModelID, testHFRepo, "main", "", "")
@@ -621,7 +621,7 @@ func TestRunOchreCredentialsSet_RoundTripsAndDefaultsToPlatformAccount(t *testin
 	defer verifyConn.Close()
 	verifyJS := testutil.NewJetStream(t, verifyConn)

-	credStore := gateway_bedrock.NewCredentialStore(verifyJS, masterKey, 1, nil)
+	credStore := gateway_bedrock.NewCredentialStore(verifyJS, masterKey, nil)
 	token, ok, err := credStore.Resolve(context.Background(), utils.GlobalAccountID, vendorHuggingFace)
 	require.NoError(t, err)
 	require.True(t, ok)
@@ -42,7 +42,7 @@ const (
 func newWeightsStoreForTest(t *testing.T) *gateway_bedrock.WeightsStore {
 	t.Helper()
 	_, nc, _ := testutil.StartTestJetStream(t)
-	return gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, nc), 1)
+	return gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, nc))
 }

 // explodingObjectStore fails the test immediately if any of its methods is
@@ -775,7 +775,7 @@ func TestRunOchreWeightsList_PrintsStagedModels(t *testing.T) {
 	}
 	stubConnect(t, cfg, nc, nil)

-	seedStore := gateway_bedrock.NewWeightsStore(js, 1)
+	seedStore := gateway_bedrock.NewWeightsStore(js)
 	require.NoError(t, seedStore.PutWeights(context.Background(), selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001"))

 	out := captureStdout(t, func() { runOchreWeightsList(nil, nil) })
@@ -822,7 +822,7 @@ func TestRunOchreWeightsRemove_RoundTripPrintsSnapshotAndDropsEntry(t *testing.T
 	ns, nc, js := testutil.StartTestJetStream(t)
 	stubConnect(t, fakeClusterConfig(), nc, nil)

-	seedStore := gateway_bedrock.NewWeightsStore(js, 1)
+	seedStore := gateway_bedrock.NewWeightsStore(js)
 	require.NoError(t, seedStore.PutWeights(context.Background(), selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001"))

 	cmd := newOchreWeightsRemoveTestCmd(t, selfHostModelID)
@@ -835,7 +835,7 @@ func TestRunOchreWeightsRemove_RoundTripPrintsSnapshotAndDropsEntry(t *testing.T
 	verifyNC, err := nats.Connect(ns.ClientURL())
 	require.NoError(t, err)
 	t.Cleanup(func() { verifyNC.Close() })
-	verifyStore := gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, verifyNC), 1)
+	verifyStore := gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, verifyNC))

 	_, ok, err := verifyStore.GetWeights(context.Background(), selfHostModelID)
 	require.NoError(t, err)
@@ -2,6 +2,8 @@ package cmd

 import (
 	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"net/netip"
@@ -9,11 +11,13 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"

 	"github.com/mulgadc/northstar/pkg/backend"
 	nsconfig "github.com/mulgadc/northstar/pkg/config"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	"github.com/mulgadc/spinifex/spinifex/formation"
 	handlers_dns "github.com/mulgadc/spinifex/spinifex/handlers/dns"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
@@ -734,21 +738,45 @@ func TestSpinifexTomlTemplate_ExternalPoolStaticSource(t *testing.T) {
 	assert.Contains(t, content, `gateway     = "192.168.1.1"`)
 }

-// Formation joiners must receive PoolBindBridge from the init node so the
+// Formation joiners must receive BindBridge from the init node so the
 // cluster-wide config reaches every chassis intact.
 func TestApplyNetworkConfig_PropagatesPoolBindBridge(t *testing.T) {
 	settings := &admin.ConfigSettings{}
 	nc := &formation.NetworkConfig{
-		ExternalMode:   "pool",
-		PoolName:       "wan",
-		PoolSource:     "dhcp",
-		PoolBindBridge: "br-wan",
-		PoolPrefixLen:  24,
+		ExternalMode: "pool",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "dhcp", BindBridge: "br-wan", PrefixLen: 24,
+		}},
 	}
 	applyNetworkConfig(settings, nc)
 	require.Len(t, settings.Pools, 1)
 	assert.Equal(t, "dhcp", settings.Pools[0].Source)
 	assert.Equal(t, "br-wan", settings.Pools[0].BindBridge)
+	assert.Empty(t, settings.BridgeMode, "bridged modes stay auto-detected, only nat is pinned")
+}
+
+// Routed NAT renders two pools, and the transit one carries a gateway-LRP range
+// that pool mode has no equivalent for. A joiner that received only the first,
+// or lost the range, would allocate VPC gateway addresses from nowhere.
+func TestApplyNetworkConfig_PropagatesEveryPool(t *testing.T) {
+	settings := &admin.ConfigSettings{}
+	nc := &formation.NetworkConfig{
+		ExternalMode: "nat",
+		Pools: []admin.PoolData{
+			{Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
+				GwLrpRangeStart: "100.127.0.16", GwLrpRangeEnd: "100.127.0.254"},
+			{Name: "wan", Source: "static", Start: "192.168.1.150", End: "192.168.1.200",
+				Gateway: "192.168.1.1", PrefixLen: 24},
+		},
+	}
+	applyNetworkConfig(settings, nc)
+	require.Len(t, settings.Pools, 2)
+	assert.Equal(t, "nat", settings.BridgeMode,
+		"a joiner that loses bridge_mode falls back to autodetection for a mode init pinned")
+	assert.Equal(t, "nat-transit", settings.Pools[0].Name)
+	assert.Equal(t, "100.127.0.16", settings.Pools[0].GwLrpRangeStart)
+	assert.Equal(t, "100.127.0.254", settings.Pools[0].GwLrpRangeEnd)
+	assert.Equal(t, "wan", settings.Pools[1].Name)
 }

 func renderSingleNodePredastore(t *testing.T, settings admin.ConfigSettings) string {
@@ -1288,7 +1316,218 @@ func TestAMIVolumeSizeGiB(t *testing.T) {
 	}
 }

+func TestValidateChecksumFlags(t *testing.T) {
+	tests := []struct {
+		name       string
+		checksum   string
+		unset      bool
+		file       string
+		skipVerify bool
+		wantErr    string
+	}{
+		{name: "no checksum", unset: true, file: "img.raw"},
+		{name: "checksum with file", checksum: "SHA256SUMS", file: "img.raw"},
+		{name: "empty checksum value", checksum: "", file: "img.raw", wantErr: "--checksum requires a sums file path"},
+		{name: "blank checksum value", checksum: "  ", file: "img.raw", wantErr: "--checksum requires a sums file path"},
+		{name: "checksum without file", checksum: "SHA256SUMS", wantErr: "--checksum requires --file"},
+		{name: "checksum with skip-verify", checksum: "SHA256SUMS", file: "img.raw", skipVerify: true,
+			wantErr: "mutually exclusive"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateChecksumFlags(tt.checksum, !tt.unset, tt.file, tt.skipVerify)
+			if tt.wantErr == "" {
+				assert.NoError(t, err)
+				return
+			}
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tt.wantErr)
+		})
+	}
+}
+
+func TestSourceDigestMode(t *testing.T) {
+	tests := []struct {
+		name       string
+		localFile  string
+		checksum   string
+		skipVerify bool
+		want       string
+	}{
+		{name: "file with checksum", localFile: "img.raw", checksum: "SHA256SUMS", want: ebsmetadata.DigestOperator},
+		{name: "catalog download verified", want: ebsmetadata.DigestCatalog},
+		{name: "catalog download skip-verify", skipVerify: true, want: ebsmetadata.DigestUnverified},
+		{name: "file without checksum", localFile: "img.raw", want: ebsmetadata.DigestUnverified},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, sourceDigestMode(tt.localFile, tt.checksum, tt.skipVerify))
+		})
+	}
+}
+
+func TestImportSourceResolveDigest(t *testing.T) {
+	imgBytes := []byte("import-source-fixture")
+	sum := sha256.Sum256(imgBytes)
+	imgHex := hex.EncodeToString(sum[:])
+	dir := t.TempDir()
+	img := filepath.Join(dir, "image.raw")
+	require.NoError(t, os.WriteFile(img, imgBytes, 0o600))
+	sums := filepath.Join(dir, "SHA256SUMS")
+
+	t.Run("operator verified", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		got, err := importSource{
+			imageFile: img, localFile: img,
+			checksumPath: sums, checksumAlgo: "sha256", expectedDigest: imgHex,
+		}.resolveDigest(&out, &errOut)
+		require.NoError(t, err)
+		assert.Equal(t, ebsmetadata.ImageDigest{
+			Algorithm: "sha256", Value: imgHex, Verification: ebsmetadata.DigestOperator,
+			Source: "SHA256SUMS", Filename: "image.raw",
+		}, got)
+		assert.Contains(t, out.String(), "Verified image checksum (sha256)")
+		assert.Empty(t, errOut.String())
+	})
+
+	t.Run("operator mismatch names the file and sums file", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		_, err := importSource{
+			imageFile: img, localFile: img,
+			checksumPath: sums, checksumAlgo: "sha256", expectedDigest: strings.Repeat("0", 64),
+		}.resolveDigest(&out, &errOut)
+		require.ErrorIs(t, err, utils.ErrChecksumMismatch)
+		assert.Contains(t, errOut.String(), "file:     "+img)
+		assert.Contains(t, errOut.String(), "checksum: "+sums)
+		assert.Contains(t, errOut.String(), imgHex)
+		assert.FileExists(t, img, "the operator's file must be left in place")
+	})
+
+	t.Run("local file without checksum is recorded unverified", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		got, err := importSource{imageFile: img, localFile: img}.resolveDigest(&out, &errOut)
+		require.NoError(t, err)
+		assert.Equal(t, ebsmetadata.ImageDigest{
+			Algorithm: "sha256", Value: imgHex, Verification: ebsmetadata.DigestUnverified, Filename: "image.raw",
+		}, got)
+		assert.Equal(t, "Importing "+img+" without checksum validation (pass --checksum <sums-file> to verify)\n",
+			errOut.String())
+	})
+
+	t.Run("catalog skip-verify is recorded unverified", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		got, err := importSource{imageFile: img, imageName: "debian-13-x86_64", skipVerify: true}.resolveDigest(&out, &errOut)
+		require.NoError(t, err)
+		assert.Equal(t, ebsmetadata.DigestUnverified, got.Verification)
+		assert.Equal(t, imgHex, got.Value)
+		assert.Contains(t, errOut.String(), "--skip-verify set")
+	})
+
+	t.Run("catalog entry without checksum is refused", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		_, err := importSource{imageFile: img, imageName: "debian-13-x86_64"}.resolveDigest(&out, &errOut)
+		require.Error(t, err)
+		assert.Contains(t, errOut.String(), "missing Checksum/ChecksumDigest/ChecksumType")
+	})
+
+	// Oracle publishes its digests inline on a web page and ships no sums file,
+	// so a pinned digest is the only way its entries can import verified.
+	t.Run("catalog pinned digest verifies without fetching a sums file", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		got, err := importSource{
+			imageFile: img, imageName: "oracle-10.1-x86_64",
+			image: utils.Images{ChecksumType: "sha256", ChecksumDigest: imgHex},
+		}.resolveDigest(&out, &errOut)
+		require.NoError(t, err)
+		assert.Equal(t, ebsmetadata.ImageDigest{
+			Algorithm: "sha256", Value: imgHex, Source: catalogPinnedDigestSource,
+			Verification: ebsmetadata.DigestCatalog, Filename: "image.raw",
+		}, got)
+		assert.Contains(t, out.String(), "pinned catalog digest")
+	})
+
+	t.Run("catalog pinned digest mismatch is refused and names the url", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		wrong := strings.Repeat("a", 64)
+		_, err := importSource{
+			imageFile: img, imageName: "oracle-10.1-x86_64",
+			image: utils.Images{ChecksumType: "sha256", ChecksumDigest: wrong, URL: "https://example.invalid/ol.qcow2"},
+		}.resolveDigest(&out, &errOut)
+		require.ErrorIs(t, err, utils.ErrChecksumMismatch)
+		assert.Contains(t, errOut.String(), catalogPinnedDigestSource)
+		assert.Contains(t, errOut.String(), "https://example.invalid/ol.qcow2")
+		assert.FileExists(t, img, "the downloaded file must be left in place")
+	})
+
+	t.Run("unreadable image fails the unverified hash", func(t *testing.T) {
+		var out, errOut bytes.Buffer
+		missing := filepath.Join(dir, "missing.raw")
+		_, err := importSource{imageFile: missing, localFile: missing}.resolveDigest(&out, &errOut)
+		require.Error(t, err)
+		assert.Contains(t, errOut.String(), "Could not hash image")
+	})
+}
+
+func TestImagesDescribeCmd_FlagSchema(t *testing.T) {
+	imageIDFlag := imagesDescribeCmd.Flags().Lookup("image-id")
+	require.NotNil(t, imageIDFlag, "--image-id must be defined")
+	assert.Equal(t, []string{"true"}, imageIDFlag.Annotations[cobraRequiredAnnotation],
+		"--image-id must be marked required")
+}
+
+func TestPrintAMIDescription(t *testing.T) {
+	base := ebsmetadata.AMI{
+		ImageID:         "ami-1",
+		Name:            "ami-debian-13-x86_64",
+		ImageOwnerAlias: "system",
+		CreationDate:    time.Date(2026, 9, 23, 1, 2, 3, 0, time.UTC),
+		BootMode:        "uefi",
+		Architecture:    "x86_64",
+	}
+
+	t.Run("digest recorded", func(t *testing.T) {
+		meta := base
+		meta.SourceDigest = &ebsmetadata.ImageDigest{
+			Algorithm: "sha512", Value: "abc123", Verification: ebsmetadata.DigestOperator,
+			Source: "SHA512SUMS", Filename: "debian-13-generic-amd64.tar.xz",
+		}
+		var buf bytes.Buffer
+		printAMIDescription(&buf, meta)
+		out := buf.String()
+		assert.Contains(t, out, "Image ID:       ami-1")
+		assert.Contains(t, out, "Created:        2026-09-23T01:02:03Z")
+		assert.Contains(t, out, "Boot mode:      uefi")
+		assert.Contains(t, out, "Source digest:  sha512:abc123")
+		assert.Contains(t, out, "Verification:   operator (SHA512SUMS)")
+		assert.Contains(t, out, "Source file:    debian-13-generic-amd64.tar.xz")
+	})
+
+	t.Run("unverified digest has no source", func(t *testing.T) {
+		meta := base
+		meta.SourceDigest = &ebsmetadata.ImageDigest{
+			Algorithm: "sha256", Value: "def456", Verification: ebsmetadata.DigestUnverified,
+			Filename: "image.raw",
+		}
+		var buf bytes.Buffer
+		printAMIDescription(&buf, meta)
+		assert.Contains(t, buf.String(), "Verification:   unverified\n")
+	})
+
+	t.Run("digest not recorded", func(t *testing.T) {
+		var buf bytes.Buffer
+		printAMIDescription(&buf, base)
+		out := buf.String()
+		assert.Contains(t, out, "Source digest:  not recorded (imported before digest recording)")
+		assert.NotContains(t, out, "Verification:")
+	})
+}
+
 func TestCheckJoinPreconditions(t *testing.T) {
+	withFakeProc(t, nil)
+	noStore := filepath.Join(t.TempDir(), "nats", "jetstream")
+
 	writeToml := func(t *testing.T) string {
 		t.Helper()
 		dir := t.TempDir()
@@ -1299,13 +1538,13 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	}

 	t.Run("uninitialized node joins", func(t *testing.T) {
-		if err := checkJoinPreconditions(t.TempDir(), false); err != nil {
+		if err := checkJoinPreconditions(t.TempDir(), noStore, false); err != nil {
 			t.Fatalf("want nil, got %v", err)
 		}
 	})

 	t.Run("initialized node is refused", func(t *testing.T) {
-		err := checkJoinPreconditions(writeToml(t), false)
+		err := checkJoinPreconditions(writeToml(t), noStore, false)
 		if err == nil {
 			t.Fatal("want refusal, got nil")
 		}
@@ -1317,7 +1556,7 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	// The guidance is the whole point of the guard: an operator must be able to
 	// tell what a forced join would destroy without reading the source.
 	t.Run("guidance names what is discarded", func(t *testing.T) {
-		for _, want := range []string{"CA certificate", "master key", "viperblock key", "--force"} {
+		for _, want := range []string{"CA certificate", "master key", "viperblock key", "JetStream store", "--force"} {
 			if !strings.Contains(joinDiscardsIdentityMsg, want) {
 				t.Errorf("guidance missing %q", want)
 			}
@@ -1325,10 +1564,27 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	})

 	t.Run("force overrides", func(t *testing.T) {
-		if err := checkJoinPreconditions(writeToml(t), true); err != nil {
+		if err := checkJoinPreconditions(writeToml(t), noStore, true); err != nil {
 			t.Fatalf("want nil with force, got %v", err)
 		}
 	})
+
+	// A store with streams and no config is still data the join would destroy.
+	t.Run("uninitialized node with streams is refused", func(t *testing.T) {
+		err := checkJoinPreconditions(t.TempDir(), seedStore(t, "KV_a"), false)
+		if err == nil || !strings.Contains(err.Error(), "JetStream store") {
+			t.Fatalf("want JetStream refusal, got %v", err)
+		}
+	})
+
+	// --force accepts the loss of data, never a wipe under a live server.
+	t.Run("running NATS is refused even with force", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"4242": "/usr/local/bin/spx\x00service\x00nats\x00start\x00"})
+		err := checkJoinPreconditions(writeToml(t), seedStore(t, "KV_a"), true)
+		if !errors.Is(err, errNATSRunning) {
+			t.Fatalf("want errNATSRunning, got %v", err)
+		}
+	})
 }

 func TestJoinRetryable(t *testing.T) {
@@ -167,7 +167,7 @@ func runClusterShutdown(cmd *cobra.Command, args []string) {
 	fmt.Printf("Timeout per phase: %s\n\n", timeout)

 	// Write cluster shutdown marker to KV
-	jsm, err := daemon.NewJetStreamManager(nc, 1)
+	jsm, err := daemon.NewJetStreamManager(nc)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: failed to create JetStream manager: %v\n", err)
 	} else {
@@ -4,8 +4,10 @@
 package cmd

 import (
+	"bytes"
 	"errors"
 	"strconv"
+	"strings"
 	"time"

 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
@@ -146,3 +148,28 @@ var PrintAccountTable = printAccountTable

 // AccountSummaries exposes accountSummaries for testing.
 var AccountSummaries = accountSummaries
+
+// CompareKVDigests runs "kv compare" over digest JSON documents and returns
+// whether any stream diverged, plus the printed report.
+func CompareKVDigests(docs ...string) (bool, string, error) {
+	var nodes []nodeDigest
+	for i, doc := range docs {
+		got, err := loadNodeDigests(strings.NewReader(doc), "doc"+strconv.Itoa(i))
+		if err != nil {
+			return false, "", err
+		}
+		nodes = append(nodes, got...)
+	}
+	res, err := compareNodeDigests(nodes)
+	if err != nil {
+		return false, "", err
+	}
+	var buf bytes.Buffer
+	writeCompareResult(&buf, nodes, res)
+	return res.divergent(), buf.String(), nil
+}
+
+// AdminInitFlagDefault returns the registered default of an admin init flag.
+func AdminInitFlagDefault(name string) string {
+	return adminInitCmd.Flags().Lookup(name).DefValue
+}
@@ -0,0 +1,139 @@
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// jetStreamStoreDir is where the NATS service keeps its JetStream data under a
+// node's data directory. The nats.conf template sets store_dir to
+// <data_dir>/nats/, and nats-server appends "jetstream" itself.
+func jetStreamStoreDir(dataDir string) string {
+	return filepath.Join(dataDir, "nats", "jetstream")
+}
+
+// localStreams lists the streams a JetStream store holds on disk, as
+// "<account>/<stream>". A missing store holds nothing.
+func localStreams(storeDir string) ([]string, error) {
+	accounts, err := os.ReadDir(storeDir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read JetStream store %s: %w", storeDir, err)
+	}
+	var streams []string
+	for _, acc := range accounts {
+		if !acc.IsDir() {
+			continue
+		}
+		entries, err := os.ReadDir(filepath.Join(storeDir, acc.Name(), "streams"))
+		if errors.Is(err, os.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read streams for account %s: %w", acc.Name(), err)
+		}
+		for _, e := range entries {
+			if e.IsDir() {
+				streams = append(streams, acc.Name()+"/"+e.Name())
+			}
+		}
+	}
+	sort.Strings(streams)
+	return streams, nil
+}
+
+// procRoot is overridable in tests so the process scan can run against a fake
+// /proc tree.
+var procRoot = "/proc"
+
+// natsRunningLocally reports whether this host runs the Spinifex NATS service,
+// found as a process whose command line is `spx service nats ...`. Removing a
+// store under a live server races JetStream, which rewrites its index files.
+func natsRunningLocally() (bool, error) {
+	entries, err := os.ReadDir(procRoot)
+	if err != nil {
+		return false, fmt.Errorf("scan processes: %w", err)
+	}
+	for _, e := range entries {
+		if !e.IsDir() || strings.TrimLeft(e.Name(), "0123456789") != "" {
+			continue
+		}
+		raw, err := os.ReadFile(filepath.Join(procRoot, e.Name(), "cmdline"))
+		if err != nil || len(raw) == 0 {
+			continue
+		}
+		if isSpxNATSService(bytes.Split(bytes.TrimRight(raw, "\x00"), []byte{0})) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// isSpxNATSService matches `spx service nats` with any leading path on spx.
+func isSpxNATSService(argv [][]byte) bool {
+	return len(argv) >= 3 &&
+		filepath.Base(string(argv[0])) == "spx" &&
+		string(argv[1]) == "service" &&
+		string(argv[2]) == "nats"
+}
+
+// errNATSRunning is returned whenever a store would be discarded with the NATS
+// service still up. No flag overrides it: the wipe would race the live server.
+var errNATSRunning = errors.New("the NATS service is still running on this node; stop it first (sudo systemctl stop spinifex.target, then confirm `pgrep -af 'spx service'` prints nothing)")
+
+// checkStoreDiscardable fails if the NATS service is running locally.
+func checkStoreDiscardable() error {
+	running, err := natsRunningLocally()
+	if err != nil {
+		return err
+	}
+	if running {
+		return errNATSRunning
+	}
+	return nil
+}
+
+// discardJetStreamStore removes a node's JetStream store before it enters a
+// multi-node cluster. nats-server adopts a same-named local stream and raft
+// catch-up never repairs the prefix it already holds.
+func discardJetStreamStore(storeDir string) error {
+	if err := checkStoreDiscardable(); err != nil {
+		return err
+	}
+	if err := os.RemoveAll(storeDir); err != nil {
+		return fmt.Errorf("remove JetStream store %s: %w", storeDir, err)
+	}
+	if _, err := os.Stat(storeDir); !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("JetStream store %s still present after removal", storeDir)
+	}
+	return nil
+}
+
+// checkInitJetStreamStore refuses a multi-node init over a store holding streams
+// when discard was turned off. Keeping it as a seed is not safe: leadership
+// ignores who holds data, and unassigned local streams are deleted as orphans.
+func checkInitJetStreamStore(storeDir string, nodes int, discard bool) error {
+	if nodes < 2 {
+		return nil
+	}
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return err
+	}
+	if len(streams) == 0 {
+		return nil
+	}
+	if !discard {
+		return fmt.Errorf("this node's JetStream store %s holds %d stream(s) from before formation; "+
+			"a multi-node cluster cannot adopt them consistently. "+
+			"re-run without --discard-jetstream=false to remove them once every node has joined", storeDir, len(streams))
+	}
+	return checkStoreDiscardable()
+}
@@ -0,0 +1,140 @@
+//test:in-package: swaps the unexported procRoot for a fake /proc tree and asserts on
+// the store helpers the init and join commands call.
+
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// withFakeProc points the process scan at a temp tree holding the given
+// pid -> cmdline entries (NUL-separated argv) for the duration of the test.
+func withFakeProc(t *testing.T, procs map[string]string) {
+	t.Helper()
+	root := t.TempDir()
+	for pid, cmdline := range procs {
+		dir := filepath.Join(root, pid)
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "cmdline"), []byte(cmdline), 0o644))
+	}
+	prev := procRoot
+	procRoot = root
+	t.Cleanup(func() { procRoot = prev })
+}
+
+// seedStore builds a JetStream store with the named streams under account $G,
+// laid out the way nats-server writes it, and returns the store path.
+func seedStore(t *testing.T, streams ...string) string {
+	t.Helper()
+	store := filepath.Join(t.TempDir(), "nats", "jetstream")
+	for _, s := range streams {
+		dir := filepath.Join(store, "$G", "streams", s, "msgs")
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "1.blk"), []byte("x"), 0o600))
+	}
+	return store
+}
+
+func TestJetStreamStoreDir(t *testing.T) {
+	assert.Equal(t, "/var/lib/spinifex/nats/jetstream", jetStreamStoreDir("/var/lib/spinifex"))
+}
+
+func TestLocalStreams(t *testing.T) {
+	t.Run("missing store holds nothing", func(t *testing.T) {
+		got, err := localStreams(filepath.Join(t.TempDir(), "absent"))
+		require.NoError(t, err)
+		assert.Empty(t, got)
+	})
+
+	t.Run("lists streams across accounts, sorted", func(t *testing.T) {
+		store := seedStore(t, "KV_b", "KV_a")
+		require.NoError(t, os.MkdirAll(filepath.Join(store, "$SYS", "_js_", "_meta_"), 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(store, "$G", "streams", "stray-file"), nil, 0o600))
+
+		got, err := localStreams(store)
+		require.NoError(t, err)
+		assert.Equal(t, []string{"$G/KV_a", "$G/KV_b"}, got)
+	})
+}
+
+func TestNATSRunningLocally(t *testing.T) {
+	tests := []struct {
+		name  string
+		procs map[string]string
+		want  bool
+	}{
+		{"no processes", nil, false},
+		{"installed service", map[string]string{"10": "/usr/local/bin/spx\x00service\x00nats\x00start\x00"}, true},
+		{"bare spx on PATH", map[string]string{"10": "spx\x00service\x00nats\x00start\x00"}, true},
+		// The join itself is an spx process and must not count.
+		{"spx admin join", map[string]string{"10": "spx\x00admin\x00join\x00--force\x00"}, false},
+		{"other spx service", map[string]string{"10": "spx\x00service\x00predastore\x00start\x00"}, false},
+		{"nats-server binary", map[string]string{"10": "nats-server\x00service\x00nats\x00"}, false},
+		{"kernel thread, empty cmdline", map[string]string{"2": ""}, false},
+		{"non-pid entry ignored", map[string]string{"self": "spx\x00service\x00nats\x00"}, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withFakeProc(t, tt.procs)
+			got, err := natsRunningLocally()
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestDiscardJetStreamStore(t *testing.T) {
+	t.Run("removes the store", func(t *testing.T) {
+		withFakeProc(t, nil)
+		store := seedStore(t, "KV_a")
+		require.NoError(t, discardJetStreamStore(store))
+		_, err := os.Stat(store)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+	})
+
+	t.Run("missing store is not an error", func(t *testing.T) {
+		withFakeProc(t, nil)
+		assert.NoError(t, discardJetStreamStore(filepath.Join(t.TempDir(), "absent")))
+	})
+
+	t.Run("refuses under a running NATS and leaves the store", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"7": "spx\x00service\x00nats\x00start\x00"})
+		store := seedStore(t, "KV_a")
+		assert.ErrorIs(t, discardJetStreamStore(store), errNATSRunning)
+		_, err := os.Stat(store)
+		assert.NoError(t, err)
+	})
+}
+
+func TestCheckInitJetStreamStore(t *testing.T) {
+	withFakeProc(t, nil)
+
+	t.Run("single-node init keeps its store", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 1, false))
+	})
+
+	t.Run("multi-node init over an empty store", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(filepath.Join(t.TempDir(), "absent"), 3, false))
+	})
+
+	t.Run("multi-node init over streams is refused without the flag", func(t *testing.T) {
+		err := checkInitJetStreamStore(seedStore(t, "KV_a", "KV_b"), 4, false)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "2 stream(s)")
+		assert.Contains(t, err.Error(), "--discard-jetstream")
+	})
+
+	t.Run("multi-node init over streams proceeds with the flag", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 4, true))
+	})
+
+	t.Run("flag does not override a running NATS", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"7": "spx\x00service\x00nats\x00start\x00"})
+		assert.ErrorIs(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 4, true), errNATSRunning)
+	})
+}
@@ -0,0 +1,44 @@
+//test:in-package — the embedded nats.conf template it checks is unexported,
+// and exporting it would widen the package's API for a test's convenience.
+
+package cmd
+
+import (
+	"strings"
+	"testing"
+
+	natssvc "github.com/mulgadc/spinifex/spinifex/services/nats"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestNATSConfServerNameMatchesTheHelper ties the rendered server_name to
+// nats.ServerName. JetStream reports stream peers by server name, so anything
+// mapping a replica back to the node holding it reads the prefix from that
+// helper — and a template that drifted from it would break the mapping silently,
+// reporting a healthy cluster as one whose buckets name no node at all.
+func TestNATSConfServerNameMatchesTheHelper(t *testing.T) {
+	var line string
+	for l := range strings.SplitSeq(natsConfTemplate, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(l), "server_name:") {
+			line = strings.TrimSpace(l)
+			break
+		}
+	}
+	require.NotEmpty(t, line, "nats.conf no longer sets server_name")
+
+	got := strings.TrimSpace(strings.TrimPrefix(line, "server_name:"))
+	assert.Equal(t, natssvc.ServerName("{{.Node}}"), got)
+
+	node, ok := natssvc.NodeFromServerName(natssvc.ServerName("node2"))
+	assert.True(t, ok)
+	assert.Equal(t, "node2", node)
+
+	_, ok = natssvc.NodeFromServerName("node2")
+	assert.False(t, ok, "a bare node name is not a server name")
+
+	// The bare prefix is a name we could not have generated, and mapping it to the
+	// empty node would put a phantom peer into anything counting nodes.
+	_, ok = natssvc.NodeFromServerName(natssvc.ServerNamePrefix)
+	assert.False(t, ok, "the prefix with no node after it is not a server name")
+}
@@ -1025,12 +1025,15 @@ var vpcdStartCmd = &cobra.Command{
 			AZ:                      nodeConfig.AZ,
 			NorthstarBaseDomain:     handlers_dns.ResolveBaseDomain(&nodeConfig),
 			NorthstarInternalDomain: handlers_dns.ResolveInternalDomain(&nodeConfig),
+			ServicesDomain:          clusterConfig.AWS.ServicesDomain,
 			ResolverNameservers:     handlers_dns.ResolverNameserverIPs(clusterConfig),
 			NATExemptCIDRs:          clusterConfig.Network.NATExemptCIDRs,
 			IPSecEnabled:            clusterConfig.Network.IPSecEnabled,
 			UnderlayMTU:             clusterConfig.Network.UnderlayMTU,
 			BlockedWANPorts:         clusterConfig.Network.ResolvedBlockedWANPorts(),
 			EgressBlockExemptVPCs:   clusterConfig.Network.EgressBlockExemptVPCs,
+			IMDSHostMetaIP:          clusterConfig.Network.IMDSHostMetaIP,
+			IMDSHostDNSIP:           clusterConfig.Network.IMDSHostDNSIP,
 		})
 		if err != nil {
 			return fmt.Errorf("create vpcd service: %w", err)
@@ -10,7 +10,7 @@ email = "{{.OperatorEmail}}"

 [aws]
 region = "{{.Region}}"
-internal_suffix = "spinifex.internal"
+services_domain = "services.internal"

 [network]
 ipsec_enabled = {{.IPSecEnabled}}
@@ -289,31 +289,25 @@ Configs are generated by `spx admin init` (leader) or `spx admin join` (follower

 ```
 spinifex/
+├── cmd/
+│   ├── spinifex/            # spx CLI: service entry points and `spx admin`
+│   ├── installer/           # Platform installer
+│   └── ...                  # In-guest agents (lb, ecs, rds, eks helpers) and dev tools
 ├── spinifex/
-│   ├── services/                # Long-running services (awsgw, spinifex, viperblockd,
-│   │                            #   predastore, nats, spinifexui)
-│   ├── gateway/                 # AWS API surface
-│   │   ├── gateway.go           # chi router, SigV4 auth, throttling
-│   │   ├── auth.go              # SigV4 verification + IAM lookup
-│   │   ├── ec2.go               # EC2 action dispatcher
-│   │   ├── ec2/                 # Per-resource handlers (instance, volume, vpc, ...)
-│   │   ├── elbv2/, iam/, policy/
-│   ├── handlers/                # NATS-side service implementations (ec2, elbv2, iam, ...)
-│   ├── daemon/
-│   │   ├── daemon.go            # Daemon entry, NATS subscription table
-│   │   └── daemon_handlers_*.go # One file per resource family
-│   ├── admin/                   # Cluster bootstrap (init/join, master.key, CA, tokens)
-│   ├── config/                  # Configuration structures
-│   ├── instancetypes/           # t3.*, m5.*, sys.* definitions
-│   ├── lbagent/                 # In-VM agent for ELB target health
-│   ├── vm/                      # VM instance representation
-│   ├── vpcd/                    # VPC daemon (OVN/OVS network programming)
-│   └── qmp/                     # QEMU Machine Protocol client
-└── cmd/
-    ├── spinifex/                # spx CLI
-    ├── lb-agent/                # ELB target agent binary
-    ├── installer/               # Platform installer
-    └── ...                      # ECS agent, ECR credential provider, EKS helper binaries
+│   ├── services/            # Long-running services (awsgw, spinifex, nats, predastore, ...)
+│   ├── gateway/             # AWS API surface: routing, SigV4 auth, per-service dispatch
+│   ├── handlers/            # NATS-side service implementations (ec2, iam, eks, rds, ...)
+│   ├── daemon/              # Per-node daemon and its NATS subscription table
+│   ├── vm/, qmp/            # VM lifecycle and the QEMU Machine Protocol client
+│   ├── vpcd/, network/      # VPC daemon and the layered OVN/OVS network stack
+│   ├── admin/               # Cluster bootstrap (init/join, master.key, CA, tokens)
+│   └── config/              # Configuration structures
+├── internal/                # In-guest SigV4 clients and the AWS Smithy models
+├── build/                   # systemd units, node helper scripts, microVM build
+├── scripts/                 # Install, dev-environment, image-build scripts
+└── tests/
+    ├── integration/         # Gateway router against embedded NATS JetStream
+    └── e2e/                 # Live-cluster suites (see docs/testing/e2e-coverage.md)
 ```

 ## Development
@@ -2,7 +2,11 @@

 This directory contains the Spinifex documentation. This is available in individual markdown files, and a generated website to aid navigation and browsing documentation at: [https://docs.mulgadc.com](https://docs.mulgadc.com/)

-The [AWS model operation coverage](aws-model-operation-coverage.md) report is generated from Spinifex's dispatch tables and the pinned AWS API models. Run `make aws-model-coverage` to regenerate and print it.
+The [AWS API coverage](coverage/README.md) pages are generated from Spinifex's dispatch tables and the pinned AWS API models, one page per service. Run `make aws-model-coverage` to regenerate them and print a summary. Only `pages.json` and the `intro.md` files under `coverage/` are hand-written; every `README.md` there is generated and will be overwritten.
+
+The same run writes [`coverage/coverage.json`](coverage/coverage.json), a versioned machine-readable model-versus-dispatch inventory. It is generated and must not be edited. Its operation states are structural: `implemented` means a registered action has neither a stub nor an explicit refusal. It is not a behavioural compatibility claim; conformance evidence and capability profiles establish that separately.
+
+A published page names the operations Spinifex serves, and nothing else. It carries no percentage, no unimplemented operation and no operation the platform will never offer: the same run writes those to [`coverage/internal-report.md`](coverage/internal-report.md), which no slug in the docs site's configuration points at, so it stays internal. A `notApplicable` declaration in `pages.json` therefore publishes nothing; it keeps a permanent absence out of the report's candidate work.

 ## Frontmatter

@@ -0,0 +1,233 @@
+---
+title: "Host Firewall"
+seoTitle: "Host Firewall and Cluster Membership — Spinifex Docs"
+description: "Arm the Spinifex host firewall, narrow SSH to your management networks, and learn how the peer-scoped cluster plane tracks membership changes on its own."
+category: "Admin"
+tags:
+  - security
+  - firewall
+  - nftables
+  - hardening
+resources:
+  - title: "Network Connections"
+    url: "/docs/network-connections"
+  - title: "Multi-Node Install"
+    url: "/docs/install-multi-node"
+  - title: "Setting Up Your Cluster"
+    url: "/docs/setting-up-your-cluster"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+---
+
+# Host Firewall
+
+> Arm the host firewall, narrow SSH, and keep the cluster plane off the public internet.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [What the policy allows](#what-the-policy-allows)
+- [Cluster membership](#cluster-membership)
+- [Is it on?](#is-it-on)
+- [Narrowing SSH access](#narrowing-ssh-access)
+- [Turning it off and on around cluster changes](#turning-it-off-and-on-around-cluster-changes)
+- [The files](#the-files)
+- [Checking it](#checking-it)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Spinifex ships an optional host firewall — an nftables input policy that defaults to drop and divides the node's ports into two groups:
+
+| Group | Ports | Who can reach them |
+|---|---|---|
+| **Public** | SSH, 443, 3000 (console), 8443 (S3), 9999 (AWS gateway), 53 (DNS) | anyone |
+| **Internal** | OVN, NATS, formation, Geneve and the rest of the cluster plane | **cluster members only** |
+
+The internal group is the point. Before this existed, OVN and NATS were reachable from the public internet on a WAN-facing node.
+
+**"Cluster members" is not a list you maintain.** Each node works it out from the cluster it belongs to and rewrites its own rules whenever membership changes — you never edit the peer list by hand.
+
+The policy governs inbound traffic only. Outbound is untouched, because the metadata reply path egresses under per-tap policy routing and a filter there would break it.
+
+## Instructions
+
+## What the Policy Allows
+
+The public plane is open to any source, IPv4 only — an accept with no source scope would otherwise also serve on any global IPv6 address the node picks up from an upstream router advertisement:
+
+| Ports | Serving |
+|---|---|
+| TCP 443 | Held for the TLS edge; no listener yet, so the rollout needs no firewall change |
+| TCP 3000 | Web console |
+| TCP 8443 | Predastore S3 gate |
+| TCP 9999 | AWS gateway — also how every guest agent (load balancer, EKS, ECS, RDS) reaches its control plane, since they speak SigV4 rather than NATS |
+| TCP and UDP 53 | Northstar authoritative DNS. Open on every node deliberately: a node serving no public zone still answers here, which is the accepted cost of not templating the policy per node |
+
+SSH is public too, but scoped by a variable you own — see [Narrowing SSH access](#narrowing-ssh-access).
+
+The cluster plane is peer-scoped. On a single-NIC node the planes collapse onto one address, which is why these rules match peer addresses rather than an interface or a CIDR:
+
+| Ports | Serving |
+|---|---|
+| TCP 4222, 4248 | NATS client and cluster routes |
+| TCP 4432 | Formation, then the daemon cluster manager |
+| TCP and UDP 5300 | Internal DNS |
+| TCP 6641, 6642 | OVN northbound and southbound |
+| TCP 6643, 6644 | OVN database clustering |
+| TCP 8660 | Predastore's admin listener |
+| UDP 6660, 7660 | Object shards and metadata consensus |
+| UDP 6081, 500, 4500, and ESP | Geneve, IKE, NAT-T and IPsec |
+
+Two of those are peer-scoped for reasons worth knowing. Outside a formation window, 4432 is the daemon cluster manager, whose routes report node topology, service inventory and running instances to anyone who asks. And 8660 is Predastore's unauthenticated `/healthz` and `/readyz`, which name the peers a gate cannot reach.
+
+**The formation port needs nothing from you.** `spx admin init` opens 4432 to any source for the length of the formation window and closes it again afterwards, because a node dialling in to join is not a peer yet. It is one slot rather than a list, so it cannot be grown into a general-purpose hole. The handshake behind it is TLS 1.3 with a bearer token.
+
+Beyond those groups the policy accepts what the platform cannot work without: established and related connections, loopback, the ICMP types PMTUD needs under a Geneve overlay, broadcast DHCP replies for the uplink and external-pool leases, and guest traffic to the metadata and VPC DNS endpoints on `169.254.169.254` and `169.254.169.253`. The guest rules are scoped to those two addresses and their ports rather than accepting everything arriving on a guest port, which would expose every wildcard-bound host service to any tenant.
+
+## Cluster Membership
+
+The daemon resolves who the node's peers are from the cluster it belongs to, writes them into `peers.nft`, and reloads the policy. A node that has not resolved membership yet is left reachable rather than cut off from a cluster it cannot name: the apply helper fails without touching the ruleset when the peer file is missing.
+
+One consequence matters in practice. **A node only recognises the members of the cluster it currently belongs to.** During formation the nodes do not yet know each other, so internal traffic between them is dropped — which is why forming or expanding a cluster starts by turning the policy off.
+
+## Is It On?
+
+| How the node was installed | Firewall |
+|---|---|
+| From the ISO | **on** |
+| Binary installer (`curl \| bash`) or `setup.sh` | **off** |
+| `setup.sh --firewall=on` | **on** |
+
+The binary installer defaults to off deliberately: it runs on servers that already have an operating system and services on them, and switching on a default-deny policy uninvited could cut off something Spinifex knows nothing about. The ISO is an appliance on hardware we define, so there the policy is the product.
+
+**For production, turn it on** — either at install time:
+
+```bash
+curl -fsSL https://install.mulgadc.com | bash -s -- --firewall=on
+```
+
+or afterwards, by setting it in `/etc/spinifex/spinifex.toml` and restarting the daemon:
+
+```toml
+[network]
+firewall_enabled = true
+```
+
+Before you do, check what else the machine is serving. Anything listening on a port outside the public group above stops accepting new connections.
+
+## Narrowing SSH Access
+
+SSH is accepted from **every source** by default. That is deliberate — arming the policy should not lock an operator out of the node they are arming it from — but it is the first thing to change on a production cluster.
+
+The sources live in `/etc/spinifex/firewall/custom.nft`, which the installer creates once and never rewrites, so an edit here survives upgrades, re-installs and node resets:
+
+```
+define trusted_ssh_peers = { 10.0.0.0/8, 192.168.1.0/24 }
+```
+
+Check the change before applying it, and check the policy file rather than this one — on its own `custom.nft` parses as a bare define and proves nothing, because the rule that uses the variable lives elsewhere:
+
+```bash
+sudo nft -c -f /etc/spinifex/firewall/spinifex.nft
+sudo nft -f /etc/spinifex/firewall/spinifex.nft
+```
+
+Never leave the variable empty. An undefined or empty set fails the whole ruleset, not just its own rule.
+
+The ports come from sshd's own configuration, re-read on every `setup.sh` run, so moving sshd to a non-standard port is picked up by an upgrade rather than needing an edit here. IPv4 only: sshd reached over IPv6 is not matched by this rule.
+
+## Turning It Off and On Around Cluster Changes
+
+Turn the firewall off while you form or expand a cluster, and on again once it is up:
+
+```bash
+# Off — before forming or expanding a cluster. Run on every node.
+sudo /usr/local/lib/spinifex/spinifex-firewall-apply disable
+
+# On — once the cluster is formed and verified. Run on every node.
+sudo systemctl restart spinifex-daemon
+```
+
+Restarting the daemon is what re-arms it: the node rebuilds its peer list from the cluster it is now part of, reloads the rules, and re-enables the boot-time unit so the policy survives a reboot. It also happens on its own within five minutes if you would rather wait.
+
+`disable` removes Spinifex's own table and the peer file and leaves every other nftables table alone. It is idempotent, and it is the recovery path — it runs even on a node with no policy installed, so it cannot leave you with a node you can neither arm nor disarm.
+
+> [!WARNING]
+> Do not flush the whole nftables ruleset to clear the policy. The VPC daemon writes SNAT and per-elastic-IP forward rules into the `ip nat` and `ip filter` tables and reinstalls them only when it starts, so `nft flush ruleset` breaks elastic IPs silently until the next `systemctl restart spinifex-vpcd`. Use `disable`, which only ever touches `inet spinifex_filter`.
+
+## The Files
+
+Everything lives under `/etc/spinifex/firewall/`. Which files are safe to edit follows from who writes them:
+
+| File | Written by | Survives an upgrade |
+|---|---|---|
+| `spinifex.nft` | `setup.sh` — the policy itself | No. Edits are overwritten |
+| `local.nft` | `setup.sh` — the sshd ports it detected | No. Refreshed on every run |
+| `peers.nft` | The daemon — cluster and encap peer addresses | No. Rewritten on every membership change |
+| `open-ports.nft` | The apply helper — the formation window only | No. Reset closed on install and upgrade |
+| `custom.nft` | **You** — `trusted_ssh_peers` and anything else you add | **Yes.** Created once, never rewritten |
+| `mode` | `setup.sh` — whether the install path armed the policy | No |
+
+The daemon cannot write `/etc` itself, and a blanket `nft` grant would be root-equivalent, so changes go through `/usr/local/lib/spinifex/spinifex-firewall-apply` — a root helper that accepts a fixed set of verbs and re-validates every address handed to it.
+
+## Checking It
+
+```bash
+sudo nft list table inet spinifex_filter
+```
+
+The peer list is an nft variable, expanded when the rules load, so it does not appear under a name of its own. Look instead at the `ip saddr { ... }` addresses on the cluster-plane rules — the ones accepting 4222, 6641, 6642 and the rest.
+
+Every node's addresses should be there. On a multi-NIC node that means its WAN, LAN and VPC addresses, so expect several entries per node. A missing node means its cluster traffic is being dropped.
+
+Dropped packets are logged, rate-limited to five a minute, so this tells you whether a connection problem is the firewall or something else:
+
+```bash
+sudo journalctl -k | grep 'spinifex-fw drop'
+```
+
+An empty log with a connection still failing means the policy is not the cause. A node with no table loaded at all is not armed:
+
+```bash
+systemctl is-enabled spinifex-firewall.service
+```
+
+## Troubleshooting
+
+### A Join Hangs Instead of Failing
+
+A hang rather than a quick refusal means packets are being dropped. Check the drop log on the node running init, and confirm the formation port opened — `spx admin init` prints `⚠️ Could not open port 4432 in the host firewall` when it could not:
+
+```bash
+sudo journalctl -k | grep 'spinifex-fw drop'
+```
+
+On ISO-installed nodes the usual cause is that each node's policy still trusts only itself, because that is the whole cluster as far as it knows. Turn the firewall off on **every** node, retry, and re-arm once the cluster is verified.
+
+### A Node Is Missing from the Peer List
+
+The daemon rewrites `peers.nft` from the cluster it belongs to, so a missing node means that node is not in the membership this one resolved. Confirm what the cluster thinks first, then restart the daemon to force a rebuild:
+
+```bash
+spx get nodes
+sudo systemctl restart spinifex-daemon
+```
+
+### One of My Own Services Stopped Answering
+
+Arming the policy closes every port outside the public group. Add what you need to `custom.nft`, which is the only file an upgrade leaves alone, and check the policy before applying it:
+
+```bash
+sudo nft -c -f /etc/spinifex/firewall/spinifex.nft
+```
+
+### Elastic IPs Stopped Working
+
+Something flushed the whole nftables ruleset, taking the VPC daemon's SNAT and forward rules with it. They are reinstalled on start:
+
+```bash
+sudo systemctl restart spinifex-vpcd
+```
@@ -0,0 +1,250 @@
+---
+title: "Predastore Distributed Storage"
+seoTitle: "How Predastore Distributed Storage Works — Spinifex Docs"
+description: "How Spinifex stores objects across a cluster: gate, blob and meta nodes, Reed-Solomon erasure coding, placement, degraded writes, repair and encryption at rest."
+category: "Admin"
+tags:
+  - storage
+  - s3
+  - predastore
+  - erasure coding
+  - durability
+resources:
+  - title: "Predastore Repository"
+    url: "https://github.com/mulgadc/predastore"
+  - title: "Viperblock (EBS)"
+    url: "https://github.com/mulgadc/viperblock"
+  - title: "Multi-Node Install"
+    url: "/docs/install-multi-node"
+  - title: "Network Connections"
+    url: "/docs/network-connections"
+---
+
+# Predastore Distributed Storage
+
+> How Spinifex stores objects across a cluster, and what a server failure costs.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Hosts and nodes](#hosts-and-nodes)
+- [The configuration file](#the-configuration-file)
+- [Erasure coding and durability](#erasure-coding-and-durability)
+- [Placement](#placement)
+- [The write path](#the-write-path)
+- [The read path](#the-read-path)
+- [Repair](#repair)
+- [Encryption at rest](#encryption-at-rest)
+- [What a server failure costs](#what-a-server-failure-costs)
+- [Checking it](#checking-it)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Predastore is the S3-compatible object store underneath a Spinifex cluster, and almost everything durable ends up in it:
+
+- **Machine images** — the AMIs instances launch from
+- **Block storage** — EBS volume data and snapshots, written through [Viperblock](https://github.com/mulgadc/viperblock), which uses Predastore as its S3 backend
+- **User data** — cloud-init configuration and system artifacts
+- **DNS zone files** — the zones Northstar serves
+- **Anything your own workloads put in a bucket** through the S3 API
+
+This is background reading. You do not configure any of it by hand: `spx admin init` and `spx admin join` build the topology from the servers that actually form the cluster, and every machine gets the same file with its own host ID recorded in `spinifex.toml`.
+
+## Instructions
+
+## Hosts and Nodes
+
+A cluster is two levels.
+
+A **host** is one machine: one Predastore process owning that machine's data directory, TLS identity and encryption key. A **node** is a role pinned to a host, running inside that process on its own port. Every server in a Spinifex cluster is one host carrying three nodes:
+
+| Role | Port | Purpose |
+|---|---|---|
+| `gate` | TCP 8443 | Serves the S3 API — SigV4, IAM, erasure coding, placement. Every server runs one, so any of them answers an S3 request. |
+| `blob` | UDP 6660 | Holds erasure-coded object shards. One per machine. |
+| `meta` | UDP 7660 | Member of the Raft quorum over global state — buckets and the object index. |
+
+Ports have to be unique within a host but not across the cluster, so every machine uses the same three. That is why the port surface does not widen as the cluster grows — see [Network requirements](/docs/install-multi-node#network-requirements).
+
+Which transport a pair of nodes uses follows from where they sit, not from a mode flag. Nodes on the same machine talk over an in-process pipe and open no socket at all; nodes on different machines talk over QUIC, TLS 1.3, authenticated by the cluster CA. A single-server install is therefore one process listening on 8443 and nothing else.
+
+Nothing dials a gate, so a gate binds no cluster-plane listener. Its port is the S3 port and nothing more.
+
+## The Configuration File
+
+The whole cluster's storage topology lives in `/etc/spinifex/predastore/predastore.toml`, identical on every server apart from which host the local process is told to be.
+
+Each server is one `[[host]]` block carrying three `[[host.node]]` blocks:
+
+```toml
+[[host]]
+id = 1
+bind_addr = "10.2.0.2"                                   # cluster plane: raft and blob
+addr = "10.2.0.2"                                        # what the other hosts dial
+data_dir = "/var/lib/spinifex/predastore"
+tls_cert = "/etc/spinifex/server.pem"
+tls_key = "/etc/spinifex/server.key"
+encryption_key = "/etc/spinifex/predastore/encryption.key"
+admin_port = 8660                                        # /healthz, /readyz, /repair
+
+[[host.node]]
+id = 1
+role = "gate"
+port = 8443
+bind_addr = "0.0.0.0"                                    # public plane; only a gate may set this
+```
+
+The two bind addresses are the point. `bind_addr` on the host is the cluster plane — Raft consensus and shard traffic listen there and nowhere else, at the address peers dial rather than a wildcard, so on a multi-homed machine replication never reaches the public interface. The gate binds the public plane separately, because S3 is meant to be reachable.
+
+## Erasure Coding and Durability
+
+The erasure code is cluster configuration, not a per-request choice, and it is chosen at formation from the number of servers — each machine contributes exactly one blob node:
+
+| Servers | Code | What it survives |
+|---|---|---|
+| 1 | `RS(1,0)` | Nothing at this layer. One shard on one node; redundancy is the pool's job — ZFS raidz or a mirror under the data directory. |
+| 2 | `RS(1,1)` | One server's shards, but the metadata quorum has no majority to lose. |
+| 3 or more | `RS(2,1)` | Any one server's shards, with the metadata quorum intact. |
+
+An object is encoded end to end into `K` data and `M` parity shards with no intermediate chunking. Each shard is roughly `⌈size / K⌉` bytes, and any `K` of the `K+M` shards rebuild the object.
+
+`data + parity` must not exceed the number of blob nodes, because a stripe is spread over distinct nodes and a narrower cluster would fail every write. `RS(2,1)` surviving the loss of any one server is the reason [three servers is the recommended minimum](/docs/install-multi-node).
+
+## Placement
+
+Which nodes hold an object's shards comes from a consistent hash ring over the blob nodes, keyed by the object hash. `Nodes(hash, K+M)` returns the placement in shard order — shard `i` goes to the node at position `i` — so every gate in the cluster derives the same placement from the same object without consulting anything.
+
+The gate then **records** that placement in the metadata plane rather than trusting the ring at read time. A ring that has since changed shape does not lose objects placed under the old one.
+
+## The Write Path
+
+The gate authenticates the request, resolves the bucket, hashes the object name, places it on the ring, encodes the body into data and parity shards, streams each shard to its blob node in parallel, and records the placement.
+
+Three properties matter operationally:
+
+- **The write is two-phase.** Each shard put makes the bytes durable and invisible. The placement record landing in Raft is the commit point, and the shards are published only afterwards — so an interrupted writer costs a round trip rather than the object. Shards are always durable before metadata: a crash between the two leaves shards nothing points at, rather than metadata pointing at data that was never written.
+- **Every write mints an epoch**, stamped on every shard of the stripe and into the record. The record names one generation, and a shard either belongs to it or does not. That is what makes a half-completed overwrite a reconstruction rather than an object spliced from two generations.
+- **One server down does not refuse writes.** The write is acknowledged once `K` shards are durable, and the response carries `X-Spx-Degraded-Write: <n>`. `K` is the floor rather than `K+1` because any `K` of the `K+M` reconstruct — at `RS(2,1)` one more buys no redundancy and would refuse three quarters of writes with a single server down.
+
+A shard its owner will not take is written to the next node along the ring instead of being given up on — **hinted handoff** — and the response carries `X-Spx-Handoff: <n>`. The stripe is then complete and the object is as redundant as a full-width write; what is outstanding is only that the shards are not all where the record says. The holder is derived rather than recorded, so there is no hint to store, replicate or lose. Handoff is skipped when the cluster has no node to spare, since a second shard of one object on a node that already holds one is not redundancy.
+
+If any node reported storage pressure the response also carries `X-Predastore-Pool-Pressure: nearfull`; a node that refused outright surfaces as `507 InsufficientStorage`.
+
+## The Read Path
+
+The gate reads the recorded placement, fetches the data shards, and joins them. Parity is never touched on the happy path. Only if the join fails does it refetch including parity and reconstruct, and a read that had to reconstruct says so in `X-Spx-Degraded: <n>` — a cost report, not an error.
+
+Every shard read names the record's epoch, and a node holding another generation answers `epoch-mismatch`. **The gate counts that exactly as a missing shard**, so the read either rebuilds the object the record names or fails. It never returns plausible nonsense.
+
+A shard its owner cannot serve is looked for on the handoff node before it is counted as failed, since a write that could not reach the owner will have put it there.
+
+A `Range` request that lands entirely inside one data shard takes a fast path — a single ranged shard read — because Reed-Solomon splits data sequentially. Anything wider reconstructs the object and slices it.
+
+## Repair
+
+Degraded writes trade redundancy for availability, and repair is what pays that back. It restores shards a blob node owns but does not hold at the generation its object's record names, and it runs on every server with a local blob node.
+
+Two things are worth knowing about it:
+
+- **It carries no correctness weight.** A read compares each shard's epoch against the record it has already loaded and discards a stale one whether or not repair has ever run. What repair restores is redundancy — the number of further losses an object survives — and never the object.
+- **It rebuilds onto the owner**, so it closes the gap only once the missing server returns. At `RS(2,1)` an object written while a server is down holds no parity for the duration of the outage. That is the accepted position: refusing writes meanwhile is the worse failure.
+
+Every pass scans the placement records from the beginning, asks each holder which generation it has, and rebuilds the ones that disagree. A shard that was handed off is returned rather than recomputed — cheaper by a decode — and the holder's copy is released only after the owner's is published. If fewer than `K` peers hold the record's generation the rebuild fails and says so, rather than inventing a shard from a mixture of generations.
+
+## Encryption at Rest
+
+Every fragment is sealed independently under AES-256-GCM, with a deterministic nonce and additional data reconstructed at read time from the on-disk header plus per-data-directory state. There is no unencrypted code path: the process refuses to start without an encryption key.
+
+| Property | What it means |
+|---|---|
+| **Confidentiality** | Disk-level access yields ciphertext and tags. Plaintext is never written. |
+| **Authenticated integrity** | GCM is the sole integrity authority; there is no separate checksum. Corruption, tampering and a wrong key all surface as one integrity error. |
+| **Position binding** | The additional data binds each fragment to its slot, so swapping fragments between objects fails the tag. |
+| **Cross-directory defence** | A per-data-directory store ID enters the nonce, so splicing a fragment from one data directory into another fails the tag. |
+
+The key is one 32-byte AES-256 key per cluster, and **every server must be given the same file** — fragments sealed under one key cannot be opened under another. `spx admin init` generates it and `spx admin join` distributes it, which is also why joining a server that has already been in service discards its own key and orphans everything sealed under it. The loader is fail-closed on permissions: a group- or other-readable key file is rejected outright, with no override.
+
+## What a Server Failure Costs
+
+| What fails | Reads | Writes |
+|---|---|---|
+| One gate (of three or more) | Unaffected — any gate answers for the cluster | Unaffected |
+| One blob node, `RS(2,1)` | Served by reconstruction, reported in `X-Spx-Degraded` | Accepted at the `K` floor, reported in `X-Spx-Degraded-Write`, or placed complete via handoff |
+| More than `M` blob nodes | Objects placed on them are lost | Refused |
+| Metadata quorum | Serve whatever the surviving replicas last applied | Stopped entirely |
+
+Reads try the cached Raft leader first, then every replica, and report not-found only once every replica has answered not-found — a replica that has not applied a key yet must not be mistaken for absence. Writes go through the leader, and a replica that cannot commit redirects the client to the one that can.
+
+Replacing a server restores capacity, not the shards that were on the old one: repair rebuilds onto the owner it expects, so plan a replacement as a return of that node rather than as re-replication onto a new one.
+
+## Checking It
+
+Each host runs an unauthenticated admin listener on port 8660, bound to the cluster plane rather than the S3 address. It serves counters and readiness, never addresses or keys:
+
+```bash
+curl -s http://$(hostname -I | awk '{print $1}'):8660/readyz
+curl -s http://$(hostname -I | awk '{print $1}'):8660/repair
+```
+
+`/readyz` reports the most recent probe cycle for the nodes in that process, and answers `503` until one has completed — reporting ready on no information would send traffic to a process that has never answered the question. `/healthz` answers for the process itself: reaching the handler is the whole check.
+
+That the gate is serving is an S3 request like any other:
+
+```bash
+export AWS_PROFILE=spinifex
+aws s3 ls
+```
+
+To see whether a write or read was degraded, ask for the response headers:
+
+```bash
+aws s3api put-object --bucket predastore --key probe --body /etc/hostname --debug 2>&1 | grep -i 'x-spx-\|x-predastore-'
+```
+
+And `spx get nodes` should list `predastore` in `SERVICES` on every node — a short service list on one server means something failed to start there.
+
+## Troubleshooting
+
+### S3 Answers but Every Write Fails
+
+The gate is up and the metadata quorum is not. Check that a majority of servers are running `spinifex-predastore`:
+
+```bash
+spx get nodes
+sudo systemctl is-active spinifex-predastore
+```
+
+Reads keep serving what the surviving replicas last applied, which is why this presents as a write-only outage.
+
+### 507 InsufficientStorage, or `nearfull` on Every Response
+
+A blob node is out of space, or close enough that compaction has been kicked. Free space in the data directory on each server:
+
+```bash
+df -h /var/lib/spinifex/predastore
+```
+
+Capacity here is per server, not pooled: a stripe needs its shard on a specific node, so one full server refuses writes for the objects the ring places on it.
+
+### Reads Fail with an Integrity Error
+
+GCM is the only integrity authority, so corruption, tampering and a wrong encryption key are indistinguishable. Confirm the key first, since it is by far the most likely of the three — every server must hold the identical file:
+
+```bash
+sudo sha256sum /etc/spinifex/predastore/encryption.key
+```
+
+Compare the digest across servers. A mismatch means that server was initialized separately rather than joined, and the fragments it sealed cannot be opened by the cluster.
+
+### Degraded Reads That Never Clear
+
+`X-Spx-Degraded` on reads long after an outage means repair has not restored the shards. Repair rebuilds onto the node that owns the shard, so it cannot finish while that node is absent:
+
+```bash
+curl -s http://$(hostname -I | awk '{print $1}'):8660/repair
+```
+
+`"enabled": false` means the sweep is off on that host. Otherwise bring the missing server back and let a pass complete — the sweep runs every five minutes by default.
@@ -1,7 +1,7 @@
 ---
 title: "Setting Up Your Cluster"
 seoTitle: "Set Up Your First Spinifex Cluster — Spinifex Docs"
-description: "Import an AMI, create an SSH key pair and a VPC with a public subnet, then launch your first EC2 instance and connect to it on a fresh Spinifex cluster."
+description: "Import an AMI and the RDS, ECS and EKS base images, create an SSH key pair and a VPC with a public subnet, then launch your first EC2 instance on a fresh Spinifex cluster."
 category: "Admin"
 tags:
  - setup
@@ -17,24 +17,26 @@ resources:

 # Setting Up Your Cluster

-> Import an AMI, create a VPC with a public subnet, and launch your first instance.
+> Import an AMI and the system base images, create a VPC with a public subnet, and launch your first instance.

 ## Table of Contents

 - [1. Import an AMI](#1-import-an-ami)
+  - [Option C: Import the system base images](#option-c-import-the-system-base-images)
 - [2. Create an SSH Key](#2-create-an-ssh-key)
 - [3. Create a VPC and Public Subnet](#3-create-a-vpc-and-public-subnet)
 - [4. Launch an Instance](#4-launch-an-instance)
 - [5. Connect via SSH](#5-connect-via-ssh)
 - [6. Managing Instances](#6-managing-instances)
 - [7. Launching the Web UI](#7-launching-the-web-ui)
+- [8. Secure the Install](#8-secure-the-install)
 - [Troubleshooting](#troubleshooting)

 ---

 ## Overview

-This guide walks through the first steps on a freshly installed cluster: importing an AMI, creating an SSH key and a VPC with a public subnet, launching an instance, and connecting to it.
+This guide walks through the first steps on a freshly installed cluster: importing an AMI and the system base images RDS, ECS and EKS need, creating an SSH key and a VPC with a public subnet, launching an instance, and connecting to it.

 ## Instructions

@@ -72,7 +74,7 @@ spx admin images import --name ubuntu-26.04-x86_64
 ### Option B: Import a local image file

 ```bash
-spx admin images import --file ~/images/ubuntu-26.04.img --distro ubuntu --version 26.04 --arch x86_64 --boot-mode uefi
+spx admin images import --file ~/images/ubuntu-26.04-server-cloudimg-amd64.img --checksum ~/images/SHA256SUMS --distro ubuntu --version 26.04 --arch x86_64 --boot-mode uefi
 ```

 Verify the import and note the AMI ID:
@@ -81,6 +83,41 @@ Verify the import and note the AMI ID:
 AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
 ```

+### Option C: Import the system base images
+
+**RDS, ECS and EKS each launch from an image of their own, and none of them is imported for you.** Until these are in the catalog, `CreateDBInstance`, `CreateCluster` and `CreateNodegroup` all fail at the point they go looking for a node image — so a cluster without them has those three services present in the API and unusable in practice. Import them once, now, and the install is functional from the start.
+
+Run these one at a time rather than in parallel, so predastore is not taking four concurrent uploads:
+
+```bash
+for img in spinifex-rds-postgres spinifex-rds-mariadb spinifex-ecs-node spinifex-eks-node; do
+    sudo spx admin images import --name "$img" --config /etc/spinifex/spinifex.toml
+done
+```
+
+| Image | Serves | Contents |
+| --- | --- | --- |
+| `spinifex-rds-postgres` | RDS, `--engine postgres` | Alpine 3.24.1 + PostgreSQL 18 + `rds-init` |
+| `spinifex-rds-mariadb` | RDS, `--engine mariadb` | Alpine 3.24.1 + MariaDB 11.8 + `rds-init` |
+| `spinifex-ecs-node` | ECS container instances | Alpine 3.21.7 + containerd + ecs-agent |
+| `spinifex-eks-node` | EKS control plane and nodegroups | Alpine 3.21.7 + K3s v1.32.5 + eks-token-webhook |
+
+All four boot **BIOS**, unlike the distro images, which boot UEFI. That is a property of how they are built and nothing you need to pass.
+
+If you run GPU workloads, add `spinifex-ecs-node-gpu` and `spinifex-eks-node-gpu` as well — Ubuntu 26.04 with the NVIDIA driver, selected automatically for a GPU instance type. They boot UEFI and they are large, so import them only if you have GPUs.
+
+> [!IMPORTANT]
+> **These register under generic names, and that is correct.** `describe-images` shows `spinifex-eks-node` and `spinifex-ecs-node` both as `ami-alpine-3.21.7-x86_64`, and the two RDS images both as `ami-alpine-3.24.1-x86_64`. Nothing resolves them by name: ECS and EKS find theirs by the `spinifex:managed-by=ecs|eks` tag, and RDS by `spinifex:managed-by=rds` plus the `engine` and `engine-version` tags. **Verify the tags, not the names** — four rows with duplicate-looking names is what success looks like:
+>
+> ```bash
+> aws ec2 describe-images \
+>   --filters 'Name=tag-key,Values=spinifex:managed-by' \
+>   --query 'Images[].[ImageId,State,Tags[?Key==`spinifex:managed-by`].Value|[0],Tags[?Key==`engine`].Value|[0]]' \
+>   --output text
+> ```
+
+`spx admin images list --config /etc/spinifex/spinifex.toml` on a node shows the whole catalog, including the images not imported here.
+
 ## 2. Create an SSH Key

 ### Option A: Import an existing key
@@ -314,15 +351,16 @@ On first load, your browser will show a TLS warning — Spinifex generates a sel

 1. Accept the warning to reach the login page (exact wording varies by browser — e.g. Chrome: *Advanced → Proceed to ...*, Firefox: *Advanced → Accept the Risk and Continue*).
 2. On the login page, click **Download Certificate** and save `spinifex-ca.pem` to your machine.
-3. Install the certificate as a **trusted root** on your workstation:
-
-   - **macOS:** open `spinifex-ca.pem` in Keychain Access → *System* keychain → set *Trust* to **Always Trust**.
-   - **Linux:** `sudo cp spinifex-ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates`
-   - **Windows:** double-click the file → *Install Certificate* → *Local Machine* → *Trusted Root Certification Authorities*.
-   - **Browser-only (Firefox):** *Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import* and tick *Trust this CA to identify websites*.
-
+3. Install the certificate as a **trusted root** on your workstation, following the steps for your platform below.
 4. Restart your browser and reload `https://YOUR_NODE_IP:3000`. The padlock should now show a valid certificate.

+Step 3, by platform:
+
+- **macOS:** open `spinifex-ca.pem` in Keychain Access → *System* keychain → set *Trust* to **Always Trust**.
+- **Linux:** `sudo cp spinifex-ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates`
+- **Windows:** double-click the file → *Install Certificate* → *Local Machine* → *Trusted Root Certification Authorities*.
+- **Browser-only (Firefox):** *Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import* and tick *Trust this CA to identify websites*.
+
 > **Why this is required:** the UI logs in by reading your AWS credentials through a trusted TLS channel. Browsers refuse to send credentials over an untrusted connection, so the certificate must be installed as trusted — temporary "Proceed anyway" exceptions won't work for login.

 ### Log In with AWS Credentials
@@ -339,6 +377,22 @@ At the Spinifex login screen, paste the **Access Key ID** and **Secret Access Ke

 Once logged in, you have browser-based access to every Spinifex feature: launch and manage instances, attach EBS volumes, browse S3 buckets, configure VPCs and security groups, and manage IAM users and keys — all backed by the same AWS-compatible control plane the CLI uses.

+## 8. Secure the Install
+
+Your cluster works. Before it carries anything real, close the host down.
+
+**Arm the host firewall.** A binary install leaves it off, because it lands on a machine that was already doing something. Armed, it keeps OVN, NATS and the rest of the cluster plane reachable by cluster members only, and everything outside the public group closed:
+
+```toml
+# /etc/spinifex/spinifex.toml
+[network]
+firewall_enabled = true
+```
+
+**Narrow SSH.** It is accepted from every source by default, so that arming the policy cannot lock you out of the node you are arming it from. Replace `0.0.0.0/0` with your management networks in `/etc/spinifex/firewall/custom.nft` — the one file an upgrade never overwrites.
+
+[Host Firewall](/docs/host-firewall) covers both, what the policy allows, and how each node tracks cluster membership without a peer list you maintain.
+
 ## Additional Options

 ### Private Subnets (No Public IP)
@@ -109,7 +109,7 @@ spx admin images list
 spx admin images import --name debian-13-arm64
 ```

-Catalog imports verify the image against the catalog-declared SHA-256/SHA-512 digest before extraction. Use `--file` to import operator-supplied media (verification skipped — operator is responsible for integrity), or `--force` to re-download after a checksum mismatch.
+Catalog imports verify the image against the catalog-declared SHA-256/SHA-512 digest before extraction. Use `--file` with `--checksum <sums-file>` to import operator-supplied media verified against the sums file it was published with, or `--force` to re-download after a checksum mismatch. A `--file` import without `--checksum` proceeds unverified and prints a notice. Every import records the source file's digest on the AMI; `spx admin images describe --image-id <ami>` shows it.

 ### EKS node image

@@ -37,20 +37,6 @@ For operators who want to review migrations before they are applied, a manual up
 > [!WARNING]
 > **Swapping the `spx` binary alone is not an upgrade.** Systemd unit files (`KillMode`, `TimeoutStopSec`, drain ordering, and similar) are written once at install time and never re-asserted just because a new binary is in place — a node "upgraded" by replacing `/usr/local/bin/spx` directly keeps running whatever units it was first installed with, which can silently disagree with the new binary's behaviour. Re-running the installer always reinstalls units unconditionally, so it is unaffected. `spx admin upgrade` now reconciles units too, so it closes this gap for operators who update the binary by hand. See [Checking for Unit Drift](#checking-for-unit-drift).

-> [!WARNING]
-> **Object storage is exempt when upgrading from v1.15.0 or earlier to v1.16.0.** Predastore's configuration schema and on-disk layout changed with the object storage cutover in v1.16.0, and no migration converts an installation from before it. `spx admin upgrade` will not report anything pending for `predastore.toml`, and updating the binary over such an installation leaves Predastore unable to start. Those clusters have to be re-initialised from scratch, which discards their stored objects — export anything you need first. Upgrades between v1.16.0 and later releases are unaffected.
-
-> [!WARNING]
-> **AMI metadata is exempt when upgrading from v1.16.0 or earlier to the release carrying the EBS-provider decoupling.** AMI metadata moved to `ebsmetadata` documents, and the legacy path that read it from `ami-<id>/config.json` was removed rather than migrated. There is no prefix scan, no fallback and no backfill at daemon start, so an AMI imported before the change becomes invisible to the control plane afterwards: `describe-images --image-ids` answers `InvalidAMIID.NotFound` and launches fail with `AMI has no snapshot ID, cannot perform zero-copy clone`. As with object storage above, `spx admin upgrade` reports nothing pending, because the gap is in stored data rather than in a config file.
->
-> **Re-import affected AMIs after upgrading**, using [`spx admin images import`](/docs/spinifex-admin-cli), which writes metadata in the new location. If the source images are no longer available, the installation has to be re-initialised. Verify before you rely on it — an AMI is only healthy if it resolves by ID, not merely if it appears in the list:
->
-> ```bash
-> aws ec2 describe-images --image-ids <ami-id>
-> ```
->
-> This warning covers AMI metadata specifically, which is what has been observed. Check any other imported state you depend on before upgrading a cluster you cannot rebuild.
-
 ## Instructions

 ## Step 1. Re-run the Installer
@@ -1,577 +0,0 @@
-# AWS model operation coverage
-
-Generated from the cached `aws-sdk-go v1.55.8` `api-2.json` models and Spinifex's authoritative gateway dispatch tables.
-
-Implemented means a modelled operation is registered to a real handler. Stub and deliberately unsupported handlers are reported separately. Shape conformance does not imply behavioural conformance.
-
-| Service | API version | Modelled | Registered | Implemented | Stubbed | Unsupported | Missing | Extra |
-|---|---:|---:|---:|---:|---:|---:|---:|---:|
-| acm | 2015-12-08 | 15 | 9 | 9 | 0 | 0 | 6 | 0 |
-| ec2 | 2016-11-15 | 625 | 121 | 121 | 0 | 0 | 504 | 0 |
-| ecr | 2015-09-21 | 47 | 39 | 19 | 13 | 7 | 11 | 3 |
-| ecs | 2014-11-13 | 56 | 39 | 31 | 5 | 0 | 20 | 3 |
-| elasticloadbalancingv2 | 2015-12-01 | 46 | 37 | 33 | 0 | 0 | 13 | 4 |
-| iam | 2010-05-08 | 159 | 75 | 75 | 0 | 0 | 84 | 0 |
-| rds | 2014-10-31 | 162 | 43 | 26 | 0 | 12 | 124 | 5 |
-| s3 | 2006-03-01 | 99 | — | — | — | — | — | — |
-| sts | 2011-06-15 | 8 | 4 | 4 | 0 | 0 | 4 | 0 |
-
-## acm
-
-Implements **9 of 15** modelled operations (60.0%).
-
-<details><summary>Implemented (9)</summary>
-
-`AddTagsToCertificate`, `DeleteCertificate`, `DescribeCertificate`, `GetCertificate`, `ImportCertificate`, `ListCertificates`, `ListTagsForCertificate`, `RemoveTagsFromCertificate`
-
-`RequestCertificate`
-
-
-</details>
-
-<details><summary>Missing from dispatch (6)</summary>
-
-`ExportCertificate`, `GetAccountConfiguration`, `PutAccountConfiguration`, `RenewCertificate`, `ResendValidationEmail`, `UpdateCertificateOptions`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>
-
-## ec2
-
-Implements **121 of 625** modelled operations (19.4%).
-
-<details><summary>Implemented (121)</summary>
-
-`AllocateAddress`, `AssociateAddress`, `AssociateIamInstanceProfile`, `AssociateRouteTable`, `AttachInternetGateway`, `AttachNetworkInterface`, `AttachVolume`, `AuthorizeSecurityGroupEgress`
-
-`AuthorizeSecurityGroupIngress`, `CancelCapacityReservation`, `CancelSpotInstanceRequests`, `CopyImage`, `CopySnapshot`, `CreateCapacityReservation`, `CreateEgressOnlyInternetGateway`, `CreateImage`
-
-`CreateInternetGateway`, `CreateKeyPair`, `CreateLaunchTemplate`, `CreateLaunchTemplateVersion`, `CreateNatGateway`, `CreateNetworkInterface`, `CreatePlacementGroup`, `CreateRoute`
-
-`CreateRouteTable`, `CreateSecurityGroup`, `CreateSnapshot`, `CreateSubnet`, `CreateTags`, `CreateVolume`, `CreateVpc`, `DeleteEgressOnlyInternetGateway`
-
-`DeleteInternetGateway`, `DeleteKeyPair`, `DeleteLaunchTemplate`, `DeleteLaunchTemplateVersions`, `DeleteNatGateway`, `DeleteNetworkInterface`, `DeletePlacementGroup`, `DeleteRoute`
-
-`DeleteRouteTable`, `DeleteSecurityGroup`, `DeleteSnapshot`, `DeleteSubnet`, `DeleteTags`, `DeleteVolume`, `DeleteVpc`, `DeregisterImage`
-
-`DescribeAccountAttributes`, `DescribeAddresses`, `DescribeAddressesAttribute`, `DescribeAvailabilityZones`, `DescribeCapacityReservations`, `DescribeEgressOnlyInternetGateways`, `DescribeIamInstanceProfileAssociations`, `DescribeImageAttribute`
-
-`DescribeImages`, `DescribeInstanceAttribute`, `DescribeInstanceCreditSpecifications`, `DescribeInstanceStatus`, `DescribeInstanceTypes`, `DescribeInstances`, `DescribeInternetGateways`, `DescribeKeyPairs`
-
-`DescribeLaunchTemplateVersions`, `DescribeLaunchTemplates`, `DescribeNatGateways`, `DescribeNetworkInterfaces`, `DescribePlacementGroups`, `DescribeRegions`, `DescribeRouteTables`, `DescribeSecurityGroupRules`
-
-`DescribeSecurityGroups`, `DescribeSnapshots`, `DescribeSpotInstanceRequests`, `DescribeSubnets`, `DescribeTags`, `DescribeVolumeStatus`, `DescribeVolumes`, `DescribeVolumesModifications`
-
-`DescribeVpcAttribute`, `DescribeVpcs`, `DetachInternetGateway`, `DetachNetworkInterface`, `DetachVolume`, `DisableEbsEncryptionByDefault`, `DisableSerialConsoleAccess`, `DisassociateAddress`
-
-`DisassociateIamInstanceProfile`, `DisassociateRouteTable`, `EnableEbsEncryptionByDefault`, `EnableSerialConsoleAccess`, `GetConsoleOutput`, `GetEbsEncryptionByDefault`, `GetPasswordData`, `GetSerialConsoleAccessStatus`
-
-`ImportKeyPair`, `ModifyImageAttribute`, `ModifyInstanceAttribute`, `ModifyInstanceMetadataOptions`, `ModifyLaunchTemplate`, `ModifyNetworkInterfaceAttribute`, `ModifySubnetAttribute`, `ModifyVolume`
-
-`ModifyVpcAttribute`, `MonitorInstances`, `RebootInstances`, `RegisterImage`, `ReleaseAddress`, `ReplaceIamInstanceProfileAssociation`, `ReplaceRoute`, `ReplaceRouteTableAssociation`
-
-`RequestSpotInstances`, `ResetImageAttribute`, `RevokeSecurityGroupEgress`, `RevokeSecurityGroupIngress`, `RunInstances`, `StartInstances`, `StopInstances`, `TerminateInstances`
-
-`UnmonitorInstances`
-
-
-</details>
-
-<details><summary>Missing from dispatch (504)</summary>
-
-`AcceptAddressTransfer`, `AcceptReservedInstancesExchangeQuote`, `AcceptTransitGatewayMulticastDomainAssociations`, `AcceptTransitGatewayPeeringAttachment`, `AcceptTransitGatewayVpcAttachment`, `AcceptVpcEndpointConnections`, `AcceptVpcPeeringConnection`, `AdvertiseByoipCidr`
-
-`AllocateHosts`, `AllocateIpamPoolCidr`, `ApplySecurityGroupsToClientVpnTargetNetwork`, `AssignIpv6Addresses`, `AssignPrivateIpAddresses`, `AssignPrivateNatGatewayAddress`, `AssociateClientVpnTargetNetwork`, `AssociateDhcpOptions`
-
-`AssociateEnclaveCertificateIamRole`, `AssociateInstanceEventWindow`, `AssociateIpamByoasn`, `AssociateIpamResourceDiscovery`, `AssociateNatGatewayAddress`, `AssociateSubnetCidrBlock`, `AssociateTransitGatewayMulticastDomain`, `AssociateTransitGatewayPolicyTable`
-
-`AssociateTransitGatewayRouteTable`, `AssociateTrunkInterface`, `AssociateVpcCidrBlock`, `AttachClassicLinkVpc`, `AttachVerifiedAccessTrustProvider`, `AttachVpnGateway`, `AuthorizeClientVpnIngress`, `BundleInstance`
-
-`CancelBundleTask`, `CancelCapacityReservationFleets`, `CancelConversionTask`, `CancelExportTask`, `CancelImageLaunchPermission`, `CancelImportTask`, `CancelReservedInstancesListing`, `CancelSpotFleetRequests`
-
-`ConfirmProductInstance`, `CopyFpgaImage`, `CreateCapacityReservationFleet`, `CreateCarrierGateway`, `CreateClientVpnEndpoint`, `CreateClientVpnRoute`, `CreateCoipCidr`, `CreateCoipPool`
-
-`CreateCustomerGateway`, `CreateDefaultSubnet`, `CreateDefaultVpc`, `CreateDhcpOptions`, `CreateFleet`, `CreateFlowLogs`, `CreateFpgaImage`, `CreateInstanceConnectEndpoint`
-
-`CreateInstanceEventWindow`, `CreateInstanceExportTask`, `CreateIpam`, `CreateIpamExternalResourceVerificationToken`, `CreateIpamPool`, `CreateIpamResourceDiscovery`, `CreateIpamScope`, `CreateLocalGatewayRoute`
-
-`CreateLocalGatewayRouteTable`, `CreateLocalGatewayRouteTableVirtualInterfaceGroupAssociation`, `CreateLocalGatewayRouteTableVpcAssociation`, `CreateManagedPrefixList`, `CreateNetworkAcl`, `CreateNetworkAclEntry`, `CreateNetworkInsightsAccessScope`, `CreateNetworkInsightsPath`
-
-`CreateNetworkInterfacePermission`, `CreatePublicIpv4Pool`, `CreateReplaceRootVolumeTask`, `CreateReservedInstancesListing`, `CreateRestoreImageTask`, `CreateSnapshots`, `CreateSpotDatafeedSubscription`, `CreateStoreImageTask`
-
-`CreateSubnetCidrReservation`, `CreateTrafficMirrorFilter`, `CreateTrafficMirrorFilterRule`, `CreateTrafficMirrorSession`, `CreateTrafficMirrorTarget`, `CreateTransitGateway`, `CreateTransitGatewayConnect`, `CreateTransitGatewayConnectPeer`
-
-`CreateTransitGatewayMulticastDomain`, `CreateTransitGatewayPeeringAttachment`, `CreateTransitGatewayPolicyTable`, `CreateTransitGatewayPrefixListReference`, `CreateTransitGatewayRoute`, `CreateTransitGatewayRouteTable`, `CreateTransitGatewayRouteTableAnnouncement`, `CreateTransitGatewayVpcAttachment`
-
-`CreateVerifiedAccessEndpoint`, `CreateVerifiedAccessGroup`, `CreateVerifiedAccessInstance`, `CreateVerifiedAccessTrustProvider`, `CreateVpcEndpoint`, `CreateVpcEndpointConnectionNotification`, `CreateVpcEndpointServiceConfiguration`, `CreateVpcPeeringConnection`
-
-`CreateVpnConnection`, `CreateVpnConnectionRoute`, `CreateVpnGateway`, `DeleteCarrierGateway`, `DeleteClientVpnEndpoint`, `DeleteClientVpnRoute`, `DeleteCoipCidr`, `DeleteCoipPool`
-
-`DeleteCustomerGateway`, `DeleteDhcpOptions`, `DeleteFleets`, `DeleteFlowLogs`, `DeleteFpgaImage`, `DeleteInstanceConnectEndpoint`, `DeleteInstanceEventWindow`, `DeleteIpam`
-
-`DeleteIpamExternalResourceVerificationToken`, `DeleteIpamPool`, `DeleteIpamResourceDiscovery`, `DeleteIpamScope`, `DeleteLocalGatewayRoute`, `DeleteLocalGatewayRouteTable`, `DeleteLocalGatewayRouteTableVirtualInterfaceGroupAssociation`, `DeleteLocalGatewayRouteTableVpcAssociation`
-
-`DeleteManagedPrefixList`, `DeleteNetworkAcl`, `DeleteNetworkAclEntry`, `DeleteNetworkInsightsAccessScope`, `DeleteNetworkInsightsAccessScopeAnalysis`, `DeleteNetworkInsightsAnalysis`, `DeleteNetworkInsightsPath`, `DeleteNetworkInterfacePermission`
-
-`DeletePublicIpv4Pool`, `DeleteQueuedReservedInstances`, `DeleteSpotDatafeedSubscription`, `DeleteSubnetCidrReservation`, `DeleteTrafficMirrorFilter`, `DeleteTrafficMirrorFilterRule`, `DeleteTrafficMirrorSession`, `DeleteTrafficMirrorTarget`
-
-`DeleteTransitGateway`, `DeleteTransitGatewayConnect`, `DeleteTransitGatewayConnectPeer`, `DeleteTransitGatewayMulticastDomain`, `DeleteTransitGatewayPeeringAttachment`, `DeleteTransitGatewayPolicyTable`, `DeleteTransitGatewayPrefixListReference`, `DeleteTransitGatewayRoute`
-
-`DeleteTransitGatewayRouteTable`, `DeleteTransitGatewayRouteTableAnnouncement`, `DeleteTransitGatewayVpcAttachment`, `DeleteVerifiedAccessEndpoint`, `DeleteVerifiedAccessGroup`, `DeleteVerifiedAccessInstance`, `DeleteVerifiedAccessTrustProvider`, `DeleteVpcEndpointConnectionNotifications`
-
-`DeleteVpcEndpointServiceConfigurations`, `DeleteVpcEndpoints`, `DeleteVpcPeeringConnection`, `DeleteVpnConnection`, `DeleteVpnConnectionRoute`, `DeleteVpnGateway`, `DeprovisionByoipCidr`, `DeprovisionIpamByoasn`
-
-`DeprovisionIpamPoolCidr`, `DeprovisionPublicIpv4PoolCidr`, `DeregisterInstanceEventNotificationAttributes`, `DeregisterTransitGatewayMulticastGroupMembers`, `DeregisterTransitGatewayMulticastGroupSources`, `DescribeAddressTransfers`, `DescribeAggregateIdFormat`, `DescribeAwsNetworkPerformanceMetricSubscriptions`
-
-`DescribeBundleTasks`, `DescribeByoipCidrs`, `DescribeCapacityBlockOfferings`, `DescribeCapacityReservationFleets`, `DescribeCarrierGateways`, `DescribeClassicLinkInstances`, `DescribeClientVpnAuthorizationRules`, `DescribeClientVpnConnections`
-
-`DescribeClientVpnEndpoints`, `DescribeClientVpnRoutes`, `DescribeClientVpnTargetNetworks`, `DescribeCoipPools`, `DescribeConversionTasks`, `DescribeCustomerGateways`, `DescribeDhcpOptions`, `DescribeElasticGpus`
-
-`DescribeExportImageTasks`, `DescribeExportTasks`, `DescribeFastLaunchImages`, `DescribeFastSnapshotRestores`, `DescribeFleetHistory`, `DescribeFleetInstances`, `DescribeFleets`, `DescribeFlowLogs`
-
-`DescribeFpgaImageAttribute`, `DescribeFpgaImages`, `DescribeHostReservationOfferings`, `DescribeHostReservations`, `DescribeHosts`, `DescribeIdFormat`, `DescribeIdentityIdFormat`, `DescribeImportImageTasks`
-
-`DescribeImportSnapshotTasks`, `DescribeInstanceConnectEndpoints`, `DescribeInstanceEventNotificationAttributes`, `DescribeInstanceEventWindows`, `DescribeInstanceTopology`, `DescribeInstanceTypeOfferings`, `DescribeIpamByoasn`, `DescribeIpamExternalResourceVerificationTokens`
-
-`DescribeIpamPools`, `DescribeIpamResourceDiscoveries`, `DescribeIpamResourceDiscoveryAssociations`, `DescribeIpamScopes`, `DescribeIpams`, `DescribeIpv6Pools`, `DescribeLocalGatewayRouteTableVirtualInterfaceGroupAssociations`, `DescribeLocalGatewayRouteTableVpcAssociations`
-
-`DescribeLocalGatewayRouteTables`, `DescribeLocalGatewayVirtualInterfaceGroups`, `DescribeLocalGatewayVirtualInterfaces`, `DescribeLocalGateways`, `DescribeLockedSnapshots`, `DescribeMacHosts`, `DescribeManagedPrefixLists`, `DescribeMovingAddresses`
-
-`DescribeNetworkAcls`, `DescribeNetworkInsightsAccessScopeAnalyses`, `DescribeNetworkInsightsAccessScopes`, `DescribeNetworkInsightsAnalyses`, `DescribeNetworkInsightsPaths`, `DescribeNetworkInterfaceAttribute`, `DescribeNetworkInterfacePermissions`, `DescribePrefixLists`
-
-`DescribePrincipalIdFormat`, `DescribePublicIpv4Pools`, `DescribeReplaceRootVolumeTasks`, `DescribeReservedInstances`, `DescribeReservedInstancesListings`, `DescribeReservedInstancesModifications`, `DescribeReservedInstancesOfferings`, `DescribeScheduledInstanceAvailability`
-
-`DescribeScheduledInstances`, `DescribeSecurityGroupReferences`, `DescribeSnapshotAttribute`, `DescribeSnapshotTierStatus`, `DescribeSpotDatafeedSubscription`, `DescribeSpotFleetInstances`, `DescribeSpotFleetRequestHistory`, `DescribeSpotFleetRequests`
-
-`DescribeSpotPriceHistory`, `DescribeStaleSecurityGroups`, `DescribeStoreImageTasks`, `DescribeTrafficMirrorFilterRules`, `DescribeTrafficMirrorFilters`, `DescribeTrafficMirrorSessions`, `DescribeTrafficMirrorTargets`, `DescribeTransitGatewayAttachments`
-
-`DescribeTransitGatewayConnectPeers`, `DescribeTransitGatewayConnects`, `DescribeTransitGatewayMulticastDomains`, `DescribeTransitGatewayPeeringAttachments`, `DescribeTransitGatewayPolicyTables`, `DescribeTransitGatewayRouteTableAnnouncements`, `DescribeTransitGatewayRouteTables`, `DescribeTransitGatewayVpcAttachments`
-
-`DescribeTransitGateways`, `DescribeTrunkInterfaceAssociations`, `DescribeVerifiedAccessEndpoints`, `DescribeVerifiedAccessGroups`, `DescribeVerifiedAccessInstanceLoggingConfigurations`, `DescribeVerifiedAccessInstances`, `DescribeVerifiedAccessTrustProviders`, `DescribeVolumeAttribute`
-
-`DescribeVpcClassicLink`, `DescribeVpcClassicLinkDnsSupport`, `DescribeVpcEndpointConnectionNotifications`, `DescribeVpcEndpointConnections`, `DescribeVpcEndpointServiceConfigurations`, `DescribeVpcEndpointServicePermissions`, `DescribeVpcEndpointServices`, `DescribeVpcEndpoints`
-
-`DescribeVpcPeeringConnections`, `DescribeVpnConnections`, `DescribeVpnGateways`, `DetachClassicLinkVpc`, `DetachVerifiedAccessTrustProvider`, `DetachVpnGateway`, `DisableAddressTransfer`, `DisableAwsNetworkPerformanceMetricSubscription`
-
-`DisableFastLaunch`, `DisableFastSnapshotRestores`, `DisableImage`, `DisableImageBlockPublicAccess`, `DisableImageDeprecation`, `DisableImageDeregistrationProtection`, `DisableIpamOrganizationAdminAccount`, `DisableSnapshotBlockPublicAccess`
-
-`DisableTransitGatewayRouteTablePropagation`, `DisableVgwRoutePropagation`, `DisableVpcClassicLink`, `DisableVpcClassicLinkDnsSupport`, `DisassociateClientVpnTargetNetwork`, `DisassociateEnclaveCertificateIamRole`, `DisassociateInstanceEventWindow`, `DisassociateIpamByoasn`
-
-`DisassociateIpamResourceDiscovery`, `DisassociateNatGatewayAddress`, `DisassociateSubnetCidrBlock`, `DisassociateTransitGatewayMulticastDomain`, `DisassociateTransitGatewayPolicyTable`, `DisassociateTransitGatewayRouteTable`, `DisassociateTrunkInterface`, `DisassociateVpcCidrBlock`
-
-`EnableAddressTransfer`, `EnableAwsNetworkPerformanceMetricSubscription`, `EnableFastLaunch`, `EnableFastSnapshotRestores`, `EnableImage`, `EnableImageBlockPublicAccess`, `EnableImageDeprecation`, `EnableImageDeregistrationProtection`
-
-`EnableIpamOrganizationAdminAccount`, `EnableReachabilityAnalyzerOrganizationSharing`, `EnableSnapshotBlockPublicAccess`, `EnableTransitGatewayRouteTablePropagation`, `EnableVgwRoutePropagation`, `EnableVolumeIO`, `EnableVpcClassicLink`, `EnableVpcClassicLinkDnsSupport`
-
-`ExportClientVpnClientCertificateRevocationList`, `ExportClientVpnClientConfiguration`, `ExportImage`, `ExportTransitGatewayRoutes`, `GetAssociatedEnclaveCertificateIamRoles`, `GetAssociatedIpv6PoolCidrs`, `GetAwsNetworkPerformanceData`, `GetCapacityReservationUsage`
-
-`GetCoipPoolUsage`, `GetConsoleScreenshot`, `GetDefaultCreditSpecification`, `GetEbsDefaultKmsKeyId`, `GetFlowLogsIntegrationTemplate`, `GetGroupsForCapacityReservation`, `GetHostReservationPurchasePreview`, `GetImageBlockPublicAccessState`
-
-`GetInstanceMetadataDefaults`, `GetInstanceTpmEkPub`, `GetInstanceTypesFromInstanceRequirements`, `GetInstanceUefiData`, `GetIpamAddressHistory`, `GetIpamDiscoveredAccounts`, `GetIpamDiscoveredPublicAddresses`, `GetIpamDiscoveredResourceCidrs`
-
-`GetIpamPoolAllocations`, `GetIpamPoolCidrs`, `GetIpamResourceCidrs`, `GetLaunchTemplateData`, `GetManagedPrefixListAssociations`, `GetManagedPrefixListEntries`, `GetNetworkInsightsAccessScopeAnalysisFindings`, `GetNetworkInsightsAccessScopeContent`
-
-`GetReservedInstancesExchangeQuote`, `GetSecurityGroupsForVpc`, `GetSnapshotBlockPublicAccessState`, `GetSpotPlacementScores`, `GetSubnetCidrReservations`, `GetTransitGatewayAttachmentPropagations`, `GetTransitGatewayMulticastDomainAssociations`, `GetTransitGatewayPolicyTableAssociations`
-
-`GetTransitGatewayPolicyTableEntries`, `GetTransitGatewayPrefixListReferences`, `GetTransitGatewayRouteTableAssociations`, `GetTransitGatewayRouteTablePropagations`, `GetVerifiedAccessEndpointPolicy`, `GetVerifiedAccessGroupPolicy`, `GetVpnConnectionDeviceSampleConfiguration`, `GetVpnConnectionDeviceTypes`
-
-`GetVpnTunnelReplacementStatus`, `ImportClientVpnClientCertificateRevocationList`, `ImportImage`, `ImportInstance`, `ImportSnapshot`, `ImportVolume`, `ListImagesInRecycleBin`, `ListSnapshotsInRecycleBin`
-
-`LockSnapshot`, `ModifyAddressAttribute`, `ModifyAvailabilityZoneGroup`, `ModifyCapacityReservation`, `ModifyCapacityReservationFleet`, `ModifyClientVpnEndpoint`, `ModifyDefaultCreditSpecification`, `ModifyEbsDefaultKmsKeyId`
-
-`ModifyFleet`, `ModifyFpgaImageAttribute`, `ModifyHosts`, `ModifyIdFormat`, `ModifyIdentityIdFormat`, `ModifyInstanceCapacityReservationAttributes`, `ModifyInstanceCreditSpecification`, `ModifyInstanceEventStartTime`
-
-`ModifyInstanceEventWindow`, `ModifyInstanceMaintenanceOptions`, `ModifyInstanceMetadataDefaults`, `ModifyInstancePlacement`, `ModifyIpam`, `ModifyIpamPool`, `ModifyIpamResourceCidr`, `ModifyIpamResourceDiscovery`
-
-`ModifyIpamScope`, `ModifyLocalGatewayRoute`, `ModifyManagedPrefixList`, `ModifyPrivateDnsNameOptions`, `ModifyReservedInstances`, `ModifySecurityGroupRules`, `ModifySnapshotAttribute`, `ModifySnapshotTier`
-
-`ModifySpotFleetRequest`, `ModifyTrafficMirrorFilterNetworkServices`, `ModifyTrafficMirrorFilterRule`, `ModifyTrafficMirrorSession`, `ModifyTransitGateway`, `ModifyTransitGatewayPrefixListReference`, `ModifyTransitGatewayVpcAttachment`, `ModifyVerifiedAccessEndpoint`
-
-`ModifyVerifiedAccessEndpointPolicy`, `ModifyVerifiedAccessGroup`, `ModifyVerifiedAccessGroupPolicy`, `ModifyVerifiedAccessInstance`, `ModifyVerifiedAccessInstanceLoggingConfiguration`, `ModifyVerifiedAccessTrustProvider`, `ModifyVolumeAttribute`, `ModifyVpcEndpoint`
-
-`ModifyVpcEndpointConnectionNotification`, `ModifyVpcEndpointServiceConfiguration`, `ModifyVpcEndpointServicePayerResponsibility`, `ModifyVpcEndpointServicePermissions`, `ModifyVpcPeeringConnectionOptions`, `ModifyVpcTenancy`, `ModifyVpnConnection`, `ModifyVpnConnectionOptions`
-
-`ModifyVpnTunnelCertificate`, `ModifyVpnTunnelOptions`, `MoveAddressToVpc`, `MoveByoipCidrToIpam`, `ProvisionByoipCidr`, `ProvisionIpamByoasn`, `ProvisionIpamPoolCidr`, `ProvisionPublicIpv4PoolCidr`
-
-`PurchaseCapacityBlock`, `PurchaseHostReservation`, `PurchaseReservedInstancesOffering`, `PurchaseScheduledInstances`, `RegisterInstanceEventNotificationAttributes`, `RegisterTransitGatewayMulticastGroupMembers`, `RegisterTransitGatewayMulticastGroupSources`, `RejectTransitGatewayMulticastDomainAssociations`
-
-`RejectTransitGatewayPeeringAttachment`, `RejectTransitGatewayVpcAttachment`, `RejectVpcEndpointConnections`, `RejectVpcPeeringConnection`, `ReleaseHosts`, `ReleaseIpamPoolAllocation`, `ReplaceNetworkAclAssociation`, `ReplaceNetworkAclEntry`
-
-`ReplaceTransitGatewayRoute`, `ReplaceVpnTunnel`, `ReportInstanceStatus`, `RequestSpotFleet`, `ResetAddressAttribute`, `ResetEbsDefaultKmsKeyId`, `ResetFpgaImageAttribute`, `ResetInstanceAttribute`
-
-`ResetNetworkInterfaceAttribute`, `ResetSnapshotAttribute`, `RestoreAddressToClassic`, `RestoreImageFromRecycleBin`, `RestoreManagedPrefixListVersion`, `RestoreSnapshotFromRecycleBin`, `RestoreSnapshotTier`, `RevokeClientVpnIngress`
-
-`RunScheduledInstances`, `SearchLocalGatewayRoutes`, `SearchTransitGatewayMulticastGroups`, `SearchTransitGatewayRoutes`, `SendDiagnosticInterrupt`, `StartNetworkInsightsAccessScopeAnalysis`, `StartNetworkInsightsAnalysis`, `StartVpcEndpointServicePrivateDnsVerification`
-
-`TerminateClientVpnConnections`, `UnassignIpv6Addresses`, `UnassignPrivateIpAddresses`, `UnassignPrivateNatGatewayAddress`, `UnlockSnapshot`, `UpdateSecurityGroupRuleDescriptionsEgress`, `UpdateSecurityGroupRuleDescriptionsIngress`, `WithdrawByoipCidr`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>
-
-## ecr
-
-Implements **19 of 47** modelled operations (40.4%).
-
-<details><summary>Implemented (19)</summary>
-
-`BatchDeleteImage`, `BatchGetImage`, `CreateRepository`, `DeleteLifecyclePolicy`, `DeleteRepository`, `DeleteRepositoryPolicy`, `DescribeImages`, `DescribeRepositories`
-
-`GetAuthorizationToken`, `GetLifecyclePolicy`, `GetLifecyclePolicyPreview`, `GetRepositoryPolicy`, `ListImages`, `ListTagsForResource`, `PutImage`, `PutImageTagMutability`
-
-`PutLifecyclePolicy`, `SetRepositoryPolicy`, `StartLifecyclePolicyPreview`
-
-
-</details>
-
-<details><summary>Missing from dispatch (11)</summary>
-
-`CreatePullThroughCacheRule`, `CreateRepositoryCreationTemplate`, `DeletePullThroughCacheRule`, `DeleteRegistryPolicy`, `DeleteRepositoryCreationTemplate`, `DescribeImageReplicationStatus`, `DescribePullThroughCacheRules`, `DescribeRepositoryCreationTemplates`
-
-`UpdatePullThroughCacheRule`, `UpdateRepositoryCreationTemplate`, `ValidatePullThroughCacheRule`
-
-
-</details>
-
-<details><summary>Registered stubs (13)</summary>
-
-`BatchCheckLayerAvailability`, `CompleteLayerUpload`, `DescribeRegistry`, `GetDownloadUrlForLayer`, `GetRegistryPolicy`, `InitiateLayerUpload`, `ListRepositories`, `PutRegistryPolicy`
-
-`PutReplicationConfiguration`, `ReplicateImage`, `TagResource`, `UntagResource`, `UploadLayerPart`
-
-
-</details>
-
-<details><summary>Deliberately unsupported (7)</summary>
-
-`BatchGetRepositoryScanningConfiguration`, `DescribeImageScanFindings`, `GetImageScanningConfiguration`, `GetRegistryScanningConfiguration`, `PutImageScanningConfiguration`, `PutRegistryScanningConfiguration`, `StartImageScan`
-
-
-</details>
-
-<details><summary>Registered outside the pinned model (3)</summary>
-
-`GetImageScanningConfiguration`, `ListRepositories`, `ReplicateImage`
-
-
-</details>
-
-## ecs
-
-Implements **31 of 56** modelled operations (55.4%).
-
-<details><summary>Implemented (31)</summary>
-
-`CreateCapacityProvider`, `CreateCluster`, `CreateService`, `DeleteCapacityProvider`, `DeleteCluster`, `DeleteService`, `DeregisterContainerInstance`, `DeregisterTaskDefinition`
-
-`DescribeCapacityProviders`, `DescribeClusters`, `DescribeContainerInstances`, `DescribeServices`, `DescribeTaskDefinition`, `DescribeTasks`, `ListClusters`, `ListContainerInstances`
-
-`ListServices`, `ListTagsForResource`, `ListTaskDefinitions`, `ListTasks`, `PutClusterCapacityProviders`, `RegisterContainerInstance`, `RegisterTaskDefinition`, `RunTask`
-
-`StartTask`, `StopTask`, `SubmitTaskStateChange`, `TagResource`, `UntagResource`, `UpdateContainerInstancesState`, `UpdateService`
-
-
-</details>
-
-<details><summary>Missing from dispatch (20)</summary>
-
-`CreateTaskSet`, `DeleteAccountSetting`, `DeleteAttributes`, `DeleteTaskDefinitions`, `DeleteTaskSet`, `DescribeTaskSets`, `DiscoverPollEndpoint`, `ExecuteCommand`
-
-`GetTaskProtection`, `ListAttributes`, `PutAccountSettingDefault`, `PutAttributes`, `SubmitAttachmentStateChanges`, `SubmitContainerStateChange`, `UpdateCapacityProvider`, `UpdateClusterSettings`
-
-`UpdateContainerAgent`, `UpdateServicePrimaryTaskSet`, `UpdateTaskProtection`, `UpdateTaskSet`
-
-
-</details>
-
-<details><summary>Registered stubs (5)</summary>
-
-`ListAccountSettings`, `ListServicesByNamespace`, `ListTaskDefinitionFamilies`, `PutAccountSetting`, `UpdateCluster`
-
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (3)</summary>
-
-`PollAssignments`, `ProvisionCapacity`, `ReportTaskGPU`
-
-
-</details>
-
-## elasticloadbalancingv2
-
-Implements **33 of 46** modelled operations (71.7%).
-
-<details><summary>Implemented (33)</summary>
-
-`AddListenerCertificates`, `AddTags`, `CreateListener`, `CreateLoadBalancer`, `CreateRule`, `CreateTargetGroup`, `DeleteListener`, `DeleteLoadBalancer`
-
-`DeleteRule`, `DeleteTargetGroup`, `DeregisterTargets`, `DescribeListenerCertificates`, `DescribeListeners`, `DescribeLoadBalancerAttributes`, `DescribeLoadBalancers`, `DescribeRules`
-
-`DescribeSSLPolicies`, `DescribeTags`, `DescribeTargetGroupAttributes`, `DescribeTargetGroups`, `DescribeTargetHealth`, `ModifyListener`, `ModifyLoadBalancerAttributes`, `ModifyRule`
-
-`ModifyTargetGroup`, `ModifyTargetGroupAttributes`, `RegisterTargets`, `RemoveListenerCertificates`, `RemoveTags`, `SetIpAddressType`, `SetRulePriorities`, `SetSecurityGroups`
-
-`SetSubnets`
-
-
-</details>
-
-<details><summary>Missing from dispatch (13)</summary>
-
-`AddTrustStoreRevocations`, `CreateTrustStore`, `DeleteSharedTrustStoreAssociation`, `DeleteTrustStore`, `DescribeAccountLimits`, `DescribeTrustStoreAssociations`, `DescribeTrustStoreRevocations`, `DescribeTrustStores`
-
-`GetResourcePolicy`, `GetTrustStoreCaCertificatesBundle`, `GetTrustStoreRevocationContent`, `ModifyTrustStore`, `RemoveTrustStoreRevocations`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (4)</summary>
-
-`DescribeListenerAttributes`, `GetLBConfig`, `LBAgentHeartbeat`, `ModifyListenerAttributes`
-
-
-</details>
-
-## iam
-
-Implements **75 of 159** modelled operations (47.2%).
-
-<details><summary>Implemented (75)</summary>
-
-`AddRoleToInstanceProfile`, `AddUserToGroup`, `AttachGroupPolicy`, `AttachRolePolicy`, `AttachUserPolicy`, `CreateAccessKey`, `CreateGroup`, `CreateInstanceProfile`
-
-`CreateOpenIDConnectProvider`, `CreatePolicy`, `CreateRole`, `CreateUser`, `DeleteAccessKey`, `DeleteGroup`, `DeleteGroupPolicy`, `DeleteInstanceProfile`
-
-`DeleteOpenIDConnectProvider`, `DeletePolicy`, `DeleteRole`, `DeleteRolePolicy`, `DeleteUser`, `DeleteUserPolicy`, `DetachGroupPolicy`, `DetachRolePolicy`
-
-`DetachUserPolicy`, `GetAccountSummary`, `GetGroup`, `GetGroupPolicy`, `GetInstanceProfile`, `GetOpenIDConnectProvider`, `GetPolicy`, `GetPolicyVersion`
-
-`GetRole`, `GetRolePolicy`, `GetUser`, `GetUserPolicy`, `ListAccessKeys`, `ListAttachedGroupPolicies`, `ListAttachedRolePolicies`, `ListAttachedUserPolicies`
-
-`ListGroupPolicies`, `ListGroups`, `ListGroupsForUser`, `ListInstanceProfileTags`, `ListInstanceProfiles`, `ListInstanceProfilesForRole`, `ListOpenIDConnectProviderTags`, `ListOpenIDConnectProviders`
-
-`ListPolicies`, `ListPolicyTags`, `ListPolicyVersions`, `ListRolePolicies`, `ListRoleTags`, `ListRoles`, `ListUserPolicies`, `ListUserTags`
-
-`ListUsers`, `PutGroupPolicy`, `PutRolePolicy`, `PutUserPolicy`, `RemoveRoleFromInstanceProfile`, `RemoveUserFromGroup`, `TagInstanceProfile`, `TagOpenIDConnectProvider`
-
-`TagPolicy`, `TagRole`, `TagUser`, `UntagInstanceProfile`, `UntagOpenIDConnectProvider`, `UntagPolicy`, `UntagRole`, `UntagUser`
-
-`UpdateAccessKey`, `UpdateAssumeRolePolicy`, `UpdateRole`
-
-
-</details>
-
-<details><summary>Missing from dispatch (84)</summary>
-
-`AddClientIDToOpenIDConnectProvider`, `ChangePassword`, `CreateAccountAlias`, `CreateLoginProfile`, `CreatePolicyVersion`, `CreateSAMLProvider`, `CreateServiceLinkedRole`, `CreateServiceSpecificCredential`
-
-`CreateVirtualMFADevice`, `DeactivateMFADevice`, `DeleteAccountAlias`, `DeleteAccountPasswordPolicy`, `DeleteLoginProfile`, `DeletePolicyVersion`, `DeleteRolePermissionsBoundary`, `DeleteSAMLProvider`
-
-`DeleteSSHPublicKey`, `DeleteServerCertificate`, `DeleteServiceLinkedRole`, `DeleteServiceSpecificCredential`, `DeleteSigningCertificate`, `DeleteUserPermissionsBoundary`, `DeleteVirtualMFADevice`, `EnableMFADevice`
-
-`GenerateCredentialReport`, `GenerateOrganizationsAccessReport`, `GenerateServiceLastAccessedDetails`, `GetAccessKeyLastUsed`, `GetAccountAuthorizationDetails`, `GetAccountPasswordPolicy`, `GetContextKeysForCustomPolicy`, `GetContextKeysForPrincipalPolicy`
-
-`GetCredentialReport`, `GetLoginProfile`, `GetMFADevice`, `GetOrganizationsAccessReport`, `GetSAMLProvider`, `GetSSHPublicKey`, `GetServerCertificate`, `GetServiceLastAccessedDetails`
-
-`GetServiceLastAccessedDetailsWithEntities`, `GetServiceLinkedRoleDeletionStatus`, `ListAccountAliases`, `ListEntitiesForPolicy`, `ListMFADeviceTags`, `ListMFADevices`, `ListPoliciesGrantingServiceAccess`, `ListSAMLProviderTags`
-
-`ListSAMLProviders`, `ListSSHPublicKeys`, `ListServerCertificateTags`, `ListServerCertificates`, `ListServiceSpecificCredentials`, `ListSigningCertificates`, `ListVirtualMFADevices`, `PutRolePermissionsBoundary`
-
-`PutUserPermissionsBoundary`, `RemoveClientIDFromOpenIDConnectProvider`, `ResetServiceSpecificCredential`, `ResyncMFADevice`, `SetDefaultPolicyVersion`, `SetSecurityTokenServicePreferences`, `SimulateCustomPolicy`, `SimulatePrincipalPolicy`
-
-`TagMFADevice`, `TagSAMLProvider`, `TagServerCertificate`, `UntagMFADevice`, `UntagSAMLProvider`, `UntagServerCertificate`, `UpdateAccountPasswordPolicy`, `UpdateGroup`
-
-`UpdateLoginProfile`, `UpdateOpenIDConnectProviderThumbprint`, `UpdateRoleDescription`, `UpdateSAMLProvider`, `UpdateSSHPublicKey`, `UpdateServerCertificate`, `UpdateServiceSpecificCredential`, `UpdateSigningCertificate`
-
-`UpdateUser`, `UploadSSHPublicKey`, `UploadServerCertificate`, `UploadSigningCertificate`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>
-
-## rds
-
-Implements **26 of 162** modelled operations (16.0%).
-
-<details><summary>Implemented (26)</summary>
-
-`AddTagsToResource`, `CreateDBInstance`, `CreateDBParameterGroup`, `CreateDBSnapshot`, `CreateDBSubnetGroup`, `DeleteDBInstance`, `DeleteDBParameterGroup`, `DeleteDBSnapshot`
-
-`DeleteDBSubnetGroup`, `DescribeDBEngineVersions`, `DescribeDBInstanceAutomatedBackups`, `DescribeDBInstances`, `DescribeDBParameterGroups`, `DescribeDBParameters`, `DescribeDBSnapshots`, `DescribeDBSubnetGroups`
-
-`DescribeEvents`, `DescribeOrderableDBInstanceOptions`, `ListTagsForResource`, `ModifyDBInstance`, `ModifyDBParameterGroup`, `RebootDBInstance`, `RemoveTagsFromResource`, `RestoreDBInstanceFromDBSnapshot`
-
-`StartDBInstance`, `StopDBInstance`
-
-
-</details>
-
-<details><summary>Missing from dispatch (124)</summary>
-
-`AddRoleToDBCluster`, `AddRoleToDBInstance`, `AddSourceIdentifierToSubscription`, `ApplyPendingMaintenanceAction`, `AuthorizeDBSecurityGroupIngress`, `BacktrackDBCluster`, `CancelExportTask`, `CopyDBClusterParameterGroup`
-
-`CopyDBClusterSnapshot`, `CopyDBParameterGroup`, `CopyDBSnapshot`, `CopyOptionGroup`, `CreateBlueGreenDeployment`, `CreateCustomDBEngineVersion`, `CreateDBClusterEndpoint`, `CreateDBClusterParameterGroup`
-
-`CreateDBClusterSnapshot`, `CreateDBProxy`, `CreateDBProxyEndpoint`, `CreateDBSecurityGroup`, `CreateDBShardGroup`, `CreateEventSubscription`, `CreateGlobalCluster`, `CreateIntegration`
-
-`CreateTenantDatabase`, `DeleteBlueGreenDeployment`, `DeleteCustomDBEngineVersion`, `DeleteDBClusterAutomatedBackup`, `DeleteDBClusterEndpoint`, `DeleteDBClusterParameterGroup`, `DeleteDBClusterSnapshot`, `DeleteDBInstanceAutomatedBackup`
-
-`DeleteDBProxy`, `DeleteDBProxyEndpoint`, `DeleteDBSecurityGroup`, `DeleteDBShardGroup`, `DeleteEventSubscription`, `DeleteGlobalCluster`, `DeleteIntegration`, `DeleteTenantDatabase`
-
-`DeregisterDBProxyTargets`, `DescribeAccountAttributes`, `DescribeBlueGreenDeployments`, `DescribeCertificates`, `DescribeDBClusterAutomatedBackups`, `DescribeDBClusterBacktracks`, `DescribeDBClusterEndpoints`, `DescribeDBClusterParameterGroups`
-
-`DescribeDBClusterParameters`, `DescribeDBClusterSnapshotAttributes`, `DescribeDBClusterSnapshots`, `DescribeDBLogFiles`, `DescribeDBProxies`, `DescribeDBProxyEndpoints`, `DescribeDBProxyTargetGroups`, `DescribeDBProxyTargets`
-
-`DescribeDBRecommendations`, `DescribeDBSecurityGroups`, `DescribeDBShardGroups`, `DescribeDBSnapshotAttributes`, `DescribeDBSnapshotTenantDatabases`, `DescribeEngineDefaultClusterParameters`, `DescribeEngineDefaultParameters`, `DescribeEventCategories`
-
-`DescribeEventSubscriptions`, `DescribeExportTasks`, `DescribeGlobalClusters`, `DescribeIntegrations`, `DescribeOptionGroupOptions`, `DescribePendingMaintenanceActions`, `DescribeReservedDBInstances`, `DescribeReservedDBInstancesOfferings`
-
-`DescribeSourceRegions`, `DescribeTenantDatabases`, `DescribeValidDBInstanceModifications`, `DisableHttpEndpoint`, `DownloadDBLogFilePortion`, `EnableHttpEndpoint`, `FailoverGlobalCluster`, `ModifyActivityStream`
-
-`ModifyCertificates`, `ModifyCurrentDBClusterCapacity`, `ModifyCustomDBEngineVersion`, `ModifyDBClusterEndpoint`, `ModifyDBClusterParameterGroup`, `ModifyDBClusterSnapshotAttribute`, `ModifyDBProxy`, `ModifyDBProxyEndpoint`
-
-`ModifyDBProxyTargetGroup`, `ModifyDBRecommendation`, `ModifyDBShardGroup`, `ModifyDBSnapshot`, `ModifyDBSnapshotAttribute`, `ModifyDBSubnetGroup`, `ModifyEventSubscription`, `ModifyGlobalCluster`
-
-`ModifyIntegration`, `ModifyTenantDatabase`, `PromoteReadReplicaDBCluster`, `PurchaseReservedDBInstancesOffering`, `RebootDBCluster`, `RebootDBShardGroup`, `RegisterDBProxyTargets`, `RemoveFromGlobalCluster`
-
-`RemoveRoleFromDBCluster`, `RemoveRoleFromDBInstance`, `RemoveSourceIdentifierFromSubscription`, `ResetDBClusterParameterGroup`, `ResetDBParameterGroup`, `RestoreDBClusterFromS3`, `RestoreDBClusterFromSnapshot`, `RestoreDBClusterToPointInTime`
-
-`RestoreDBInstanceFromS3`, `RevokeDBSecurityGroupIngress`, `StartActivityStream`, `StartDBCluster`, `StartDBInstanceAutomatedBackupsReplication`, `StartExportTask`, `StopActivityStream`, `StopDBCluster`
-
-`StopDBInstanceAutomatedBackupsReplication`, `SwitchoverBlueGreenDeployment`, `SwitchoverGlobalCluster`, `SwitchoverReadReplica`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (12)</summary>
-
-`CreateDBCluster`, `CreateDBInstanceReadReplica`, `CreateOptionGroup`, `DeleteDBCluster`, `DeleteOptionGroup`, `DescribeDBClusters`, `DescribeOptionGroups`, `FailoverDBCluster`
-
-`ModifyDBCluster`, `ModifyOptionGroup`, `PromoteReadReplica`, `RestoreDBInstanceToPointInTime`
-
-
-</details>
-
-<details><summary>Registered outside the pinned model (5)</summary>
-
-`AcknowledgeDBBootstrap`, `GetDBBootstrapConfig`, `PollDBCommands`, `RegisterDBInstance`, `SubmitDBStateChange`
-
-
-</details>
-
-## s3
-
-Operation coverage is not enumerable: Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.
-
-## sts
-
-Implements **4 of 8** modelled operations (50.0%).
-
-<details><summary>Implemented (4)</summary>
-
-`AssumeRole`, `AssumeRoleWithWebIdentity`, `GetCallerIdentity`, `GetSessionToken`
-
-
-</details>
-
-<details><summary>Missing from dispatch (4)</summary>
-
-`AssumeRoleWithSAML`, `DecodeAuthorizationMessage`, `GetAccessKeyInfo`, `GetFederationToken`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>
@@ -136,6 +136,21 @@ aws ec2 run-instances \
  --key-name spinifex-key
 ```

+### Multi-GPU Instances
+
+The GPU count comes from the instance type, exactly as it does in EC2. AWS multi-GPU types pair eight GPUs with a very large host — `p4d.24xlarge` is 96 vCPU and 1152 GiB, — so an edge rig with eight cards and 256 GiB cannot admit them even though it has the GPUs. For those hosts Spinifex offers its own `gpu.*` family, which is not an EC2 family:
+
+```bash
+aws ec2 run-instances \
+  --image-id $GPU_AMI \
+  --instance-type gpu.8x4c \
+  --key-name spinifex-key
+```
+
+`gpu.<count>x<vcpu>c` is that many GPUs at that many vCPU each, so `gpu.8x4c` is eight GPUs on 32 vCPU and 128 GiB. Counts run in three tiers — `2c`, `4c` and `8c` — all at 4 GiB of memory per vCPU. The GPU model and VRAM reported in `DescribeInstanceTypes` are whatever the node discovered, so a `gpu.8x4c` on an 8×RTX 3090 box advertises eight RTX 3090s at 24 GiB each.
+
+A node holding more than one distinct GPU model offers single-GPU types only, and logs each withheld type by name. Two devices count as the same model when their reported name and VRAM match, which keeps a mixed-SKU H100 node fully usable.
+
 To verify the GPU is visible from inside the instance, SSH in and run:

 ```bash
@@ -161,6 +161,12 @@ Both support the same AWS features: public subnets, Elastic IPs, security groups

 ## `nat` — Shared SNAT (Simple)

+**`nat` is a supported mode, not a fallback.** It is covered by the nightly e2e suite (cells 19 and 30, single- and multi-node) and it is the only mode a cloud-hosted node can run, so the OCI deployment depends on it. Read "specialised" below as "fewer deployments need it", never as "less finished".
+
+**Start with `pool` unless one of the reasons below applies.** Pool mode is the default, is what most of this document describes, and is what the AWS feature set was built against. `nat` is the specialised mode: it exists for uplinks that cannot be bridged, and it reaches AWS parity through host-side plumbing rather than by putting VM addresses on the wire.
+
+Reach for `nat` when the uplink refuses to carry anything but its own MAC and its own addresses — WiFi and WWAN, and **any cloud-hosted node**, where the VNIC drops a frame whose source address is not registered to it. On such a host `nat` is not a preference, it is the only mode that works at all; pool mode's localnet needs to announce VM addresses by ARP and a cloud VNIC will not carry them. See [oci-integration](../../oci-integration/README.md).
+
 All VMs share a single external IP for outbound SNAT. By default there are no public IPs, no Elastic IPs, and no inbound from WAN — all subnets behave as private subnets with internet access. On routed-NAT nodes, adding a public pool restores full public IP parity (see below).

 > **Limitation (routed-NAT v1):** System instances (ECS/EKS/load-balancer
@@ -244,6 +250,24 @@ gateway     = "192.168.1.1"
 prefix_len  = 24
 ```

+### NAT gateways in NAT mode
+
+`CreateNatGateway` works here too, and draws its public address from the same public pool an EIP does. On the OVN side it is an `snat` row on the VPC router rewriting the private subnet's CIDR to that address — not the `dnat_and_snat` an EIP gets — so it is centralised on the VPC's gateway chassis by construction and there is no port or MAC to bind it to.
+
+**That address needs the same host plumbing as an EIP**, and on a routed-NAT node it is the host that delivers it: the `/32` route steering it into OVN, the proxy-ARP entry answering for it on the uplink, and the FORWARD accepts. Without them an egressing private instance leaves with a source address the host holds no route back to, while the masquerade rule — which matches the transit `100.127.0.0/24` alone — never sees the packet. Spinifex binds it when the gateway is created and re-asserts it on the host EIP pass, so no operator step is involved; `ip route show dev spx-nat-host` should list the gateway's public address alongside any EIPs.
+
+### NAT mode on more than one node
+
+`spx admin init --external-mode=nat --nodes N` works for N > 1, and each public IP is delivered by the node running its instance rather than by one gateway node. Three things make that true, and all three are automatic:
+
+- The `dnat_and_snat` row carries `external_mac` and `logical_port`, so OVN processes it on the instance's own chassis — the same mechanism pool mode has always used. It is safe here because `ext-shared` is a localnet, so the transit veth is node-local and only the node running the instance can answer for the address on it.
+- Each node installs a host `/32` for its own instances' addresses and leaves the rest alone, asking local OVS which ENIs are plugged in here. Run `ip route show dev spx-nat-host` on each node: exactly one node should hold each public address.
+- Every node's `spx-nat-host` carries the same MAC, `02:00:64:7f:00:01`. OVN keeps one MAC binding for the transit next hop across the whole cluster, so a per-node address would send every chassis's return traffic to whichever node wrote the binding last. `setup-ovn.sh` sets it and vpcd re-asserts it on every start.
+
+Default egress — an instance with no public IP — is still centralised on the VPC's gateway chassis, which is unchanged and is what the gateway LRP's transit address is for.
+
+**On a cloud host the address must also be registered on that node's VNIC.** The allocator runs per node and registers on the local VNIC, which is the right one exactly because the traffic now leaves there. `oci_vnic_iface` is the setting; `oci_vnic_id` is only needed to pin a pool to a specific VNIC and should not be set for this.
+
 **Caveat — reaching an EIP from the spinifex host itself.** Host-sourced traffic enters OVN from the transit net, which is exempt from NAT (that is what makes the jumpbox pattern work) — so a host connection to an EIP that carries the transit source IP would skip DNAT. Spinifex stamps the EIP route with the uplink's LAN IP as source to avoid this, but if no uplink address can be determined, connect to the instance's **private IP** from the host instead. Other machines on the LAN are unaffected.

 ## Disabled (Empty/Omitted)
@@ -263,6 +287,9 @@ VPC networking is overlay-only. No external connectivity. Instances can only com
 | DescribeInstances shows public IP | Yes             | Yes           | With public pool  | No       |
 | Admin must reserve IP range       | Yes             | No            | Only static pool  | No       |
 | Needs router DHCP                 | No              | Yes           | Optional          | No       |
+| Works on a non-bridgeable uplink  | No              | No            | Yes               | n/a      |
+| NAT gateways                      | Yes             | Yes           | With public pool  | No       |
+| More than one node                | Yes             | Yes           | Yes               | Yes      |

 If you start with `nat` and later need public subnets: on a bridgeable uplink switch to `pool` and define a range (or use `source = "dhcp"`); on a routed-NAT node just add a public pool alongside `nat-transit` — no data migration needed.

@@ -460,8 +487,8 @@ external_mode = "pool"    # "pool", "nat", or "" (disabled)

 | Value          | Behavior                                                          |
 | -------------- | ----------------------------------------------------------------- |
-| `"pool"`       | Full public networking — public subnets, auto-assign, Elastic IPs |
-| `"nat"`        | Outbound-only SNAT — all VMs share one external IP                |
+| `"pool"`       | **Default.** Full public networking — public subnets, auto-assign, Elastic IPs |
+| `"nat"`        | Outbound-only SNAT — all VMs share one external IP. Add a public pool for full parity. For uplinks that cannot be bridged, including every cloud host |
 | `""` / omitted | Overlay-only — no external connectivity                           |

 ## IP Pools: network.external_pools
@@ -842,18 +869,8 @@ one entry covers every security group in that VPC.
 own `spinifex.toml`, and there is no live distribution of it. Two properties of
 the current implementation make the operator responsible for consistency:

- **One node writes the ACLs.** SG reconcile runs on a single CAS-elected vpcd
-  leader, which programs the shared OVN northbound DB. Whichever node holds the
-  lease is the one whose `blocked_ports_wan` / `egress_block_exempt_vpcs` is in
-  force. Leadership moves on restart or crash, so if the node configs disagree,
-  the effective policy **changes when the leader changes** and the drift pass
-  flaps the ACLs between the two states. Edit the value **identically on every
-  node**.
- **It is read at vpcd startup, not hot-reloaded.** The policy is built once when
-  vpcd starts; editing the TOML does nothing until vpcd restarts. Deploy the
-  config change to all nodes and restart vpcd cluster-wide (take the target down
-  and confirm no `spx` process survives — a selective single-service restart is
-  not reliable on the shared binary).
+- **One node writes the ACLs.** SG reconcile runs on a single CAS-elected vpcd leader, which programs the shared OVN northbound DB. Whichever node holds the lease is the one whose `blocked_ports_wan` / `egress_block_exempt_vpcs` is in force. Leadership moves on restart or crash, so if the node configs disagree, the effective policy **changes when the leader changes** and the drift pass flaps the ACLs between the two states. Edit the value **identically on every node**.
+- **It is read at vpcd startup, not hot-reloaded.** The policy is built once when vpcd starts; editing the TOML does nothing until vpcd restarts. Deploy the config change to all nodes and restart vpcd cluster-wide (take the target down and confirm no `spx` process survives — a selective single-service restart is not reliable on the shared binary).

 This is the current implementation and is deliberately minimal. A future revision
 will move the exemption into shared cluster state (so a single edit propagates
@@ -933,7 +950,7 @@ firewall_enabled = true

 Before enabling it, check what else the machine is serving — anything listening
 outside the public port group stops accepting new connections. See
-[Firewall and Cluster Membership](/docs/install-multi-node#firewall-and-cluster-membership)
+[Host Firewall](/docs/host-firewall)
 for the full port policy and the cluster-formation steps.

 ### Verifying it
@@ -146,12 +146,11 @@ Repositories can be managed as code with the standard `aws_ecr_repository` resou

 ```hcl
 provider "aws" {
-  region                      = "ap-southeast-2"
-  skip_credentials_validation = true
-  skip_requesting_account_id  = true
+  region = "ap-southeast-2"

  endpoints {
    ecr = "https://<your-spinifex-gateway>"
+    sts = "https://<your-spinifex-gateway>"
  }
 }

@@ -82,12 +82,11 @@ Before creating a cluster you need the supporting infrastructure in place. The T
 - **Spinifex running**, with the AWS CLI configured for the `spinifex` profile (see [Installing Spinifex](/docs/install)) and `kubectl` installed.
 - **The `eks-node` image** registered on the cluster. The console blocks cluster creation until it is present.
 - **A VPC with at least one subnet.** The control plane launches in your first subnet; workers run in the subnets you pass to the node group.
- **Egress for the workers.** Workers must reach the internet to pull container images:
-  - **Public workers** — an **Internet Gateway** with a default route (`0.0.0.0/0 → IGW`) and public IPs.
-  - **Private workers** — a **NAT Gateway** in a public subnet with a default route (`0.0.0.0/0 → NAT`) from the private subnets.
- **Two IAM roles:**
-  - A **cluster role** trusted by `eks.amazonaws.com` with `AmazonEKSClusterPolicy` attached.
-  - A **node role** trusted by `ec2.amazonaws.com` with `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly` attached (the last lets workers pull images from [ECR](/docs/ecr)).
+- **Egress for the workers.** Workers must reach the internet to pull container images.
+- **Public workers** need an **Internet Gateway** with a default route (`0.0.0.0/0 → IGW`) and public IPs.
+- **Private workers** need a **NAT Gateway** in a public subnet with a default route (`0.0.0.0/0 → NAT`) from the private subnets.
+- **A cluster role** trusted by `eks.amazonaws.com` with `AmazonEKSClusterPolicy` attached.
+- **A node role** trusted by `ec2.amazonaws.com` with `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly` attached (the last lets workers pull images from [ECR](/docs/ecr)).

 ## Instructions

@@ -273,4 +272,4 @@ The workbooks form a ladder. **eks-quickstart** is a NodePort demo; **eks-https-

 A complete end-to-end example of Spinifex EKS in production: a GPU-accelerated AI inference platform running on a Supermicro X14 2U with two NVIDIA RTX Pro 6000 Blackwell GPUs. The guide provisions an EKS cluster with two GPU worker nodes via VFIO PCIe passthrough, deploys an OpenAI-compatible LLM API (llama.cpp / Llama 3.2 3B) and a real-time YOLO object-detection stream on separate GPU nodes, and wires them together behind an ALB using IAM, ECR, ACM, and EBS — all with standard AWS tooling and a single `AWS_PROFILE` swap.

-→ [EKS AI Platform Reference Architecture](/docs/RTX-Pro-6000-EKS)
+→ [EKS AI Platform Reference Architecture](/hardware/supermicro/rtx-pro-6000)
@@ -0,0 +1,34 @@
+---
+title: "AWS API Coverage"
+seoTitle: "Which AWS API Operations Spinifex Implements — Spinifex Docs"
+description: "The AWS API operations Spinifex serves, generated from its gateway dispatch tables and the pinned AWS SDK service models on each build, service by service."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - api
+  - operations
+---
+
+# AWS API Coverage
+
+## Overview
+
+The platform serves **397 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
+
+| Service | Operations |
+|---|---:|
+| [ACM](/coverage/acm) | 9 |
+| [EC2](/coverage/ec2) | 127 |
+| [ECR](/coverage/ecr) | 22 |
+| [ECS](/coverage/ecs) | 31 |
+| [EKS](/coverage/eks) | 36 |
+| [ELBv2](/coverage/elbv2) | 36 |
+| [IAM](/coverage/iam) | 79 |
+| [RDS](/coverage/rds) | 26 |
+| [S3](/coverage/s3) | 26 |
+| [STS](/coverage/sts) | 5 |
+| **Total** | **397** |
@@ -0,0 +1,43 @@
+---
+title: "ACM API Coverage"
+seoTitle: "AWS Certificate Manager API Coverage — Spinifex Docs"
+description: "The AWS Certificate Manager API operations Spinifex implements, generated from the gateway dispatch tables and the pinned AWS service model on each build."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - acm
+  - certificates
+  - tls
+---
+
+# ACM API Coverage
+
+## Overview
+
+Spinifex implements **9 operations** in the ACM `2015-12-08` API model.
+
+### Import and issuance
+
+Spinifex both stores externally-issued certificates and issues its own for load balancer listener references. Certificates are account-scoped, and a delete is refused while any listener still references the ARN — there is no force flag, matching AWS.
+
+`RequestCertificate` mints an ARN immediately and returns `PENDING_VALIDATION`; it never issues inline except against a tenant private CA, which has no domain to validate. The validation mode is derived from deployment state rather than configured: the DNS provider API where a credential exists, a manual TXT record where the platform hosts the zone, and a private CA otherwise — the only option for a deployment with no publicly delegated domain.
+
+Terraform's canonical certificate, DNS record and validation flow works unmodified in every mode. Where Spinifex owns the record write it emits no `ResourceRecord`, so iterating the validation options yields zero records and the validation resource still blocks correctly by polling until the certificate is issued.
+
+### Operations
+
+| Operation |
+|---|
+| `AddTagsToCertificate` |
+| `DeleteCertificate` |
+| `DescribeCertificate` |
+| `GetCertificate` |
+| `ImportCertificate` |
+| `ListCertificates` |
+| `ListTagsForCertificate` |
+| `RemoveTagsFromCertificate` |
+| `RequestCertificate` |
@@ -0,0 +1,7 @@
+### Import and issuance
+
+Spinifex both stores externally-issued certificates and issues its own for load balancer listener references. Certificates are account-scoped, and a delete is refused while any listener still references the ARN — there is no force flag, matching AWS.
+
+`RequestCertificate` mints an ARN immediately and returns `PENDING_VALIDATION`; it never issues inline except against a tenant private CA, which has no domain to validate. The validation mode is derived from deployment state rather than configured: the DNS provider API where a credential exists, a manual TXT record where the platform hosts the zone, and a private CA otherwise — the only option for a deployment with no publicly delegated domain.
+
+Terraform's canonical certificate, DNS record and validation flow works unmodified in every mode. Where Spinifex owns the record write it emits no `ResourceRecord`, so iterating the validation options yields zero records and the validation resource still blocks correctly by polling until the certificate is issued.
@@ -0,0 +1,157 @@
+---
+title: "EC2 API Coverage"
+seoTitle: "Amazon EC2 API Coverage on Spinifex — Spinifex Docs"
+description: "The Amazon EC2 API operations Spinifex implements, covering instances, EBS volumes, VPC networking, tags, security groups and the rest of the compute surface."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - ec2
+  - compute
+  - vpc
+---
+
+# EC2 API Coverage
+
+## Overview
+
+Spinifex implements **127 operations** in the EC2 `2016-11-15` API model.
+
+### Spot Instances Are a Mock
+
+Spot Instance Requests are a mock over the on-demand `RunInstances` path. A request synchronously launches real VMs on the operator's own compute and is then reported `active` and `fulfilled`. There is no spot market: no bidding, no price rejection, no interruption and no reclamation, and instances are never taken back.
+
+### Operations
+
+| Operation |
+|---|
+| `AllocateAddress` |
+| `AssociateAddress` |
+| `AssociateIamInstanceProfile` |
+| `AssociateRouteTable` |
+| `AttachInternetGateway` |
+| `AttachNetworkInterface` |
+| `AttachVolume` |
+| `AuthorizeSecurityGroupEgress` |
+| `AuthorizeSecurityGroupIngress` |
+| `CancelCapacityReservation` |
+| `CancelSpotInstanceRequests` |
+| `CopyImage` |
+| `CopySnapshot` |
+| `CreateCapacityReservation` |
+| `CreateEgressOnlyInternetGateway` |
+| `CreateImage` |
+| `CreateInternetGateway` |
+| `CreateKeyPair` |
+| `CreateLaunchTemplate` |
+| `CreateLaunchTemplateVersion` |
+| `CreateNatGateway` |
+| `CreateNetworkInterface` |
+| `CreatePlacementGroup` |
+| `CreateRoute` |
+| `CreateRouteTable` |
+| `CreateSecurityGroup` |
+| `CreateSnapshot` |
+| `CreateSubnet` |
+| `CreateTags` |
+| `CreateVolume` |
+| `CreateVpc` |
+| `DeleteEgressOnlyInternetGateway` |
+| `DeleteInternetGateway` |
+| `DeleteKeyPair` |
+| `DeleteLaunchTemplate` |
+| `DeleteLaunchTemplateVersions` |
+| `DeleteNatGateway` |
+| `DeleteNetworkInterface` |
+| `DeletePlacementGroup` |
+| `DeleteRoute` |
+| `DeleteRouteTable` |
+| `DeleteSecurityGroup` |
+| `DeleteSnapshot` |
+| `DeleteSubnet` |
+| `DeleteTags` |
+| `DeleteVolume` |
+| `DeleteVpc` |
+| `DeregisterImage` |
+| `DescribeAccountAttributes` |
+| `DescribeAddresses` |
+| `DescribeAddressesAttribute` |
+| `DescribeAvailabilityZones` |
+| `DescribeCapacityReservations` |
+| `DescribeEgressOnlyInternetGateways` |
+| `DescribeIamInstanceProfileAssociations` |
+| `DescribeImageAttribute` |
+| `DescribeImages` |
+| `DescribeInstanceAttribute` |
+| `DescribeInstanceCreditSpecifications` |
+| `DescribeInstanceStatus` |
+| `DescribeInstanceTypeOfferings` |
+| `DescribeInstanceTypes` |
+| `DescribeInstances` |
+| `DescribeInternetGateways` |
+| `DescribeKeyPairs` |
+| `DescribeLaunchTemplateVersions` |
+| `DescribeLaunchTemplates` |
+| `DescribeNatGateways` |
+| `DescribeNetworkInterfaces` |
+| `DescribePlacementGroups` |
+| `DescribeRegions` |
+| `DescribeRouteTables` |
+| `DescribeSecurityGroupRules` |
+| `DescribeSecurityGroups` |
+| `DescribeSnapshots` |
+| `DescribeSpotInstanceRequests` |
+| `DescribeSubnets` |
+| `DescribeTags` |
+| `DescribeVolumeStatus` |
+| `DescribeVolumes` |
+| `DescribeVolumesModifications` |
+| `DescribeVpcAttribute` |
+| `DescribeVpcEndpoints` |
+| `DescribeVpcs` |
+| `DetachInternetGateway` |
+| `DetachNetworkInterface` |
+| `DetachVolume` |
+| `DisableEbsEncryptionByDefault` |
+| `DisableSerialConsoleAccess` |
+| `DisassociateAddress` |
+| `DisassociateIamInstanceProfile` |
+| `DisassociateRouteTable` |
+| `EnableEbsEncryptionByDefault` |
+| `EnableSerialConsoleAccess` |
+| `GetConsoleOutput` |
+| `GetEbsEncryptionByDefault` |
+| `GetPasswordData` |
+| `GetSecurityGroupsForVpc` |
+| `GetSerialConsoleAccessStatus` |
+| `ImportKeyPair` |
+| `ModifyImageAttribute` |
+| `ModifyInstanceAttribute` |
+| `ModifyInstanceMetadataOptions` |
+| `ModifyLaunchTemplate` |
+| `ModifyNetworkInterfaceAttribute` |
+| `ModifySecurityGroupRules` |
+| `ModifySubnetAttribute` |
+| `ModifyVolume` |
+| `ModifyVpcAttribute` |
+| `MonitorInstances` |
+| `RebootInstances` |
+| `RegisterImage` |
+| `ReleaseAddress` |
+| `ReplaceIamInstanceProfileAssociation` |
+| `ReplaceRoute` |
+| `ReplaceRouteTableAssociation` |
+| `RequestSpotInstances` |
+| `ResetImageAttribute` |
+| `RevokeSecurityGroupEgress` |
+| `RevokeSecurityGroupIngress` |
+| `RunInstances` |
+| `StartInstances` |
+| `StopInstances` |
+| `TerminateInstances` |
+| `UnmonitorInstances` |
+| `UpdateSecurityGroupRuleDescriptionsEgress` |
+| `UpdateSecurityGroupRuleDescriptionsIngress` |
@@ -0,0 +1,3 @@
+### Spot Instances Are a Mock
+
+Spot Instance Requests are a mock over the on-demand `RunInstances` path. A request synchronously launches real VMs on the operator's own compute and is then reported `active` and `fulfilled`. There is no spot market: no bidding, no price rejection, no interruption and no reclamation, and instances are never taken back.
@@ -0,0 +1,56 @@
+---
+title: "ECR API Coverage"
+seoTitle: "Elastic Container Registry API Coverage — Spinifex Docs"
+description: "The Amazon ECR API operations Spinifex implements, alongside the OCI distribution endpoint that carries the image layers for every repository it serves."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - ecr
+  - containers
+  - registry
+---
+
+# ECR API Coverage
+
+## Overview
+
+Spinifex implements **22 operations** in the ECR `2015-09-21` API model.
+
+### Two endpoints, one registry
+
+Repository metadata is served over the AWS API on the gateway endpoint. Image data moves over the OCI Distribution `/v2/` endpoint on that same host, authenticated by the bearer token `GetAuthorizationToken` mints for `docker login`.
+
+The split explains the stubs below. The layer-transfer operations — `BatchCheckLayerAvailability`, `InitiateLayerUpload`, `UploadLayerPart`, `CompleteLayerUpload` and `GetDownloadUrlForLayer` — are registered stubs because the `/v2/` endpoint carries that traffic instead. A client using `docker` or any OCI-compatible tool never calls them.
+
+Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to.
+
+### Operations
+
+| Operation |
+|---|
+| `BatchDeleteImage` |
+| `BatchGetImage` |
+| `CreateRepository` |
+| `DeleteLifecyclePolicy` |
+| `DeleteRepository` |
+| `DeleteRepositoryPolicy` |
+| `DescribeImages` |
+| `DescribeRepositories` |
+| `GetAuthorizationToken` |
+| `GetLifecyclePolicy` |
+| `GetLifecyclePolicyPreview` |
+| `GetRepositoryPolicy` |
+| `ListImages` |
+| `ListTagsForResource` |
+| `PutImage` |
+| `PutImageScanningConfiguration` |
+| `PutImageTagMutability` |
+| `PutLifecyclePolicy` |
+| `SetRepositoryPolicy` |
+| `StartLifecyclePolicyPreview` |
+| `TagResource` |
+| `UntagResource` |
@@ -0,0 +1,7 @@
+### Two endpoints, one registry
+
+Repository metadata is served over the AWS API on the gateway endpoint. Image data moves over the OCI Distribution `/v2/` endpoint on that same host, authenticated by the bearer token `GetAuthorizationToken` mints for `docker login`.
+
+The split explains the stubs below. The layer-transfer operations — `BatchCheckLayerAvailability`, `InitiateLayerUpload`, `UploadLayerPart`, `CompleteLayerUpload` and `GetDownloadUrlForLayer` — are registered stubs because the `/v2/` endpoint carries that traffic instead. A client using `docker` or any OCI-compatible tool never calls them.
+
+Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to.
@@ -0,0 +1,61 @@
+---
+title: "ECS API Coverage"
+seoTitle: "Elastic Container Service API Coverage — Spinifex Docs"
+description: "The Amazon ECS API operations Spinifex implements, covering clusters, services, tasks, container instances and the task definitions they are launched from."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - ecs
+  - containers
+  - orchestration
+---
+
+# ECS API Coverage
+
+## Overview
+
+Spinifex implements **31 operations** in the ECS `2014-11-13` API model.
+
+### No Fargate
+
+Clusters, services and tasks run on EC2 container instances. There is no Fargate launch type, and no operation below provides one.
+
+### Operations
+
+| Operation |
+|---|
+| `CreateCapacityProvider` |
+| `CreateCluster` |
+| `CreateService` |
+| `DeleteCapacityProvider` |
+| `DeleteCluster` |
+| `DeleteService` |
+| `DeregisterContainerInstance` |
+| `DeregisterTaskDefinition` |
+| `DescribeCapacityProviders` |
+| `DescribeClusters` |
+| `DescribeContainerInstances` |
+| `DescribeServices` |
+| `DescribeTaskDefinition` |
+| `DescribeTasks` |
+| `ListClusters` |
+| `ListContainerInstances` |
+| `ListServices` |
+| `ListTagsForResource` |
+| `ListTaskDefinitions` |
+| `ListTasks` |
+| `PutClusterCapacityProviders` |
+| `RegisterContainerInstance` |
+| `RegisterTaskDefinition` |
+| `RunTask` |
+| `StartTask` |
+| `StopTask` |
+| `SubmitTaskStateChange` |
+| `TagResource` |
+| `UntagResource` |
+| `UpdateContainerInstancesState` |
+| `UpdateService` |
@@ -0,0 +1,3 @@
+### No Fargate
+
+Clusters, services and tasks run on EC2 container instances. There is no Fargate launch type, and no operation below provides one.
@@ -0,0 +1,62 @@
+---
+title: "EKS API Coverage"
+seoTitle: "Elastic Kubernetes Service API Coverage — Spinifex Docs"
+description: "The Amazon EKS API operations Spinifex implements, covering clusters, nodegroups, add-ons, access entries and the identity providers they authenticate."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - eks
+  - kubernetes
+  - containers
+---
+
+# EKS API Coverage
+
+## Overview
+
+Spinifex implements **36 operations** in the EKS `2017-11-01` API model.
+
+### Operations
+
+| Operation |
+|---|
+| `AssociateAccessPolicy` |
+| `AssociateIdentityProviderConfig` |
+| `CreateAccessEntry` |
+| `CreateAddon` |
+| `CreateCluster` |
+| `CreateNodegroup` |
+| `DeleteAccessEntry` |
+| `DeleteAddon` |
+| `DeleteCluster` |
+| `DeleteNodegroup` |
+| `DescribeAccessEntry` |
+| `DescribeAddon` |
+| `DescribeAddonVersions` |
+| `DescribeCluster` |
+| `DescribeIdentityProviderConfig` |
+| `DescribeNodegroup` |
+| `DescribeUpdate` |
+| `DisassociateAccessPolicy` |
+| `DisassociateIdentityProviderConfig` |
+| `ListAccessEntries` |
+| `ListAccessPolicies` |
+| `ListAddons` |
+| `ListAssociatedAccessPolicies` |
+| `ListClusters` |
+| `ListIdentityProviderConfigs` |
+| `ListNodegroups` |
+| `ListTagsForResource` |
+| `ListUpdates` |
+| `TagResource` |
+| `UntagResource` |
+| `UpdateAccessEntry` |
+| `UpdateAddon` |
+| `UpdateClusterConfig` |
+| `UpdateClusterVersion` |
+| `UpdateNodegroupConfig` |
+| `UpdateNodegroupVersion` |
@@ -0,0 +1,66 @@
+---
+title: "ELBv2 API Coverage"
+seoTitle: "Elastic Load Balancing v2 API Coverage — Spinifex Docs"
+description: "The Elastic Load Balancing v2 API operations Spinifex implements, for both the Application and Network Load Balancers it serves, listeners and target groups."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - elbv2
+  - load balancing
+  - networking
+---
+
+# ELBv2 API Coverage
+
+## Overview
+
+Spinifex implements **36 operations** in the ELBv2 `2015-12-01` API model.
+
+### Two data planes
+
+The data plane is a system-managed load balancer VM, launched automatically during `CreateLoadBalancer`. Application Load Balancers run HAProxy for the L7 surface — rules, fixed responses and redirects over HTTP and HTTPS. Network Load Balancers run nginx `stream` for the L4 surface — TCP, UDP, TLS and TCP_UDP — because HAProxy cannot load-balance UDP. The agent selects the engine from the configuration the control plane delivers, so the choice follows the load balancer type and is not separately configurable.
+
+### Operations
+
+| Operation |
+|---|
+| `AddListenerCertificates` |
+| `AddTags` |
+| `CreateListener` |
+| `CreateLoadBalancer` |
+| `CreateRule` |
+| `CreateTargetGroup` |
+| `DeleteListener` |
+| `DeleteLoadBalancer` |
+| `DeleteRule` |
+| `DeleteTargetGroup` |
+| `DeregisterTargets` |
+| `DescribeAccountLimits` |
+| `DescribeListenerAttributes` |
+| `DescribeListenerCertificates` |
+| `DescribeListeners` |
+| `DescribeLoadBalancerAttributes` |
+| `DescribeLoadBalancers` |
+| `DescribeRules` |
+| `DescribeSSLPolicies` |
+| `DescribeTags` |
+| `DescribeTargetGroupAttributes` |
+| `DescribeTargetGroups` |
+| `DescribeTargetHealth` |
+| `ModifyListener` |
+| `ModifyListenerAttributes` |
+| `ModifyLoadBalancerAttributes` |
+| `ModifyRule` |
+| `ModifyTargetGroup` |
+| `ModifyTargetGroupAttributes` |
+| `RegisterTargets` |
+| `RemoveListenerCertificates` |
+| `RemoveTags` |
+| `SetIpAddressType` |
+| `SetRulePriorities` |
+| `SetSecurityGroups` |
+| `SetSubnets` |
@@ -0,0 +1,3 @@
+### Two data planes
+
+The data plane is a system-managed load balancer VM, launched automatically during `CreateLoadBalancer`. Application Load Balancers run HAProxy for the L7 surface — rules, fixed responses and redirects over HTTP and HTTPS. Network Load Balancers run nginx `stream` for the L4 surface — TCP, UDP, TLS and TCP_UDP — because HAProxy cannot load-balance UDP. The agent selects the engine from the configuration the control plane delivers, so the choice follows the load balancer type and is not separately configurable.
@@ -0,0 +1,109 @@
+---
+title: "IAM API Coverage"
+seoTitle: "AWS IAM API Operation Coverage on Spinifex — Spinifex Docs"
+description: "The AWS IAM API operations Spinifex implements, covering users, roles, policies, groups, instance profiles and the access keys that authenticate them."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - iam
+  - identity
+  - policies
+---
+
+# IAM API Coverage
+
+## Overview
+
+Spinifex implements **79 operations** in the IAM `2010-05-08` API model.
+
+### Scope
+
+All IAM operations are account-scoped. The root user of an account bypasses policy evaluation entirely, as it does on AWS.
+
+### Operations
+
+| Operation |
+|---|
+| `AddRoleToInstanceProfile` |
+| `AddUserToGroup` |
+| `AttachGroupPolicy` |
+| `AttachRolePolicy` |
+| `AttachUserPolicy` |
+| `CreateAccessKey` |
+| `CreateGroup` |
+| `CreateInstanceProfile` |
+| `CreateOpenIDConnectProvider` |
+| `CreatePolicy` |
+| `CreatePolicyVersion` |
+| `CreateRole` |
+| `CreateUser` |
+| `DeleteAccessKey` |
+| `DeleteGroup` |
+| `DeleteGroupPolicy` |
+| `DeleteInstanceProfile` |
+| `DeleteOpenIDConnectProvider` |
+| `DeletePolicy` |
+| `DeletePolicyVersion` |
+| `DeleteRole` |
+| `DeleteRolePolicy` |
+| `DeleteUser` |
+| `DeleteUserPolicy` |
+| `DetachGroupPolicy` |
+| `DetachRolePolicy` |
+| `DetachUserPolicy` |
+| `GetAccountSummary` |
+| `GetGroup` |
+| `GetGroupPolicy` |
+| `GetInstanceProfile` |
+| `GetOpenIDConnectProvider` |
+| `GetPolicy` |
+| `GetPolicyVersion` |
+| `GetRole` |
+| `GetRolePolicy` |
+| `GetUser` |
+| `GetUserPolicy` |
+| `ListAccessKeys` |
+| `ListAttachedGroupPolicies` |
+| `ListAttachedRolePolicies` |
+| `ListAttachedUserPolicies` |
+| `ListEntitiesForPolicy` |
+| `ListGroupPolicies` |
+| `ListGroups` |
+| `ListGroupsForUser` |
+| `ListInstanceProfileTags` |
+| `ListInstanceProfiles` |
+| `ListInstanceProfilesForRole` |
+| `ListOpenIDConnectProviderTags` |
+| `ListOpenIDConnectProviders` |
+| `ListPolicies` |
+| `ListPolicyTags` |
+| `ListPolicyVersions` |
+| `ListRolePolicies` |
+| `ListRoleTags` |
+| `ListRoles` |
+| `ListUserPolicies` |
+| `ListUserTags` |
+| `ListUsers` |
+| `PutGroupPolicy` |
+| `PutRolePolicy` |
+| `PutUserPolicy` |
+| `RemoveRoleFromInstanceProfile` |
+| `RemoveUserFromGroup` |
+| `SetDefaultPolicyVersion` |
+| `TagInstanceProfile` |
+| `TagOpenIDConnectProvider` |
+| `TagPolicy` |
+| `TagRole` |
+| `TagUser` |
+| `UntagInstanceProfile` |
+| `UntagOpenIDConnectProvider` |
+| `UntagPolicy` |
+| `UntagRole` |
+| `UntagUser` |
+| `UpdateAccessKey` |
+| `UpdateAssumeRolePolicy` |
+| `UpdateRole` |
@@ -0,0 +1,3 @@
+### Scope
+
+All IAM operations are account-scoped. The root user of an account bypasses policy evaluation entirely, as it does on AWS.
@@ -0,0 +1,296 @@
+{
+  "category": "Coverage",
+  "basePath": "/coverage",
+  "index": {
+    "name": "All services",
+    "title": "AWS API Coverage",
+    "seoTitle": "Which AWS API Operations Spinifex Implements — Spinifex Docs",
+    "description": "The AWS API operations Spinifex serves, generated from its gateway dispatch tables and the pinned AWS SDK service models on each build, service by service.",
+    "tags": [
+      "aws",
+      "compatibility",
+      "coverage",
+      "api",
+      "operations"
+    ]
+  },
+  "services": {
+    "acm": {
+      "slug": "acm",
+      "name": "ACM",
+      "title": "ACM API Coverage",
+      "seoTitle": "AWS Certificate Manager API Coverage — Spinifex Docs",
+      "description": "The AWS Certificate Manager API operations Spinifex implements, generated from the gateway dispatch tables and the pinned AWS service model on each build.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "acm",
+        "certificates",
+        "tls"
+      ]
+    },
+    "ec2": {
+      "slug": "ec2",
+      "name": "EC2",
+      "title": "EC2 API Coverage",
+      "seoTitle": "Amazon EC2 API Coverage on Spinifex — Spinifex Docs",
+      "description": "The Amazon EC2 API operations Spinifex implements, covering instances, EBS volumes, VPC networking, tags, security groups and the rest of the compute surface.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "ec2",
+        "compute",
+        "vpc"
+      ],
+      "notApplicable": {
+        "AttachClassicLinkVpc": "ec2-classic",
+        "CancelReservedInstancesListing": "ri-marketplace",
+        "ConfirmProductInstance": "marketplace",
+        "CopyFpgaImage": "fpga",
+        "CreateCarrierGateway": "aws-edge",
+        "CreateFpgaImage": "fpga",
+        "CreateReservedInstancesListing": "ri-marketplace",
+        "DeleteCarrierGateway": "aws-edge",
+        "DeleteFpgaImage": "fpga",
+        "DescribeAggregateIdFormat": "id-format",
+        "DescribeAwsNetworkPerformanceMetricSubscriptions": "aws-backbone",
+        "DescribeCarrierGateways": "aws-edge",
+        "DescribeClassicLinkInstances": "ec2-classic",
+        "DescribeElasticGpus": "retired-accelerators",
+        "DescribeFpgaImageAttribute": "fpga",
+        "DescribeFpgaImages": "fpga",
+        "DescribeIdFormat": "id-format",
+        "DescribeIdentityIdFormat": "id-format",
+        "DescribeMovingAddresses": "ec2-classic",
+        "DescribePrincipalIdFormat": "id-format",
+        "DescribeReservedInstancesListings": "ri-marketplace",
+        "DescribeSpotPriceHistory": "spot-price",
+        "DescribeVpcClassicLink": "ec2-classic",
+        "DescribeVpcClassicLinkDnsSupport": "ec2-classic",
+        "DetachClassicLinkVpc": "ec2-classic",
+        "DisableAwsNetworkPerformanceMetricSubscription": "aws-backbone",
+        "DisableVpcClassicLink": "ec2-classic",
+        "DisableVpcClassicLinkDnsSupport": "ec2-classic",
+        "EnableAwsNetworkPerformanceMetricSubscription": "aws-backbone",
+        "EnableVpcClassicLink": "ec2-classic",
+        "EnableVpcClassicLinkDnsSupport": "ec2-classic",
+        "GetAwsNetworkPerformanceData": "aws-backbone",
+        "ModifyFpgaImageAttribute": "fpga",
+        "ModifyIdFormat": "id-format",
+        "ModifyIdentityIdFormat": "id-format",
+        "MoveAddressToVpc": "ec2-classic",
+        "ReportInstanceStatus": "aws-support",
+        "ResetFpgaImageAttribute": "fpga",
+        "RestoreAddressToClassic": "ec2-classic"
+      },
+      "notes": {
+        "aws-backbone": "These report performance across AWS's own global network between its regions and zones.",
+        "aws-edge": "Carrier gateways route to a telecommunications carrier's 5G network in an AWS Wavelength Zone.",
+        "aws-support": "Reporting an instance as impaired files a report with AWS support about AWS hardware.",
+        "ec2-classic": "EC2-Classic is the pre-VPC network AWS retired in 2022. Every instance here is in a VPC.",
+        "fpga": "FPGA images are built and ingested through AWS's own AFI pipeline for its F1 instances.",
+        "id-format": "These switched resource IDs between the short and long forms during AWS's 2016 migration, which has long since finished.",
+        "marketplace": "Product codes belong to AWS Marketplace, which has no equivalent here.",
+        "retired-accelerators": "Elastic Graphics was retired by AWS, and attached nothing but an AWS-hosted accelerator while it existed.",
+        "ri-marketplace": "The Reserved Instance Marketplace resells a reservation to another AWS customer, so AWS is the counterparty.",
+        "spot-price": "On owned hardware there is no spot-to-on-demand price differential, so any figure would be invented."
+      }
+    },
+    "ecr": {
+      "slug": "ecr",
+      "name": "ECR",
+      "title": "ECR API Coverage",
+      "seoTitle": "Elastic Container Registry API Coverage — Spinifex Docs",
+      "description": "The Amazon ECR API operations Spinifex implements, alongside the OCI distribution endpoint that carries the image layers for every repository it serves.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "ecr",
+        "containers",
+        "registry"
+      ]
+    },
+    "ecs": {
+      "slug": "ecs",
+      "name": "ECS",
+      "title": "ECS API Coverage",
+      "seoTitle": "Elastic Container Service API Coverage — Spinifex Docs",
+      "description": "The Amazon ECS API operations Spinifex implements, covering clusters, services, tasks, container instances and the task definitions they are launched from.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "ecs",
+        "containers",
+        "orchestration"
+      ],
+      "internal": [
+        "PollAssignments",
+        "ProvisionCapacity",
+        "ReportTaskGPU"
+      ]
+    },
+    "eks": {
+      "slug": "eks",
+      "name": "EKS",
+      "title": "EKS API Coverage",
+      "seoTitle": "Elastic Kubernetes Service API Coverage — Spinifex Docs",
+      "description": "The Amazon EKS API operations Spinifex implements, covering clusters, nodegroups, add-ons, access entries and the identity providers they authenticate.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "eks",
+        "kubernetes",
+        "containers"
+      ],
+      "internal": [
+        "GetRecoveryDirective",
+        "ListInternalAddons",
+        "PublishInternal",
+        "WebhookTokenReview"
+      ]
+    },
+    "elasticloadbalancingv2": {
+      "slug": "elbv2",
+      "name": "ELBv2",
+      "title": "ELBv2 API Coverage",
+      "seoTitle": "Elastic Load Balancing v2 API Coverage — Spinifex Docs",
+      "description": "The Elastic Load Balancing v2 API operations Spinifex implements, for both the Application and Network Load Balancers it serves, listeners and target groups.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "elbv2",
+        "load balancing",
+        "networking"
+      ],
+      "internal": [
+        "GetLBConfig",
+        "LBAgentHeartbeat"
+      ]
+    },
+    "iam": {
+      "slug": "iam",
+      "name": "IAM",
+      "title": "IAM API Coverage",
+      "seoTitle": "AWS IAM API Operation Coverage on Spinifex — Spinifex Docs",
+      "description": "The AWS IAM API operations Spinifex implements, covering users, roles, policies, groups, instance profiles and the access keys that authenticate them.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "iam",
+        "identity",
+        "policies"
+      ],
+      "notApplicable": {
+        "CreateServiceLinkedRole": "service-linked-roles",
+        "CreateServiceSpecificCredential": "codecommit",
+        "DeleteSSHPublicKey": "codecommit",
+        "DeleteServiceLinkedRole": "service-linked-roles",
+        "DeleteServiceSpecificCredential": "codecommit",
+        "DeleteSigningCertificate": "signing-certificates",
+        "GenerateOrganizationsAccessReport": "organizations",
+        "GetOrganizationsAccessReport": "organizations",
+        "GetSSHPublicKey": "codecommit",
+        "GetServiceLinkedRoleDeletionStatus": "service-linked-roles",
+        "ListSSHPublicKeys": "codecommit",
+        "ListServiceSpecificCredentials": "codecommit",
+        "ListSigningCertificates": "signing-certificates",
+        "ResetServiceSpecificCredential": "codecommit",
+        "UpdateSSHPublicKey": "codecommit",
+        "UpdateServiceSpecificCredential": "codecommit",
+        "UpdateSigningCertificate": "signing-certificates",
+        "UploadSSHPublicKey": "codecommit",
+        "UploadSigningCertificate": "signing-certificates"
+      },
+      "notes": {
+        "codecommit": "CodeCommit credentials and SSH keys; the service does not exist here.",
+        "organizations": "AWS Organizations is not offered, so there is no organization to report on.",
+        "service-linked-roles": "Service-linked roles exist for AWS service principals, which this platform has none of.",
+        "signing-certificates": "X.509 signing certificates are an EC2-Classic SOAP credential, retired by AWS."
+      }
+    },
+    "rds": {
+      "slug": "rds",
+      "name": "RDS",
+      "title": "RDS API Coverage",
+      "seoTitle": "Relational Database Service API Coverage — Spinifex Docs",
+      "description": "The Amazon RDS API operations Spinifex implements, for the managed PostgreSQL and MariaDB engines it offers, covering instances, snapshots and parameters.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "rds",
+        "databases",
+        "postgresql",
+        "mariadb"
+      ],
+      "internal": [
+        "AcknowledgeDBBootstrap",
+        "GetDBBootstrapConfig",
+        "PollDBCommands",
+        "RegisterDBInstance",
+        "SubmitDBStateChange"
+      ],
+      "notApplicable": {
+        "CreateOptionGroup": "option-groups",
+        "DeleteOptionGroup": "option-groups",
+        "DescribeOptionGroups": "option-groups",
+        "ModifyOptionGroup": "option-groups"
+      },
+      "notes": {
+        "option-groups": "Option groups configure engine add-ons for engines this platform does not offer, such as Oracle and SQL Server."
+      }
+    },
+    "s3": {
+      "slug": "s3",
+      "name": "S3",
+      "title": "S3 API Coverage",
+      "seoTitle": "Amazon S3 API Coverage on Spinifex — Spinifex Docs",
+      "description": "The Amazon S3 API operations Predastore serves on the platform's S3 endpoint, covering buckets, objects, multipart uploads and the policies that guard them.",
+      "implementedBy": "Predastore",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "s3",
+        "storage",
+        "objects"
+      ],
+      "notApplicable": {
+        "CreateSession": "express-one-zone",
+        "GetBucketAccelerateConfiguration": "accelerate",
+        "GetBucketRequestPayment": "request-payment",
+        "PutBucketAccelerateConfiguration": "accelerate",
+        "PutBucketRequestPayment": "request-payment",
+        "WriteGetObjectResponse": "object-lambda"
+      },
+      "notes": {
+        "accelerate": "Transfer acceleration routes uploads over AWS edge locations, which an on-premise deployment has none of.",
+        "express-one-zone": "Sessions authenticate S3 Express One Zone directory buckets, a storage class this platform does not offer.",
+        "object-lambda": "S3 Object Lambda rewrites a response from a Lambda function, and Lambda is not offered.",
+        "request-payment": "Requester Pays shifts transfer charges to the caller, which needs AWS billing behind it."
+      }
+    },
+    "sts": {
+      "slug": "sts",
+      "name": "STS",
+      "title": "STS API Coverage",
+      "seoTitle": "AWS Security Token Service API Coverage — Spinifex Docs",
+      "description": "The AWS STS API operations Spinifex implements, covering role assumption, session tokens and web identity federation (IRSA), with those it does not offer.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "sts",
+        "identity",
+        "credentials"
+      ]
+    }
+  }
+}
@@ -0,0 +1,100 @@
+---
+title: "RDS API Coverage"
+seoTitle: "Relational Database Service API Coverage — Spinifex Docs"
+description: "The Amazon RDS API operations Spinifex implements, for the managed PostgreSQL and MariaDB engines it offers, covering instances, snapshots and parameters."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - rds
+  - databases
+  - postgresql
+  - mariadb
+---
+
+# RDS API Coverage
+
+## Overview
+
+Spinifex implements **26 operations** in the RDS `2014-10-31` API model.
+
+### Engines
+
+Spinifex offers PostgreSQL and MariaDB. Each DB instance is one dedicated system-owned VM running the engine directly, launched from a platform AMI and hidden from the customer's EC2 API. `Engine` is fixed at create: there is no in-place engine change, no cross-engine snapshot restore and no migration between the two.
+
+`mysql` is not an accepted engine and is not an alias for `mariadb`. MariaDB is offered under its own AWS engine name, exactly as AWS RDS offers it, so a client — including Terraform's `aws_db_instance` — must set `engine = "mariadb"`.
+
+Engine versions are pinned per engine. An `EngineVersion` naming anything but the pin is rejected, including a narrower minor version, because the AMI makes no promise about which minor it carries.
+
+### The endpoint is private
+
+The engine is reached over a customer-account ENI injected into a subnet of the DB subnet group, so the endpoint is reachable from inside the VPC only. The DB VM has other NICs that no customer security group governs, and the engine binds none of them — the port is not open there at all, rather than open and gated.
+
+### TLS is required by default
+
+Both engines enforce encrypted connections by default: `rds.force_ssl` on PostgreSQL and `require_secure_transport` on MariaDB. For MariaDB this is a deliberate divergence from AWS, which leaves it off. Both are boolean, modifiable and dynamic, so setting either to `0` in a parameter group restores plaintext without a reboot.
+
+### Rejected parameters
+
+A parameter whose omission would create a false safety, security or availability guarantee is rejected with `InvalidParameterValue` rather than silently dropped.
+
+| Parameter | Why it is rejected |
+|-----------|--------------------|
+| `MultiAZ=true` | Single-AZ platform; a standby would not exist |
+| `PubliclyAccessible=true` | The endpoint is a private VPC address |
+| `StorageEncrypted=false` | Unencrypted storage is not offered |
+| `EnableIAMDatabaseAuthentication` | IAM database authentication is not implemented |
+| `Iops`, `StorageThroughput`, `StorageType` ≠ `gp3` | Provisioned performance classes are not implemented |
+| `KmsKeyId`, `TdeCredentialArn` | Storage is encrypted with the cluster key, not a customer-managed one |
+| `AvailabilityZone` | The platform exposes a single zone |
+| `AvailabilityZoneGroup` (orderable options) | It selects a zone or local-zone group, and naming a zone is already refused |
+| `DBSecurityGroups` | EC2-Classic security groups — use `VpcSecurityGroupIds` |
+| `DBClusterIdentifier`, `DBClusterSnapshotIdentifier` | Clustered engines are not offered |
+| `EnableCloudwatchLogsExports` | Log export is not implemented |
+| `EngineVersion` other than the engine's pin, `Engine` on modify | No in-place engine or version change |
+| `Engine=mysql` (and Aurora engines) | Oracle MySQL is not offered; `mariadb` is a distinct engine, not an alias for it |
+| `NewDBInstanceIdentifier` | The identifier is the DNS label and the KV key |
+| `DBPortNumber`, `DBSubnetGroupName` on modify | Both would move the endpoint |
+| `MaxAllocatedStorage` | Storage autoscaling is not implemented |
+| `ManageMasterUserPassword`, `RotateMasterUserPassword` | Secrets Manager integration is not offered |
+| `CACertificateIdentifier` | The serving certificate is minted from the cluster CA |
+| `Domain`, `DomainFqdn` | Active Directory domain join is not offered |
+| `OptionGroupName` | Option groups are not offered |
+| `CustomIamInstanceProfile` | The DB VM's instance profile is platform-owned |
+| `EnableCustomerOwnedIp` | An Outposts feature |
+| `ForceFailover` (reboot) | No standby to fail over to |
+| `DBSnapshotIdentifier` (stop) | Snapshot-on-stop is not implemented |
+
+### Operations
+
+| Operation |
+|---|
+| `AddTagsToResource` |
+| `CreateDBInstance` |
+| `CreateDBParameterGroup` |
+| `CreateDBSnapshot` |
+| `CreateDBSubnetGroup` |
+| `DeleteDBInstance` |
+| `DeleteDBParameterGroup` |
+| `DeleteDBSnapshot` |
+| `DeleteDBSubnetGroup` |
+| `DescribeDBEngineVersions` |
+| `DescribeDBInstanceAutomatedBackups` |
+| `DescribeDBInstances` |
+| `DescribeDBParameterGroups` |
+| `DescribeDBParameters` |
+| `DescribeDBSnapshots` |
+| `DescribeDBSubnetGroups` |
+| `DescribeEvents` |
+| `DescribeOrderableDBInstanceOptions` |
+| `ListTagsForResource` |
+| `ModifyDBInstance` |
+| `ModifyDBParameterGroup` |
+| `RebootDBInstance` |
+| `RemoveTagsFromResource` |
+| `RestoreDBInstanceFromDBSnapshot` |
+| `StartDBInstance` |
+| `StopDBInstance` |
@@ -0,0 +1,46 @@
+### Engines
+
+Spinifex offers PostgreSQL and MariaDB. Each DB instance is one dedicated system-owned VM running the engine directly, launched from a platform AMI and hidden from the customer's EC2 API. `Engine` is fixed at create: there is no in-place engine change, no cross-engine snapshot restore and no migration between the two.
+
+`mysql` is not an accepted engine and is not an alias for `mariadb`. MariaDB is offered under its own AWS engine name, exactly as AWS RDS offers it, so a client — including Terraform's `aws_db_instance` — must set `engine = "mariadb"`.
+
+Engine versions are pinned per engine. An `EngineVersion` naming anything but the pin is rejected, including a narrower minor version, because the AMI makes no promise about which minor it carries.
+
+### The endpoint is private
+
+The engine is reached over a customer-account ENI injected into a subnet of the DB subnet group, so the endpoint is reachable from inside the VPC only. The DB VM has other NICs that no customer security group governs, and the engine binds none of them — the port is not open there at all, rather than open and gated.
+
+### TLS is required by default
+
+Both engines enforce encrypted connections by default: `rds.force_ssl` on PostgreSQL and `require_secure_transport` on MariaDB. For MariaDB this is a deliberate divergence from AWS, which leaves it off. Both are boolean, modifiable and dynamic, so setting either to `0` in a parameter group restores plaintext without a reboot.
+
+### Rejected parameters
+
+A parameter whose omission would create a false safety, security or availability guarantee is rejected with `InvalidParameterValue` rather than silently dropped.
+
+| Parameter | Why it is rejected |
+|-----------|--------------------|
+| `MultiAZ=true` | Single-AZ platform; a standby would not exist |
+| `PubliclyAccessible=true` | The endpoint is a private VPC address |
+| `StorageEncrypted=false` | Unencrypted storage is not offered |
+| `EnableIAMDatabaseAuthentication` | IAM database authentication is not implemented |
+| `Iops`, `StorageThroughput`, `StorageType` ≠ `gp3` | Provisioned performance classes are not implemented |
+| `KmsKeyId`, `TdeCredentialArn` | Storage is encrypted with the cluster key, not a customer-managed one |
+| `AvailabilityZone` | The platform exposes a single zone |
+| `AvailabilityZoneGroup` (orderable options) | It selects a zone or local-zone group, and naming a zone is already refused |
+| `DBSecurityGroups` | EC2-Classic security groups — use `VpcSecurityGroupIds` |
+| `DBClusterIdentifier`, `DBClusterSnapshotIdentifier` | Clustered engines are not offered |
+| `EnableCloudwatchLogsExports` | Log export is not implemented |
+| `EngineVersion` other than the engine's pin, `Engine` on modify | No in-place engine or version change |
+| `Engine=mysql` (and Aurora engines) | Oracle MySQL is not offered; `mariadb` is a distinct engine, not an alias for it |
+| `NewDBInstanceIdentifier` | The identifier is the DNS label and the KV key |
+| `DBPortNumber`, `DBSubnetGroupName` on modify | Both would move the endpoint |
+| `MaxAllocatedStorage` | Storage autoscaling is not implemented |
+| `ManageMasterUserPassword`, `RotateMasterUserPassword` | Secrets Manager integration is not offered |
+| `CACertificateIdentifier` | The serving certificate is minted from the cluster CA |
+| `Domain`, `DomainFqdn` | Active Directory domain join is not offered |
+| `OptionGroupName` | Option groups are not offered |
+| `CustomIamInstanceProfile` | The DB VM's instance profile is platform-owned |
+| `EnableCustomerOwnedIp` | An Outposts feature |
+| `ForceFailover` (reboot) | No standby to fail over to |
+| `DBSnapshotIdentifier` (stop) | Snapshot-on-stop is not implemented |
@@ -0,0 +1,62 @@
+---
+title: "S3 API Coverage"
+seoTitle: "Amazon S3 API Coverage on Spinifex — Spinifex Docs"
+description: "The Amazon S3 API operations Predastore serves on the platform's S3 endpoint, covering buckets, objects, multipart uploads and the policies that guard them."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - s3
+  - storage
+  - objects
+---
+
+# S3 API Coverage
+
+## Overview
+
+Predastore implements **26 operations** in the S3 `2006-03-01` API model.
+
+### Predastore serves this endpoint
+
+S3 is the one surface the AWS gateway does not answer itself. Object storage runs on [Predastore](https://github.com/mulgadc/predastore), which serves the S3 REST API directly over its own endpoint.
+
+### Routed is not the same as conforming
+
+This page says an operation is routed to a handler. It does not say the handler's behaviour matches S3 in every case.
+
+That behaviour is measured separately, against the `ceph/s3-tests` suite Ceph RGW, MinIO and Garage are all validated with, and the results are published in Predastore's [S3 compatibility report](https://github.com/mulgadc/predastore/blob/dev/docs/S3-COMPATIBILITY.md).
+
+### Operations
+
+| Operation |
+|---|
+| `AbortMultipartUpload` |
+| `CompleteMultipartUpload` |
+| `CopyObject` |
+| `CreateBucket` |
+| `CreateMultipartUpload` |
+| `DeleteBucket` |
+| `DeleteBucketTagging` |
+| `DeleteObject` |
+| `DeleteObjects` |
+| `GetBucketLocation` |
+| `GetBucketTagging` |
+| `GetBucketVersioning` |
+| `GetObject` |
+| `HeadBucket` |
+| `HeadObject` |
+| `ListBuckets` |
+| `ListMultipartUploads` |
+| `ListObjectVersions` |
+| `ListObjects` |
+| `ListObjectsV2` |
+| `ListParts` |
+| `PutBucketTagging` |
+| `PutBucketVersioning` |
+| `PutObject` |
+| `UploadPart` |
+| `UploadPartCopy` |
@@ -0,0 +1,9 @@
+### Predastore serves this endpoint
+
+S3 is the one surface the AWS gateway does not answer itself. Object storage runs on [Predastore](https://github.com/mulgadc/predastore), which serves the S3 REST API directly over its own endpoint.
+
+### Routed is not the same as conforming
+
+This page says an operation is routed to a handler. It does not say the handler's behaviour matches S3 in every case.
+
+That behaviour is measured separately, against the `ceph/s3-tests` suite Ceph RGW, MinIO and Garage are all validated with, and the results are published in Predastore's [S3 compatibility report](https://github.com/mulgadc/predastore/blob/dev/docs/S3-COMPATIBILITY.md).
@@ -0,0 +1,41 @@
+---
+title: "STS API Coverage"
+seoTitle: "AWS Security Token Service API Coverage — Spinifex Docs"
+description: "The AWS STS API operations Spinifex implements, covering role assumption, session tokens and web identity federation (IRSA), with those it does not offer."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - sts
+  - identity
+  - credentials
+---
+
+# STS API Coverage
+
+## Overview
+
+Spinifex implements **5 operations** in the STS `2011-06-15` API model.
+
+### Trust policies
+
+Trust policies are validated at write time rather than silently narrowed at assume time. `NotPrincipal`, `NotAction`, empty-string `Action` elements and empty `Principal` blocks are all rejected as malformed.
+
+`Condition` blocks are rejected except on `AssumeRoleWithWebIdentity` with `StringEquals`, which is the shape IRSA needs and which Spinifex evaluates at assume time against the token's issuer, subject and audience. Anything wider is refused rather than accepted and ignored, because an accepted-but-unevaluated condition is a silent over-grant.
+
+### Parameters that are refused, not ignored
+
+Several inputs the model describes are deliberately rejected rather than accepted as no-ops: inline session policies and policy ARNs, session tags, and MFA serial numbers and token codes. Each of them would otherwise appear to restrict or strengthen a session that in fact carries the role's full permissions.
+
+### Operations
+
+| Operation |
+|---|
+| `AssumeRole` |
+| `AssumeRoleWithWebIdentity` |
+| `GetAccessKeyInfo` |
+| `GetCallerIdentity` |
+| `GetSessionToken` |
@@ -0,0 +1,9 @@
+### Trust policies
+
+Trust policies are validated at write time rather than silently narrowed at assume time. `NotPrincipal`, `NotAction`, empty-string `Action` elements and empty `Principal` blocks are all rejected as malformed.
+
+`Condition` blocks are rejected except on `AssumeRoleWithWebIdentity` with `StringEquals`, which is the shape IRSA needs and which Spinifex evaluates at assume time against the token's issuer, subject and audience. Anything wider is refused rather than accepted and ignored, because an accepted-but-unevaluated condition is a silent over-grant.
+
+### Parameters that are refused, not ignored
+
+Several inputs the model describes are deliberately rejected rather than accepted as no-ops: inline session policies and policy ARNs, session tags, and MFA serial numbers and token codes. Each of them would otherwise appear to restrict or strengthen a session that in fact carries the role's full permissions.
@@ -19,11 +19,9 @@ resources:
  - title: "GPU Passthrough"
    url: "/docs/gpu-passthrough"
  - title: "Cisco UCS: AWS-compatible cloud at the edge (companion document)"
-    url: "/docs/cisco-ucs-platform-benchmark"
+    url: "/hardware/cisco/platform-benchmark"
  - title: "Vision pipeline (companion document)"
-    url: "/docs/cisco-ucs-vision-pipeline"
-  - title: "Benchmark and workload materials (CISCO-refarch)"
-    url: "https://github.com/tomnewton-mulga/CISCO-refarch"
+    url: "/hardware/cisco/vision-pipeline"
  - title: "vLLM"
    url: "https://docs.vllm.ai"
  - title: "Qwen2.5-7B-Instruct"
@@ -45,7 +43,7 @@ cluster, comparing Intel AMX-accelerated CPU serving on an `m8i.2xlarge`-class i
 against NVIDIA L4 GPU serving on a `g6.2xlarge`-class instance — a real,
 production-representative LLM-serving stack, with the two accelerators measured independently at matched model, version, and serving configuration.

-**Companion architectures:** [Cisco UCS: AWS-compatible cloud at the edge](/docs/cisco-ucs-platform-benchmark) · [Spinifex Vision Pipeline on Cisco UCS](/docs/cisco-ucs-vision-pipeline)
+**Companion architectures:** [Cisco UCS: AWS-compatible cloud at the edge](/hardware/cisco/platform-benchmark) · [Spinifex Vision Pipeline on Cisco UCS](/hardware/cisco/vision-pipeline)

 ### Platform

@@ -93,10 +91,7 @@ sudo spx admin gpu enable

 - One `m8i.2xlarge` (`cpu1`) and one `g6.2xlarge` (`gpu`) instance provisioned and reachable — the `gpu` instance must land on the node with passthrough configured; see [Launching Instances](/docs/launching-instances) for the full provisioning workflow including VPC, key pair, and security group setup
 - ~20 GB free disk per instance for model weights
- Python 3.14, `pip install vllm` (GPU instance) or the CPU wheel with
-  `--extra-index-url https://download.pytorch.org/whl/cpu` (CPU instance) —
-  vLLM 0.26.0 / PyTorch 2.11.0 on both, `torch+cu130` on the GPU instance,
-  `torch+cpu` on the CPU instance
+- Python 3.14, `pip install vllm` (GPU instance) or the CPU wheel with `--extra-index-url https://download.pytorch.org/whl/cpu` (CPU instance) — vLLM 0.26.0 / PyTorch 2.11.0 on both, `torch+cu130` on the GPU instance, `torch+cpu` on the CPU instance

 ## Instructions

@@ -129,16 +124,8 @@ VLLM_USE_FLASHINFER_SAMPLER=0 vllm serve ./Qwen2.5-7B-Instruct \
 Two environment-specific workarounds were needed on this cluster, worth recording for
 anyone reproducing this:

- **cpu1**: vLLM's compiled extension (`vllm/_C.abi3.so`) shipped with an executable-stack
-  ELF flag (`GNU_STACK` = RWE) that this guest kernel refuses to `mprotect`
-  (`cannot enable executable stack as shared object requires: Invalid argument`),
-  crashing the server at import time. Fixed with `patchelf --clear-execstack
-  vllm/_C.abi3.so` — a one-time, host-local binary patch, not a vLLM or model issue.
- **gpu**: this instance has no CUDA toolkit (`nvcc`) installed, only the driver/runtime —
-  fine for running pre-built PyTorch/CUDA kernels, but vLLM's default sampler
-  (FlashInfer) JIT-compiles a kernel on first use and fails without `nvcc`.
-  `VLLM_USE_FLASHINFER_SAMPLER=0` falls back to vLLM's native PyTorch sampler, which
-  needs no compilation step.
+- **cpu1**: vLLM's compiled extension (`vllm/_C.abi3.so`) shipped with an executable-stack ELF flag (`GNU_STACK` = RWE) that this guest kernel refuses to `mprotect` (`cannot enable executable stack as shared object requires: Invalid argument`), crashing the server at import time. Fixed with `patchelf --clear-execstack vllm/_C.abi3.so` — a one-time, host-local binary patch, not a vLLM or model issue.
+- **gpu**: this instance has no CUDA toolkit (`nvcc`) installed, only the driver/runtime — fine for running pre-built PyTorch/CUDA kernels, but vLLM's default sampler (FlashInfer) JIT-compiles a kernel on first use and fails without `nvcc`. `VLLM_USE_FLASHINFER_SAMPLER=0` falls back to vLLM's native PyTorch sampler, which needs no compilation step.

 ### 3. Confirm Intel AMX is executing for this workload

@@ -181,9 +168,9 @@ across the full matrix. Figures below are the mean of each cell's 3 repeats (med
 metric per run). Raw per-run JSON results are available in the
 [LLM serving repository](https://github.com/mulgadc/cisco-ucs-llm-serving).

-<img src="../../../.github/assets/images/cisco-ucs-llm-serving/tpot-vs-concurrency.png" alt="Median time per output token versus concurrency, CPU AMX versus GPU, by input length">
+<img src="../../../../.github/assets/images/cisco-ucs-llm-serving/tpot-vs-concurrency.png" alt="Median time per output token versus concurrency, CPU AMX versus GPU, by input length">

-<img src="../../../.github/assets/images/cisco-ucs-llm-serving/throughput-vs-concurrency.png" alt="Total token throughput versus concurrency, CPU AMX versus GPU, by input length">
+<img src="../../../../.github/assets/images/cisco-ucs-llm-serving/throughput-vs-concurrency.png" alt="Total token throughput versus concurrency, CPU AMX versus GPU, by input length">

 | Engine | Input tok | Concurrency | TTFT p50 (ms) | TPOT p50 (ms) | E2E p50 (ms) | Output tok/s | Total tok/s |
 |---|---:|---:|---:|---:|---:|---:|---:|
--- a/Show More
+++ b/Show More