release: update go.mod for v1.15.0

This reverts commit c2b80a9b93.

.githooks/pre-commit

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-branch=$(git rev-parse --abbrev-ref HEAD)
-
-# Only enforce preflight on dev and main
-if [ "$branch" != "dev" ] && [ "$branch" != "main" ]; then
-  exit 0
-fi
-
-# Only run preflight if Go files were staged
-if git diff --cached --name-only | grep -q '\.go$'; then
-  echo "Go files staged on $branch — running make preflight..."
-  make preflight
-else
-  echo "No Go files staged, skipping preflight."
-fi

.github/actions/e2e-analyze/README.md

@@ -70,7 +70,7 @@ When the report format changes intentionally, regenerate the snapshots:
 go test ./.github/actions/e2e-analyze/ -update
 ```
 
-## Per-failure bundles (Stage 2)
+## Per-failure bundles
 
 After parsing, the action materialises a bundle directory per failure
 under `<log-dir>/analysis/NN-<TestName>/`. The report links to it from

.github/actions/e2e-analyze/analyze.go

@@ -1,6 +1,6 @@
 // Package main implements the e2e-analyze GitHub Action.
 //
-// Stage 1: cluster failures from go-junit-report's junit-*.xml by error signature,
+// Cluster failures from go-junit-report's junit-*.xml by error signature,
 // pick the earliest non-cascade failure per suite as the likely root cause,
 // and render a human-readable report.
 //
@@ -73,8 +73,14 @@ type Failure struct {
 	// `test-<suite>.start` file written by the workflow before the suite
 	// runs) can recompute StartAt without re-parsing the XML.
 	OffsetFromSuiteStart time.Duration
+	// SuiteSpan is the suite's own wall-clock span, set only when the suite
+	// ran its tests in parallel. Overlapping tests make the cumulative
+	// offset above pure fiction — it lands minutes past the real failure —
+	// so consumers widen the journal window to the suite instead of
+	// slicing a plausible-looking window around the wrong minute.
+	SuiteSpan time.Duration
 	// BundlePath is the relative path (from log-dir) to the per-failure
-	// bundle written by Stage 2's WriteBundles. Empty when bundles were
+	// bundle written by WriteBundles. Empty when bundles were
 	// not generated (e.g. unit tests that call Render directly).
 	BundlePath string
 }
@@ -88,6 +94,12 @@ type SuiteReport struct {
 	Root      *Failure    // nil if no failures
 	Cascades  []Failure   // everything else that failed in this suite
 	Buckets   [][]Failure // failures grouped by signature (root's bucket first)
+	// EndedAt is go-junit-report's <testsuite timestamp>, which records when
+	// the XML was produced — i.e. when the suite finished, not when it began.
+	EndedAt time.Time
+	// TopLevelTotal is the summed duration of the parentless tests. Longer than
+	// the suite's wall clock means they overlapped.
+	TopLevelTotal time.Duration
 }
 
 // Report is the full analysis across every junit file.
@@ -119,6 +131,28 @@ var cascadeMarkers = []string{
 	"Expected value not to be nil",
 }
 
+// teardownPatterns match the content a test logs from t.Cleanup or from the
+// harness's on-failure diagnostics — i.e. after it has already failed. They are
+// the LAST file:line lines in the failure body, so extractErrorLine's
+// keep-the-last rule picks them over the assertion unless they are skipped.
+// Anchored deliberately: a fragment loose enough to match an assertion would
+// blank out the one line the report exists to show.
+var teardownPatterns = []*regexp.Regexp{
+	regexp.MustCompile(`^DB instance \S+ is gone$`),
+	regexp.MustCompile(`^db diagnostics \S+:`),
+}
+
+// isTeardownLine reports whether a candidate error line was emitted after the
+// failure it would otherwise be mistaken for.
+func isTeardownLine(content string) bool {
+	for _, re := range teardownPatterns {
+		if re.MatchString(content) {
+			return true
+		}
+	}
+	return false
+}
+
 // noiseReplacers normalise dynamic identifiers so that "ten failures, same
 // bug" bucket together. Pattern ordering matters: longer/more-specific
 // patterns must precede the catch-alls.
@@ -206,11 +240,14 @@ func extractErrorLine(body string) string {
 		if m := goFileLineRe.FindStringSubmatch(l); m != nil {
 			// Keep the LAST occurrence: t.Log / harness traces emit
 			// progress lines from the same file:line pattern before the
-			// failing assertion finally fires.
-			goFileLine = strings.TrimSpace(m[1])
+			// failing assertion finally fires. Cleanup runs after it, though,
+			// so its lines are last of all and have to be skipped.
+			if content := strings.TrimSpace(m[1]); !isTeardownLine(content) {
+				goFileLine = content
+			}
 			continue
 		}
-		if fileHintRe.MatchString(l) {
+		if fileHintRe.MatchString(l) && !isTeardownLine(l) {
 			lastTestLine = l
 		}
 	}
@@ -254,11 +291,19 @@ func extractFileHint(body string) string {
 	if m := errorTraceRe.FindStringSubmatch(body); m != nil {
 		return m[1]
 	}
-	all := fileHintRe.FindAllString(body, -1)
-	if len(all) == 0 {
-		return ""
+	// Scanned line by line rather than across the body so a cleanup line's own
+	// file:line does not become the site the report points a reader at.
+	hint := ""
+	for _, raw := range strings.Split(body, "\n") {
+		l := strings.TrimSpace(raw)
+		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isTeardownLine(strings.TrimSpace(m[1])) {
+			continue
+		}
+		if all := fileHintRe.FindAllString(l, -1); len(all) > 0 {
+			hint = all[len(all)-1]
+		}
 	}
-	return all[len(all)-1]
+	return hint
 }
 
 // signature normalises an error line so different runs of the same root
@@ -313,6 +358,47 @@ func computeLeafSet(cases []junitTC) map[string]bool {
 	return leaf
 }
 
+// computeRolledUpSet returns the testcase names whose failure is already
+// reported by a failing descendant. go-junit-report gives such a parent a
+// testcase of its own, and its body is the parent's t.Log output rather than an
+// assertion — a cleanup line like "DB instance … is gone", which passing tests
+// emit too, reads to extractErrorLine exactly like a failure site.
+func computeRolledUpSet(cases []junitTC) map[string]bool {
+	rolledUp := make(map[string]bool, len(cases))
+	for _, tc := range cases {
+		if tc.Failure == nil && tc.Error == nil {
+			continue
+		}
+		// Mark every ancestor: the failure belongs to the deepest case that
+		// carries it, not to the tests it is nested under.
+		name := tc.Name
+		for {
+			i := strings.LastIndex(name, "/")
+			if i <= 0 {
+				break
+			}
+			name = name[:i]
+			rolledUp[name] = true
+		}
+	}
+	return rolledUp
+}
+
+// topLevelTotal sums the durations of the tests with no parent, which are the
+// units t.Parallel overlaps. Compared against the suite's wall clock it says
+// whether the suite ran its tests concurrently — see ApplySuiteStartFiles.
+// suite.Time cannot answer that: go-junit-report sets it to the sum of every
+// testcase, parents included, so it is never less than any subset of itself.
+func topLevelTotal(cases []junitTC) time.Duration {
+	total := 0.0
+	for _, tc := range cases {
+		if !strings.Contains(tc.Name, "/") {
+			total += tc.Time
+		}
+	}
+	return time.Duration(total * float64(time.Second))
+}
+
 // parseStartTime turns a junit timestamp attr (RFC3339) into a time.Time;
 // zero value if unparseable.
 func parseStartTime(s string) time.Time {
@@ -359,6 +445,9 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 		// summing every testcase double-counts. Pre-compute the leaf set so
 		// only leaves contribute to the suite's running offset.
 		isLeaf := computeLeafSet(suite.Cases)
+		rolledUp := computeRolledUpSet(suite.Cases)
+		rep.EndedAt = start
+		rep.TopLevelTotal += topLevelTotal(suite.Cases)
 		cumul := 0.0
 		for _, tc := range suite.Cases {
 			rep.Total++
@@ -376,11 +465,12 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 				continue
 			}
 
-			// Skip the synthetic parent-test "Failed" stub that go-junit-report
-			// emits for any Go test whose subtests failed. It has an empty body
-			// and would otherwise outrank real failures by XML order.
+			// Skip the parent-test entry go-junit-report emits for any Go test
+			// whose subtests failed: it is a rollup of failures already counted
+			// below it, and it would outrank them by XML order. An empty body is
+			// the same stub in the case where the parent logged nothing.
 			trimmed := strings.TrimSpace(stripANSI(body))
-			if trimmed == "" {
+			if trimmed == "" || rolledUp[tc.Name] {
 				if leaf {
 					cumul += tc.Time
 				}
@@ -462,14 +552,24 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 	return rep, nil
 }
 
-// Render writes the markdown report (Stage 1).
+// Render writes the markdown report.
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
 
-	totalFail := 0
+	totalFail, totalTests := 0, 0
 	for _, s := range r.Suites {
 		totalFail += s.FailCount
+		totalTests += s.Total
+	}
+
+	// A cell that dies before the suites run emits junit with no testcases at
+	// all. Reporting that as clean turns "nothing ran" into "nothing wrong", so
+	// name it and point above the suites.
+	if totalTests == 0 {
+		b.WriteString("⚠️ No tests ran — no testcases in any junit file. " +
+			"The failure is upstream of the suites (provisioning, bootstrap, or reimage); check the job log.\n")
+		return b.String()
 	}
 	if totalFail == 0 {
 		b.WriteString("✅ No failures across any suite.\n")
@@ -484,12 +584,22 @@ func Render(r Report) string {
 		fmt.Fprintf(&b, "### Suite `%s`: %d failed, 1 root cause likely\n\n", s.Label, s.FailCount)
 
 		if s.Root != nil {
-			b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			// XML order is completion order, so "earliest" only means earliest
+			// in time for a suite whose tests ran one after another.
+			if s.Root.SuiteSpan > 0 {
+				b.WriteString("**First non-cascade failure** (the suite ran in parallel, so this is report order, not time order)\n\n")
+			} else {
+				b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			}
 			fmt.Fprintf(&b, "- Test: `%s`\n", s.Root.Name)
-			if !s.Root.StartAt.IsZero() {
+			switch {
+			case s.Root.SuiteSpan > 0:
+				fmt.Fprintf(&b, "- Start: unknown, the suite ran in parallel (duration %.1fs)\n",
+					s.Root.Duration)
+			case !s.Root.StartAt.IsZero():
 				fmt.Fprintf(&b, "- Start: %s (duration %.1fs)\n",
 					s.Root.StartAt.Format("15:04:05"), s.Root.Duration)
-			} else {
+			default:
 				fmt.Fprintf(&b, "- Duration: %.1fs\n", s.Root.Duration)
 			}
 			if s.Root.Error != "" {

.github/actions/e2e-analyze/analyze_test.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 )
 
 // updateGolden lets `go test ./.github/actions/e2e-analyze/ -update` rewrite
@@ -124,6 +125,22 @@ func TestExtractErrorLine_FallsBackToLastTestLine(t *testing.T) {
 	}
 }
 
+// A test's cleanup logs after it has failed, so its lines are the last in the
+// body — and the cleanup line here is one passing tests emit too.
+func TestExtractErrorLine_SkipsCleanupLoggedAfterTheFailure(t *testing.T) {
+	body := `    failure_test.go:42: DB instance rds-e2e-failure-1 reached status available
+    failure_test.go:60: EventuallyErr: condition not met within 4m0s: rds-e2e-failure-1 status=available want=failed
+    rds.go:364: db diagnostics rds-e2e-failure-1: no VM to capture a console from (<nil>)
+    main_test.go:325: DB instance rds-e2e-failure-1 is gone`
+	got := extractErrorLine(body)
+	if !strings.HasPrefix(got, "EventuallyErr:") {
+		t.Errorf("extractErrorLine = %q, want the assertion at failure_test.go:60", got)
+	}
+	if hint := extractFileHint(body); hint != "failure_test.go:60" {
+		t.Errorf("extractFileHint = %q, want failure_test.go:60", hint)
+	}
+}
+
 func TestExtractFileHint_PicksLastMatch(t *testing.T) {
 	body := "ec2helpers.go:50: setup\n    vpc_test.go:227: Eventually: condition not met"
 	got := extractFileHint(body)
@@ -157,6 +174,116 @@ func TestParseFile_SkipsEmptyParentFailure(t *testing.T) {
 	}
 }
 
+func TestParseFile_SkipsParentRolledUpBySubtests(t *testing.T) {
+	// A parent that logs during cleanup gets a non-empty body, and a t.Log like
+	// "DB instance … is gone" — which passing tests emit too — reads exactly
+	// like an assertion site. Its subtest holds the real failure.
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="2">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    main_test.go:325: DB instance rds-e2e-backup-1 is gone
+]]></failure></testcase>
+  <testcase name="TestX/RealFailure" time="0.5"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 {
+		t.Fatalf("FailCount = %d, want 1 (the parent rollup should be skipped)", sr.FailCount)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX/RealFailure" {
+		t.Fatalf("expected the subtest as root, got %+v", sr.Root)
+	}
+}
+
+// A parent that fails on its own after its subtests passed is a real failure,
+// and dropping it would lose the only record of it.
+func TestParseFile_KeepsParentWithNoFailingSubtest(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    foo_test.go:42:
+        Messages: the parent asserted after its subtests
+]]></failure></testcase>
+  <testcase name="TestX/Passing" time="0.5"></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX" {
+		t.Fatalf("expected the parent as root, got %+v", sr.Root)
+	}
+}
+
+// suiteFixture writes a junit file plus its .start sidecar into a temp log dir
+// and returns the parsed, start-file-corrected report for one suite.
+func suiteFixture(t *testing.T, start string, xml []byte) SuiteReport {
+	t.Helper()
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "test-x.start"), []byte(start), 0o644); err != nil {
+		t.Fatalf("write start file: %v", err)
+	}
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	rep := Report{Suites: []SuiteReport{sr}}
+	ApplySuiteStartFiles(&rep, dir)
+	return rep.Suites[0]
+}
+
+// Parallel tests overlap, so summing the ones before a failure puts its start
+// minutes past where it really was — and the journal slice cut from that window
+// misses the failure entirely. The tell is the top-level tests taking longer in
+// total than the suite's own wall clock.
+func TestApplySuiteStartFiles_DropsTheOffsetWhenTheSuiteRanInParallel(t *testing.T) {
+	// timestamp is when go-junit-report wrote the XML, i.e. the suite's end.
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestSlow" time="100.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 2*time.Minute {
+		t.Errorf("SuiteSpan = %s, want 2m0s to mark the suite parallel", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 0, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start %s rather than a summed offset", s.Root.StartAt, want)
+	}
+}
+
+// The sequential case must keep its offsets: they are the only thing that makes
+// a per-failure journal slice narrow enough to read.
+func TestApplySuiteStartFiles_KeepsTheOffsetWhenTheSuiteRanSequentially(t *testing.T) {
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestFirst" time="30.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 0 {
+		t.Errorf("SuiteSpan = %s, want 0 for a sequential suite", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 30, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start plus 30s of preceding tests %s", s.Root.StartAt, want)
+	}
+}
+
 func TestParseFile_CascadeFallbackWhenAllCascades(t *testing.T) {
 	// If every failure is a cascade marker, the earliest one becomes root
 	// so the report isn't blank.
@@ -189,3 +316,25 @@ func TestRender_NoFailuresIsClean(t *testing.T) {
 		t.Errorf("expected zero-failure banner, got:\n%s", out)
 	}
 }
+
+// A cell that fails in provisioning writes junit with no testcases. Rendering
+// the clean banner there reports "nothing ran" as "nothing wrong".
+func TestRender_NoTestsRanIsNotClean(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		suites []SuiteReport
+	}{
+		{"empty junit file", []SuiteReport{{File: "junit-baremetal.xml", Label: "baremetal", Total: 0, FailCount: 0}}},
+		{"no junit files at all", nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			out := Render(Report{Title: "E2E failure analysis", Suites: tc.suites})
+			if strings.Contains(out, "No failures across any suite") {
+				t.Errorf("clean banner rendered when no test ran, got:\n%s", out)
+			}
+			if !strings.Contains(out, "No tests ran") {
+				t.Errorf("expected a no-tests-ran warning, got:\n%s", out)
+			}
+		})
+	}
+}

.github/actions/e2e-analyze/bundle.go

@@ -17,9 +17,9 @@ import (
 )
 
 // bundlePad is the per-failure window padding either side of the
-// [StartAt, StartAt+Duration] testcase span. Five seconds matches the
-// plan doc: long enough to catch the daemon's pre-test setup and
-// post-test cleanup chatter, short enough to keep the slice focused.
+// [StartAt, StartAt+Duration] testcase span. Five seconds catches the
+// daemon's pre-test setup and post-test cleanup chatter while keeping
+// the slice focused.
 const bundlePad = 5 * time.Second
 
 // nameSanitiseRe replaces characters unsafe in directory names with `_`.
@@ -58,21 +58,57 @@ func ApplySuiteStartFiles(rep *Report, logDir string) {
 			fmt.Fprintf(os.Stderr, "::warning::e2e-analyze: parse %s: %v\n", startPath, err)
 			continue
 		}
-		if s.Root != nil {
-			s.Root.StartAt = ts.Add(s.Root.OffsetFromSuiteStart)
-		}
-		for ci := range s.Cascades {
-			s.Cascades[ci].StartAt = ts.Add(s.Cascades[ci].OffsetFromSuiteStart)
-		}
-		for bi := range s.Buckets {
-			for fi := range s.Buckets[bi] {
-				f := &s.Buckets[bi][fi]
-				f.StartAt = ts.Add(f.OffsetFromSuiteStart)
+		// The start file is the first wall clock the analyzer has, so it is also
+		// the first chance to tell a parallel suite from a sequential one — and
+		// a parallel one's per-test offsets are fiction, since overlapping tests
+		// never started at the sum of the ones before them.
+		span := suiteSpanIfParallel(s, ts)
+		for _, f := range collectFailures(s) {
+			f.SuiteSpan = span
+			if span > 0 {
+				f.StartAt = ts
+				continue
 			}
+			f.StartAt = ts.Add(f.OffsetFromSuiteStart)
 		}
 	}
 }
 
+// suiteSpanIfParallel returns the suite's wall-clock span when its top-level
+// tests overlapped, and zero when they ran one after another. Consumers read a
+// non-zero span as "no per-test offset is derivable from this suite".
+func suiteSpanIfParallel(s *SuiteReport, start time.Time) time.Duration {
+	if s.EndedAt.IsZero() || s.TopLevelTotal == 0 {
+		return 0
+	}
+	wall := s.EndedAt.Sub(start)
+	// The 5% margin keeps per-testcase rounding from calling a sequential
+	// suite parallel; real overlap runs to several times the wall clock.
+	if wall <= 0 || s.TopLevelTotal <= time.Duration(float64(wall)*1.05) {
+		return 0
+	}
+	return wall
+}
+
+// collectFailures returns every failure in the report by pointer. Root, cascades
+// and buckets hold independent copies of the same failures, so a field that has
+// to reach the bundle writer must be set on all three.
+func collectFailures(s *SuiteReport) []*Failure {
+	var all []*Failure
+	if s.Root != nil {
+		all = append(all, s.Root)
+	}
+	for ci := range s.Cascades {
+		all = append(all, &s.Cascades[ci])
+	}
+	for bi := range s.Buckets {
+		for fi := range s.Buckets[bi] {
+			all = append(all, &s.Buckets[bi][fi])
+		}
+	}
+	return all
+}
+
 // WriteBundles materialises per-failure bundles under <logDir>/analysis/.
 // Mutates rep in-place so each Failure.BundlePath holds the relative
 // path the renderer will link from the summary.
@@ -143,7 +179,11 @@ func writeOneBundle(logDir, base, suiteLabel string, f *Failure) {
 
 	writeFailureSummary(dir, suiteLabel, f)
 
-	if !f.StartAt.IsZero() {
+	// A parallel suite offers no derivable per-test window, and the narrowing is
+	// the whole point of the slice: cutting one from the wrong minute leaves out
+	// the lines that explain the failure, and widening it to the suite just
+	// copies the artifact's own journal into every bundle.
+	if !f.StartAt.IsZero() && f.SuiteSpan == 0 {
 		start := f.StartAt.Add(-bundlePad)
 		end := f.StartAt.Add(time.Duration(f.Duration * float64(time.Second))).Add(bundlePad)
 		journalSrc := filepath.Join(logDir, "spinifex-journal.log")
@@ -188,7 +228,11 @@ func writeFailureSummary(dir, suiteLabel string, f *Failure) {
 	var b strings.Builder
 	fmt.Fprintf(&b, "Suite:    %s\n", suiteLabel)
 	fmt.Fprintf(&b, "Test:     %s\n", f.Name)
-	if !f.StartAt.IsZero() {
+	switch {
+	case f.SuiteSpan > 0:
+		b.WriteString("Start:    unknown, the suite ran in parallel; " +
+			"no journal window is derivable — read ../../spinifex-journal.log\n")
+	case !f.StartAt.IsZero():
 		fmt.Fprintf(&b, "Start:    %s\n", f.StartAt.Format(time.RFC3339))
 	}
 	fmt.Fprintf(&b, "Duration: %.3fs\n", f.Duration)

.github/actions/e2e-analyze/main.go

@@ -45,7 +45,7 @@ func main() {
 		report.Suites = append(report.Suites, sr)
 	}
 
-	// Stage 2: prefer authoritative per-suite start times written by the
+	// Prefer authoritative per-suite start times written by the
 	// workflow over junit-report's conversion-time `timestamp` attribute,
 	// then materialise per-failure bundles so Render can link them.
 	if logDir != "" {

.github/actions/e2e-analyze/slice.go

@@ -1,6 +1,6 @@
 package main
 
-// Stage 2: per-failure log slicers used by bundle.go to materialise time-window /
+// Per-failure log slicers used by bundle.go to materialise time-window /
 // testcase-window log excerpts next to each named failure.
 
 import (

.github/actions/otel-job-id/action.yml

@@ -0,0 +1,52 @@
+name: Resolve OTEL job id
+description: >-
+  Append ci.job_id to SPINIFEX_OTEL_ATTRS so telemetry emitted by this job can
+  be pivoted straight back to its GitHub job log. Must run before the step that
+  bakes SPINIFEX_OTEL_ATTRS into a VM's telemetry.env, and requires
+  `actions: read` on the calling job.
+
+inputs:
+  job-name:
+    description: >-
+      Rendered name of the calling job, exactly as it appears in the Actions UI.
+      github.job is only the YAML key and no context exposes the numeric id, so
+      the id has to be looked up by name.
+    required: true
+  token:
+    description: Token used for the Actions jobs API lookup.
+    required: false
+    default: ${{ github.token }}
+
+runs:
+  using: composite
+  steps:
+    - name: Resolve CI job id (best-effort)
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.token }}
+        JOB_NAME: ${{ inputs.job-name }}
+        REPO: ${{ github.repository }}
+        RUN_ID: ${{ github.run_id }}
+      run: |
+        # Never fail the caller: on any lookup error or name mismatch ci.job_id
+        # is simply omitted and the run proceeds with the labels it already has.
+        set +e
+
+        if [ -z "${SPINIFEX_OTEL_ATTRS:-}" ]; then
+          echo "SPINIFEX_OTEL_ATTRS is unset or empty — telemetry is disabled, nothing to label"
+          exit 0
+        fi
+
+        # env.JOB_NAME rather than shell interpolation into the jq filter: job
+        # names carry quotes, slashes and non-ASCII separators that would
+        # otherwise break the expression.
+        JOB_ID=$(gh api "repos/${REPO}/actions/runs/${RUN_ID}/jobs" --paginate \
+          --jq '.jobs[] | select(.name == env.JOB_NAME) | .id' 2>/dev/null | head -n1)
+
+        if [ -n "$JOB_ID" ]; then
+          echo "SPINIFEX_OTEL_ATTRS=${SPINIFEX_OTEL_ATTRS},ci.job_id=${JOB_ID}" >> "$GITHUB_ENV"
+          echo "resolved ci.job_id=${JOB_ID} for job '${JOB_NAME}'"
+        else
+          echo "::warning::could not resolve numeric job id for '${JOB_NAME}' — continuing without ci.job_id"
+        fi
+        exit 0

.github/assets/architecture.svg

@@ -1,225 +1,114 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="900" height="620" viewBox="0 0 900 620">
-  <defs>
-    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
-      <stop offset="0%" style="stop-color:#0a0a0a"/>
-      <stop offset="100%" style="stop-color:#0d1117"/>
-    </linearGradient>
-    <linearGradient id="bar-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#161b22"/>
-      <stop offset="45%" style="stop-color:#1f2028"/>
-      <stop offset="100%" style="stop-color:#161b22"/>
-    </linearGradient>
-    <linearGradient id="client-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#3fb95022"/>
-      <stop offset="100%" style="stop-color:#3fb95008"/>
-    </linearGradient>
-    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#DC262633"/>
-      <stop offset="100%" style="stop-color:#7F1D1D14"/>
-    </linearGradient>
-    <linearGradient id="nats-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#F87171"/>
-      <stop offset="50%" style="stop-color:#DC2626"/>
-      <stop offset="100%" style="stop-color:#7F1D1D"/>
-    </linearGradient>
-  </defs>
-
-  <rect width="900" height="620" fill="url(#bg)" rx="12"/>
-
-  <!-- Title bar -->
-  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">SPINIFEX ARCHITECTURE</text>
-  <text x="24" y="52" font-family="'Segoe UI', system-ui, sans-serif" font-size="16" fill="#f5f5f5" font-weight="500">AWS SDK → Gateway → NATS → Service daemons</text>
-  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">stateless · message-driven · horizontally scalable</text>
-
-  <!-- ═══ CLIENT LAYER ════════════════════════════════════ -->
-  <rect x="16" y="72" width="868" height="78" rx="8" fill="url(#client-grad)" stroke="#3fb950" stroke-width="1" stroke-opacity="0.5"/>
-  <text x="32" y="93" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" font-weight="700" letter-spacing="1">CLIENTS</text>
-  <text x="32" y="108" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">Standard AWS tooling — unmodified, no shims, no custom SDK</text>
-
-  <g transform="translate(32, 120)" font-family="'SF Mono', ui-monospace, monospace" font-size="10">
-    <rect x="0" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="43" y="15" fill="#7ee787" text-anchor="middle">aws-cli</text>
-
-    <rect x="94" y="0" width="96" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="142" y="15" fill="#7ee787" text-anchor="middle">Terraform</text>
-
-    <rect x="198" y="0" width="76" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="236" y="15" fill="#7ee787" text-anchor="middle">boto3</text>
-
-    <rect x="282" y="0" width="98" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="331" y="15" fill="#7ee787" text-anchor="middle">aws-sdk-go</text>
-
-    <rect x="388" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="431" y="15" fill="#7ee787" text-anchor="middle">kubectl</text>
-
-    <rect x="482" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="525" y="15" fill="#7ee787" text-anchor="middle">Packer</text>
-
-    <rect x="576" y="0" width="120" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="636" y="15" fill="#7ee787" text-anchor="middle">your application</text>
-  </g>
-
-  <!-- ═══ HTTPS + SigV4 connector ═══════════════════════════ -->
-  <line x1="450" y1="150" x2="450" y2="178" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
-  <rect x="390" y="154" width="120" height="18" rx="9" fill="#0a0a0a" stroke="#DC2626" stroke-width="0.8" stroke-opacity="0.5"/>
-  <text x="450" y="166" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#F87171" text-anchor="middle">HTTPS · SigV4 · :9999</text>
-
-  <!-- ═══ AWS GATEWAY ════════════════════════════════════ -->
-  <rect x="16" y="178" width="868" height="68" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.75"/>
-  <text x="32" y="200" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" font-weight="700" letter-spacing="1">AWS GATEWAY</text>
-  <text x="32" y="215" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">spinifex/services/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
-
-  <g transform="translate(32, 224)" font-family="'SF Mono', ui-monospace, monospace" font-size="9">
-    <rect x="0" y="0" width="78" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="39" y="11" fill="#FCA5A5" text-anchor="middle">TLS 1.3</text>
-
-    <rect x="86" y="0" width="78" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="125" y="11" fill="#FCA5A5" text-anchor="middle">SigV4 auth</text>
-
-    <rect x="172" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="221" y="11" fill="#FCA5A5" text-anchor="middle">policy eval</text>
-
-    <rect x="278" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="327" y="11" fill="#FCA5A5" text-anchor="middle">req routing</text>
-
-    <rect x="384" y="0" width="88" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="428" y="11" fill="#FCA5A5" text-anchor="middle">rate-limit</text>
-
-    <rect x="480" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="529" y="11" fill="#FCA5A5" text-anchor="middle">audit log</text>
-  </g>
-
-  <!-- ═══ NATS BUS ═══════════════════════════════════════ -->
-  <line x1="450" y1="246" x2="450" y2="272" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
-  <rect x="16" y="272" width="868" height="34" rx="6" fill="url(#nats-grad)" opacity="0.92"/>
-  <text x="32" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#f5f5f5" font-weight="700" letter-spacing="1.5">NATS BUS</text>
-  <text x="140" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#fee2e2" opacity="0.95">ec2.*   ebs.*   s3.*   vpc.*   iam.*</text>
-  <text x="876" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#fee2e2" text-anchor="end" opacity="0.9">queue groups · fan-out · request/reply</text>
-
-  <!-- ═══ DAEMON FANOUT CONNECTORS ═══════════════════════ -->
-  <g stroke="#DC2626" stroke-width="0.9" stroke-dasharray="3,3" opacity="0.55" fill="none">
-    <line x1="100" y1="306" x2="100" y2="328"/>
-    <line x1="275" y1="306" x2="275" y2="328"/>
-    <line x1="450" y1="306" x2="450" y2="328"/>
-    <line x1="625" y1="306" x2="625" y2="328"/>
-    <line x1="800" y1="306" x2="800" y2="328"/>
-  </g>
-
-  <!-- ═══ SERVICE DAEMONS ROW ═══════════════════════════ -->
-  <!-- EC2 -->
-  <g transform="translate(28, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">ec2.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">RunInstances</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">TerminateInstances</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">QMP · cloud-init</text>
-  </g>
-
-  <!-- EBS -->
-  <g transform="translate(204, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">ebs.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">CreateVolume</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">AttachVolume</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">snapshots · WAL</text>
-  </g>
-
-  <!-- S3 -->
-  <g transform="translate(380, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">s3.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">PutObject · GetObject</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Multipart · presigned</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">erasure-coded</text>
-  </g>
-
-  <!-- VPC -->
-  <g transform="translate(556, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">vpc.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Subnets · SGs · RTs</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">ALB · NAT · IGW</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">OVN · OVS</text>
-  </g>
-
-  <!-- IAM -->
-  <g transform="translate(732, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">iam.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Users · Roles · Keys</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Policy evaluation</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">audit trail</text>
-  </g>
-
-  <!-- ═══ DAEMON → BACKEND CONNECTORS ═══════════════════ -->
-  <g stroke="#71717a" stroke-width="0.8" stroke-dasharray="2,3" opacity="0.55" fill="none">
-    <line x1="100" y1="410" x2="100" y2="440"/>
-    <line x1="276" y1="410" x2="276" y2="440"/>
-    <line x1="452" y1="410" x2="452" y2="440"/>
-    <line x1="628" y1="410" x2="628" y2="440"/>
-    <line x1="804" y1="410" x2="804" y2="440"/>
-  </g>
-
-  <!-- ═══ BACKEND RESOURCES BAR ═════════════════════════ -->
-  <rect x="16" y="440" width="868" height="26" rx="5" fill="url(#bar-grad)"/>
-  <text x="450" y="458" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle" letter-spacing="1.5">BACKEND RESOURCES · local hardware · no cloud dependency</text>
-
-  <!-- ═══ BACKEND ROW ═══════════════════════════════════ -->
-  <!-- QEMU/KVM -->
-  <g transform="translate(28, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">QEMU / KVM</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">hypervisor</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">host kernel · virtio</text>
-  </g>
-
-  <!-- Viperblock -->
-  <g transform="translate(204, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Viperblock</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">block storage</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">NVMe WAL · NBD</text>
-  </g>
-
-  <!-- Predastore -->
-  <g transform="translate(380, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Predastore</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">object storage</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">Reed–Solomon</text>
-  </g>
-
-  <!-- OVN/OVS -->
-  <g transform="translate(556, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">OVN / OVS</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">SDN fabric</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">overlays · flow rules</text>
-  </g>
-
-  <!-- IAM store -->
-  <g transform="translate(732, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Identity store</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">users · keys · rules</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">local · audit-logged</text>
-  </g>
-
-  <!-- ═══ FOOTER NOTES ══════════════════════════════════ -->
-  <g transform="translate(16, 558)">
-    <rect width="868" height="48" rx="6" fill="#0a0a0a" stroke="#27272a" stroke-width="0.8"/>
-    <text x="24" y="20" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">
-      <tspan fill="#F87171" font-weight="600">Queue groups</tspan> load-balance ops across daemon replicas.
-      <tspan fill="#F87171" font-weight="600"> Fan-out</tspan> (no queue group) dispatches describe calls to every node.
-    </text>
-    <text x="24" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">
-      Daemons are <tspan fill="#F87171" font-weight="600">stateless</tspan> — scale horizontally by starting more.
-      No etcd, no Kubernetes, no external control plane. Systemd units, a NATS cluster, and your hardware.
-    </text>
-  </g>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="770" viewBox="0 0 1020 770" role="img" aria-label="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus; stateless ec2/ebs/s3/vpc/iam daemons answer and drive local backends (QEMU/KVM, Viperblock, Predastore, OVN/OVS, identity store).">
+<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
+<rect x="0.0" y="0.0" width="1020.0" height="770.0" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="28.0" y="42.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ARCHITECTURE</text>
+<text x="992.0" y="42.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">stateless · message-driven · horizontally scalable</text>
+<text x="28.0" y="72.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">AWS SDK → Gateway → NATS → service daemons</text>
+<rect x="28.0" y="92.0" width="964.0" height="92.0" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48.0" y="119.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">CLIENTS</text>
+<text x="48.0" y="140.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#5B636C" font-weight="400" text-anchor="start">Standard AWS tooling, unmodified. No shims, no custom SDK.</text>
+<rect x="46.0" y="153.0" width="68.2" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="80.1" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli</text>
+<rect x="123.2" y="153.0" width="81.4" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="163.9" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Terraform</text>
+<rect x="213.6" y="153.0" width="55.0" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="241.1" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">boto3</text>
+<rect x="277.6" y="153.0" width="88.0" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="321.6" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-sdk-go</text>
+<rect x="374.6" y="153.0" width="68.2" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="408.7" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">kubectl</text>
+<rect x="451.8" y="153.0" width="61.6" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="482.6" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Packer</text>
+<rect x="522.4" y="153.0" width="74.8" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="559.8" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">your app</text>
+<line x1="510.0" y1="186.0" x2="510.0" y2="192.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="429.9" y="191.0" width="160.3" height="22.0" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.4"/>
+<text x="510.0" y="205.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="400" text-anchor="middle" fill-opacity="0.9">HTTPS · SigV4 · :9999</text>
+<line x1="510.0" y1="212.0" x2="510.0" y2="220.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28.0" y="224.0" width="964.0" height="94.0" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="48.0" y="251.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="700" text-anchor="start" letter-spacing="1.5">AWS GATEWAY</text>
+<text x="48.0" y="272.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">spinifex/services/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
+<rect x="46.0" y="287.0" width="68.2" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="80.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">TLS 1.3</text>
+<rect x="123.2" y="287.0" width="88.0" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="167.2" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">SigV4 auth</text>
+<rect x="220.2" y="287.0" width="94.6" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="267.5" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">policy eval</text>
+<rect x="323.8" y="287.0" width="94.6" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="371.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">req routing</text>
+<rect x="427.4" y="287.0" width="88.0" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="471.4" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">rate-limit</text>
+<rect x="524.4" y="287.0" width="81.4" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="565.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">audit log</text>
+<line x1="510.0" y1="320.0" x2="510.0" y2="332.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28.0" y="334.0" width="964.0" height="52.0" rx="12" fill="url(#nats)" fill-opacity="1.0" stroke="none" stroke-width="1" stroke-opacity="1.0"/>
+<rect x="28.0" y="334.0" width="964.0" height="52.0" rx="12" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55"/>
+<text x="50.0" y="365.6" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">NATS BUS</text>
+<text x="178.0" y="364.6" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#F5F3EE" font-weight="400" text-anchor="start" fill-opacity="0.92">ec2.*  ebs.*  s3.*  vpc.*  iam.*</text>
+<text x="972.0" y="363.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F5F3EE" font-weight="400" text-anchor="end" fill-opacity="0.9">queue groups · fan-out · request/reply</text>
+<line x1="119.6" y1="388.0" x2="119.6" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="314.8" y1="388.0" x2="314.8" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="510.0" y1="388.0" x2="510.0" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="705.2" y1="388.0" x2="705.2" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="900.4" y1="388.0" x2="900.4" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<rect x="28.0" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="44.0" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">ec2.* daemon</text>
+<line x1="44.0" y1="448.0" x2="195.2" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="44.0" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">RunInstances</text>
+<text x="44.0" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">TerminateInstances</text>
+<text x="44.0" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">QMP · cloud-init</text>
+<rect x="223.2" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="239.2" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">ebs.* daemon</text>
+<line x1="239.2" y1="448.0" x2="390.4" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="239.2" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">CreateVolume</text>
+<text x="239.2" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">AttachVolume</text>
+<text x="239.2" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">snapshots · WAL</text>
+<rect x="418.4" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="434.4" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">s3.* daemon</text>
+<line x1="434.4" y1="448.0" x2="585.6" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="434.4" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">PutObject · GetObject</text>
+<text x="434.4" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Multipart · presigned</text>
+<text x="434.4" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">erasure-coded</text>
+<rect x="613.6" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="629.6" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">vpc.* daemon</text>
+<line x1="629.6" y1="448.0" x2="780.8" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="629.6" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Subnets · SGs · RTs</text>
+<text x="629.6" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">ALB · NAT · IGW</text>
+<text x="629.6" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">OVN · OVS</text>
+<rect x="808.8" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="824.8" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">iam.* daemon</text>
+<line x1="824.8" y1="448.0" x2="976.0" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="824.8" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Users · Roles · Keys</text>
+<text x="824.8" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Policy evaluation</text>
+<text x="824.8" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">audit trail</text>
+<line x1="119.6" y1="516.0" x2="119.6" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="314.8" y1="516.0" x2="314.8" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="510.0" y1="516.0" x2="510.0" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="705.2" y1="516.0" x2="705.2" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="900.4" y1="516.0" x2="900.4" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<rect x="28.0" y="530.0" width="964.0" height="30.0" rx="9" fill="#1E242B" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="510.0" y="548.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle" letter-spacing="1">BACKEND RESOURCES  ·  local hardware  ·  no cloud dependency</text>
+<rect x="28.0" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="119.6" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">QEMU / KVM</text>
+<text x="119.6" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">hypervisor</text>
+<text x="119.6" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">host kernel · virtio</text>
+<rect x="223.2" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="314.8" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">Viperblock</text>
+<text x="314.8" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">block storage</text>
+<text x="314.8" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">NVMe WAL · NBD</text>
+<rect x="418.4" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="510.0" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">Predastore</text>
+<text x="510.0" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">object storage</text>
+<text x="510.0" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Reed-Solomon</text>
+<rect x="613.6" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="705.2" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">OVN / OVS</text>
+<text x="705.2" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">SDN fabric</text>
+<text x="705.2" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">overlays · flow rules</text>
+<rect x="808.8" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="900.4" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">Identity store</text>
+<text x="900.4" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">users · keys · rules</text>
+<text x="900.4" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">local · audit-logged</text>
+<rect x="28.0" y="670.0" width="964.0" height="72.0" rx="11" fill="#181D23" fill-opacity="0.6" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48.0" y="697.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400"><tspan fill="#EE4B12" font-weight="700">Queue groups</tspan><tspan fill="#5B636C"> load-balance ops across daemon replicas. </tspan><tspan fill="#EE4B12" font-weight="700">Fan-out</tspan><tspan fill="#5B636C"> (no queue group) dispatches describe calls to every node.</tspan></text>
+<text x="48.0" y="720.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400"><tspan fill="#5B636C">Daemons are </tspan><tspan fill="#EE4B12" font-weight="700">stateless</tspan><tspan fill="#5B636C">: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Systemd units, a NATS cluster, and your hardware.</tspan></text>
+</svg>

.github/assets/platform.svg

@@ -1,144 +1,84 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="900" height="560" viewBox="0 0 900 560">
-  <defs>
-    <linearGradient id="pf-bg" x1="0%" y1="0%" x2="100%" y2="100%">
-      <stop offset="0%" style="stop-color:#0a0a0a"/>
-      <stop offset="100%" style="stop-color:#0d1117"/>
-    </linearGradient>
-    <linearGradient id="hw-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#52525b;stop-opacity:0.18"/>
-      <stop offset="100%" style="stop-color:#52525b;stop-opacity:0.06"/>
-    </linearGradient>
-    <linearGradient id="os-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#a855f7;stop-opacity:0.18"/>
-      <stop offset="100%" style="stop-color:#a855f7;stop-opacity:0.05"/>
-    </linearGradient>
-    <linearGradient id="spx-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#DC2626;stop-opacity:0.28"/>
-      <stop offset="100%" style="stop-color:#7F1D1D;stop-opacity:0.1"/>
-    </linearGradient>
-    <linearGradient id="app-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#3fb950;stop-opacity:0.18"/>
-      <stop offset="100%" style="stop-color:#3fb950;stop-opacity:0.05"/>
-    </linearGradient>
-  </defs>
-
-  <rect width="900" height="560" fill="url(#pf-bg)" rx="12"/>
-
-  <!-- Subtle mesh dots background -->
-  <g opacity="0.12" fill="#7F1D1D">
-    <circle cx="40" cy="40" r="1"/><circle cx="120" cy="30" r="1"/><circle cx="220" cy="50" r="1"/>
-    <circle cx="320" cy="25" r="1"/><circle cx="440" cy="45" r="1"/><circle cx="560" cy="30" r="1"/>
-    <circle cx="680" cy="50" r="1"/><circle cx="780" cy="30" r="1"/><circle cx="860" cy="45" r="1"/>
-    <circle cx="60" cy="530" r="1"/><circle cx="160" cy="520" r="1"/><circle cx="260" cy="540" r="1"/>
-    <circle cx="380" cy="525" r="1"/><circle cx="500" cy="540" r="1"/><circle cx="620" cy="520" r="1"/>
-    <circle cx="740" cy="540" r="1"/><circle cx="840" cy="525" r="1"/>
-  </g>
-
-  <!-- Title -->
-  <text x="450" y="32" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" text-anchor="middle" letter-spacing="2.8">THE MULGA PLATFORM</text>
-  <text x="450" y="58" font-family="'Segoe UI', system-ui, sans-serif" font-size="20" font-weight="600" fill="#f5f5f5" text-anchor="middle">AWS-style workflows, on your own stack</text>
-
-  <!-- Layer 4: Applications (top) -->
-  <g transform="translate(60, 95)">
-    <rect width="780" height="88" rx="10" fill="url(#app-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
-    <text x="20" y="26" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" letter-spacing="1.5" font-weight="600">LAYER 4 · APPLICATIONS &amp; TOOLING</text>
-    <text x="20" y="48" font-family="'Segoe UI', sans-serif" font-size="15" fill="#f5f5f5" font-weight="500">Your workloads — unmodified AWS tooling</text>
-
-    <g transform="translate(20, 60)" font-family="'Segoe UI', sans-serif" font-size="10.5">
-      <rect x="0" y="0" width="70" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="35" y="14" fill="#7ee787" text-anchor="middle">aws-cli</text>
-
-      <rect x="78" y="0" width="78" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="117" y="14" fill="#7ee787" text-anchor="middle">Terraform</text>
-
-      <rect x="164" y="0" width="88" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="208" y="14" fill="#7ee787" text-anchor="middle">AWS SDKs</text>
-
-      <rect x="260" y="0" width="88" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="304" y="14" fill="#7ee787" text-anchor="middle">Kubernetes</text>
-
-      <rect x="356" y="0" width="70" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="391" y="14" fill="#7ee787" text-anchor="middle">Packer</text>
-
-      <rect x="434" y="0" width="98" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="483" y="14" fill="#7ee787" text-anchor="middle">Your Services</text>
-    </g>
-  </g>
-
-  <!-- Down arrow -->
-  <g transform="translate(450, 188)" opacity="0.45">
-    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
-    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
-  </g>
-
-  <!-- Layer 3: Spinifex (highlighted) -->
-  <g transform="translate(60, 215)">
-    <rect width="780" height="140" rx="10" fill="url(#spx-grad)" stroke="#DC2626" stroke-width="1.5" stroke-opacity="0.75"/>
-    <text x="20" y="26" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" letter-spacing="1.5" font-weight="600">LAYER 3 · SPINIFEX · AWS ORCHESTRATION</text>
-    <text x="20" y="50" font-family="'Segoe UI', sans-serif" font-size="15" fill="#f5f5f5" font-weight="500">AWS Gateway → NATS → Service Daemons · SigV4, IAM, VPC-aware</text>
-
-    <g transform="translate(20, 66)" font-family="'Segoe UI', sans-serif">
-      <!-- EC2 -->
-      <rect x="0" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="59" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">EC2</text>
-      <text x="59" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">QEMU/KVM VMs</text>
-      <text x="59" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">cloud-init · AMIs</text>
-
-      <!-- EBS -->
-      <rect x="126" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="185" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">EBS</text>
-      <text x="185" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">Viperblock</text>
-      <text x="185" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">WAL · NBD · snapshots</text>
-
-      <!-- S3 -->
-      <rect x="252" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="311" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">S3</text>
-      <text x="311" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">Predastore</text>
-      <text x="311" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Reed–Solomon EC</text>
-
-      <!-- VPC -->
-      <rect x="378" y="0" width="118" height="58" rx="6" fill="#7F1D1D" fill-opacity="0.22" stroke="#F87171" stroke-opacity="0.55" stroke-width="1"/>
-      <text x="437" y="18" font-size="13" fill="#FCA5A5" text-anchor="middle" font-weight="600">VPC</text>
-      <text x="437" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">OVN / OVS</text>
-      <text x="437" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">subnets · SG · ALB</text>
-
-      <!-- IAM -->
-      <rect x="504" y="0" width="118" height="58" rx="6" fill="#7F1D1D" fill-opacity="0.22" stroke="#F87171" stroke-opacity="0.55" stroke-width="1"/>
-      <text x="563" y="18" font-size="13" fill="#FCA5A5" text-anchor="middle" font-weight="600">IAM</text>
-      <text x="563" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">SigV4 · policies</text>
-      <text x="563" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">users · roles · keys</text>
-
-      <!-- Control -->
-      <rect x="630" y="0" width="110" height="58" rx="6" fill="#27272a" fill-opacity="0.8" stroke="#71717a" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="685" y="18" font-size="13" fill="#e4e4e7" text-anchor="middle" font-weight="600">Control</text>
-      <text x="685" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">NATS bus</text>
-      <text x="685" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">spx · daemons</text>
-    </g>
-  </g>
-
-  <!-- Down arrow -->
-  <g transform="translate(450, 360)" opacity="0.45">
-    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
-    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
-  </g>
-
-  <!-- Layer 2: OS -->
-  <g transform="translate(60, 387)">
-    <rect width="780" height="62" rx="10" fill="url(#os-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.5"/>
-    <text x="20" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" letter-spacing="1.5" font-weight="600">LAYER 2 · OPERATING SYSTEM</text>
-    <text x="20" y="44" font-family="'Segoe UI', sans-serif" font-size="14" fill="#f5f5f5">Debian / Ubuntu LTS · systemd · KVM · Linux bridge · headless autoinstall</text>
-  </g>
-
-  <!-- Down arrow -->
-  <g transform="translate(450, 454)" opacity="0.45">
-    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
-    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
-  </g>
-
-  <!-- Layer 1: Hardware -->
-  <g transform="translate(60, 481)">
-    <rect width="780" height="62" rx="10" fill="url(#hw-grad)" stroke="#71717a" stroke-width="1.2" stroke-opacity="0.6"/>
-    <text x="20" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#a1a1aa" letter-spacing="1.5" font-weight="600">LAYER 1 · HARDWARE</text>
-    <text x="20" y="44" font-family="'Segoe UI', sans-serif" font-size="14" fill="#f5f5f5">Single node · multi-node cluster · edge appliance · air-gapped rack · commodity x86 / ARM</text>
-  </g>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="664" viewBox="0 0 1000 664" role="img" aria-label="The Spinifex stack: applications and AWS tooling on top, the Spinifex AWS-compatible service layer, running on standard Linux on CPU/GPU hardware, deployable to Neocloud, on-prem or edge.">
+<rect x="0.0" y="0.0" width="1000.0" height="664.0" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<rect x="28.0" y="204.0" width="950.0" height="438.0" rx="16" fill="none" fill-opacity="1.0" stroke="#9AA1A8" stroke-width="1" stroke-opacity="0.12" stroke-dasharray="2 4"/>
+<rect x="25.0" y="201.0" width="950.0" height="438.0" rx="16" fill="none" fill-opacity="1.0" stroke="#9AA1A8" stroke-width="1" stroke-opacity="0.2" stroke-dasharray="2 4"/>
+<rect x="22.0" y="198.0" width="950.0" height="438.0" rx="16" fill="#181D23" fill-opacity="0.25" stroke="#9AA1A8" stroke-width="1.3" stroke-opacity="0.45" stroke-dasharray="2 4"/>
+<text x="36.0" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">Apps</text>
+<rect x="136.0" y="34.0" width="836.0" height="54.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="66.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="15" fill="#F5F3EE" font-weight="700" text-anchor="start">Your infrastructure &amp; apps</text>
+<rect x="516.7" y="51.0" width="89.3" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.16" stroke="none" stroke-width="1" stroke-opacity="1.0"/>
+<text x="561.4" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="500" text-anchor="middle">AI training</text>
+<rect x="613.0" y="51.0" width="76.7" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.16" stroke="none" stroke-width="1" stroke-opacity="1.0"/>
+<text x="651.4" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="500" text-anchor="middle">Inference</text>
+<rect x="696.7" y="51.0" width="79.0" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="736.2" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Web &amp; APIs</text>
+<rect x="782.7" y="51.0" width="72.7" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="819.1" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Databases</text>
+<rect x="862.4" y="51.0" width="91.6" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="908.2" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Batch &amp; data</text>
+<text x="36.0" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">Tooling</text>
+<rect x="136.0" y="102.0" width="836.0" height="54.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="134.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="15" fill="#F5F3EE" font-weight="700" text-anchor="start">AWS tooling</text>
+<rect x="600.9" y="119.0" width="64.1" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="633.0" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.85">aws-cli</text>
+<rect x="672.0" y="119.0" width="76.7" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="710.4" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.85">Terraform</text>
+<rect x="755.7" y="119.0" width="64.1" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="787.8" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.85">kubectl</text>
+<rect x="826.8" y="119.0" width="66.4" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="860.0" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">AWS SDKs</text>
+<rect x="900.2" y="119.0" width="53.8" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="927.1" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Packer</text>
+<text x="484.0" y="184.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">↓  AWS API · SigV4</text>
+<text x="624.0" y="184.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">RESPONSE  ↑</text>
+<text x="136.0" y="231.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="15" fill="#F5F3EE" font-weight="700" text-anchor="start">One portable stack.</text>
+<text x="136.0" y="249.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="start">Runs the same on a single node, cluster, edge, or air-gapped rack.</text>
+<rect x="752.3" y="222.0" width="72.4" height="22.0" rx="6" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="788.5" y="236.5" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="500" text-anchor="middle">NEOCLOUD</text>
+<rect x="832.7" y="222.0" width="66.1" height="22.0" rx="6" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="865.7" y="236.5" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="500" text-anchor="middle">ON-PREM</text>
+<rect x="906.8" y="222.0" width="47.2" height="22.0" rx="6" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="930.4" y="236.5" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="500" text-anchor="middle">EDGE</text>
+<text x="152.0" y="295.6" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">AWS orchestration</text>
+<text x="956.0" y="293.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.9">Gateway → NATS → daemons</text>
+<rect x="150.0" y="312.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="164.0" y="333.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">EC2</text>
+<text x="164.0" y="349.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Compute</text>
+<text x="164.0" y="363.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">QEMU / KVM</text>
+<rect x="422.7" y="312.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="436.7" y="333.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">EBS</text>
+<text x="436.7" y="349.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Block storage</text>
+<text x="436.7" y="363.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">Viperblock</text>
+<rect x="695.3" y="312.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="709.3" y="333.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">S3</text>
+<text x="709.3" y="349.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Object storage</text>
+<text x="709.3" y="363.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">Predastore</text>
+<rect x="150.0" y="382.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="164.0" y="403.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">VPC</text>
+<text x="164.0" y="419.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Networking</text>
+<text x="164.0" y="433.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">OVN / OVS</text>
+<rect x="422.7" y="382.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="436.7" y="403.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">IAM</text>
+<text x="436.7" y="419.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Identity</text>
+<text x="436.7" y="433.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">SigV4 · policies</text>
+<rect x="695.3" y="382.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="709.3" y="403.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">EKS</text>
+<text x="709.3" y="419.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Kubernetes</text>
+<text x="709.3" y="433.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">K8s-conformant</text>
+<rect x="150.0" y="452.0" width="808.0" height="32.0" rx="8" fill="#14181D" fill-opacity="0.6" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0" stroke-dasharray="3 3"/>
+<text x="164.0" y="472.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">+ ALB · ECS · ECR · RDS and more of the core AWS services</text>
+<text x="944.0" y="471.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#EE4B12" font-weight="400" text-anchor="end">See all →</text>
+<text x="36.0" y="386.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="500" text-anchor="start">Spinifex</text>
+<rect x="136.0" y="268.0" width="836.0" height="230.0" rx="12" fill="#EE4B12" fill-opacity="0.07" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="36.0" y="536.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">OS</text>
+<rect x="136.0" y="510.0" width="836.0" height="46.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="537.9" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="700" text-anchor="start">Operating system</text>
+<text x="952.0" y="536.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="end">Debian / Ubuntu LTS · systemd · KVM · Linux bridge · Open vSwitch</text>
+<text x="36.0" y="596.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">Hardware</text>
+<rect x="136.0" y="568.0" width="413.0" height="50.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="597.9" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="700" text-anchor="start">CPU</text>
+<text x="533.0" y="596.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="end">Intel · AMD · ARM</text>
+<rect x="559.0" y="568.0" width="413.0" height="50.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="579.0" y="597.9" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="700" text-anchor="start">GPU</text>
+<text x="956.0" y="596.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#EE4B12" font-weight="400" text-anchor="end">NVIDIA · AMD</text>
+</svg>

.github/resolve-sibling-refs.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Resolve which branch of each sibling repo (viperblock, predastore) the CI
+# workspace should build against. A cross-repo change is developed on a branch
+# of the SAME name in each repo; build against that branch when it exists so the
+# set can be tested together, and fall back to dev otherwise. This mirrors the
+# resolution the e2e path already does in
+# mulga/scripts/tofu-cluster/build-install-artifacts.sh, so unit CI and e2e CI
+# agree on how sibling branches are selected.
+#
+# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref> and
+# predastore=<ref> to $GITHUB_OUTPUT for the checkout steps to consume.
+set -euo pipefail
+
+: "${BRANCH:?BRANCH must be set to the triggering branch name}"
+: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
+
+for repo in viperblock predastore; do
+  if git ls-remote --exit-code --heads \
+      "https://github.com/mulgadc/${repo}.git" "$BRANCH" >/dev/null 2>&1; then
+    ref="$BRANCH"
+  else
+    ref="dev"
+  fi
+  echo "${repo}: building against ${ref}"
+  echo "${repo}=${ref}" >>"$GITHUB_OUTPUT"
+done

.github/scripts/suite-sets.sh

@@ -0,0 +1,62 @@
+# shellcheck shell=bash
+# shellcheck disable=SC2034  # every variable here is consumed by the sourcing workflow
+# Source from a workflow `run:` block:
+#   . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+#
+# Which e2e suites each topology can run. Sourced by every workflow that drives
+# suites (e2e.yml, e2e-suites.yml, e2e-nightly.yml) so the lists cannot drift
+# apart — they previously disagreed three ways.
+#
+# Eligibility is a property of the suite, not of the workflow: a suite belongs
+# to a topology when all of its assertions are reachable there. lb is
+# multi-only for exactly that reason — on a single node every internet-facing
+# subtest skips for want of a peer, so a single-node lb run reports green
+# having exercised a fraction of the suite.
+
+# Suites runnable against a one-node environment.
+#
+# partialblock gates merges because it earned it, not because it is new: on one
+# node, same binary and workload, it fails against the pre-fix engine (1758 of
+# 8192 halves reverted a generation) and passes against the fixed one (0 of
+# 8192). A suite that has never been shown to go red for the reason it claims
+# does not belong here.
+# rds is single-only for the opposite reason to lb: v1 has no multi-AZ
+# behaviour, so a second node adds no assertion. Its DB VMs are capped at four
+# concurrent by the suite's own semaphore, which is what keeps it inside the
+# job's 50-minute budget alongside the suites above.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
+
+# Suites runnable against a multi-node environment.
+E2E_SUITES_MULTI="multinode lb cert"
+
+# grep -xE alternation form, for narrowing a requested set to the eligible one.
+E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
+E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
+
+# e2e-nightly runs a narrower set than the two sets above, deliberately: its
+# ~19 cells each have a 35-minute budget and exist to prove that every
+# install / network / host-OS permutation boots and serves. Suite breadth is
+# the PR gate's job, and it already runs on every push. These are subsets, and
+# e2e_suites_assert_subset below enforces that.
+E2E_SUITES_NIGHTLY_SINGLE="single cert"
+E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
+
+# Fail loudly when a narrowed list names a suite its topology cannot run. This
+# is the guard that stops the lists drifting again: a suite added to a nightly
+# subset without also being declared eligible above stops the job here rather
+# than silently skipping half its assertions in CI.
+#   e2e_suites_assert_subset <label> <subset> <eligible-set>
+e2e_suites_assert_subset() {
+  local label="$1" subset="$2" eligible="$3" suite found eligible_suite
+  for suite in ${subset}; do
+    found=0
+    for eligible_suite in ${eligible}; do
+      if [ "${suite}" = "${eligible_suite}" ]; then found=1; break; fi
+    done
+    if [ "${found}" -ne 1 ]; then
+      echo "::error::${label} names suite '${suite}', which is not eligible for that topology (eligible: ${eligible}). Fix the list in .github/scripts/suite-sets.sh." >&2
+      return 1
+    fi
+  done
+  return 0
+}

.github/workflows/apt-packages.yml

@@ -0,0 +1,48 @@
+name: Apt Packages
+
+# Path-filtered like image-scripts.yml: this resolves setup.sh's runtime package
+# list against each supported suite's real indices, so a renamed, dropped, or
+# ambiguously virtual package fails here rather than on a user's machine.
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  apt_packages:
+    name: Resolve apt list (${{ matrix.image }})
+    runs-on: ubuntu-26.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # The suites the nightly E2E matrix actually installs on. Ubuntu 22.04 is
+        # accepted by detect_os but has never been exercised and does not resolve.
+        image:
+          - ubuntu:24.04
+          - ubuntu:26.04
+          - debian:13
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Resolve runtime package list
+        run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}

.github/workflows/e2e-baremetal.yml

@@ -9,6 +9,11 @@ on:
         required: false
         type: string
         default: ""
+      otel-attrs:
+        description: "OTEL_RESOURCE_ATTRIBUTES value for this run, e.g. ci.run_id=123,ci.cell=20 (caller-specific, so it's an input rather than a workflow-level default)"
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -16,6 +21,10 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  # Same central ES/APM sink as the other four e2e workflows. Bare-metal had
+  # no default here before — this cell's telemetry never reached the sink at
+  # all (not just missing ci.cell), see bm-bootstrap.sh's telemetry.env step.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
 
 jobs:
   e2e_baremetal:
@@ -28,6 +37,9 @@ jobs:
     env:
       BM_ISO_CACHE: /var/cache/spinifex-iso-builder
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-20-${{ github.run_id }}
+      # Caller-supplied (workflow_call input); empty for a manual
+      # workflow_dispatch run, which bm-bootstrap.sh treats as "no attrs".
+      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs }}
     steps:
       - name: Clean Workspace
         run: |

.github/workflows/e2e-ddil.yml

@@ -1,8 +1,7 @@
 name: E2E DDIL
 
-# Phase 1 trigger: workflow_dispatch only. Nightly cron + on-main-merge
-# land in mulga-siv-5.8 once ≥4 scenarios are asserting. See
-# docs/development/improvements/ddil-e2e-test-harness.md §4 / §Dependencies.
+# This workflow is manually triggered while the DDIL scenarios are being
+# validated.
 on:
   workflow_dispatch:
     inputs:
@@ -13,6 +12,9 @@ on:
 
 permissions:
   contents: read
+  # actions: read lets the otel-job-id action look up this job's numeric id via
+  # the Actions API — github.job is only the YAML key, no context exposes it.
+  actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
@@ -28,6 +30,10 @@ jobs:
     timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
+      # Job-level env wins over the workflow-level default. ci.cell=ddil marks
+      # this workflow's telemetry so a DDIL run's partition-induced errors are
+      # never mistaken for a healthy multi-node run's in the sink.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=ddil
     steps:
       - name: Clean Workspace
         run: sudo rm -rf "$GITHUB_WORKSPACE"/*
@@ -72,6 +78,13 @@ jobs:
           echo "${BIN_DIR}" >> "${GITHUB_PATH}"
           "${BIN_DIR}/gotestsum" --version
 
+      # Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
+      # the VMs, so a telemetry doc points back at this job's log directly.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: DDIL Real Multi-Node
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
@@ -208,3 +221,12 @@ jobs:
           quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
           ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-ddil-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 30

.github/workflows/e2e-nightly.yml

@@ -11,12 +11,16 @@ on:
 
 permissions:
   contents: read
+  # actions: read lets the "Resolve CI job id" step below look up this job's
+  # numeric id via the Actions API — github.job is only the YAML key, no
+  # context exposes the id itself.
+  actions: read
 
 # Scheduled runs are loaded from the default branch (main), so the schedule
 # trigger must stay in sync on main for cron to fire.
 env:
   GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
-  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
+  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
   # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
   # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
@@ -32,11 +36,11 @@ jobs:
       matrix:
         include:
           - { cell: 1,  install: binary, topo: single,     net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
+          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
+          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
           - { cell: 8,  install: binary, topo: real-multi, net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
+          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
+          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
           - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
           - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
           - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }
@@ -46,6 +50,11 @@ jobs:
     env:
       TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
+      # Job-level env wins over the workflow-level default above, so telemetry
+      # from this cell carries ci.cell/ci.matrix. Nine cells run under one
+      # run id and every cell names its hosts node1/node2/node3, so without
+      # this every cell's telemetry collapses into one unattributable bucket.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}
 
     steps:
       # blank or scheduled run = full matrix.
@@ -92,6 +101,16 @@ jobs:
         with:
           path: spinifex
 
+      # Nine cells share one run id and every cell names its hosts
+      # node1/node2/node3, so ci.cell above separates them and ci.job_id here
+      # links each cell's telemetry to the job log that produced it. Runs
+      # before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
+      - name: Resolve CI job id
+        if: steps.gate.outputs.selected == 'true'
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
+
       - name: Build Install Artifacts
         if: steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
@@ -109,10 +128,13 @@ jobs:
         run: |
           export TF_VAR_git_branch="${GIT_BRANCH}"
 
+          # Match the family, not a pinned version: an exact match silently stops
+          # overriding the image when the matrix bumps release, dropping the cell
+          # back onto the debian gold.
           TOFU_VAR_ARGS=""
-          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
-            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
-          fi
+          case "$MATRIX_HOST" in
+            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
+          esac
 
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
@@ -147,13 +169,20 @@ jobs:
         if: steps.gate.outputs.selected == 'true' && matrix.topo == 'single' && matrix.extra == ''
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          set -euo pipefail
+          # Nightly runs a documented subset of the single-node set; the assert
+          # fails the cell if that subset ever names a suite this topology
+          # cannot fully exercise.
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          e2e_suites_assert_subset "e2e-nightly single" "${E2E_SUITES_NIGHTLY_SINGLE}" "${E2E_SUITES_SINGLE}"
+
           INVENTORY=$(tofu output -json inventory)
           WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
           SSH_KEY="${SSH_KEY/#\~/$HOME}"
           ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
             "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
-             for SUITE in single cert lb; do \
+             for SUITE in ${E2E_SUITES_NIGHTLY_SINGLE}; do \
                echo \"::group::suite \$SUITE\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
@@ -190,6 +219,11 @@ jobs:
         if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          set -euo pipefail
+          # See the single-node step: same subset guard, multi-node set.
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          e2e_suites_assert_subset "e2e-nightly multi" "${E2E_SUITES_NIGHTLY_MULTI}" "${E2E_SUITES_MULTI}"
+
           INVENTORY=$(tofu output -json inventory)
           NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
@@ -204,7 +238,7 @@ jobs:
 
           ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
             "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
-             for SUITE in multinode cert lb; do \
+             for SUITE in ${E2E_SUITES_NIGHTLY_MULTI}; do \
                echo \"::group::suite \$SUITE\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
@@ -339,15 +373,26 @@ jobs:
         env:
           MATRIX_HOST: ${{ matrix.host }}
         run: |
+          # Must mirror the Provision step's override, or destroy runs against a
+          # different image path than apply did.
           TOFU_VAR_ARGS=""
-          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
-            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
-          fi
+          case "$MATRIX_HOST" in
+            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
+          esac
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
   # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
   # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
   # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
@@ -364,4 +409,8 @@ jobs:
     uses: ./.github/workflows/e2e-baremetal.yml
     with:
       ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+      # A reusable-workflow call gets none of this workflow's context (env:
+      # above, matrix.cell) for free — cell-20 needs its own attrs string.
+      # There's no matrix leg here, just the one fixed baremetal cell.
+      otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
     secrets: inherit

.github/workflows/e2e-suites.yml

@@ -1,13 +1,14 @@
-# E2E (per-suite blocks) — EXPERIMENTAL, workflow_dispatch only.
+# E2E per-suite blocks. Each suite appears as its own node in the Actions
+# graph:
 #
-# Restructures the looped e2e.yml into env-grouped job blocks so each suite
-# surfaces as its own node in the Actions graph (mulga-siv-283):
-#
-#   single_provision ─┬─ single_suite (matrix: single, cert, lb, eks, iam)
+#   single_provision ─┬─ single_suite (matrix: the single-node-eligible suites)
 #                     └─ single_teardown (if: always)
-#   multi_provision  ─┬─ multi_suite  (matrix: multinode)
+#   multi_provision  ─┬─ multi_suite  (matrix: the multi-node-eligible suites)
 #                     └─ multi_teardown (if: always)
 #
+# Both matrices come from .github/scripts/suite-sets.sh, the one definition of
+# which suites a topology can run, shared with e2e.yml and e2e-nightly.yml.
+#
 # Constraint: tofu state is runner-filesystem-local, so *_provision and
 # *_teardown must execute on the same self-hosted runner (one runner per label
 # today). Teardown reuses the persisted mulga/ tofu-cluster workspace and does
@@ -25,9 +26,13 @@ on:
   workflow_dispatch:
     inputs:
       single_suites:
-        description: "Single-node suites to run as matrix legs"
+        description: "Single-node suites to run as matrix legs (empty = every single-node-eligible suite, per .github/scripts/suite-sets.sh)"
         required: false
-        default: "single cert lb eks iam"
+        default: ""
+      multi_suites:
+        description: "Multi-node suites to run as matrix legs (empty = every multi-node-eligible suite, per .github/scripts/suite-sets.sh)"
+        required: false
+        default: ""
       run_multinode:
         description: "Also run the multinode env block"
         type: boolean
@@ -40,6 +45,10 @@ on:
 
 permissions:
   contents: read
+  # actions: read lets the otel-job-id action look up a provision job's numeric
+  # id via the Actions API — github.job is only the YAML key, no context
+  # exposes it.
+  actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
@@ -64,6 +73,13 @@ jobs:
       matrix: ${{ steps.suites.outputs.matrix }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
+      # This job (not single_suite) is where "Bootstrap via Binary Install"
+      # bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env, before the
+      # suite matrix even exists — single_suite's legs share this one
+      # already-booted VM, so daemon telemetry can't be scoped any finer
+      # than "single env" per suite leg. ci.cell=single distinguishes it
+      # from multi_provision's VMs within the same run id.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -107,7 +123,7 @@ jobs:
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -115,12 +131,22 @@ jobs:
       - name: Re-enable Go toolchain auto-download
         run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
 
+      # Default and allow-list both come from the shared definition, so this
+      # workflow cannot drift from e2e.yml about what a single node can run.
       - name: Compute suite matrix
         id: suites
+        env:
+          REQUESTED: ${{ inputs.single_suites }}
         run: |
-          LIST="${{ inputs.single_suites || 'single cert lb eks iam' }}"
+          set -euo pipefail
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          LIST="${REQUESTED:-${E2E_SUITES_SINGLE}}"
+          e2e_suites_assert_subset "single_suites" "${LIST}" "${E2E_SUITES_SINGLE}"
           echo "list=${LIST}" >> "$GITHUB_OUTPUT"
-          JSON=$(printf '%s\n' $LIST | jq -R . | jq -cs .)
+          # Unquoted on purpose: the space-separated list must word-split into
+          # one JSON element per suite.
+          # shellcheck disable=SC2086
+          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
           echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
 
       - name: Reclaim docker build cache
@@ -130,6 +156,14 @@ jobs:
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 
+      # ci.job_id names the provision job, not the suite job that later drives
+      # the env: the suite legs share this env and its baked telemetry.env, so
+      # the provisioner is the only job the daemon telemetry can belong to.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: Single env · provision
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
@@ -267,6 +301,15 @@ jobs:
             "$VM":/tmp/start-net-capture.sh
           ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
   single_suite:
     name: Single env · suite
     needs: single_provision
@@ -285,7 +328,7 @@ jobs:
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -467,6 +510,15 @@ jobs:
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
   # ---------------------------------------------------------------------------
   # Multinode env
   # ---------------------------------------------------------------------------
@@ -479,8 +531,13 @@ jobs:
       node1_ip: ${{ steps.cluster.outputs.node1_ip }}
       ssh_key: ${{ steps.cluster.outputs.ssh_key }}
       nodes_csv: ${{ steps.cluster.outputs.nodes_csv }}
+      suites: ${{ steps.suites.outputs.list }}
+      matrix: ${{ steps.suites.outputs.matrix }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
+      # See single_provision above — daemon telemetry is baked in here, at
+      # provision time, before the multi_suite matrix exists.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=multi
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -521,7 +578,7 @@ jobs:
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -536,6 +593,12 @@ jobs:
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 
+      # See the matching note on single_provision.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: Multi env · provision
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
@@ -583,14 +646,36 @@ jobs:
           echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
           echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
 
-      - name: Build multinode binary on runner
+      # Mirrors single_provision's step: the multi leg previously ran multinode
+      # alone, silently dropping the lb and cert coverage that only a
+      # multi-node env can complete.
+      - name: Compute suite matrix
+        id: suites
+        env:
+          REQUESTED: ${{ inputs.multi_suites }}
+        run: |
+          set -euo pipefail
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          LIST="${REQUESTED:-${E2E_SUITES_MULTI}}"
+          e2e_suites_assert_subset "multi_suites" "${LIST}" "${E2E_SUITES_MULTI}"
+          echo "list=${LIST}" >> "$GITHUB_OUTPUT"
+          # Unquoted on purpose: the space-separated list must word-split into
+          # one JSON element per suite.
+          # shellcheck disable=SC2086
+          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
+          echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
+
+      - name: Build multinode binaries on runner
         working-directory: spinifex/tests/e2e
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          banner "build multinode, teardown"
+          # teardown always builds: the run-scoped sweep binary runs in the
+          # teardown job, it is never a selectable suite leg.
+          SUITES_BUILD="${{ steps.suites.outputs.list }} teardown"
+          banner "build ${SUITES_BUILD// /, }"
           group_open "go test -c"
-          make e2e-build SUITES_BUILD="multinode teardown"
+          make e2e-build SUITES_BUILD="${SUITES_BUILD}"
           group_close
           ok "$(basename _bin/multinode.test) $(stat -c '%s' _bin/multinode.test | numfmt --to=iec)"
 
@@ -617,6 +702,15 @@ jobs:
             ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
           done
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
   multi_suite:
     name: Multi env · suite
     needs: multi_provision
@@ -625,7 +719,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: [multinode]
+        suite: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
     env:
       NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
       SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
@@ -636,7 +730,7 @@ jobs:
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -789,6 +883,15 @@ jobs:
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
   e2e_suites_status:
     name: E2E suite blocks · summary
     if: always()

.github/workflows/e2e.yml

@@ -15,7 +15,7 @@ on:
   workflow_dispatch:
     inputs:
       suites:
-        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam')"
+        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam e2e-rds')"
         required: false
         default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
       test_run:
@@ -26,10 +26,13 @@ on:
 permissions:
   contents: read
   statuses: write
+  # actions: read lets the otel-job-id action look up a job's numeric id via
+  # the Actions API — github.job is only the YAML key, no context exposes it.
+  actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
-  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam' }}
+  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam e2e-partialblock' }}
   TEST_RUN: ${{ inputs.test_run || '' }}
   # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
   # empty string disables telemetry entirely.
@@ -37,54 +40,28 @@ env:
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
-  # Shadow-mode suite selector. Computes which suites the diff *would* run and logs it to the job summary.
-  # Does NOT gate or override the real SUITES env yet — it runs alongside the full matrix so we
-  # can compare predicted vs actual before flipping it on.
-  select_suites:
-    name: Select Suites (shadow)
-    runs-on: ubuntu-26.04
-    timeout-minutes: 5
-    steps:
-      - uses: actions/checkout@v7
-        with:
-          fetch-depth: 0
-      - uses: actions/setup-go@v6
-        with:
-          go-version-file: go.mod
-          cache: false
-      - name: Compute targeted suite selection
-        run: |
-          set -euo pipefail
-          BASE="${{ github.event.before }}"
-          HEAD="${{ github.sha }}"
-          # First push to a branch reports an all-zero before-SHA; fall back to
-          # the first parent so the diff is still meaningful.
-          if [ -z "$BASE" ] || [ "$BASE" = "0000000000000000000000000000000000000000" ]; then
-            BASE="${HEAD}~1"
-          fi
-          echo "## Targeted suite selection (shadow)" >> "$GITHUB_STEP_SUMMARY"
-          echo '```' >> "$GITHUB_STEP_SUMMARY"
-          echo "base=$BASE head=$HEAD" >> "$GITHUB_STEP_SUMMARY"
-          go run ./tests/e2e/select-suites/cmd/select-suites \
-            --manifest docs/service-interfaces.yaml \
-            --base-sha "$BASE" --head-sha "$HEAD" --repo-root . \
-            2>>"$GITHUB_STEP_SUMMARY" \
-            | tee -a "$GITHUB_STEP_SUMMARY"
-          echo '```' >> "$GITHUB_STEP_SUMMARY"
-          echo "Shadow only — actual run uses SUITES='${SUITES}'." >> "$GITHUB_STEP_SUMMARY"
-
   e2e_single:
     name: Single-Node E2E
-    # Skip when the dispatch input requests no in-VM suites (e.g.
-    # suites='e2e-multinode' only). Push triggers and the default SUITES list
-    # both include single/cert/lb so the inputs.suites null-check keeps the
-    # job running on plain PR pushes.
-    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-iam')
+    # The dispatch check gates on exactly the suites this job's Run step
+    # allow-lists, i.e. E2E_SUITES_SINGLE in .github/scripts/suite-sets.sh —
+    # an `if:` expression cannot source that file, so this one list is
+    # hand-mirrored and must be updated with it. Gating on a suite the job
+    # won't run — lb, which only executes in the multinode job — would
+    # provision a tofu VM and pay the Spinifex install only to run nothing and
+    # tear down. Push triggers keep running via the null-check on
+    # inputs.suites.
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
     runs-on: [self-hosted, ci-single]
     timeout-minutes: 50
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
+      # Job-level env wins over the workflow-level default. This job and
+      # e2e_multi share one run id and both name their hosts node1/node2/node3,
+      # so without ci.cell their telemetry collapses into one bucket. This job
+      # is where "Bootstrap via Binary Install" bakes SPINIFEX_OTEL_ATTRS into
+      # the VM's telemetry.env, so the label has to be set here.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -134,7 +111,7 @@ jobs:
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -149,13 +126,21 @@ jobs:
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 
+      # Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
+      # the VM, so a telemetry doc points back at this job's log directly.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: Single-Node E2E
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
           ./build-install-artifacts.sh "${GIT_BRANCH}" \
             "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
             "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar"
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
 
       - name: Provision VM
         working-directory: mulga/scripts/tofu-cluster
@@ -265,6 +250,54 @@ jobs:
           EOF
           ok "kubectl installed on VM"
 
+      # CreateDBInstance boots the DB VM from the rds-postgres AMI, so that image
+      # must be in the VM's image store before the suite runs — same split as the
+      # eks-node pair above: build on the runner (needs source + network), import
+      # on the VM (needs predastore/NATS). The suite's psql client needs no step
+      # here: it runs inside a client VM that installs postgresql-client from its
+      # own user-data, because a private-only endpoint is unreachable from the
+      # runner and from the cluster host alike.
+      - name: Build rds-postgres AMI (libguestfs, runner)
+        if: contains(env.SUITES, 'e2e-rds')
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          banner "build rds-postgres AMI (libguestfs — no host devices)"
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
+          export LIBGUESTFS_BACKEND=direct
+          sudo chmod 0644 /boot/vmlinuz-* || true
+          group_open "make build-rds-postgres-image"
+          make build-rds-postgres-image
+          group_close
+          ok "raw: $(du -h /tmp/rds-postgres-image-build/rds-postgres-alpine.raw | cut -f1)"
+
+      - name: Import rds-postgres AMI on VM
+        if: contains(env.SUITES, 'e2e-rds')
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+            spinifex/scripts/images/rds-postgres/manifest.conf)"
+          # SYSTEM_TAG carries three tags, not one: managed-by marks it a system
+          # image and engine/engine-version are what an EngineVersion request
+          # resolves against. --tag is repeatable, so expand rather than pass the
+          # whole string as a single value.
+          TAG_ARGS=""
+          for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          banner "import rds-postgres AMI to VM image store"
+          scp "${SSH_OPTS[@]}" \
+            /tmp/rds-postgres-image-build/rds-postgres-alpine.raw \
+            "$VM":/tmp/rds-postgres-alpine.raw
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "spx admin images import --file /tmp/rds-postgres-alpine.raw \
+               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+               --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
+          ok "rds-postgres AMI imported"
+
       - name: Build E2E test binaries on runner
         working-directory: spinifex/tests/e2e
         run: |
@@ -312,14 +345,23 @@ jobs:
         id: e2e
         run: |
           set +e
-          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(multinode)$' | tr '\n' ' ')
+          # Allow-list, not a deny-list: a suite runs here only if named. Deny-lists
+          # silently fan a newly added suite out to every job, which is how lb ended up
+          # running twice per e2e run. The allow-list itself is shared with the other
+          # e2e workflows so the three cannot disagree about what a topology can run;
+          # suite-sets.sh also carries the per-suite rationale.
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_SINGLE_RE}" | tr '\n' ' ')
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no in-VM suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
             exit 0
           fi
           if [ -n "${TEST_RUN}" ]; then
-            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+            # The value is expanded into a command that a remote shell re-parses over
+            # ssh, so single-quote it — otherwise regex metacharacters like ( | ) $ are
+            # taken as shell syntax on the VM and bash fails before the test binary runs.
+            TEST_RUN_FLAG="-test.run='${TEST_RUN}'"
           else
             TEST_RUN_FLAG=""
           fi
@@ -466,6 +508,15 @@ jobs:
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
       - name: Fail job if any suite failed
         if: always()
         run: |
@@ -477,12 +528,19 @@ jobs:
 
   e2e_multi:
     name: Multi-Node E2E
-    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode')
+    # The dispatch check gates on exactly the suites this job's Run step
+    # allow-lists (multinode|lb|cert) so lb — which only runs here — still
+    # provisions the 3-node env when requested on its own, and the env stays
+    # unprovisioned when none of them are asked for.
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-cert')
     runs-on: [self-hosted, ci-multi]
     timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
+      # See the matching note on e2e_single: ci.cell is what separates this
+      # job's 3-node telemetry from the single-node job's under one run id.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=multi
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -532,7 +590,7 @@ jobs:
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -547,13 +605,20 @@ jobs:
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 
+      # See the matching note on e2e_single.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: Multi-Node E2E
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
           ./build-install-artifacts.sh "${GIT_BRANCH}" \
             "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
             "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar"
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
 
       - name: Provision VMs
         working-directory: mulga/scripts/tofu-cluster
@@ -688,14 +753,18 @@ jobs:
         id: e2e
         run: |
           set +e
-          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks|iam)$' | tr '\n' ' ')
+          # Allow-list, not a deny-list — see the single-node job for why.
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_MULTI_RE}" | tr '\n' ' ')
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
             exit 0
           fi
           if [ -n "${TEST_RUN}" ]; then
-            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+            # Single-quote for the remote shell — see the single-node job for the
+            # metacharacter-injection reason.
+            TEST_RUN_FLAG="-test.run='${TEST_RUN}'"
           else
             TEST_RUN_FLAG=""
           fi
@@ -881,6 +950,15 @@ jobs:
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
       - name: Fail job if any suite failed
         if: always()
         run: |
@@ -917,3 +995,136 @@ jobs:
             -f context=e2e \
             -f description="$DESC" \
             -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+  # Emits one summary doc per run to the central Elastic sink, keyed by
+  # labels.ci_run_id, so Kibana can list "did run X pass, and where are its
+  # artifacts" without hand-aggregating traces-apm*/logs-apm.error*. This is
+  # a separate self-hosted job (not a step on e2e_status) because e2e_status
+  # runs on GitHub-hosted ubuntu-26.04, which has no route to the private
+  # 192.168.1.13 sink, and moving e2e_status off GH-hosted would risk the PR
+  # status-report check.
+  e2e_es_summary:
+    name: ES CI Run Summary
+    needs: [e2e_single, e2e_multi]
+    if: always()
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 5
+    permissions:
+      actions: read
+    env:
+      # Repo var overrides; defaults to the LAN Elastic sink reachable from
+      # ci-single/ci-multi runners. Anonymous ES requests are superuser on
+      # this host, so no credentials are needed for the POST.
+      MULGA_ES_SINK: ${{ vars.MULGA_ES_SINK || '192.168.1.13:9200' }}
+    steps:
+      # Either leg may not have run (e.g. a workflow_dispatch that only
+      # requested single-node suites), so a missing artifact is tolerated —
+      # the aggregation script below treats an absent suite-status.txt as
+      # "no failures from that leg".
+      - name: Download single-node artifacts
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          name: e2e-artifacts-${{ github.run_id }}
+          path: ${{ github.workspace }}/es-summary-single
+
+      - name: Download multi-node artifacts
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          name: e2e-multi-artifacts-${{ github.run_id }}
+          path: ${{ github.workspace }}/es-summary-multi
+
+      - name: Build and post CI run summary
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          RUN_ID: ${{ github.run_id }}
+          RUN_ATTEMPT: ${{ github.run_attempt }}
+          WORKFLOW: ${{ github.workflow }}
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+          SHA: ${{ github.sha }}
+          SERVER_URL: ${{ github.server_url }}
+          SINGLE_RESULT: ${{ needs.e2e_single.result }}
+          MULTI_RESULT: ${{ needs.e2e_multi.result }}
+          SINGLE_DIR: ${{ github.workspace }}/es-summary-single
+          MULTI_DIR: ${{ github.workspace }}/es-summary-multi
+        run: |
+          set -euo pipefail
+
+          SINGLE_STATUS="${SINGLE_DIR}/suite-status.txt"
+          MULTI_STATUS="${MULTI_DIR}/suite-status.txt"
+
+          # Collect "<suite>: FAILED" lines from both legs' suite-status.txt
+          # (either may be absent — a leg that didn't run, or a passing leg
+          # that never wrote the file). Accumulate as newline-separated text
+          # rather than a bash array so this stays correct under `set -u`
+          # even when nothing failed.
+          FAILED_SUITES=""
+          for f in "$SINGLE_STATUS" "$MULTI_STATUS"; do
+            [ -f "$f" ] || continue
+            while IFS= read -r line; do
+              case "$line" in
+                *": FAILED") FAILED_SUITES="${FAILED_SUITES}${line%: FAILED}"$'\n' ;;
+              esac
+            done < "$f"
+          done
+
+          FAILED_JSON=$(printf '%s' "$FAILED_SUITES" | jq -R -s 'split("\n") | map(select(length > 0))')
+          FAILED_COUNT=$(printf '%s' "$FAILED_JSON" | jq 'length')
+
+          # Mirror e2e_status's single_ok/multi_ok logic exactly so the PR
+          # status and this ES doc can never disagree on overall pass/fail.
+          single_ok=true
+          case "$SINGLE_RESULT" in success | skipped) ;; *) single_ok=false ;; esac
+          multi_ok=true
+          case "$MULTI_RESULT" in success | skipped) ;; *) multi_ok=false ;; esac
+
+          STATE=success
+          if [ "$single_ok" != true ] || [ "$multi_ok" != true ] || [ "$FAILED_COUNT" -gt 0 ]; then
+            STATE=failure
+          fi
+
+          RUN_STARTED_AT=$(gh api "repos/${REPO}/actions/runs/${RUN_ID}" --jq .run_started_at)
+          START_EPOCH=$(date -u -d "${RUN_STARTED_AT}" +%s)
+          NOW_EPOCH=$(date -u +%s)
+          DURATION=$((NOW_EPOCH - START_EPOCH))
+
+          DOC=$(jq -n \
+            --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+            --arg run_id "${RUN_ID}" \
+            --arg workflow "${WORKFLOW}" \
+            --arg branch "${BRANCH}" \
+            --arg sha "${SHA}" \
+            --arg run_attempt "${RUN_ATTEMPT}" \
+            --arg state "${STATE}" \
+            --arg single_result "${SINGLE_RESULT}" \
+            --arg multi_result "${MULTI_RESULT}" \
+            --argjson duration "${DURATION}" \
+            --arg artifact_url "${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}" \
+            --argjson failed_suites "${FAILED_JSON}" \
+            '{
+              "@timestamp": $ts,
+              labels: { ci_run_id: $run_id },
+              ci: { workflow: $workflow, branch: $branch, sha: $sha, run_attempt: $run_attempt },
+              state: $state,
+              single_result: $single_result,
+              multi_result: $multi_result,
+              failed_suites: $failed_suites,
+              duration_seconds: $duration,
+              artifact_url: $artifact_url
+            }')
+
+          echo "$DOC" | jq .
+
+          HTTP_CODE=$(curl -sS -o /tmp/es-summary-resp.json -w '%{http_code}' \
+            -X POST "http://${MULGA_ES_SINK}/mulga-ci-runs/_doc" \
+            -H 'Content-Type: application/json' \
+            -d "${DOC}")
+
+          if [ "$HTTP_CODE" != "200" ] && [ "$HTTP_CODE" != "201" ]; then
+            echo "::error::ES POST to ${MULGA_ES_SINK}/mulga-ci-runs failed (HTTP ${HTTP_CODE})"
+            cat /tmp/es-summary-resp.json
+            exit 1
+          fi
+          echo "CI run summary doc posted to ${MULGA_ES_SINK}/mulga-ci-runs (HTTP ${HTTP_CODE})"

.github/workflows/gpu-e2e.yml

@@ -0,0 +1,345 @@
+# GPU E2E — drives wattle's persistent, shared-host Spinifex stack over SSH to
+# exercise GPU passthrough (raw EC2, ECS task GPU exposure, EKS pod GPU
+# exposure).
+#
+# ---------------------------------------------------------------------------
+# ONE-TIME HOST-SIDE SETUP — NOT part of this repo, a human must do this:
+#
+#   1. Register ONE self-hosted GitHub Actions runner on a banksia host with
+#      the dedicated label `ci-gpu` (in addition to `self-hosted`). Do
+#      NOT add it to the `ci-single` / `ci-multi` pool and do NOT give any
+#      other runner the `ci-gpu` label — a second labelled runner would
+#      let two GPU jobs race the single RTX A1000. With exactly one runner
+#      carrying the label, `runs-on: [self-hosted, ci-gpu]` alone forces
+#      serialization; the `concurrency:` block below queues same-group runs
+#      when that runner is already busy with a live job.
+#   2. Drop the wattle SSH private key on that runner's filesystem and wire
+#      it into this repo as `secrets.WATTLE_SSH_KEY` (raw private key
+#      contents). Optionally override `vars.WATTLE_SSH_HOST` /
+#      `vars.WATTLE_SSH_USER` if they differ from the defaults below.
+#   3. Confirm the runner user can drive wattle's control plane over SSH. The
+#      job calls `spx` (read-only checks) and the `aws` CLI with the already-
+#      configured `spinifex` profile directly — no wrapper scripts, no install
+#      on wattle (tf-user is already in the `spinifex` group there):
+#        ssh tf-user@192.168.1.13 'AWS_PROFILE=spinifex aws ec2 describe-regions'
+#   4. Manual-work mutex: an agent doing hands-on GPU testing on wattle can
+#      block CI by touching a sentinel file in the RUNNER's home directory
+#      (NOT on wattle):
+#        touch ~/.ci-gpu-paused     # blocks CI
+#        rm ~/.ci-gpu-paused        # resumes CI
+#      or by setting the repo/environment variable `CI_GPU_PAUSED=true`
+#      (no runner filesystem access needed). Either one causes this workflow
+#      to skip cleanly (job succeeds, all work steps skipped) instead of
+#      queuing behind manual work indefinitely.
+#
+# ---------------------------------------------------------------------------
+
+name: GPU E2E
+
+on:
+  workflow_dispatch:
+    inputs:
+      test_run:
+        description: "Optional -test.run regex scoped to tests/e2e/gpu (empty = run all GPU tests)"
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+
+# Single dedicated runner (see setup note 1 above) + this group together
+# enforce single-flight on the one RTX A1000: a second dispatch has nowhere
+# else to land (unique label) and, if it somehow overlaps in scheduling,
+# queues here rather than cancelling/stomping the in-flight run.
+concurrency:
+  group: ci-gpu
+  cancel-in-progress: false
+
+env:
+  # OTLP export to the central Elastic sink (wattle is the sink itself — see
+  # Part A.2). Repo variable overrides; carried for correlation even though
+  # the GPU suite's own daemon-side spans are emitted by wattle's already-
+  # configured obs-agent, not by this job.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  # ci.cell=gpu keeps this workflow's telemetry separable from the tofu-env
+  # workflows'. Nothing in this job bakes these attrs into a daemon (see the
+  # note above), so the label only takes effect once wattle's obs-agent is
+  # given them; there is deliberately no ci.job_id lookup for the same reason.
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=gpu
+  WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
+  WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
+  TEST_RUN: ${{ inputs.test_run || '' }}
+
+jobs:
+  gpu_e2e:
+    name: GPU E2E (wattle)
+    runs-on: [self-hosted, ci-gpu]
+    timeout-minutes: 75
+    env:
+      ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
+    steps:
+      # Manual-work mutex gate — see setup note 4 above. Checked BEFORE any
+      # SSH to wattle so a paused run touches the box zero times.
+      - name: Manual-work mutex gate
+        id: gate
+        env:
+          REPO_VAR_PAUSED: ${{ vars.CI_GPU_PAUSED }}
+        run: |
+          set -euo pipefail
+          PAUSED=false
+          if [ -f "$HOME/.ci-gpu-paused" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu runner paused via ~/.ci-gpu-paused sentinel on the runner — skipping this run (manual GPU work in progress on wattle). Remove the file to resume CI."
+          fi
+          if [ "$(echo "${REPO_VAR_PAUSED:-}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu paused via repo/environment variable CI_GPU_PAUSED=true — skipping this run."
+          fi
+          echo "paused=${PAUSED}" >> "$GITHUB_OUTPUT"
+
+      - name: Paused — nothing to do
+        if: steps.gate.outputs.paused == 'true'
+        run: echo "ci-gpu is paused; job exits cleanly without touching wattle. See gate step above for the reason."
+
+      - name: Clean Workspace
+        if: steps.gate.outputs.paused != 'true'
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/checkout@v7
+        if: steps.gate.outputs.paused != 'true'
+
+      - uses: actions/setup-go@v7
+        if: steps.gate.outputs.paused != 'true'
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Install wattle SSH key
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          install -m 0600 /dev/null "${RUNNER_TEMP}/wattle-ci"
+          echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
+          echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
+
+      # Healthcheck: confirm wattle's persistent AWS-compat control plane is
+      # actually up before spending time building/shipping the test binary.
+      - name: Healthcheck — describe-regions
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
+            'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
+
+      # Ensure the base EC2 GPU AMI is present — the raw-EC2 GPU passthrough
+      # test is the always-on baseline, so CI self-provisions it rather than
+      # depending on a manual import. `--name` selects the catalog entry (pulls
+      # the prebuilt image from iso.mulgadc.com); `--ami-name` is REQUIRED to
+      # name the AMI ubuntu-26.04-nvidia-gpu-x86_64 — without it the import
+      # falls back to the generic ami-<distro>-<version>-<arch>, which the
+      # tests/e2e/gpu filter would never match. Idempotent — skipped once
+      # wattle's persistent loop-fs already has the named AMI (the 4.4G source
+      # qcow2 is cached in the images dir, so a re-import wouldn't re-download).
+      # `spx admin images import` writes local spinifex storage, so it needs
+      # the `spinifex` group active (sg); aws-CLI calls hit the gateway over
+      # HTTPS and don't.
+      - name: Ensure base GPU AMI present (import-if-absent)
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$BASE" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            ID=$(present)
+            if [ -n "$ID" ] && [ "$ID" != "None" ]; then
+              echo "base GPU AMI already present: $BASE = $ID"; exit 0
+            fi
+            echo "base GPU AMI $BASE absent — importing prebuilt from catalog..."
+            sg spinifex -c "AWS_PROFILE=spinifex spx admin images import --name $BASE --ami-name $BASE"
+            for i in 1 2 3 4 5 6; do
+              ID=$(present)
+              [ -n "$ID" ] && [ "$ID" != "None" ] && break
+              sleep 5
+            done
+            [ -n "$ID" ] && [ "$ID" != "None" ] || { echo "import ran but $BASE still not available" >&2; exit 1; }
+            echo "imported base GPU AMI: $BASE = $ID"
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      # Preflight — the base GPU AMI is REQUIRED: the setup step above just
+      # ensured it, so a miss here means the import silently failed. Hard-fail
+      # rather than run a false-green all-skipped suite. ECS/EKS node AMIs are
+      # opt-in tiers (heavier — node AMIs + cluster spin-up); their tests
+      # self-skip when absent, so only warn.
+      - name: Preflight — verify GPU AMIs
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$1" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            ID=$(present "$BASE")
+            if [ -z "$ID" ] || [ "$ID" = "None" ]; then
+              echo "required base GPU AMI $BASE missing — cannot run the baseline EC2 GPU test" >&2
+              exit 1
+            fi
+            echo "required: $BASE = $ID"
+            for OPT in spinifex-ecs-node-gpu spinifex-eks-node spinifex-eks-node-gpu; do
+              OID=$(present "$OPT")
+              if [ -z "$OID" ] || [ "$OID" = "None" ]; then
+                echo "::warning::$OPT not imported — its GPU test(s) will skip (opt-in tier)"
+              else
+                echo "opt-in: $OPT = $OID"
+              fi
+            done
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      # Same build command CI's other e2e workflows use to cross-compile a
+      # suite into a shippable binary (tests/e2e/Makefile e2e-build target,
+      # SUITES_BUILD scoped to just gpu). Building on the runner keeps go
+      # toolchain / module cache off wattle entirely.
+      - name: Build GPU e2e test binary on runner
+        if: steps.gate.outputs.paused != 'true'
+        working-directory: tests/e2e
+        run: |
+          set -euo pipefail
+          make e2e-build SUITES_BUILD=gpu
+          ls -la _bin/gpu.test
+
+      - name: Install go-junit-report on runner
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Ship GPU e2e test binary to wattle
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
+          scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
+
+      # Same invocation shape e2e.yml uses to run a pre-built suite binary
+      # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
+      # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
+      # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
+      # the advertised gateway, so the override is required here. The harness
+      # should prefer spinifex.toml's advertised bridge.
+      - name: Run GPU e2e suite on wattle
+        id: e2e
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          if [ -n "${TEST_RUN}" ]; then
+            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          else
+            TEST_RUN_FLAG=""
+          fi
+          ssh -tt ${{ env.ssh_opts }} "$VM" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-gpu-e2e-bin; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_WAN_IP='${WATTLE_SSH_HOST}' \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./gpu.test -test.v -test.timeout=60m ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-gpu.log'; \
+             exit \${PIPESTATUS[0]}"
+          E2E_RC=$?
+          echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect artifacts from wattle
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
+
+      - name: Convert test log to JUnit XML
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log)
+            SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: ./.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "GPU E2E results"
+
+      - name: Analyze e2e failures
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: ./.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "GPU E2E failure analysis"
+
+      - uses: actions/upload-artifact@v7
+        if: always() && steps.gate.outputs.paused != 'true'
+        with:
+          name: gpu-e2e-artifacts-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      # Cleanup is NEVER a host teardown: it never runs setup-ovn.sh,
+      # dev-reset, or cluster-reset. It only reconciles EC2-API-level state —
+      # terminate any GPU-typed instance still pending/running (the Go
+      # suite's own t.Cleanup already does this on normal pass/fail exit;
+      # this is the backstop for a hard `-test.timeout` kill or a workflow
+      # cancellation, which skip deferred Go cleanups). Termination triggers
+      # gpuManager.Release() on wattle, returning the A1000 to the pool for
+      # the next queued run or manual test.
+      - name: Cleanup — terminate leftover GPU instances, release GPU
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          REMOTE='
+            set +e
+            GPU_TYPES=$(AWS_PROFILE=spinifex aws ec2 describe-instance-types \
+              --query "InstanceTypes[?GpuInfo!=null].InstanceType" --output text | tr "\t" ",")
+            if [ -z "$GPU_TYPES" ]; then
+              echo "no GPU instance types advertised; nothing to reconcile"
+              exit 0
+            fi
+            IDS=$(AWS_PROFILE=spinifex aws ec2 describe-instances \
+              --filters "Name=instance-type,Values=${GPU_TYPES}" "Name=instance-state-name,Values=pending,running" \
+              --query "Reservations[].Instances[].InstanceId" --output text)
+            if [ -z "$IDS" ] || [ "$IDS" = "None" ]; then
+              echo "no leftover GPU instances — GPU already released"
+              exit 0
+            fi
+            echo "terminating leftover GPU instance(s): $IDS"
+            AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids $IDS
+          '
+          ssh ${{ env.ssh_opts }} "$VM" "$REMOTE"
+
+      - name: Fail job if GPU suite failed
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::GPU e2e suite failed (rc=${{ steps.e2e.outputs.rc }}) — see analysis above; raw ES/APM traces at ${SPINIFEX_OTLP_ENDPOINT} (ci.run_id=${{ github.run_id }})"
+            exit 1
+          fi

.github/workflows/image-scripts.yml

@@ -0,0 +1,39 @@
+name: Image Scripts
+
+# Separate from spinifex.yml on purpose: scripts/images/ and images/ are shell
+# and mkosi assets baked into system images, not Go, so they get their own
+# path-filtered gate instead of running on every Go contributor's push (see
+# ui.yml for the same pattern applied to the frontend).
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  image_scripts:
+    name: Image Scripts
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Run image script tests + shellcheck
+        run: make test-images

.github/workflows/publish-eks-ami.yml

@@ -1,18 +1,34 @@
 name: Publish EKS + ECS node AMIs
 
-# Builds the eks-node and ecs-agent system AMIs (libguestfs, on the runner) and
-# publishes each compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket
-# that backs iso.mulgadc.com/system-ami/. Operators then install them on any
-# cluster with:
+# Builds the system AMIs (libguestfs, on the runner) and publishes each
+# compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
+# iso.mulgadc.com/system-ami/. Four images ship:
+#   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
+#   spinifex-ecs-node       Alpine ECS container instance
+#   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
+#   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
+# Operators then install them on any cluster with:
 #   spx admin images import --name spinifex-eks-node
 #   spx admin images import --name spinifex-ecs-node
+#   spx admin images import --name spinifex-eks-node-gpu
+#   spx admin images import --name spinifex-ecs-node-gpu
 #
 # Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
-# action, not something every push should trigger.
+# action, not something every push should trigger. The `variant` input scopes
+# which images build — the Ubuntu GPU legs are 16G and much slower (NVIDIA
+# driver download + DKMS build), so publishing only the CPU AMIs stays cheap.
 
 on:
   workflow_dispatch:
     inputs:
+      variant:
+        description: "Which images to build"
+        type: choice
+        default: all
+        options:
+          - all
+          - cpu
+          - gpu
       dry_run:
         description: "Build + convert only; skip the R2 upload"
         type: boolean
@@ -26,9 +42,37 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  publish:
-    name: Build + publish eks-node + ecs-agent AMIs
+  matrix:
+    name: Resolve image matrix
     runs-on: [ self-hosted, ci-single ]
+    outputs:
+      images: ${{ steps.gen.outputs.images }}
+    steps:
+      - name: Select images for variant
+        id: gen
+        env:
+          VARIANT: ${{ inputs.variant }}
+        run: |
+          set -euo pipefail
+          cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
+          gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
+          case "${VARIANT}" in
+            cpu) items="${cpu}" ;;
+            gpu) items="${gpu}" ;;
+            *)   items="${cpu},${gpu}" ;;
+          esac
+          echo "images={\"include\":[${items}]}" >> "$GITHUB_OUTPUT"
+
+  publish:
+    name: Build + publish ${{ matrix.name }}
+    needs: matrix
+    runs-on: [ self-hosted, ci-single ]
+    strategy:
+      fail-fast: false
+      # ci-single is one runner and each GPU build needs ~16G of scratch disk;
+      # serialise the legs so they never contend for /_temp or the mirror.
+      max-parallel: 1
+      matrix: ${{ fromJSON(needs.matrix.outputs.images) }}
 
     steps:
       - name: Checkout Spinifex
@@ -51,7 +95,7 @@ jobs:
           path: predastore
 
       - name: Setup Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: "spinifex/go.mod"
           # Self-hosted runner: the Go module + build cache already persists on
@@ -61,7 +105,13 @@ jobs:
           cache: false
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        # Idempotent: the ci-single runner reuses one workspace across matrix
+        # legs and runs, and checkout never scrubs the root go.work. Remove any
+        # stale file so each leg re-inits cleanly instead of failing on
+        # "go.work already exists".
+        run: |
+          rm -f go.work go.work.sum
+          go work init ./spinifex ./viperblock ./predastore
 
       - name: Install build + upload tooling
         run: |
@@ -75,7 +125,7 @@ jobs:
             rm -rf /tmp/awscliv2.zip /tmp/aws-install
           fi
 
-      - name: Build + publish eks-node to R2
+      - name: Build + publish ${{ matrix.name }} to R2
         working-directory: spinifex
         env:
           # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
@@ -84,34 +134,20 @@ jobs:
           R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
           R2_BUCKET: spinifex-iso
           DRY_RUN: ${{ inputs.dry_run }}
+          MANIFEST: ${{ matrix.manifest }}
+          # Ubuntu GPU images are 16G — a RAM-backed /tmp cannot hold the qcow2 +
+          # expanded raw, so point the build scratch at the runner's on-disk
+          # _temp. Harmless for the small Alpine legs too.
+          SYSTEM_IMAGE_BUILD_DIR: ${{ runner.temp }}/system-image-build
         run: |
           set -euo pipefail
           # libguestfs needs to read a host kernel; Ubuntu ships it mode 0600.
           # direct backend = no libvirt on the runner.
           export LIBGUESTFS_BACKEND=direct
           sudo chmod 0644 /boot/vmlinuz-* || true
+          mkdir -p "${SYSTEM_IMAGE_BUILD_DIR}"
           FLAGS=(--build)
           if [[ "${DRY_RUN}" == "true" ]]; then
             FLAGS+=(--dry-run)
           fi
-          ./scripts/publish-system-image.sh \
-            scripts/images/eks-node/manifest.conf "${FLAGS[@]}"
-
-      - name: Build + publish ecs-agent to R2
-        working-directory: spinifex
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
-          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
-          R2_BUCKET: spinifex-iso
-          DRY_RUN: ${{ inputs.dry_run }}
-        run: |
-          set -euo pipefail
-          export LIBGUESTFS_BACKEND=direct
-          sudo chmod 0644 /boot/vmlinuz-* || true
-          FLAGS=(--build)
-          if [[ "${DRY_RUN}" == "true" ]]; then
-            FLAGS+=(--dry-run)
-          fi
-          ./scripts/publish-system-image.sh \
-            scripts/images/ecs-agent/manifest.conf "${FLAGS[@]}"
+          ./scripts/publish-system-image.sh "${MANIFEST}" "${FLAGS[@]}"

.github/workflows/release.yml

@@ -32,7 +32,7 @@ jobs:
         uses: docker/setup-buildx-action@v4
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version: "1.26.5"
 
@@ -280,7 +280,7 @@ jobs:
         run: sudo make -C mulga/scripts/iso-builder clean
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: spinifex/go.mod
           cache: false

.github/workflows/segscan-oracle.yml

@@ -0,0 +1,81 @@
+# Segscan storage oracle — push-triggered (main/dev) + manual dispatch only,
+# deliberately NOT wired into spinifex.yml's pull_request path.
+#
+# tests/integration/segscan_oracle_test.go needs the mulga umbrella repo's
+# scripts/segscan checked out alongside spinifex (it is a separate Go module,
+# `package main`, and cannot be imported — see spinifex/testutil/segscanoracle).
+# spinifex.yml's jobs check out only spinifex/viperblock/predastore and run on
+# every PR; adding a second-repo checkout there would make every spinifex PR
+# depend on mulga main being healthy and pay that checkout cost on every run,
+# for one storage-accounting oracle test. Mirrors e2e.yml's push-only trigger
+# This never gates a PR before merge, matching how e2e.yml's own
+# "front gate" job works.
+name: Segscan Storage Oracle
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  segscan_oracle:
+    name: Segscan Oracle
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - name: Checkout Mulga (scripts/segscan only)
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/segscan
+          path: mulga
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore
+
+      - name: Run Segscan Oracle Test
+        env:
+          SEGSCAN_DIR: ${{ github.workspace }}/mulga/scripts/segscan
+        run: make -C spinifex test-segscan-oracle

.github/workflows/spinifex.yml

@@ -32,22 +32,29 @@ jobs:
           path: spinifex
           fetch-depth: 2
 
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
       - name: Checkout Viperblock
         uses: actions/checkout@v7
         with:
           repository: mulgadc/viperblock
-          ref: dev
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
         uses: actions/checkout@v7
         with:
           repository: mulgadc/predastore
-          ref: dev
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Setup Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
@@ -70,6 +77,55 @@ jobs:
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
 
+  # In-process integration tier (gateway.SetupRoutes over embedded NATS
+  # JetStream). Its own job so it runs in parallel with the unit tiers rather
+  # than adding to their wall time, and it still blocks a PR — unlike e2e.yml's
+  # "Integration (front gate)" job, which is push-triggered against main/dev on
+  # a self-hosted runner and so never gates a PR before merge.
+  integration_tests:
+    name: Integration Tests
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore
+
+      - name: Run Integration Tests
+        run: make -C spinifex test-integration
+
   race_detection:
     name: Race Detection
     runs-on: ubuntu-26.04
@@ -80,22 +136,29 @@ jobs:
         with:
           path: spinifex
 
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
       - name: Checkout Viperblock
         uses: actions/checkout@v7
         with:
           repository: mulgadc/viperblock
-          ref: dev
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
         uses: actions/checkout@v7
         with:
           repository: mulgadc/predastore
-          ref: dev
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Setup Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
@@ -116,22 +179,29 @@ jobs:
         with:
           path: spinifex
 
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
       - name: Checkout Viperblock
         uses: actions/checkout@v7
         with:
           repository: mulgadc/viperblock
-          ref: dev
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
         uses: actions/checkout@v7
         with:
           repository: mulgadc/predastore
-          ref: dev
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Setup Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"

.github/workflows/ui.yml

@@ -40,7 +40,7 @@ jobs:
           package_json_file: spinifex/services/spinifexui/frontend/package.json
 
       - name: Setup Node.js
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v7
         with:
           cache: "pnpm"
           cache-dependency-path: spinifex/services/spinifexui/frontend/pnpm-lock.yaml
@@ -57,3 +57,6 @@ jobs:
 
       - name: Build Frontend
         run: pnpm build
+
+      - name: Check for generated route tree drift
+        run: git diff --exit-code -- src/routeTree.gen.ts

.gitignore

@@ -37,3 +37,6 @@ Thumbs.db
 
 # Test reports
 *-report.txt
+
+# Per-worktree golangci-lint cache
+/.cache/

.golangci.yml

@@ -1,5 +1,8 @@
 version: "2"
 
+issues:
+  max-same-issues: 10
+
 formatters:
   enable:
     - gofmt
@@ -14,24 +17,29 @@ linters:
     - bidichk # checks for dangerous unicode bidi character sequences
     - bodyclose # checks whether HTTP response body is closed successfully
     - canonicalheader # checks whether net/http.Header uses canonical header keys
-    - decorder # checks declaration order of types, consts, vars, and funcs
+    - depguard # blocks imports of packages with a preferred replacement
     - durationcheck # checks for two durations multiplied together
     - embeddedstructfieldcheck # checks embedded sync.Mutex/RWMutex in structs
     - errcheck # checks for unchecked errors in type assertions and I/O
     - errchkjson # checks types passed to json encoding functions, flags omittable error checks
     - errname # checks that sentinel errors are prefixed with Err, error types suffixed with Error
+    - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
-    - forcetypeassert # flags unchecked type assertions that can panic
+    - forbidigo # blocks the legacy JetStream/KV symbols
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
+    - godot # checks that declaration comments end in a period
     - goprintffuncname # checks that printf-like functions are named with f at the end
     - gosec # inspects source code for security problems
     - govet # reports suspicious constructs (Printf alignment, shadow, etc.)
+    - iface # detects interface pollution — duplicate interface declarations
     - ineffassign # detects assignments to existing variables that are not used
+    - iotamixing # flags const blocks that mix iota with non-iota declarations
     - makezero # finds slice declarations with non-zero initial length followed by append
     - mirror # catches wrong mirror patterns in bytes/strings usage
-    - musttag # checks structs passed to marshal/unmarshal have proper tags
+    - misspell # finds commonly misspelled English words
     - modernize # modernizes code to use newer Go idioms (go fix analyzers)
+    - musttag # checks structs passed to marshal/unmarshal have proper tags
     - nakedret # finds naked returns in functions
     - nilerr # catches code that returns nil even after checking err != nil
     - nilnesserr # catches returning a different nil error variable after err != nil check
@@ -43,6 +51,8 @@ linters:
     - recvcheck # checks for receiver type consistency (pointer vs value)
     - sloglint # enforces consistent log/slog usage (key-value pairs, no globals)
     - staticcheck # extensive static analysis checks
+    - testifylint # checks common testify mistakes (assert.Len over assert.Equal on len, etc.)
+    - tparallel # detects inappropriate usage of t.Parallel()
     - unconvert # removes unnecessary type conversions
     - usestdlibvars # detects hardcoded values that should use stdlib constants
     - usetesting # catches os.TempDir() etc. in tests — should use t.TempDir()
@@ -50,14 +60,41 @@ linters:
     - whitespace # detects leading and trailing whitespace in functions
 
   settings:
+    depguard:
+      rules:
+        main:
+          deny:
+            - pkg: log$
+              desc: Use log/slog instead, see https://go.dev/blog/slog
+            - pkg: math/rand$
+              desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
+
     embeddedstructfieldcheck:
       forbid-mutex: true
 
     errcheck:
-      check-type-assertions: true
+      check-type-assertions: true # replaces forcetypeassert
       exclude-functions:
         - (*github.com/nats-io/nats.go.Subscription).Unsubscribe # cleanup-only, errors not actionable
 
+    forbidigo:
+      analyze-types: true
+      forbid:
+        - pattern: '\.JetStream$'
+          pkg: '^github\.com/nats-io/nats\.go$'
+          msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
+        - pattern: '^nats\.(JetStreamContext|KeyValue.*|KeyWatcher|Err(KeyNotFound|KeyExists|BucketNotFound|NoKeysFound|StreamNotFound))$'
+          pkg: '^github\.com/nats-io/nats\.go$'
+          msg: legacy nats KV types/sentinels are superseded; use github.com/nats-io/nats.go/jetstream.
+
+    gosec:
+      excludes:
+        - G104 # unchecked errors — errcheck is stricter
+        - G116 # bidi chars — bidichk is more thorough
+        - G117 # use of net/http serve functions (we handle TLS separately)
+        - G703 # path traversal taint analysis — false positives on config file loading
+        - G705 # XSS taint analysis — false positives on internal error responses
+
     govet:
       enable-all: true
       disable:
@@ -65,6 +102,28 @@ linters:
         - shadow # too noisy to retrofit — all 50 findings are err re-declarations
         - unusedwrite # false positives on struct field assignments before marshal/send
 
+    modernize:
+      disable:
+        - newexpr # don't replace aws.String() with new() — AWS SDK idiom
+        - stringsbuilder # don't replace small string concatenations with strings.Builder
+
+    nakedret:
+      max-func-lines: 0
+
+    nolintlint:
+      require-explanation: true
+      require-specific: true
+
+    perfsprint:
+      concat-loop: false
+      err-error: false # don't flag err.Error()
+      errorf: false # don't convert fmt.Errorf to errors.New
+      strconcat: false
+
+    reassign:
+      patterns:
+        - ".*"
+
     staticcheck:
       checks:
         - all
@@ -76,40 +135,14 @@ linters:
         - -ST1020 # godoc on exported types — not enforcing yet
         - -ST1021 # godoc on exported type groups — not enforcing yet
         - -ST1022 # godoc on exported vars/consts — not enforcing yet
-        - -SA1019 # deprecated usage (handle separately)
         - -SA4006 # value overwritten — ineffassign + wastedassign cover this better
-        - -SA9005 # ineffective break (staticcheck false positives)
+        - -SA9005 # marshaling structs with no exported fields — intended {} for empty API-response types
 
-    gosec:
-      excludes:
-        - G104 # unchecked errors — errcheck is stricter
-        - G116 # bidi chars — bidichk is more thorough
-        - G117 # use of net/http serve functions (we handle TLS separately)
-        - G703 # path traversal taint analysis — false positives on config file loading
-        - G705 # XSS taint analysis — false positives on internal error responses
-        - G706 # log injection taint analysis — false positives on structured slog usage
-
-    modernize:
+    testifylint:
       disable:
-        - newexpr # don't replace aws.String() with new() — AWS SDK idiom
-        - stringsbuilder # don't replace small string concatenations with strings.Builder
-
-    perfsprint:
-      concat-loop: false
-      err-error: false # don't flag err.Error()
-      errorf: false # don't convert fmt.Errorf to errors.New
-      strconcat: false
-
-    nakedret:
-      max-func-lines: 0
-
-    nolintlint:
-      require-explanation: true
-      require-specific: true
-
-    reassign:
-      patterns:
-        - ".*"
+        - encoded-compare # misidentifies non-JSON strings when identifiers contain "JSON"
+        - float-compare # false positives
+        - require-error # assert-on-error is deliberate where a test continues to collect further failures
 
   exclusions:
     presets:
@@ -133,3 +166,8 @@ linters:
         linters: [gosec] # subprocess execution — legitimate in VM orchestration
       - text: "G304"
         linters: [gosec] # file path from variable — necessary for config loading
+
+      # aws-sdk-go v1 is deprecated wholesale (EOL 2025-07-31); the v1->v2
+      # migration is tracked separately, so suppress its SA1019 noise for now.
+      - text: "aws-sdk-go is deprecated"
+        linters: [staticcheck]

Makefile

@@ -1,6 +1,11 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
+# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
+# path, not worktree path, so parallel worktrees of this module collide and
+# surface each other's issues. Scope it per worktree.
+export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
+
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
 ifeq ($(ARCH),x86_64)
@@ -100,6 +105,12 @@ build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd +
 import-ecs-node-image: ## Build + register the ecs-node AMI (requires a running cluster)
 	$(MAKE) build-system-image IMAGE=ecs-agent IMPORT=1
 
+build-rds-postgres-image: ## Build the spinifex-rds-postgres AMI (Alpine + PostgreSQL 18 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-postgres
+
+import-rds-postgres-image: ## Build + register the rds-postgres AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-postgres IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -120,10 +131,10 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
 
-# Preflight — runs the same checks as GitHub Actions (lint + vuln + tests).
-# Use this before committing to catch CI failures locally.
+# the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
+# integration and race tests skipped to keep quick, they run in CI.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-race test-harness
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-harness
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
@@ -133,14 +144,28 @@ test-harness:
 	@echo -e "\n....Running e2e harness unit tests...."
 	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)
 
+# In-process integration tier: the real gateway router against embedded NATS
+# JetStream, with only the daemon-side NATS subjects stubbed.
+test-integration:
+	@echo -e "\n....Running in-process integration tests...."
+	$(_Q)LOG_IGNORE=1 go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+
+# Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
+# checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
+# is not the default local or CI layout, so this is a separate target from
+# test-integration rather than folded into it. Skips itself when segscan's
+# source isn't found.
+test-segscan-oracle:
+	@echo -e "\n....Running segscan storage oracle test...."
+	$(_Q)LOG_IGNORE=1 go test -tags=integration,segscanoracle -timeout 120s ./tests/integration/... -run TestSegscanOracle $(_RACEQ)
+
 # Validate docs/service-interfaces.yaml. Schema check + cross-reference
-# of services/suites/fixtures + on-disk path existence. Subject content
-# vs source is enforced separately in Bead 5 drift lint.
+# of services/suites/fixtures + on-disk path existence.
 manifest-check:
 	@echo -e "\n....Checking service-interfaces.yaml...."
 	@go run ./tests/e2e/manifest-check/cmd/manifest-check -repo-root . -manifest docs/service-interfaces.yaml
 
-# Drift guards (Bead 5): direct-create fixture lint + NATS subject lint,
+# Drift guards: direct-create fixture lint + NATS subject lint,
 # ratcheted against tests/e2e/manifest-lint/baseline.txt. Fails only on NEW
 # drift beyond the baseline.
 manifest-lint:
@@ -154,19 +179,19 @@ manifest-lint-update:
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	LOG_IGNORE=1 go test -timeout 120s ./spinifex/...
+	GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -timeout 120s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... $(_COVQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... $(_RACEQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -174,6 +199,20 @@ test-actions:
 	@echo -e "\n....Running action tests...."
 	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
 
+# Shell suites + shellcheck for scripts/images/ helpers baked into system
+# images. Kept out of `preflight` (a dedicated CI job gates it on
+# scripts/images/** changes instead) so image-asset churn doesn't run on
+# every Go contributor's commit.
+test-images:
+	@echo -e "\n....Running scripts/images/**/*_test.sh...."
+	@for t in $$(find scripts/images -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over scripts/images/**/*.sh...."
+	shellcheck -S warning $$(find scripts/images -name '*.sh' | sort)
+	@echo "  test-images ok"
+
 # Check that new/changed code meets coverage threshold (runs tests first)
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
@@ -211,7 +250,7 @@ install-system:
 	sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
 		-o Dpkg::Options::="--force-confdef" \
 		-o Dpkg::Options::="--force-confold" \
-		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils qemu-kvm \
+		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
 		ovmf qemu-efi-aarch64 \
 		libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
 		iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
@@ -243,7 +282,7 @@ quickinstall: install-system install-go install-aws
 
 lint:
 	@echo "Running golangci-lint..."
-	$(_Q)golangci-lint run ./...
+	$(_Q)scripts/run-gate.sh golangci-lint golangci-lint run ./...
 	@echo "  golangci-lint ok"
 
 fix:
@@ -251,9 +290,15 @@ fix:
 
 govulncheck:
 	@echo "Running govulncheck..."
-	$(_Q)go tool govulncheck ./...
+	$(_Q)scripts/run-gate.sh govulncheck go tool govulncheck ./...
 	@echo "  govulncheck ok"
 
+# NilAway — advisory nil-panic analysis. Not in preflight due to false positives
+nilaway:
+	@echo "Running nilaway..."
+	$(_Q)scripts/run-gate.sh nilaway go tool nilaway -include-pkgs=github.com/mulgadc/spinifex -exclude-test-files ./...
+	@echo "  nilaway ok"
+
 # Build release tarballs — use distro-ARCH for single arch, distro for both
 distro: distro-amd64 distro-arm64
 	@echo ""
@@ -297,8 +342,8 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
-	lint fix govulncheck \
+	lint fix govulncheck nilaway \
 	distro distro-amd64 distro-arm64 distro-clean

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src=".github/assets/banner.svg" alt="Spinifex — AWS-Compatible Infrastructure Stack" width="900">
+  <img src=".github/assets/banner.svg" alt="Spinifex: the open, AWS-compatible cloud you run yourself. EC2, EBS, S3, VPC, and IAM on your infrastructure, at the edge, or on a Neocloud." width="900">
 </p>
 
 <p align="center">
@@ -11,6 +11,8 @@
 <p align="center">
   <a href="#what-is-spinifex">What is Spinifex?</a> ·
   <a href="#the-platform">Platform</a> ·
+  <a href="#three-ways-to-deploy">Deploy</a> ·
+  <a href="#aws-compatibility">AWS services</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
   <a href="#installation">Installation</a> ·
@@ -19,31 +21,90 @@
 
 ---
 
-# Spinifex: An Open Source AWS-Compatible Stack for Bare-Metal, Edge, and On-Prem Deployments
+# Spinifex: the open, AWS-compatible cloud you run yourself
 
-Spinifex developed by [Mulga Defense Corporation](https://mulgadc.com) is an open source infrastructure platform that brings the core services of AWS—like EC2, VPC, EBS, and S3—to environments where running in the cloud isn't an option. Whether you're deploying to edge sites, private data-centers, or need to operate in low-connectivity or highly contested environments, Spinifex gives you AWS-style workflows on your own hardware.
+Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your
+software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control.
+Move workloads off the hyperscalers, run AI at a fraction of the cost, or take
+the cloud to the edge, without rewriting a line. The exact build you ship to AWS,
+Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
 
 ## What is Spinifex?
 
-Spinifex replicates essential AWS primitives—virtual machines, block volumes, and object storage—using lightweight, self-contained components that are easy to deploy and integrate.
+Spinifex is an open-source infrastructure platform that recreates the AWS service
+surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises
+environments. Run cloud-native software without a hyperscaler.
 
-It's designed for developers and operators who need:
+Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering
+underneath: distributed object storage with erasure coding, block storage built
+to survive failure, and compute on bare metal. That depth is the reason your
+software runs unchanged, instead of rewritten.
 
-- Cloud-like infrastructure without the cloud
-- Minimal dependencies, full control
-- Drop-in compatibility with tools like the AWS CLI, SDKs, and Terraform
-- A secure cloud environment you control and own the entire hardware, network and software stack
+It's built for teams that need:
 
-You can run Spinifex on a few servers in a rack, a field site, or anywhere centralized cloud services aren't feasible.
+- The AWS tooling they already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
+- Full control of the stack: your hardware, your network, your data, your keys
+- A cloud that keeps running offline, through disconnection, with no external control plane
+- An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
+
+You change *where* your software runs, not *what* it is.
 
 ## The Platform
 
 <p align="center">
-  <img src=".github/assets/platform.svg" alt="The Mulga Platform — Hardware to Applications" width="900">
+  <img src=".github/assets/platform.svg" alt="The Spinifex stack: your apps and AWS tooling on top, the Spinifex AWS-compatible service layer, running on standard Linux on CPU and GPU hardware, deployable to Neocloud, on-premise, or the edge." width="900">
 </p>
 
 From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
 
+## Three ways to deploy
+
+One AWS-compatible surface, three deployment shapes. Pick the one that matches
+your reality; the platform on top is the same.
+
+### Neocloud
+
+Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and
+onto our Neocloud partner ecosystem without rewriting them. GPU capacity
+(H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the
+software.
+
+### On-premise
+
+Bring your own hardware and host Spinifex in your own data centre. A real
+multi-node HA cluster integrated with your storage and networking, with full
+control of stack, data, and jurisdiction. A predictable bill, no egress
+surprises, an auditable open-source core.
+
+### Edge
+
+Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels,
+factories, and clinics. Compute next to the data, running through disconnection,
+on hardware you own. The same AWS APIs your software already uses.
+
+## AWS compatibility
+
+Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform:
+everything you deploy on AWS deploys on Spinifex unchanged. At the edge,
+on-premise, or on a partner Neocloud.
+
+| Service | What it is | Status |
+| --- | --- | --- |
+| **EC2** | Compute | Available |
+| **EBS** | Block storage | Available |
+| **S3** | Object storage | Available |
+| **VPC** | Networking | Available |
+| **IAM** | Identity & auth | Available |
+| **ALB / NLB** | Load balancers | Available |
+| **EKS** | Kubernetes | Available |
+| **ECR** | Container registry | Available |
+| **ECS** | Container service | Available |
+| **RDS** | Databases | Available |
+| **Bedrock** | AI Deployment | Q3 2026 |
+
+Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land.
+[Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
+
 ## Core Components
 
 ### Spinifex (Compute Service – EC2 Alternative)
@@ -75,19 +136,23 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 ## Architecture at a Glance
 
 <p align="center">
-  <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture — AWS Gateway, NATS bus, service daemons" width="900">
+  <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
 </p>
 
-Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless — scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
+Every AWS API call is authenticated at the gateway, published to a NATS subject,
+and answered by whichever daemon claims it. Daemons are stateless: scale
+horizontally by starting more. No etcd, no Kubernetes, no external control plane.
+Just systemd units, a NATS cluster, and your hardware. Deep dive:
+**[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
 ## Key Features
 
-AWS-Compatible Interfaces – Provision infrastructure with awscli, Terraform, or SDKs you already use.
-
-- Designed for Control – Run on your own terms, whether that's in a datacenter, remote site, or sensitive environment.
-- Minimal Dependencies – Each service is standalone and avoids complex orchestration layers.
-- Works Offline – No reliance on centralized cloud services or external networks.
-- Open Source – Licensed under AGPL 3.0. Fork it, extend it, or deploy it as-is.
+- **AWS-compatible APIs.** Use the AWS CLI, SDKs, and Terraform you already know. Repoint your endpoint and ship; nothing to rewrite.
+- **Zero cloud dependency.** Runs entirely on your hardware. No phone-home, no control plane, no external authority. Works fully offline.
+- **Bare-metal compute.** QEMU-based with direct hardware access. No hypervisor tax, no abstraction overhead.
+- **Built-in storage.** Block and object storage included: NVMe caching, Reed-Solomon erasure coding, and replication out of the box.
+- **Edge-first architecture.** Designed for disconnected, contested, and resource-constrained environments from day one.
+- **Open core, no lock-in.** AGPL-3.0 with a commercial option. Inspect it, modify it, deploy it. The platform is yours to keep.
 
 ## Installation
 
@@ -192,6 +257,10 @@ While Spinifex is architected and implemented by experienced engineers, we lever
 
 This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.
 
+## Trademarks
+
+"AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
+
 ## License
 
 Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.

build/logrotate/spinifex

@@ -1,4 +1,8 @@
 /var/log/spinifex/*.log {
+    # The directory is 0775 root:spinifex so each service user can write its own
+    # log. logrotate refuses to rotate inside a group-writable directory unless
+    # it is told which user to rotate as.
+    su root spinifex
     daily
     rotate 14
     compress

build/scripts/nats-js-watchdog.sh

@@ -1,21 +1,32 @@
 #!/bin/sh
 # JetStream liveness watchdog for spinifex-nats.
 #
-# NATS can disable JetStream at runtime (e.g. a filesystem permission error while
-# flushing stream state) and keep the process alive serving KV as read-only. That
-# is not a process failure, so Restart=on-failure never fires and KV-backed
-# services (awsgw IAM bucket init, etc.) stall until NATS is restarted by hand.
+# NATS can stop accepting JetStream writes at runtime while the process stays
+# alive and keeps serving reads. That is not a process failure, so
+# Restart=on-failure never fires and KV-backed services (awsgw IAM bucket init,
+# etc.) stall until NATS is restarted by hand.
 #
-# This probes the NATS monitoring endpoint's JS health and restarts the service
-# only when JetStream is sustainedly unhealthy while the service is running. The
-# multi-sample window avoids restarting during transient startup/quorum 503s.
+# Two distinct latches produce that symptom, and they need different probes:
+#
+#   1. Server-wide. handleOutOfSpace calls ShutdownJetStream and sets an
+#      internal disabled flag that only a restart clears. healthz reports it.
+#   2. Per message block. filestore records a write error on the block and
+#      returns it on every later write; nothing clears it short of a restart.
+#      healthz does NOT report it, because the server-wide state is untouched —
+#      only an actual write observes it.
+#
+# So a health check alone misses case 2 entirely. Both probes run here, and a
+# sustained failure of either restarts the service. The multi-sample window
+# avoids restarting during transient startup/quorum 503s.
 
 set -eu
 
 HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
 SERVICE="spinifex-nats.service"
+SPX="${SPX_BIN:-/usr/local/bin/spx}"
 SAMPLES=3        # consecutive failing samples required before acting
 SAMPLE_GAP=2     # seconds between samples
+PROBE_TIMEOUT=5  # seconds for one canary round-trip
 
 # Only act on a running service: a stopped/activating unit is systemd's to manage,
 # and try-restart on it would either no-op or fight an in-flight (re)start.
@@ -30,15 +41,29 @@ if ! command -v curl >/dev/null 2>&1; then
     exit 0
 fi
 
-# A single 200 means JetStream is healthy — exit fast on the common case.
 js_healthy() {
     curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
 }
 
+# Round-trips a canary key through the cluster-state bucket. Absent spx this
+# degrades to the health check alone rather than restarting on a missing tool.
+js_writable() {
+    if [ ! -x "$SPX" ]; then
+        return 0
+    fi
+    "$SPX" admin node js-probe --timeout "${PROBE_TIMEOUT}s" >/dev/null 2>&1
+}
+
+failure=""
 i=1
 while [ "$i" -le "$SAMPLES" ]; do
     if js_healthy; then
-        exit 0
+        if js_writable; then
+            exit 0
+        fi
+        failure="JetStream accepts no writes (canary round-trip failed) while $HEALTH_URL reports healthy"
+    else
+        failure="JetStream unhealthy at $HEALTH_URL"
     fi
     if [ "$i" -lt "$SAMPLES" ]; then
         sleep "$SAMPLE_GAP"
@@ -46,5 +71,5 @@ while [ "$i" -le "$SAMPLES" ]; do
     i=$((i + 1))
 done
 
-echo "nats-js-watchdog: JetStream unhealthy on $SAMPLES consecutive probes ($HEALTH_URL); restarting $SERVICE" >&2
+echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2
 systemctl try-restart "$SERVICE"

build/scripts/predastore-start.sh

@@ -1,30 +0,0 @@
-#!/bin/sh
-# Helper script for spinifex-predastore.service
-# Detects host, port, backend, and node-id from spinifex.toml
-# then execs spx. Falls back to single-node defaults if no config found.
-
-CONF=/etc/spinifex/spinifex.toml
-BIND="0.0.0.0:8443"
-# -1 = co-located mode (all [[db]] peers in one process). Single-node spinifex
-# templates list 3 DB peers on 127.0.0.1; only -1 can serve that topology.
-# Multi-node spinifex.toml emits node_id = N (N >= 1) which overrides below.
-# predastore rejects NODE_ID=0 since d3a02f0 (strconv.Atoi-of-garbage guard).
-NODE_ID="-1"
-
-if [ -f "$CONF" ]; then
-    H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
-    if [ -n "$H" ]; then
-        BIND="$H"
-    fi
-
-    N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
-    if [ -n "$N" ]; then
-        NODE_ID="$N"
-    fi
-fi
-
-export SPINIFEX_PREDASTORE_HOST="${BIND%%:*}"
-export SPINIFEX_PREDASTORE_PORT="${BIND##*:}"
-export SPINIFEX_PREDASTORE_NODE_ID="$NODE_ID"
-
-exec /usr/local/bin/spx service predastore start

build/systemd/regenerate-ssh-host-keys.service

@@ -7,6 +7,9 @@ Before=ssh.service
 Type=oneshot
 ExecStart=/usr/bin/ssh-keygen -A
 RemainAfterExit=yes
+# No ExecStop: the keygen process has already exited by the time this unit
+# is stopped, so there is nothing to signal.
+TimeoutStopSec=5
 
 [Install]
 WantedBy=multi-user.target

build/systemd/spinifex-awsgw.service

@@ -14,8 +14,15 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service awsgw start
 Restart=on-failure
 RestartSec=5
+# No SIGTERM handler in this process: it exits immediately on signal, so this
+# only bounds an unexpectedly wedged process before SIGKILL.
+TimeoutStopSec=10
 
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/
 Environment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem

build/systemd/spinifex-daemon.service

@@ -7,6 +7,8 @@ Wants=openvswitch-ipsec.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
+# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see
+EnvironmentFile=-/etc/spinifex/host.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-daemon
@@ -19,13 +21,17 @@ KillMode=process
 TimeoutStopSec=15
 # RG-4: infra tier — reaped only after every guest
 OOMScoreAdjust=-500
+# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead
+MemoryHigh=75%
+MemoryMax=85%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
 # Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)
 Environment=XDG_RUNTIME_DIR=/run/spinifex
 
-# RG-9: daemon tier — privileged by necessity (GPU vfio). NoNewPrivileges omitted (RG-10: sudo ip/ovs-vsctl/dhcpcd)
+# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,
+# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.
 AmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE
 ProtectSystem=full
 PrivateDevices=no

build/systemd/spinifex-nats-watchdog.service

@@ -5,10 +5,16 @@ After=spinifex-nats.service
 
 [Service]
 Type=oneshot
-# Runs as root: the probe is a local curl to the NATS monitoring port, and the
-# remedy is `systemctl try-restart spinifex-nats` when JetStream is disabled while
-# the process stays alive (which Restart=on-failure cannot catch).
+# Runs as root: the probes are a local curl to the NATS monitoring port and a
+# canary KV round-trip via spx, and the remedy is `systemctl try-restart
+# spinifex-nats` when JetStream stops accepting writes while the process stays
+# alive (which Restart=on-failure cannot catch).
 ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
+# Sized for the probe loop's own worst case (3 samples x (5s curl + 5s canary)
+# + 2 gaps x 2s = 34s): a stop mid-cycle lets the script finish rather than
+# cutting it off. Only the failing path runs that long; a healthy node exits on
+# the first sample.
+TimeoutStopSec=45
 
 # Hardening (kept compatible with talking to PID1 over /run/systemd).
 NoNewPrivileges=yes

build/systemd/spinifex-nats.service

@@ -12,7 +12,15 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service nats start
 Restart=on-failure
 RestartSec=5
+# nats-server installs its own SIGTERM handler (JetStream store shutdown, raft
+# stepdown, client drain) with no internal deadline; every dependent service
+# stops before nats does, so size this for nats's own graceful-stop needs.
+TimeoutStopSec=20
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf
 
 # Filesystem access

build/systemd/spinifex-northstar.service

@@ -0,0 +1,61 @@
+[Unit]
+Description=Spinifex Northstar (Authoritative DNS)
+PartOf=spinifex.target
+After=spinifex-nats.service spinifex-predastore.service
+Wants=spinifex-predastore.service
+
+[Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+Type=simple
+Slice=spinifex-system.slice
+User=spinifex-northstar
+Group=spinifex
+ExecStart=/usr/local/bin/spx service northstar start
+Restart=on-failure
+RestartSec=5
+# server.Shutdown() is called with an internal 5s context; margin covers the
+# NATS unsubscribe/close and process exit that follow it.
+TimeoutStopSec=10
+OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/northstar/
+
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/northstar /etc/spinifex/spinifex.toml /etc/spinifex/ca.pem
+ReadWritePaths=/var/lib/spinifex/northstar /var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+# RG-9: exactly CAP_NET_BIND_SERVICE so the unprivileged user binds :53 (no root).
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
+[Install]
+WantedBy=spinifex.target

build/systemd/spinifex-predastore.service

@@ -14,10 +14,20 @@ Slice=spinifex-system.slice
 User=spinifex-storage
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
-ExecStart=/var/lib/spinifex/predastore-start.sh
+ExecStart=/usr/local/bin/spx service predastore start
 Restart=on-failure
 RestartSec=5
+# Matches viperblock's drain budget: viperblock's WAL flush writes into
+# predastore, and After= already stops viperblock first, but this is the
+# backstop if that ordering is ever bypassed (direct stop, future edit).
+TimeoutStopSec=30
 OOMScoreAdjust=-500
+# Legitimately the most memory-hungry infra service (S3 buffering/caching);
+# High/Max stay well above idle so normal load isn't throttled, while still
+# bounding a leak short of the slice-wide cap.
+MemoryHigh=40%
+MemoryMax=55%
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml
 Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
 Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem

build/systemd/spinifex-qmp-collector.service

@@ -16,6 +16,13 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service qmp-collector start
 Restart=on-failure
 RestartSec=5
+# Poll loop selects on ctx.Done() directly and closes NATS locally; shutdown
+# is near-instant, so this only bounds an unexpectedly wedged process.
+TimeoutStopSec=10
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 # Same runtime dir as the daemon: telemetry sockets + metadata live there

build/systemd/spinifex-shutdown.service

@@ -9,9 +9,13 @@ After=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.serv
 Type=oneshot
 RemainAfterExit=yes
 ExecStart=/bin/true
-# ExecStop only runs when the target/host goes down, not on a single-service
-# restart — so a daemon restart still leaves guests running (DDIL).
-ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s
+# ExecStop fires on ANY target stop, including the stop half of `systemctl
+# restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless
+# of whether the host is actually going down. --only-if-host-stopping drains
+# only when systemd itself is unwinding into shutdown.target; a plain restart
+# or `systemctl stop spinifex.target` falls through so QEMU/nbdkit survive
+# (KillMode=process) and the daemon reconnects to them on the way back up.
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --only-if-host-stopping
 TimeoutStopSec=180
 
 [Install]

build/systemd/spinifex-ui.service

@@ -12,16 +12,24 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service spinifex-ui start
 Restart=on-failure
 RestartSec=5
+# server.Shutdown() uses an internal 30s context (WriteTimeout=120s covers
+# long streaming proxy responses, e.g. console passthrough); add margin above it.
+TimeoutStopSec=40
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
+Environment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/
 
 # Filesystem access
 NoNewPrivileges=yes
 ProtectSystem=strict
 PrivateDevices=yes
 ReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key
-ReadWritePaths=/var/log/spinifex
+ReadWritePaths=/var/lib/spinifex/spinifex-ui /var/log/spinifex
 
 # Hardening
 ProtectHome=yes

build/systemd/spinifex-viperblock.service

@@ -15,6 +15,16 @@ ExecStart=/usr/local/bin/spx service viperblock start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=15%
+MemoryMax=25%
+# KillMode=process: nbdkit still serving a guest survives a restart here, the
+# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts
+# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.
+KillMode=process
+# Bounds this process's own WAL drain, which the plugin caps at 20s.
+TimeoutStopSec=30
 EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/

build/systemd/spinifex-vpcd.service

@@ -14,13 +14,23 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service vpcd start
 Restart=on-failure
 RestartSec=5
+# Waits for an in-flight OVN drift-reconcile pass to return before exiting;
+# no sub-timeout bounds that call today, so this covers a normal pass.
+TimeoutStopSec=20
 OOMScoreAdjust=-500
+# Per-service ceiling: bounds a leak here instead of the slice-wide cap.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/
 
 # RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no
 # longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem
-# sandbox is restored. NoNewPrivileges stays off (RG-10: sudo ip/ovs-vsctl/dhcpcd).
+# sandbox is restored.
+#
+# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and
+# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR
+# update that retires the last escalation path.
 AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
 CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
 NoNewPrivileges=no

build/systemd/spinifex.target

@@ -2,7 +2,7 @@
 Description=Spinifex Infrastructure Platform
 After=network-online.target
 Wants=network-online.target
-Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
+Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
 
 [Install]
 WantedBy=multi-user.target

cmd/dhcptest/main.go

@@ -6,7 +6,7 @@
 // Usage (must run as root — needs AF_PACKET):
 //
 //	sudo dhcptest --iface=br-wan
-//	sudo dhcptest --iface=br-wan --mac=02:00:00:7c:b7:b9 --promisc
+//	sudo dhcptest --iface=br-wan --mac=02:54:df:71:22:5c --promisc
 //	sudo dhcptest --iface=br-wan --spinifex-id=gateway-wan1
 package main
 
@@ -14,7 +14,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"log"
+	"log/slog"
 	"net"
 	"os"
 	"os/exec"
@@ -23,6 +23,7 @@ import (
 
 	"github.com/insomniacslk/dhcp/dhcpv4"
 	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
+	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
 )
 
 func main() {
@@ -36,6 +37,7 @@ func main() {
 	timeout := flag.Duration("timeout", 10*time.Second, "Per-attempt DHCP timeout")
 	retries := flag.Int("retries", 3, "Number of DORA attempts before giving up")
 	verbose := flag.Bool("verbose", false, "Print full DHCP packet contents")
+	broadcast := flag.Bool("broadcast", false, "Set the BOOTP broadcast flag, as vpcd's Acquire path does, so replies come back to ff:ff:ff:ff:ff:ff rather than unicast to the bridge MAC")
 	flag.Parse()
 
 	// Resolve the hardware address to use.
@@ -44,20 +46,25 @@ func main() {
 
 	switch {
 	case *spinifexID != "":
-		hwAddr, err = net.ParseMAC(generateMAC(*spinifexID))
+		// vpcd's own derivation, so the probe leases under the MAC the daemon
+		// would have used for this id.
+		hwAddr, err = dhcp.DeriveMAC(*spinifexID)
 		if err != nil {
-			log.Fatalf("generateMAC(%q): %v", *spinifexID, err)
+			slog.Error("Could not derive MAC", "spinifex-id", *spinifexID, "error", err)
+			os.Exit(1)
 		}
 		fmt.Printf("spinifex virtual MAC for %q: %s\n", *spinifexID, hwAddr)
 	case *macStr != "":
 		hwAddr, err = net.ParseMAC(*macStr)
 		if err != nil {
-			log.Fatalf("parse --mac %q: %v", *macStr, err)
+			slog.Error("Could not parse --mac", "mac", *macStr, "error", err)
+			os.Exit(1)
 		}
 	default:
 		hwAddr, err = ifaceMAC(*iface)
 		if err != nil {
-			log.Fatalf("get MAC for %s: %v", *iface, err)
+			slog.Error("Could not get MAC for interface", "iface", *iface, "error", err)
+			os.Exit(1)
 		}
 		fmt.Printf("using real interface MAC: %s\n", hwAddr)
 	}
@@ -83,7 +90,7 @@ func main() {
 
 	if *promisc {
 		if err := setPromisc(*iface, true); err != nil {
-			log.Printf("WARNING: failed to set promisc on %s: %v", *iface, err)
+			slog.Warn("Failed to set promisc on interface", "iface", *iface, "error", err)
 		} else {
 			fmt.Printf("set %s promisc ON\n\n", *iface)
 		}
@@ -96,25 +103,21 @@ func main() {
 		nclient4.WithRetry(*retries),
 	}
 	if *verbose {
-		opts = append(opts, nclient4.WithLogger(nclient4.ShortSummaryLogger{Printfer: log.New(os.Stderr, "[nclient4] ", 0)}))
+		opts = append(opts, nclient4.WithLogger(nclient4.ShortSummaryLogger{Printfer: slogPrintfer{}}))
 	}
 
 	client, err := nclient4.New(*iface, opts...)
 	if err != nil {
-		log.Fatalf("nclient4.New(%s): %v", *iface, err)
+		slog.Error("Could not create nclient4 client", "iface", *iface, "error", err)
+		os.Exit(1)
 	}
 	defer client.Close()
 
-	// Identity modifiers — client-id / hostname / vendor-class for upstream lease tagging.
-	var mods []dhcpv4.Modifier
-	if cid != "" {
-		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(cid))))
-	}
-	if *hostname != "" {
-		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptHostName(*hostname)))
-	}
-	if *vendorClass != "" {
-		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClassIdentifier(*vendorClass)))
+	// Identity modifiers — client-id / hostname / vendor-class for upstream lease
+	// tagging. Built by the vpcd code path so the probe cannot drift from it.
+	mods := dhcp.IdentityModifiers(cid, *hostname, *vendorClass, hwAddr)
+	if *broadcast {
+		mods = append(mods, dhcpv4.WithBroadcast(true))
 	}
 
 	fmt.Println("Sending DHCP Discover...")
@@ -141,14 +144,12 @@ func main() {
 	}
 }
 
-// generateMAC is the same deterministic hash used by vpcd's topology.go.
-// It produces a locally-administered MAC (02:00:00:xx:xx:xx) from a resource ID.
-func generateMAC(resourceID string) string {
-	h := uint32(0)
-	for _, c := range resourceID {
-		h = h*31 + uint32(c)
-	}
-	return fmt.Sprintf("02:00:00:%02x:%02x:%02x", (h>>16)&0xff, (h>>8)&0xff, h&0xff)
+// slogPrintfer adapts nclient4's Printfer to slog. nclient4 only emits through
+// it under --verbose, so the packet summaries log at info level to stay visible.
+type slogPrintfer struct{}
+
+func (slogPrintfer) Printf(format string, v ...any) {
+	slog.Info(fmt.Sprintf(format, v...), "component", "nclient4")
 }
 
 func ifaceMAC(name string) (net.HardwareAddr, error) {

cmd/ecr-credential-provider/main.go

@@ -12,17 +12,16 @@
 package main
 
 import (
-	"bufio"
 	"encoding/json"
 	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/guestenv"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 
 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
@@ -144,13 +143,7 @@ func cacheDurationFrom(expiresAt time.Time) string {
 // loadConfig reads the cloud-init env file then lets real env vars override. The
 // IMDS base is fixed (not in the env file) but overridable for tests.
 func loadConfig(envFile string) config {
-	env := parseEnvFile(envFile)
-	get := func(key string) string {
-		if v := os.Getenv(key); v != "" {
-			return v
-		}
-		return env[key]
-	}
+	get := guestenv.Load(envFile).Get
 
 	cfg := config{
 		GatewayURL: get("EKS_GATEWAY_URL"),
@@ -167,29 +160,6 @@ func loadConfig(envFile string) config {
 	return cfg
 }
 
-// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
-func parseEnvFile(path string) map[string]string {
-	out := map[string]string{}
-	f, err := os.Open(path)
-	if err != nil {
-		return out
-	}
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		key, val, ok := strings.Cut(line, "=")
-		if !ok {
-			continue
-		}
-		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
-	}
-	return out
-}
-
 // emitEmpty writes a valid CredentialProviderResponse with no auth entries.
 func emitEmpty(out io.Writer) {
 	resp := CredentialProviderResponse{

cmd/ecr-credential-provider/main_test.go

@@ -18,12 +18,14 @@ func fakeBackend(t *testing.T) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
 
-	// IMDSv2 token.
+	// IMDSv2 token. Real IMDS echoes the requested TTL back as a response
+	// header; the SDK client parses it from there, not the body.
 	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPut {
 			w.WriteHeader(http.StatusMethodNotAllowed)
 			return
 		}
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("imds-v2-token"))
 	})
 	// IMDS role name.

cmd/ecs-agent/agent.go

@@ -43,6 +43,11 @@ type Agent struct {
 	// cred serves task IAM role credentials at 169.254.170.2 (nil in unit tests).
 	cred *credEndpoint
 
+	// gpu is the local pinning ledger over the host's discovered GPU UUIDs.
+	// Always non-nil (newAgent seeds an empty one); New() reseeds it from
+	// nvidia-smi discovery.
+	gpu *gpuLedger
+
 	closers []func() error
 }
 
@@ -61,6 +66,7 @@ func newAgent(cfg config, id identity, cp controlPlane, puller ctrruntime.ImageP
 		reg:      newRegistrar(cp, id),
 		hb:       newHeartbeater(cp, id, cfg.Heartbeat),
 		netns:    newTaskNetns(execNetRunner{}),
+		gpu:      newGPULedger(id.Capacity.GPUIDs),
 	}
 }
 
@@ -84,7 +90,7 @@ func New(cfg config) (*Agent, error) {
 		InstanceID:   meta.InstanceID,
 		AZ:           meta.AZ,
 		Hostname:     host,
-		Capacity:     detectCapacity(),
+		Capacity:     detectCapacity(discoverNvidiaGPUs(execCommandRunner)),
 		AgentVersion: version,
 	}
 
@@ -153,11 +159,15 @@ func (a *Agent) Stop() error {
 	return firstErr
 }
 
-// detectCapacity reports the host's total CPU units (1 vCPU = 1024) and memory.
-func detectCapacity() bus.InstanceCapacity {
+// detectCapacity reports the host's total CPU units (1 vCPU = 1024), memory,
+// and GPU count/UUIDs from the caller's nvidia-smi discovery (nil on a
+// non-GPU host).
+func detectCapacity(gpuUUIDs []string) bus.InstanceCapacity {
 	return bus.InstanceCapacity{
 		CPU:       goruntime.NumCPU() * 1024,
 		MemoryMiB: memTotalMiB(),
+		GPU:       len(gpuUUIDs),
+		GPUIDs:    gpuUUIDs,
 	}
 }
 

cmd/ecs-agent/agent_test.go

@@ -42,7 +42,7 @@ func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
 }
 
 func TestDetectCapacity_Positive(t *testing.T) {
-	c := detectCapacity()
+	c := detectCapacity(nil)
 	if c.CPU <= 0 {
 		t.Errorf("CPU = %d, want > 0", c.CPU)
 	}
@@ -50,4 +50,17 @@ func TestDetectCapacity_Positive(t *testing.T) {
 	if c.MemoryMiB < 0 {
 		t.Errorf("MemoryMiB = %d, want >= 0", c.MemoryMiB)
 	}
+	if c.GPU != 0 || c.GPUIDs != nil {
+		t.Errorf("no GPU UUIDs given: want GPU=0/GPUIDs=nil, got %d/%v", c.GPU, c.GPUIDs)
+	}
+}
+
+func TestDetectCapacity_WithGPUUUIDs(t *testing.T) {
+	c := detectCapacity([]string{"GPU-aaa", "GPU-bbb"})
+	if c.GPU != 2 {
+		t.Errorf("GPU = %d, want 2", c.GPU)
+	}
+	if len(c.GPUIDs) != 2 {
+		t.Errorf("GPUIDs = %v, want 2 entries", c.GPUIDs)
+	}
 }

cmd/ecs-agent/config.go

@@ -1,11 +1,10 @@
 package main
 
 import (
-	"bufio"
-	"os"
 	"strconv"
-	"strings"
 	"time"
+
+	"github.com/mulgadc/spinifex/internal/guestenv"
 )
 
 const (
@@ -40,13 +39,7 @@ type config struct {
 
 // loadConfig reads the cloud-init env file then lets real env vars override.
 func loadConfig(envFile string) config {
-	env := parseEnvFile(envFile)
-	get := func(key string) string {
-		if v := os.Getenv(key); v != "" {
-			return v
-		}
-		return env[key]
-	}
+	get := guestenv.Load(envFile).Get
 
 	cfg := config{
 		GatewayURL:       get("ECS_GATEWAY_URL"),
@@ -94,26 +87,3 @@ func loadConfig(envFile string) config {
 	}
 	return cfg
 }
-
-// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
-func parseEnvFile(path string) map[string]string {
-	out := map[string]string{}
-	f, err := os.Open(path)
-	if err != nil {
-		return out
-	}
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		key, val, ok := strings.Cut(line, "=")
-		if !ok {
-			continue
-		}
-		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
-	}
-	return out
-}

cmd/ecs-agent/config_test.go

@@ -71,18 +71,3 @@ func TestLoadConfig_HeartbeatOverride(t *testing.T) {
 		t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
 	}
 }
-
-func TestParseEnvFile_SkipsBlankAndComments(t *testing.T) {
-	dir := t.TempDir()
-	p := filepath.Join(dir, "f.env")
-	if err := os.WriteFile(p, []byte("\n# c\nA=1\nnokeyval\nB = 2 \n"), 0o600); err != nil {
-		t.Fatal(err)
-	}
-	m := parseEnvFile(p)
-	if m["A"] != "1" || m["B"] != "2" {
-		t.Errorf("parse mismatch: %#v", m)
-	}
-	if _, ok := m["nokeyval"]; ok {
-		t.Errorf("line without = should be skipped")
-	}
-}

cmd/ecs-agent/controlplane.go

@@ -29,6 +29,9 @@ type controlPlane interface {
 	// assign taskIDs and stop taskIDs accepted on the previous poll. Returns the
 	// still-pending assignments and stop directives.
 	PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error)
+	// ReportTaskGPU reports the local ledger's pinned device UUIDs for a task's
+	// containers, so the control plane can populate DescribeTasks gpuIds.
+	ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error
 }
 
 // gatewayControlPlane implements controlPlane over the SigV4 ECS gateway client.
@@ -74,6 +77,11 @@ func (g *gatewayControlPlane) Register(id identity) error {
 		},
 		VersionInfo: &ecs.VersionInfo{AgentVersion: aws.String(id.AgentVersion)},
 	}
+	if len(id.Capacity.GPUIDs) > 0 {
+		in.TotalResources = append(in.TotalResources, &ecs.Resource{
+			Name: aws.String("GPU"), Type: aws.String("STRINGSET"), StringSetValue: aws.StringSlice(id.Capacity.GPUIDs),
+		})
+	}
 	body, err := json.Marshal(in)
 	if err != nil {
 		return fmt.Errorf("marshal register: %w", err)
@@ -133,3 +141,15 @@ func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssig
 	}
 	return out.Assignments, out.Stops, nil
 }
+
+func (g *gatewayControlPlane) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
+	in := &handlers_ecs.ReportTaskGPUInput{Cluster: cluster, Task: task, Containers: containers}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal report-task-gpu: %w", err)
+	}
+	if _, err := g.client.Call("ReportTaskGPU", body); err != nil {
+		return fmt.Errorf("report task gpu: %w", err)
+	}
+	return nil
+}

cmd/ecs-agent/controlplane_test.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+)
+
+// fixedCredsProvider is a static credentials.CredentialsProvider for tests
+// that only need a valid SigV4 signature, not real IMDS rotation.
+type fixedCredsProvider struct{}
+
+func (fixedCredsProvider) Retrieve(context.Context) (credentials.Credentials, error) {
+	return credentials.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
+}
+
+func newTestGatewayControlPlane(t *testing.T, srv *httptest.Server) *gatewayControlPlane {
+	t.Helper()
+	cp, err := newGatewayControlPlane(config{GatewayURL: srv.URL, Region: "us-east-1"}, fixedCredsProvider{})
+	if err != nil {
+		t.Fatalf("newGatewayControlPlane: %v", err)
+	}
+	return cp
+}
+
+// Register carries the discovered GPU UUIDs as a STRINGSET resource, matching
+// the control plane's RegisterContainerInstance GPU parsing (Epic C3).
+func TestGatewayControlPlane_Register_SendsGPUStringSet(t *testing.T) {
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	id := identity{
+		ClusterName: "default", InstanceID: "i-1",
+		Capacity: detectCapacity([]string{"GPU-aaa", "GPU-bbb"}),
+	}
+	if err := cp.Register(id); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+
+	var in ecs.RegisterContainerInstanceInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	var gpu *ecs.Resource
+	for _, r := range in.TotalResources {
+		if r.Name != nil && *r.Name == "GPU" {
+			gpu = r
+		}
+	}
+	if gpu == nil {
+		t.Fatal("no GPU resource in TotalResources")
+	}
+	if *gpu.Type != "STRINGSET" {
+		t.Errorf("GPU resource type = %q, want STRINGSET", *gpu.Type)
+	}
+	got := make([]string, len(gpu.StringSetValue))
+	for i, v := range gpu.StringSetValue {
+		got[i] = *v
+	}
+	if len(got) != 2 || got[0] != "GPU-aaa" || got[1] != "GPU-bbb" {
+		t.Errorf("GPU UUIDs = %v, want [GPU-aaa GPU-bbb]", got)
+	}
+}
+
+// A non-GPU host (no discovered UUIDs) registers with no GPU resource at all.
+func TestGatewayControlPlane_Register_NoGPUOmitsResource(t *testing.T) {
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	id := identity{ClusterName: "default", InstanceID: "i-1", Capacity: detectCapacity(nil)}
+	if err := cp.Register(id); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+
+	var in ecs.RegisterContainerInstanceInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	for _, r := range in.TotalResources {
+		if r.Name != nil && *r.Name == "GPU" {
+			t.Fatalf("unexpected GPU resource on a non-GPU host: %+v", r)
+		}
+	}
+}
+
+// ReportTaskGPU posts the per-container device UUIDs under the ReportTaskGPU
+// action, decodable as the handlers_ecs internal shape.
+func TestGatewayControlPlane_ReportTaskGPU(t *testing.T) {
+	var gotTarget string
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotTarget = r.Header.Get("X-Amz-Target")
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	err := cp.ReportTaskGPU("web", "t-1", []handlers_ecs.ContainerGPUReport{
+		{Name: "trainer", GPUIDs: []string{"GPU-aaa"}},
+	})
+	if err != nil {
+		t.Fatalf("ReportTaskGPU: %v", err)
+	}
+	if gotTarget == "" {
+		t.Fatal("no X-Amz-Target header sent")
+	}
+
+	var in handlers_ecs.ReportTaskGPUInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	if in.Cluster != "web" || in.Task != "t-1" {
+		t.Errorf("cluster/task = %q/%q, want web/t-1", in.Cluster, in.Task)
+	}
+	if len(in.Containers) != 1 || in.Containers[0].Name != "trainer" || len(in.Containers[0].GPUIDs) != 1 {
+		t.Errorf("containers = %+v, want one trainer entry with one UUID", in.Containers)
+	}
+}

cmd/ecs-agent/credendpoint.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net"
@@ -143,7 +144,7 @@ func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	creds, roleARN, err := c.fetch(r.Context(), credID)
 	if err != nil {
-		if err == errUnknownCredID {
+		if errors.Is(err, errUnknownCredID) {
 			http.Error(w, "not found", http.StatusNotFound)
 			return
 		}

cmd/ecs-agent/credentials/imds.go

@@ -3,57 +3,39 @@ package credentials
 import (
 	"context"
 	"net/http"
-	"sync"
-	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 )
 
-// refreshMargin is how long before expiry the cache is considered stale.
-const refreshMargin = 5 * time.Minute
-
-// IMDSProvider fetches instance-role credentials from IMDS and caches them until
-// they near expiry. Safe for concurrent use.
+// IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
+// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache
+// (which refetches once the real expiry passes; ec2rolecreds also caps
+// Expires to at most an hour out). Safe for concurrent use.
 type IMDSProvider struct {
-	client *http.Client
-	base   string
-
-	mu     sync.Mutex
-	cached Credentials
+	cache *aws.CredentialsCache
 }
 
 var _ CredentialsProvider = (*IMDSProvider)(nil)
 
 // NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
-// "http://169.254.169.254/latest"). A nil client uses http.DefaultClient.
+// "http://169.254.169.254/latest"). A nil client uses the SDK IMDS client's
+// default HTTP client.
 func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
-	if client == nil {
-		client = http.DefaultClient
-	}
-	return &IMDSProvider{client: client, base: base}
+	provider := imdscreds.NewProvider(client, base)
+	return &IMDSProvider{cache: aws.NewCredentialsCache(provider)}
 }
 
 // Retrieve returns cached credentials when still valid, else fetches fresh ones.
 func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	if p.cached.Valid(refreshMargin) {
-		return p.cached, nil
-	}
-	if err := ctx.Err(); err != nil {
-		return Credentials{}, err
-	}
-
-	raw, err := imdscreds.Fetch(p.client, p.base)
+	creds, err := p.cache.Retrieve(ctx)
 	if err != nil {
 		return Credentials{}, err
 	}
-	p.cached = Credentials{
-		AccessKeyID:     raw.AccessKeyID,
-		SecretAccessKey: raw.SecretAccessKey,
-		SessionToken:    raw.SessionToken,
-		Expiration:      raw.Expiration,
-	}
-	return p.cached, nil
+	return Credentials{
+		AccessKeyID:     creds.AccessKeyID,
+		SecretAccessKey: creds.SecretAccessKey,
+		SessionToken:    creds.SessionToken,
+		Expiration:      creds.Expires,
+	}, nil
 }

cmd/ecs-agent/credentials/imds_test.go

@@ -10,14 +10,19 @@ import (
 	"time"
 )
 
+// imdsStub serves the IMDSv2 token + role + credentials endpoints under the
+// real /latest prefix the SDK IMDS client always requests.
 func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		// Real IMDS echoes the requested TTL back as a response header; the SDK
+		// client parses it from there, not the body.
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("v2-token"))
 	})
-	mux.HandleFunc("/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/meta-data/iam/security-credentials/" {
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
 			_, _ = w.Write([]byte("node-role"))
 			return
 		}
@@ -33,15 +38,17 @@ func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Serv
 
 func TestRetrieve_FetchesAndCaches(t *testing.T) {
 	var hits int32
-	exp := time.Now().Add(6 * time.Hour).UTC().Round(time.Second)
+	// Kept under an hour: ec2rolecreds caps Expires to now+1h.
+	exp := time.Now().Add(30 * time.Minute).UTC().Round(time.Second)
 	srv := imdsStub(t, &hits, map[string]string{
+		"Code":            "Success",
 		"AccessKeyId":     "AKIA",
 		"SecretAccessKey": "secret",
 		"Token":           "session",
 		"Expiration":      exp.Format(time.RFC3339),
 	})
 
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	got, err := p.Retrieve(context.Background())
 	if err != nil {
 		t.Fatalf("Retrieve: %v", err)
@@ -50,7 +57,7 @@ func TestRetrieve_FetchesAndCaches(t *testing.T) {
 		t.Errorf("creds mismatch: %+v", got)
 	}
 
-	// Second call within validity window must not re-hit IMDS.
+	// Second call well within validity must not re-hit IMDS.
 	if _, err := p.Retrieve(context.Background()); err != nil {
 		t.Fatalf("Retrieve 2: %v", err)
 	}
@@ -59,15 +66,17 @@ func TestRetrieve_FetchesAndCaches(t *testing.T) {
 	}
 }
 
-func TestRetrieve_RefetchesWhenStale(t *testing.T) {
+func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
 	var hits int32
-	// Expiry inside the refresh margin → never cached, always refetched.
+	// Already expired: aws.CredentialsCache refetches once the stored
+	// credentials' real expiry has passed, regardless of ExpiryWindow.
 	srv := imdsStub(t, &hits, map[string]string{
+		"Code":            "Success",
 		"AccessKeyId":     "AKIA",
 		"SecretAccessKey": "secret",
-		"Expiration":      time.Now().Add(time.Minute).UTC().Format(time.RFC3339),
+		"Expiration":      time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
 	})
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	if _, err := p.Retrieve(context.Background()); err != nil {
 		t.Fatal(err)
 	}
@@ -75,13 +84,13 @@ func TestRetrieve_RefetchesWhenStale(t *testing.T) {
 		t.Fatal(err)
 	}
 	if hits != 2 {
-		t.Errorf("hits = %d, want 2 (stale refetch)", hits)
+		t.Errorf("hits = %d, want 2 (expired refetch)", hits)
 	}
 }
 
 func TestRetrieve_CancelledContext(t *testing.T) {
-	srv := imdsStub(t, nil, map[string]string{"AccessKeyId": "A", "SecretAccessKey": "B"})
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 	if _, err := p.Retrieve(ctx); err == nil {
@@ -89,16 +98,10 @@ func TestRetrieve_CancelledContext(t *testing.T) {
 	}
 }
 
-func TestCredentials_Valid(t *testing.T) {
-	if (Credentials{}).Valid(0) {
-		t.Error("empty creds should be invalid")
-	}
-	noExp := Credentials{AccessKeyID: "a", SecretAccessKey: "b"}
-	if !noExp.Valid(time.Hour) {
-		t.Error("zero expiry should be treated valid")
-	}
-	soon := Credentials{AccessKeyID: "a", SecretAccessKey: "b", Expiration: time.Now().Add(time.Minute)}
-	if soon.Valid(5 * time.Minute) {
-		t.Error("expiry inside margin should be invalid")
+func TestRetrieve_CodeNotSuccessErrors(t *testing.T) {
+	srv := imdsStub(t, nil, map[string]string{"Code": "Failure", "AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
+	if _, err := p.Retrieve(context.Background()); err == nil {
+		t.Fatal("expected error when Code != Success")
 	}
 }

cmd/ecs-agent/credentials/provider.go

@@ -17,18 +17,6 @@ type Credentials struct {
 	Expiration      time.Time
 }
 
-// Valid reports whether the credentials are usable now, leaving a margin so a
-// caller does not start a request with a token about to expire mid-flight.
-func (c Credentials) Valid(margin time.Duration) bool {
-	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
-		return false
-	}
-	if c.Expiration.IsZero() {
-		return true
-	}
-	return time.Now().Add(margin).Before(c.Expiration)
-}
-
 // CredentialsProvider yields current credentials, refreshing as needed. Retrieve
 // must be safe for concurrent use.
 type CredentialsProvider interface {

cmd/ecs-agent/gpu_discovery.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"os/exec"
+	"strings"
+)
+
+// commandRunner abstracts exec.Command so discovery is unit-testable without
+// shelling out. Production uses execCommandRunner; tests inject a fake.
+type commandRunner func(name string, args ...string) ([]byte, error)
+
+// execCommandRunner runs a real host command and returns its stdout.
+func execCommandRunner(name string, args ...string) ([]byte, error) {
+	return exec.Command(name, args...).Output()
+}
+
+// discoverNvidiaGPUs enumerates local NVIDIA GPU device UUIDs via nvidia-smi.
+// A missing binary or any run failure (no GPU host) yields an empty list, not
+// an error — the agent still registers with zero GPU capacity.
+func discoverNvidiaGPUs(run commandRunner) []string {
+	out, err := run("nvidia-smi", "--query-gpu=uuid", "--format=csv,noheader")
+	if err != nil {
+		return nil
+	}
+	return parseNvidiaSMIUUIDs(string(out))
+}
+
+// parseNvidiaSMIUUIDs splits nvidia-smi's newline-delimited UUID output,
+// dropping blank lines. Exported as a pure function so fixture strings can be
+// tested without exec.
+func parseNvidiaSMIUUIDs(out string) []string {
+	lines := strings.Split(strings.TrimSpace(out), "\n")
+	uuids := make([]string, 0, len(lines))
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		uuids = append(uuids, line)
+	}
+	return uuids
+}

cmd/ecs-agent/gpu_discovery_test.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func TestParseNvidiaSMIUUIDs(t *testing.T) {
+	cases := []struct {
+		name string
+		in   string
+		want []string
+	}{
+		{"empty", "", nil},
+		{"whitespace only", "   \n\n  ", nil},
+		{"single", "GPU-11111111-1111-1111-1111-111111111111\n", []string{"GPU-11111111-1111-1111-1111-111111111111"}},
+		{"multi", "GPU-aaa\nGPU-bbb\nGPU-ccc\n", []string{"GPU-aaa", "GPU-bbb", "GPU-ccc"}},
+		{"blank lines between", "GPU-aaa\n\nGPU-bbb\n", []string{"GPU-aaa", "GPU-bbb"}},
+		{"trailing whitespace per line", "GPU-aaa \r\nGPU-bbb\r\n", []string{"GPU-aaa", "GPU-bbb"}},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got := parseNvidiaSMIUUIDs(c.in)
+			if !reflect.DeepEqual(got, c.want) && (len(got) != 0 || len(c.want) != 0) {
+				t.Errorf("parseNvidiaSMIUUIDs(%q) = %v, want %v", c.in, got, c.want)
+			}
+		})
+	}
+}
+
+func TestDiscoverNvidiaGPUs_AbsentBinaryReturnsEmptyNoError(t *testing.T) {
+	run := func(name string, args ...string) ([]byte, error) {
+		return nil, errors.New(`exec: "nvidia-smi": executable file not found in $PATH`)
+	}
+	got := discoverNvidiaGPUs(run)
+	if len(got) != 0 {
+		t.Errorf("want empty list on absent nvidia-smi, got %v", got)
+	}
+}
+
+func TestDiscoverNvidiaGPUs_ParsesFixtureOutput(t *testing.T) {
+	var gotName string
+	var gotArgs []string
+	run := func(name string, args ...string) ([]byte, error) {
+		gotName = name
+		gotArgs = args
+		return []byte("GPU-aaa\nGPU-bbb\n"), nil
+	}
+	got := discoverNvidiaGPUs(run)
+	want := []string{"GPU-aaa", "GPU-bbb"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+	if gotName != "nvidia-smi" {
+		t.Errorf("command = %q, want nvidia-smi", gotName)
+	}
+	if len(gotArgs) != 2 || gotArgs[0] != "--query-gpu=uuid" || gotArgs[1] != "--format=csv,noheader" {
+		t.Errorf("args = %v, want the uuid/csv,noheader query", gotArgs)
+	}
+}
+
+func TestDiscoverNvidiaGPUs_NoDevicesReturnsEmpty(t *testing.T) {
+	run := func(name string, args ...string) ([]byte, error) {
+		return []byte(""), nil
+	}
+	got := discoverNvidiaGPUs(run)
+	if len(got) != 0 {
+		t.Errorf("want empty list, got %v", got)
+	}
+}

cmd/ecs-agent/gpu_ledger.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+// gpuLedger is the agent's local source of truth for which of the host's
+// discovered GPU UUIDs are free vs pinned to a running task's container. C4
+// reads a task's pinned UUIDs to build the CDI device list for containerd.
+type gpuLedger struct {
+	mu     sync.Mutex
+	free   []string
+	pinned map[string][]string // key ("taskID/containerName") -> uuids
+}
+
+// newGPULedger seeds the ledger with the host's discovered device UUIDs. A nil
+// or empty slice is a valid non-GPU host: every Pin then fails, which callers
+// treat as best-effort (no device, no CDI injection later).
+func newGPULedger(uuids []string) *gpuLedger {
+	free := append([]string(nil), uuids...)
+	return &gpuLedger{free: free, pinned: make(map[string][]string)}
+}
+
+// gpuKey builds the ledger key for one container's pin/release.
+func gpuKey(taskID, containerName string) string {
+	return taskID + "/" + containerName
+}
+
+// Pin reserves n free UUIDs under key, returning them. An error means fewer
+// than n devices were free; the key is left unpinned so a retry (or a
+// best-effort caller) can proceed without a partial reservation.
+func (l *gpuLedger) Pin(key string, n int) ([]string, error) {
+	if n <= 0 {
+		return nil, nil
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if len(l.free) < n {
+		return nil, fmt.Errorf("gpu ledger: requested %d GPUs, %d free", n, len(l.free))
+	}
+	uuids := append([]string(nil), l.free[:n]...)
+	l.free = l.free[n:]
+	l.pinned[key] = uuids
+	return uuids, nil
+}
+
+// ReleaseTask frees every UUID pinned under any container key belonging to
+// taskID, called once the task's terminal STOPPED state is reported.
+func (l *gpuLedger) ReleaseTask(taskID string) {
+	prefix := taskID + "/"
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	for key, uuids := range l.pinned {
+		if !strings.HasPrefix(key, prefix) {
+			continue
+		}
+		l.free = append(l.free, uuids...)
+		delete(l.pinned, key)
+	}
+}

cmd/ecs-agent/gpu_ledger_test.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"sync"
+	"testing"
+)
+
+func TestGPULedger_PinReturnsDistinctUUIDs(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b", "GPU-c"})
+	got, err := l.Pin("t-1/web", 2)
+	if err != nil {
+		t.Fatalf("Pin: %v", err)
+	}
+	if len(got) != 2 {
+		t.Fatalf("want 2 UUIDs, got %v", got)
+	}
+	if got[0] == got[1] {
+		t.Errorf("pinned UUIDs must be distinct, got %v", got)
+	}
+	if len(l.free) != 1 {
+		t.Errorf("free = %d, want 1 remaining", len(l.free))
+	}
+}
+
+func TestGPULedger_PinInsufficientFreeErrors(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a"})
+	_, err := l.Pin("t-1/web", 2)
+	if err == nil {
+		t.Fatal("want error when requesting more GPUs than free")
+	}
+	if len(l.free) != 1 {
+		t.Errorf("failed pin must not consume any device, free = %d", len(l.free))
+	}
+}
+
+func TestGPULedger_PinZeroIsNoop(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a"})
+	got, err := l.Pin("t-1/web", 0)
+	if err != nil || got != nil {
+		t.Errorf("Pin(0) = %v, %v; want nil, nil", got, err)
+	}
+}
+
+func TestGPULedger_ReleaseTaskFreesOnlyThatTasksDevices(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b", "GPU-c", "GPU-d"})
+	if _, err := l.Pin(gpuKey("t-1", "web"), 2); err != nil {
+		t.Fatalf("pin t-1: %v", err)
+	}
+	if _, err := l.Pin(gpuKey("t-2", "worker"), 1); err != nil {
+		t.Fatalf("pin t-2: %v", err)
+	}
+	if len(l.free) != 1 {
+		t.Fatalf("free = %d, want 1 before release", len(l.free))
+	}
+
+	l.ReleaseTask("t-1")
+	if len(l.free) != 3 {
+		t.Errorf("free = %d, want 3 after releasing t-1's two devices", len(l.free))
+	}
+	if _, ok := l.pinned[gpuKey("t-2", "worker")]; !ok {
+		t.Error("t-2's pin should survive releasing t-1")
+	}
+	if _, ok := l.pinned[gpuKey("t-1", "web")]; ok {
+		t.Error("t-1's pin should be gone after release")
+	}
+}
+
+// Multi-device pinning across concurrent task assigns must hand out disjoint
+// UUID sets — no two tasks may ever share a device.
+func TestGPULedger_ConcurrentPinsHandOutDistinctUUIDs(t *testing.T) {
+	const nDevices = 20
+	uuids := make([]string, nDevices)
+	for i := range uuids {
+		uuids[i] = "GPU-" + string(rune('a'+i))
+	}
+	l := newGPULedger(uuids)
+
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	seen := make(map[string]bool)
+	errs := 0
+	for i := range nDevices {
+		wg.Add(1)
+		go func(n int) {
+			defer wg.Done()
+			key := gpuKey("t-concurrent", string(rune('A'+n)))
+			got, err := l.Pin(key, 1)
+			mu.Lock()
+			defer mu.Unlock()
+			if err != nil {
+				errs++
+				return
+			}
+			for _, u := range got {
+				if seen[u] {
+					t.Errorf("UUID %s handed out to more than one concurrent pin", u)
+				}
+				seen[u] = true
+			}
+		}(i)
+	}
+	wg.Wait()
+	if errs != 0 {
+		t.Errorf("want all %d pins to succeed (exactly enough devices), got %d errors", nDevices, errs)
+	}
+	if len(seen) != nDevices {
+		t.Errorf("distinct UUIDs handed out = %d, want %d", len(seen), nDevices)
+	}
+	if len(l.free) != 0 {
+		t.Errorf("free = %d, want 0 after all devices pinned", len(l.free))
+	}
+}
+
+// Releasing a task's devices concurrently with new pins never double-hands a
+// UUID: a device is only ever free or pinned to exactly one key at a time.
+func TestGPULedger_ConcurrentPinAndReleaseNoDoubleAssignment(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b"})
+	var wg sync.WaitGroup
+	for round := range 50 {
+		wg.Add(2)
+		taskID := "t-" + string(rune('a'+round%26))
+		go func(id string) {
+			defer wg.Done()
+			_, _ = l.Pin(gpuKey(id, "web"), 1)
+		}(taskID)
+		go func(id string) {
+			defer wg.Done()
+			l.ReleaseTask(id)
+		}(taskID)
+	}
+	wg.Wait()
+	// Invariant: every currently-pinned UUID appears in exactly one pinned entry
+	// and never simultaneously in free.
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	freeSet := make(map[string]bool, len(l.free))
+	for _, u := range l.free {
+		if freeSet[u] {
+			t.Errorf("duplicate UUID %s in free list", u)
+		}
+		freeSet[u] = true
+	}
+	pinnedSet := make(map[string]bool)
+	for _, uuids := range l.pinned {
+		for _, u := range uuids {
+			if pinnedSet[u] {
+				t.Errorf("UUID %s pinned under more than one key", u)
+			}
+			if freeSet[u] {
+				t.Errorf("UUID %s is both free and pinned", u)
+			}
+			pinnedSet[u] = true
+		}
+	}
+}

cmd/ecs-agent/metadata.go

@@ -1,11 +1,14 @@
 package main
 
 import (
-	"encoding/json"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
 )
 
 // instanceMetadata is the IMDS-derived identity of the host the agent runs on.
@@ -15,83 +18,44 @@ type instanceMetadata struct {
 	AZ         string
 }
 
-// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2.
-// base is the IMDS root, e.g. "http://169.254.169.254/latest".
+// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2
+// via the SDK IMDS client. base is the IMDS root, e.g.
+// "http://169.254.169.254/latest".
 func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
-	token, err := imdsToken(client, base)
-	if err != nil {
-		return instanceMetadata{}, err
-	}
+	imdsClient := imdscreds.NewClient(client, base)
+	ctx := context.Background()
 
-	instanceID, err := imdsGet(client, base+"/meta-data/instance-id", token)
+	instanceID, err := getMetadata(ctx, imdsClient, "instance-id")
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
 	}
-	az, err := imdsGet(client, base+"/meta-data/placement/availability-zone", token)
+	az, err := getMetadata(ctx, imdsClient, "placement/availability-zone")
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
 	}
 
-	doc, err := imdsGet(client, base+"/dynamic/instance-identity/document", token)
+	doc, err := imdsClient.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
 	}
-	var ident struct {
-		AccountID string `json:"accountId"`
-	}
-	if err := json.Unmarshal([]byte(doc), &ident); err != nil {
-		return instanceMetadata{}, fmt.Errorf("parse identity document: %w", err)
-	}
 
 	return instanceMetadata{
-		AccountID:  ident.AccountID,
+		AccountID:  doc.AccountID,
 		InstanceID: strings.TrimSpace(instanceID),
 		AZ:         strings.TrimSpace(az),
 	}, nil
 }
 
-// imdsToken fetches an IMDSv2 session token.
-func imdsToken(client *http.Client, base string) (string, error) {
-	req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
+// getMetadata reads a single meta-data path and returns its body as a string.
+func getMetadata(ctx context.Context, imdsClient *imds.Client, path string) (string, error) {
+	out, err := imdsClient.GetMetadata(ctx, &imds.GetMetadataInput{Path: path})
 	if err != nil {
 		return "", err
 	}
-	req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("imds token: %w", err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
+	defer out.Content.Close()
+	body, err := io.ReadAll(out.Content)
 	if err != nil {
 		return "", err
 	}
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("imds token: status %d", resp.StatusCode)
-	}
-	return string(body), nil
-}
-
-// imdsGet performs a token-authenticated IMDS GET.
-func imdsGet(client *http.Client, url, token string) (string, error) {
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return "", err
-	}
-	if token != "" {
-		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
-	}
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("status %d", resp.StatusCode)
-	}
 	return string(body), nil
 }

cmd/ecs-agent/metadata_test.go

@@ -9,11 +9,14 @@ import (
 func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPut {
 			w.WriteHeader(http.StatusMethodNotAllowed)
 			return
 		}
+		// Real IMDS echoes the requested TTL back as a response header; the SDK
+		// client parses it from there, not the body.
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("v2-token"))
 	})
 	guard := func(w http.ResponseWriter, r *http.Request) bool {
@@ -23,17 +26,17 @@ func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 		}
 		return true
 	}
-	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte("i-0abc123"))
 		}
 	})
-	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte("us-east-1a"))
 		}
 	})
-	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte(`{"accountId":"123456789012","region":"us-east-1"}`))
 		}
@@ -45,7 +48,7 @@ func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 
 func TestFetchInstanceMetadata_OK(t *testing.T) {
 	srv := metadataStub(t, true)
-	got, err := fetchInstanceMetadata(srv.Client(), srv.URL)
+	got, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest")
 	if err != nil {
 		t.Fatalf("fetch: %v", err)
 	}
@@ -57,20 +60,23 @@ func TestFetchInstanceMetadata_OK(t *testing.T) {
 func TestFetchInstanceMetadata_TokenRequired(t *testing.T) {
 	// Server returns 405 for non-PUT token; a GET-only client would 401 downstream.
 	srv := metadataStub(t, true)
-	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err != nil {
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err != nil {
 		t.Fatalf("v2 path should succeed: %v", err)
 	}
 }
 
 func TestFetchInstanceMetadata_BadDocument(t *testing.T) {
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("t")) })
-	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
-	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
-	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("t"))
+	})
+	mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
+	mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
+	mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
 	srv := httptest.NewServer(mux)
 	t.Cleanup(srv.Close)
-	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err == nil {
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err == nil {
 		t.Fatal("expected parse error on bad identity document")
 	}
 }

cmd/ecs-agent/netns.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"hash/crc32"
+	"os"
 	"os/exec"
 	"strings"
 	"time"
@@ -21,6 +22,12 @@ const (
 	credEndpointCIDR = defaultCredEndpointIP + "/32"
 )
 
+// udhcpcScript is the config script busybox udhcpc execs to apply a lease. It
+// is the reason presence of the udhcpc binary is not enough to use it: Debian
+// and Ubuntu's busybox ship the applet without this script, and udhcpc then
+// takes the lease and exits nonzero having configured nothing.
+const udhcpcScript = "/usr/share/udhcpc/default.script"
+
 // netCmdRunner executes a host networking command. Abstracted so unit tests can
 // assert the ip/netns sequence without touching the kernel.
 type netCmdRunner interface {
@@ -43,10 +50,47 @@ type taskNetns struct {
 	run     netCmdRunner
 	nicWait time.Duration
 	poll    time.Duration
+	// Injected so the DHCP-client choice below is unit-testable on a host that
+	// has neither client installed.
+	lookPath   func(string) (string, error)
+	fileIsExec func(string) bool
 }
 
 func newTaskNetns(run netCmdRunner) *taskNetns {
-	return &taskNetns{run: run, nicWait: 30 * time.Second, poll: time.Second}
+	return &taskNetns{
+		run:        run,
+		nicWait:    30 * time.Second,
+		poll:       time.Second,
+		lookPath:   exec.LookPath,
+		fileIsExec: func(p string) bool { fi, err := os.Stat(p); return err == nil && fi.Mode()&0o111 != 0 },
+	}
+}
+
+// dhcpStep returns the in-netns DHCP command that leases the task ENI's
+// OVN-assigned address.
+//
+// This image ships on two distributions and they do not agree on a DHCP client.
+// Alpine has busybox udhcpc; Ubuntu has no udhcpc at all, and `ip netns exec`
+// exits 1 — not 127 — when it cannot exec the binary, so a hardcoded udhcpc
+// fails as an opaque "network setup failed" with the task stuck in STOPPED.
+// Resolve the client at setup time instead, preferring udhcpc so Alpine keeps
+// its existing behaviour exactly, then falling back to dhcpcd. Same order and
+// same reasoning as dhcp_oneshot() in mulga-mgmt-net.sh.
+//
+// Both invocations are one-shot: udhcpc -q exits once it has the lease and -n
+// gives up rather than backgrounding, and dhcpcd -1 exits after configuring the
+// interface. Neither leaves a daemon behind in the netns.
+func (n *taskNetns) dhcpStep(name, iface string) ([]string, error) {
+	if _, err := n.lookPath("udhcpc"); err == nil && n.fileIsExec(udhcpcScript) {
+		return []string{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"}, nil
+	}
+	// Lease state is keyed by interface name, which is unique per task here:
+	// each task ENI is a separate hot-plug, so it lands on its own PCI slot and
+	// gets its own predictable name before being moved into the netns.
+	if _, err := n.lookPath("dhcpcd"); err == nil {
+		return []string{"ip", "netns", "exec", name, "dhcpcd", "-q", "-1", "-t", "20", iface}, nil
+	}
+	return nil, fmt.Errorf("no DHCP client for %s: need udhcpc (with %s) or dhcpcd", iface, udhcpcScript)
 }
 
 func netnsName(taskID string) string { return "ecs-" + taskID }
@@ -68,11 +112,16 @@ func (n *taskNetns) Setup(taskID, mac string) (string, error) {
 	}
 	hostVeth := credVethName(taskID)
 	hostIP, taskIP := credSubnet(taskID)
+	dhcp, err := n.dhcpStep(name, iface)
+	if err != nil {
+		_ = n.Teardown(taskID)
+		return "", err
+	}
 	steps := [][]string{
 		{"ip", "link", "set", iface, "netns", name},
 		{"ip", "-n", name, "link", "set", "lo", "up"},
 		{"ip", "-n", name, "link", "set", iface, "up"},
-		{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"},
+		dhcp,
 		// Credentials path: a veth into the host netns plus a host-routed /32 to the
 		// credential endpoint, so the container reaches 169.254.170.2 without it
 		// leaking onto the task ENI's VPC route.

cmd/ecs-agent/netns_test.go

@@ -55,10 +55,27 @@ const linkWithENI = `1: lo: <LOOPBACK,UP> mtu 65536 ... link/loopback 00:00:00:0
 2: eth0: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:aa:bb:cc ...
 3: eth1: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:de:ad:01 ...`
 
+// withDHCPClients pins which DHCP clients the netns sees, so a test asserts the
+// branch it names rather than whatever happens to be installed on the machine
+// running it. Without this the DHCP step is host-dependent: an Alpine CI box and
+// an Ubuntu dev box would exercise different code from the same test.
+func withDHCPClients(n *taskNetns, present map[string]bool, scriptOK bool) {
+	n.lookPath = func(f string) (string, error) {
+		if present[f] {
+			return "/sbin/" + f, nil
+		}
+		return "", errors.New("not found")
+	}
+	n.fileIsExec = func(string) bool { return scriptOK }
+}
+
+// newTestNetns defaults to the Alpine arrangement — udhcpc plus its config
+// script — so the sequence assertions below keep describing that platform.
 func newTestNetns(f *fakeNetRunner) *taskNetns {
 	n := newTaskNetns(f)
 	n.nicWait = 50 * time.Millisecond
 	n.poll = 5 * time.Millisecond
+	withDHCPClients(n, map[string]bool{"udhcpc": true}, true)
 	return n
 }
 
@@ -142,6 +159,79 @@ func TestNetns_SetupTearsDownOnStepFailure(t *testing.T) {
 	}
 }
 
+// The image runs on two distributions with different DHCP clients, and picking
+// the wrong one fails as an opaque "network setup failed" with the task stuck
+// in STOPPED — `ip netns exec` exits 1, not 127, when it cannot exec the
+// binary. Each branch is asserted so that misresolution is a test failure.
+func TestNetns_DHCPStepPerPlatform(t *testing.T) {
+	tests := []struct {
+		name     string
+		present  map[string]bool
+		scriptOK bool
+		want     string
+		wantErr  bool
+	}{
+		{
+			name:     "alpine: udhcpc with its config script",
+			present:  map[string]bool{"udhcpc": true, "dhcpcd": true},
+			scriptOK: true,
+			want:     "ip netns exec ecs-t-001 udhcpc -i eth1 -q -n",
+		},
+		{
+			// Ubuntu ships no udhcpc at all — the case that broke awsvpc tasks.
+			name:    "ubuntu: no udhcpc, falls back to dhcpcd",
+			present: map[string]bool{"dhcpcd": true},
+			want:    "ip netns exec ecs-t-001 dhcpcd -q -1 -t 20 eth1",
+		},
+		{
+			// Debian/Ubuntu busybox ships the applet without the script, so
+			// presence of the binary alone must not select it.
+			name:     "udhcpc present but config script missing: still dhcpcd",
+			present:  map[string]bool{"udhcpc": true, "dhcpcd": true},
+			scriptOK: false,
+			want:     "ip netns exec ecs-t-001 dhcpcd -q -1 -t 20 eth1",
+		},
+		{
+			name:    "neither client: explicit error, not a silent bad command",
+			present: map[string]bool{},
+			wantErr: true,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			n := newTestNetns(&fakeNetRunner{linkOut: linkWithENI})
+			withDHCPClients(n, tc.present, tc.scriptOK)
+			got, err := n.dhcpStep("ecs-t-001", "eth1")
+			if tc.wantErr {
+				if err == nil {
+					t.Fatalf("want error with no DHCP client, got %v", got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("dhcpStep: %v", err)
+			}
+			if joined := strings.Join(got, " "); joined != tc.want {
+				t.Errorf("want %q, got %q", tc.want, joined)
+			}
+		})
+	}
+}
+
+// A missing DHCP client must not leave the half-built netns behind for the next
+// attempt to trip over, same as any other failed setup step.
+func TestNetns_SetupTearsDownWhenNoDHCPClient(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	withDHCPClients(n, map[string]bool{}, false)
+	if _, err := n.Setup("t-001", "52:54:00:de:ad:01"); err == nil {
+		t.Fatal("want error when no DHCP client is available")
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected teardown, got %v", f.joined())
+	}
+}
+
 func TestNetns_Teardown(t *testing.T) {
 	f := &fakeNetRunner{}
 	n := newTestNetns(f)

cmd/ecs-agent/reconcile.go

@@ -8,12 +8,18 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
-// reconcile re-adopts containers that survived an agent restart. It lists the
-// runtime's containers and, for each task still running in this cluster,
-// re-registers its IAM credentials, re-attaches the exit-wait, and refreshes its
-// RUNNING state on the gateway. The returned set seeds pollAssignments so the
+// reconcile re-syncs the gateway with what this instance is actually running
+// after an agent restart. It lists the runtime's containers and, for each task
+// in this cluster, either re-adopts it (still running: re-register IAM
+// credentials, re-attach the exit-wait, refresh RUNNING) or reports it STOPPED
+// and clears its leftovers. The returned set seeds pollAssignments so the
 // gateway's re-delivered assignments for these tasks are acked but not re-run.
 //
+// Reporting the dead ones is the load-bearing half. The exit-wait is cancelled
+// when the agent shuts down, so a container that dies while the agent is down
+// has nobody to report its exit; the gateway holds the task RUNNING forever,
+// its service never replaces it, and the target never recovers.
+//
 // It is best-effort: a nil runner or a List error returns an empty set and the
 // agent falls back to its pre-reconcile behaviour (poll + run).
 func (a *Agent) reconcile(ctx context.Context) map[string]bool {
@@ -27,15 +33,17 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 		return adopted
 	}
 
-	// Group this cluster's running containers back into their tasks. The Assign is
-	// reconstructed from labels — enough to re-register creds and tear down netns.
+	// Group this cluster's containers back into their tasks, running or not. The
+	// Assign is reconstructed from labels — enough to re-register creds and tear
+	// down netns.
 	type task struct {
-		as         *bus.Assign
-		containers []ctrruntime.Container
+		as      *bus.Assign
+		running []ctrruntime.Container
+		dead    []ctrruntime.Container
 	}
 	tasks := map[string]*task{}
 	for _, c := range containers {
-		if !c.Running || c.Labels[labelClusterName] != a.id.ClusterName {
+		if c.Labels[labelClusterName] != a.id.ClusterName {
 			continue
 		}
 		taskID := c.Labels[labelTaskID]
@@ -53,20 +61,50 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 			}}
 			tasks[taskID] = t
 		}
-		t.containers = append(t.containers, c)
+		if c.Running {
+			t.running = append(t.running, c)
+		} else {
+			t.dead = append(t.dead, c)
+		}
 	}
 
 	for taskID, t := range tasks {
+		// A task with nothing running is over, whatever the gateway still believes.
+		// Mixed running/dead is left to the running path: the exit-wait re-attached
+		// below reports the individual container when it is reaped.
+		if len(t.running) == 0 {
+			a.reportStoppedTask(ctx, t.as, t.dead)
+			adopted[taskID] = true
+			continue
+		}
+
 		a.registerTaskCreds(t.as)
-		statuses := make([]bus.ContainerStatus, 0, len(t.containers))
-		for _, c := range t.containers {
+		statuses := make([]bus.ContainerStatus, 0, len(t.running))
+		for _, c := range t.running {
 			name := c.Labels[labelContainerName]
 			go a.waitContainer(ctx, t.as, name, c.ID)
 			statuses = append(statuses, bus.ContainerStatus{Name: name, Status: bus.TaskStatusRunning, ContainerID: c.ID})
 		}
 		a.reportTaskState(t.as, bus.TaskStatusRunning, "", statuses)
 		adopted[taskID] = true
-		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.containers))
+		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.running))
 	}
 	return adopted
 }
+
+// reportStoppedTask clears a task's leftover containers and tells the gateway it
+// is STOPPED, so the scheduler stops counting a task this instance is not
+// running and its service can place a replacement.
+func (a *Agent) reportStoppedTask(ctx context.Context, as *bus.Assign, dead []ctrruntime.Container) {
+	statuses := make([]bus.ContainerStatus, 0, len(dead))
+	for _, c := range dead {
+		if err := a.runner.Remove(ctx, c.ID); err != nil {
+			slog.Warn("ecs-agent: reconcile remove failed", "task", as.TaskID, "container", c.ID, "err", err)
+		}
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
+		})
+	}
+	a.reportTaskState(as, bus.TaskStatusStopped, "Container exited while the agent was not running", statuses)
+	slog.Info("ecs-agent: reported stopped task", "task", as.TaskID, "containers", len(dead))
+}

cmd/ecs-agent/reconcile_test.go

@@ -72,6 +72,63 @@ func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
 	waitedFor(t, rt, containerID("t-001", "web"))
 }
 
+// A task whose container died while the agent was down must be reported STOPPED,
+// not silently skipped. The exit-wait is cancelled on agent shutdown, so nothing
+// else ever reports that exit: skipping it leaves the gateway holding the task
+// RUNNING forever, its service never places a replacement, and the load
+// balancer target stays unhealthy with no path back.
+func TestReconcile_ReportsTaskWhoseContainerDied(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-dead", "web", "default", "t-dead", testTaskRole, "52:54:00:de:ad:02", false),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusStopped); n != 1 {
+		t.Errorf("STOPPED reports for t-dead = %d, want 1", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusRunning); n != 0 {
+		t.Errorf("t-dead must not be reported RUNNING, got %d such reports", n)
+	}
+	// Seeded so a re-delivered assignment for the dead task is acked, not re-run.
+	if !adopted["t-dead"] {
+		t.Errorf("t-dead not seeded into the adopted set; got %v", adopted)
+	}
+	if want := containerID("t-dead", "web"); !slices.Contains(rt.Removed, want) {
+		t.Errorf("leftover container %s not removed; removed=%v", want, rt.Removed)
+	}
+}
+
+// A task with one live and one dead container is still running: the live half is
+// re-adopted and the re-attached exit-wait reports the dead one when it is
+// reaped, so reconcile must not declare the whole task STOPPED.
+func TestReconcile_PartiallyDeadTaskStaysRunning(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-mixed", "web", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", true),
+			adoptedContainer("t-mixed", "sidecar", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", false),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	a.reconcile(context.Background())
+
+	if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("RUNNING reports for t-mixed = %d, want 1", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusStopped); n != 0 {
+		t.Errorf("t-mixed must not be reported STOPPED while a container still runs, got %d", n)
+	}
+}
+
 // pollAssignments seeded with an adopted task acks its re-delivered assignment
 // but does not re-run it, while a genuinely new task still dispatches.
 func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
@@ -123,23 +180,35 @@ func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
 	}
 }
 
-// Containers for another cluster or with a stopped task are not adopted.
-func TestReconcile_FiltersOtherClusterAndStopped(t *testing.T) {
+// Containers belonging to another cluster are ignored entirely: this agent
+// speaks for its own cluster only, and must neither adopt nor report them.
+//
+// A stopped container of this cluster IS handled — see
+// TestReconcile_ReportsTaskWhoseContainerDied. It lands in the adopted set so
+// its re-delivered assignment is acked rather than re-run, after reconcile has
+// reported the task STOPPED.
+func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
+	cp := &fakeCP{}
 	rt := &ctrruntime.FakePuller{
 		WaitErr: errors.New("blocked"),
 		Containers: []ctrruntime.Container{
 			adoptedContainer("t-other", "web", "other", "", "", "", true),
-			adoptedContainer("t-stop", "web", "default", "", "", "", false),
+			adoptedContainer("t-other-dead", "web", "other", "", "", "", false),
 			adoptedContainer("t-ok", "web", "default", "", "", "", true),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
 
 	adopted := a.reconcile(context.Background())
 
 	if len(adopted) != 1 || !adopted["t-ok"] {
 		t.Fatalf("adopted = %v, want only t-ok", adopted)
 	}
+	for _, id := range []string{"t-other", "t-other-dead"} {
+		if n := countStatus(cp.taskStates(), id, bus.TaskStatusStopped); n != 0 {
+			t.Errorf("reported %d STOPPED states for another cluster's task %s, want 0", n, id)
+		}
+	}
 }
 
 // A nil runner or a List error degrades to an empty set without panicking.

cmd/ecs-agent/registration_test.go

@@ -7,16 +7,19 @@ import (
 	"testing"
 	"time"
 
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
-// fakeCP is a test double for the gateway control-plane. It records registers and
-// task-state reports, and replays a scripted queue of poll responses.
+// fakeCP is a test double for the gateway control-plane. It records registers,
+// task-state reports and GPU reports, and replays a scripted queue of poll
+// responses.
 type fakeCP struct {
 	mu sync.Mutex
 
 	registers    int
 	states       []bus.TaskState
+	gpuReports   []gpuReport
 	pollAcks     [][]string
 	pollStopAcks [][]string
 	pollReplies  [][]bus.Assign
@@ -27,6 +30,13 @@ type fakeCP struct {
 	submitErr   bool
 }
 
+// gpuReport is one recorded ReportTaskGPU call.
+type gpuReport struct {
+	cluster    string
+	task       string
+	containers []handlers_ecs.ContainerGPUReport
+}
+
 var _ controlPlane = (*fakeCP)(nil)
 
 func (f *fakeCP) Register(_ identity) error {
@@ -66,6 +76,25 @@ func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]
 	return out, stops, nil
 }
 
+func (f *fakeCP) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.gpuReports = append(f.gpuReports, gpuReport{cluster: cluster, task: task, containers: containers})
+	return nil
+}
+
+func (f *fakeCP) gpuReportsFor(taskID string) []gpuReport {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []gpuReport
+	for _, r := range f.gpuReports {
+		if r.task == taskID {
+			out = append(out, r)
+		}
+	}
+	return out
+}
+
 func (f *fakeCP) registerCount() int {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -102,6 +131,15 @@ func testIdentity() identity {
 	}
 }
 
+// testIdentityWithGPUs is testIdentity seeded with discovered GPU UUIDs, so
+// newAgent's ledger has real devices to pin from.
+func testIdentityWithGPUs(uuids ...string) identity {
+	id := testIdentity()
+	id.Capacity.GPU = len(uuids)
+	id.Capacity.GPUIDs = uuids
+	return id
+}
+
 func TestRegistrar_Register(t *testing.T) {
 	cp := &fakeCP{}
 	if err := newRegistrar(cp, testIdentity()).Register(); err != nil {

cmd/ecs-agent/runtime/containerd_run.go

@@ -6,12 +6,41 @@ import (
 	"sort"
 
 	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/pkg/cdi"
 	"github.com/containerd/containerd/v2/pkg/cio"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
 	"github.com/containerd/containerd/v2/pkg/oci"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
+// cdiVendorGPU is the NVIDIA CDI vendor/class prefix for whole-GPU devices, per
+// the NVIDIA CDI naming convention (nvidia.com/gpu=<uuid>).
+const cdiVendorGPU = "nvidia.com/gpu"
+
+// cdiDeviceNames maps a container's pinned GPU device UUIDs to CDI device
+// names. Empty input (non-GPU container) returns nil so no CDI opt is added.
+func cdiDeviceNames(uuids []string) []string {
+	if len(uuids) == 0 {
+		return nil
+	}
+	names := make([]string, len(uuids))
+	for i, id := range uuids {
+		names[i] = cdiVendorGPU + "=" + id
+	}
+	return names
+}
+
+// cdiSpecOpts returns the oci.SpecOpts needed to inject gpuIDs (via
+// containerd's CDI machinery) as devices into a container's OCI spec. It
+// returns nil for a non-GPU container so Run adds nothing to the opts list.
+func cdiSpecOpts(gpuIDs []string) []oci.SpecOpts {
+	devices := cdiDeviceNames(gpuIDs)
+	if len(devices) == 0 {
+		return nil
+	}
+	return []oci.SpecOpts{cdi.WithCDIDevices(devices...)}
+}
+
 var _ Runner = (*containerdPuller)(nil)
 
 // Run creates a container from an already-pulled image and starts its task. The
@@ -51,6 +80,7 @@ func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (st
 	if len(spec.Env) > 0 {
 		specOpts = append(specOpts, oci.WithEnv(envSlice(spec.Env)))
 	}
+	specOpts = append(specOpts, cdiSpecOpts(spec.GPUIDs)...)
 
 	container, err := p.client.NewContainer(ctx, id,
 		containerd.WithNewSnapshot(id+"-snapshot", image),

cmd/ecs-agent/runtime/containerd_run_test.go

@@ -0,0 +1,47 @@
+package runtime
+
+import (
+	"reflect"
+	"testing"
+)
+
+// cdiDeviceNames maps a container's pinned GPU UUIDs to CDI device names
+// (nvidia.com/gpu=<uuid>); Run wires the result into oci.WithCDIDevices when
+// non-empty, so this is the injection logic's unit-testable core.
+func TestCDIDeviceNames(t *testing.T) {
+	cases := []struct {
+		name  string
+		uuids []string
+		want  []string
+	}{
+		{name: "empty", uuids: nil, want: nil},
+		{name: "single", uuids: []string{"GPU-aaaaaaaa-1111-2222-3333-444444444444"},
+			want: []string{"nvidia.com/gpu=GPU-aaaaaaaa-1111-2222-3333-444444444444"}},
+		{name: "multiple preserves order", uuids: []string{"GPU-aaa", "GPU-bbb"},
+			want: []string{"nvidia.com/gpu=GPU-aaa", "nvidia.com/gpu=GPU-bbb"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := cdiDeviceNames(tc.uuids)
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("cdiDeviceNames(%v) = %v, want %v", tc.uuids, got, tc.want)
+			}
+		})
+	}
+}
+
+// TestCDISpecOpts_Wiring confirms Run's specOpts list picks up exactly one CDI
+// opt when GPUIDs is non-empty, and none for a non-GPU container — the wiring
+// Run relies on (a live oci.Spec is needed to unit test the opt's effect).
+func TestCDISpecOpts_Wiring(t *testing.T) {
+	if opts := cdiSpecOpts(nil); len(opts) != 0 {
+		t.Errorf("non-GPU container: want no spec opts, got %d", len(opts))
+	}
+	opts := cdiSpecOpts([]string{"GPU-aaa", "GPU-bbb"})
+	if len(opts) != 1 {
+		t.Fatalf("GPU container: want exactly one CDI spec opt, got %d", len(opts))
+	}
+	if opts[0] == nil {
+		t.Error("GPU container: spec opt is nil")
+	}
+}

cmd/ecs-agent/runtime/runtime.go

@@ -48,6 +48,12 @@ type RunSpec struct {
 	// (awsvpc task ENI). Empty keeps the container in the host (VM) netns —
 	// bridge/host mode behaviour.
 	NetnsPath string
+	// GPU is the whole-GPU count requested for this container.
+	GPU int
+	// GPUIDs are the ledger-pinned device UUIDs assigned to this container, one
+	// per GPU requested. Empty for non-GPU containers or if pinning fell short.
+	// The runner injects these as CDI devices into the OCI spec.
+	GPUIDs []string
 }
 
 // RunStatus is a finished container's outcome.

cmd/ecs-agent/taskrun.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
@@ -124,12 +125,15 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 		}
 
 		cid := containerID(as.TaskID, c.Name)
+		gpuIDs := a.pinContainerGPUs(as.TaskID, c.Name, c.GPU)
 		spec := ctrruntime.RunSpec{
 			Image:     c.Image,
 			Command:   c.Command,
 			Env:       withCredEnv(c.Environment, credID),
 			Labels:    taskLabels(as, c.Name),
 			NetnsPath: netnsPath,
+			GPU:       c.GPU,
+			GPUIDs:    gpuIDs,
 		}
 		id, err := a.runner.Run(ctx, cid, spec)
 		if err != nil {
@@ -138,7 +142,9 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 			a.reportTaskState(as, bus.TaskStatusStopped, "container start failed: "+err.Error(), statuses)
 			return
 		}
-		statuses = append(statuses, bus.ContainerStatus{Name: c.Name, Status: bus.TaskStatusRunning, ContainerID: id})
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Name, Status: bus.TaskStatusRunning, ContainerID: id, GPUIDs: gpuIDs,
+		})
 		go a.waitContainer(ctx, as, c.Name, id)
 	}
 
@@ -234,8 +240,26 @@ func withCredEnv(env map[string]string, credID string) map[string]string {
 	return out
 }
 
+// pinContainerGPUs reserves n device UUIDs for a container from the local
+// ledger, before the container is started so the runner can inject them as
+// CDI devices. A short ledger (fewer free devices than requested, e.g.
+// discovery found none) is logged and treated as best-effort: the container
+// still runs without pinned UUIDs (and without GPU access) rather than
+// failing the task.
+func (a *Agent) pinContainerGPUs(taskID, containerName string, n int) []string {
+	if n <= 0 || a.gpu == nil {
+		return nil
+	}
+	uuids, err := a.gpu.Pin(gpuKey(taskID, containerName), n)
+	if err != nil {
+		slog.Warn("ecs-agent: gpu pin short", "task", taskID, "container", containerName, "err", err)
+	}
+	return uuids
+}
+
 // reportTaskState reports a task transition through the gateway's
-// SubmitTaskStateChange action.
+// SubmitTaskStateChange action, then (RUNNING with pinned GPUs) reports the
+// per-container device UUIDs, and (STOPPED) releases them back to the ledger.
 func (a *Agent) reportTaskState(as *bus.Assign, status, reason string, containers []bus.ContainerStatus) {
 	if a.cp == nil {
 		return
@@ -253,6 +277,28 @@ func (a *Agent) reportTaskState(as *bus.Assign, status, reason string, container
 	if err := a.cp.SubmitTaskState(msg); err != nil {
 		slog.Error("ecs-agent: report task-state failed", "task", as.TaskID, "err", err)
 	}
+	a.reportTaskGPUs(as.TaskID, containers)
+	if status == bus.TaskStatusStopped && a.gpu != nil {
+		a.gpu.ReleaseTask(as.TaskID)
+	}
+}
+
+// reportTaskGPUs sends the control plane the pinned device UUIDs for any
+// container in the report that has them, so DescribeTasks can surface gpuIds.
+// A no-op when nothing was pinned (the common non-GPU-task case).
+func (a *Agent) reportTaskGPUs(taskID string, containers []bus.ContainerStatus) {
+	reports := make([]handlers_ecs.ContainerGPUReport, 0, len(containers))
+	for _, c := range containers {
+		if len(c.GPUIDs) > 0 {
+			reports = append(reports, handlers_ecs.ContainerGPUReport{Name: c.Name, GPUIDs: c.GPUIDs})
+		}
+	}
+	if len(reports) == 0 {
+		return
+	}
+	if err := a.cp.ReportTaskGPU(a.id.ClusterName, taskID, reports); err != nil {
+		slog.Error("ecs-agent: report task-gpu failed", "task", taskID, "err", err)
+	}
 }
 
 // containerID composes a containerd-valid container ID from a task + container.

cmd/ecs-agent/taskrun_test.go

@@ -70,6 +70,160 @@ func TestRunTask_PullRunReportsRunning(t *testing.T) {
 	}
 }
 
+// TestRunTask_GPUCarriedToRunSpec verifies that the AssignContainer's GPU count
+// reaches the runtime RunSpec unchanged; a
+// container with no GPU request keeps RunSpec.GPU at zero (regression).
+func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 2,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected two runs, got %+v", rt.Runs)
+	}
+	if rt.Runs[0].GPU != 0 {
+		t.Errorf("web container: want GPU=0, got %d", rt.Runs[0].GPU)
+	}
+	if rt.Runs[1].GPU != 2 {
+		t.Errorf("trainer container: want GPU=2, got %d", rt.Runs[1].GPU)
+	}
+}
+
+// TestRunTask_GPUPinsAndReportsUUIDs covers C3: a container's GPU request is
+// pinned from the agent's local ledger, carried on the RUNNING report's
+// ContainerStatus, and forwarded to the control plane via ReportTaskGPU.
+func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
+	})
+	a.runTask(context.Background(), as)
+
+	st := cp.taskStates()
+	if len(st) == 0 || st[len(st)-1].LastStatus != bus.TaskStatusRunning {
+		t.Fatalf("want RUNNING final state, got %+v", st)
+	}
+	last := st[len(st)-1]
+	var trainer *bus.ContainerStatus
+	for i := range last.Containers {
+		if last.Containers[i].Name == "trainer" {
+			trainer = &last.Containers[i]
+		}
+	}
+	if trainer == nil || len(trainer.GPUIDs) != 1 {
+		t.Fatalf("trainer container missing pinned GPU UUID: %+v", last.Containers)
+	}
+	if trainer.GPUIDs[0] != "GPU-aaa" && trainer.GPUIDs[0] != "GPU-bbb" {
+		t.Errorf("unexpected pinned UUID %q", trainer.GPUIDs[0])
+	}
+
+	reports := cp.gpuReportsFor(as.TaskID)
+	if len(reports) != 1 {
+		t.Fatalf("want one ReportTaskGPU call, got %d", len(reports))
+	}
+	if reports[0].cluster != as.ClusterName {
+		t.Errorf("report cluster = %q, want %q", reports[0].cluster, as.ClusterName)
+	}
+	if len(reports[0].containers) != 1 || reports[0].containers[0].Name != "trainer" {
+		t.Errorf("report containers = %+v, want one trainer entry", reports[0].containers)
+	}
+}
+
+// TestRunTask_GPUIDsReachRunSpec covers C4: the ledger's pinned UUIDs for a
+// container reach RunSpec.GPUIDs (not just the RunSpec.GPU count), so the
+// runner can inject them as CDI devices; a non-GPU container's RunSpec
+// carries no GPUIDs.
+func TestRunTask_GPUIDsReachRunSpec(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected two runs, got %+v", rt.Runs)
+	}
+	if len(rt.Runs[0].GPUIDs) != 0 {
+		t.Errorf("web container: want no GPUIDs, got %+v", rt.Runs[0].GPUIDs)
+	}
+	trainer := rt.Runs[1]
+	if len(trainer.GPUIDs) != 1 {
+		t.Fatalf("trainer container: want one pinned GPUID, got %+v", trainer.GPUIDs)
+	}
+	if trainer.GPUIDs[0] != "GPU-aaa" && trainer.GPUIDs[0] != "GPU-bbb" {
+		t.Errorf("unexpected pinned UUID %q", trainer.GPUIDs[0])
+	}
+}
+
+// A container with no GPU request never triggers a ReportTaskGPU call.
+func TestRunTask_NoGPUSkipsReport(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	a.runTask(context.Background(), as)
+
+	if reports := cp.gpuReportsFor(as.TaskID); len(reports) != 0 {
+		t.Errorf("want no GPU report for a non-GPU task, got %+v", reports)
+	}
+}
+
+// A GPU request exceeding the ledger's free devices (discovery found none, or
+// fewer than requested) is best-effort: the container still runs, it just
+// carries no pinned UUIDs and so no CDI devices are injected.
+func TestRunTask_GPUShortLedgerStillRuns(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt) // testIdentity() has no discovered GPU UUIDs
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 2,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected both containers to run despite the short ledger, got %+v", rt.Runs)
+	}
+	if reports := cp.gpuReportsFor(as.TaskID); len(reports) != 0 {
+		t.Errorf("want no GPU report when nothing was pinned, got %+v", reports)
+	}
+}
+
+// stopTask releases the task's pinned GPU UUIDs back to the ledger so a later
+// task on the same instance can reuse them.
+func TestStopTask_ReleasesGPUsBackToLedger(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-trainer", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "trainer",
+		}},
+	}}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, rt, nil)
+	if _, err := a.gpu.Pin(gpuKey("t-001", "trainer"), 1); err != nil {
+		t.Fatalf("seed pin: %v", err)
+	}
+	if len(a.gpu.free) != 0 {
+		t.Fatalf("setup: want ledger fully pinned, free = %v", a.gpu.free)
+	}
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(a.gpu.free) != 1 {
+		t.Errorf("want the GPU released back to the ledger, free = %v", a.gpu.free)
+	}
+}
+
 // A pull failure reports STOPPED and never starts the container.
 func TestRunTask_PullFailureReportsStopped(t *testing.T) {
 	cp := &fakeCP{}
@@ -380,7 +534,7 @@ func (sentinelResolver) Authorize(context.Context, string) (string, string, stri
 	return "", "", "", nil
 }
 
-// TestPullResolver covers siv-459: an assign with no execution role (or no
+// TestPullResolver verifies that an assign with no execution role (or no
 // credential endpoint) uses the instance resolver; with both, a distinct
 // exec-role resolver is built.
 func TestPullResolver(t *testing.T) {

cmd/eks-gateway-fetch/main.go

@@ -156,9 +156,10 @@ func emitAddonsTSV(out io.Writer, body []byte) error {
 // recoveryDirective mirrors the gateway's RecoveryDirective wire shape; duplicated
 // here to keep this tiny VM binary free of the handler package's dependency graph.
 type recoveryDirective struct {
-	Epoch    int64  `json:"epoch"`
-	Action   string `json:"action"`
-	Snapshot string `json:"snapshot"`
+	Epoch            int64  `json:"epoch"`
+	Action           string `json:"action"`
+	Snapshot         string `json:"snapshot"`
+	SnapshotRequired bool   `json:"snapshotRequired"`
 }
 
 type internalRecoveryResponse struct {
@@ -166,8 +167,10 @@ type internalRecoveryResponse struct {
 }
 
 // emitRecoveryTSV decodes the recovery directive and writes a single
-// epoch\taction\tsnapshot line the on-VM k3s-recovery agent consumes without a
-// JSON parser. An unset action defaults to "none" (steady state).
+// epoch\taction\tsnapshot\tsnapshotRequired line the on-VM k3s-recovery agent
+// consumes without a JSON parser. An unset action defaults to "none" (steady
+// state); snapshotRequired is 1 when the snapshot MUST restore (fresh DR seed
+// aborts boot on fetch failure) else 0.
 func emitRecoveryTSV(out io.Writer, body []byte) error {
 	var resp internalRecoveryResponse
 	if err := json.Unmarshal(body, &resp); err != nil {
@@ -177,7 +180,11 @@ func emitRecoveryTSV(out io.Writer, body []byte) error {
 	if d.Action == "" {
 		d.Action = "none"
 	}
-	if _, err := fmt.Fprintf(out, "%d\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot); err != nil {
+	required := "0"
+	if d.SnapshotRequired {
+		required = "1"
+	}
+	if _, err := fmt.Fprintf(out, "%d\t%s\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot, required); err != nil {
 		return err
 	}
 	return nil

cmd/installer/autoinstall/autoinstall.go

@@ -4,12 +4,14 @@
 package autoinstall
 
 import (
+	"cmp"
 	"fmt"
 	"log/slog"
 	"net"
 	"os"
 	"os/exec"
-	"sort"
+	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"syscall"
@@ -18,6 +20,10 @@ import (
 	"github.com/mulgadc/spinifex/cmd/installer/install"
 )
 
+// listDisks is the block-device scan, indirected so tests can drive selection
+// without real hardware.
+var listDisks = install.ListDisks
+
 // Load returns an install.Config built from SPINIFEX_* environment variables,
 // or nil if SPINIFEX_AUTO is not set to "1" (interactive mode).
 func Load() (*install.Config, error) {
@@ -65,9 +71,9 @@ func buildConfig() (*install.Config, error) {
 		hostname = "spinifex-node"
 	}
 
-	disk, err := resolveDisk(os.Getenv("SPINIFEX_DISK"))
+	storage, err := resolveStorage()
 	if err != nil {
-		return nil, fmt.Errorf("disk: %w", err)
+		return nil, fmt.Errorf("storage: %w", err)
 	}
 
 	wanIface, err := resolveNIC(os.Getenv("SPINIFEX_WAN_IFACE"), "")
@@ -76,73 +82,167 @@ func buildConfig() (*install.Config, error) {
 	}
 
 	cfg := &install.Config{
-		Disk:         disk,
+		Storage:      storage,
 		Hostname:     hostname,
 		RootPassword: password,
 		// Optional on the headless path — empty means no --email passed to
 		// spx admin init, which omits the operator identity from telemetry.
-		Email:        strings.TrimSpace(os.Getenv("SPINIFEX_EMAIL")),
-		WANInterface: wanIface,
-		WANDHCPMode:  strings.ToLower(os.Getenv("SPINIFEX_WAN_MODE")) != "static",
+		Email: strings.TrimSpace(os.Getenv("SPINIFEX_EMAIL")),
 	}
 
-	if !cfg.WANDHCPMode {
-		ip := os.Getenv("SPINIFEX_WAN_IP")
-		mask := os.Getenv("SPINIFEX_WAN_MASK")
-		gw := os.Getenv("SPINIFEX_WAN_GW")
-		if ip == "" || mask == "" || gw == "" {
-			return nil, fmt.Errorf("SPINIFEX_WAN_IP, SPINIFEX_WAN_MASK, SPINIFEX_WAN_GW required for static mode")
-		}
-		cfg.WANAddress = ip
-		cfg.WANMask = mask
-		cfg.WANGateway = gw
-		if dns := os.Getenv("SPINIFEX_WAN_DNS"); dns != "" {
-			cfg.WANDNS = strings.Split(dns, ",")
-		}
+	cfg.WAN, err = parseRole("WAN", wanIface)
+	if err != nil {
+		return nil, err
+	}
+	if cfg.WAN.DHCPMode {
+		cfg.WAN.Gateway = ""
+	} else if cfg.WAN.Gateway == "" {
+		return nil, fmt.Errorf("SPINIFEX_WAN_IP, SPINIFEX_WAN_MASK, SPINIFEX_WAN_GW required for static mode")
 	}
 
-	if lanIface := os.Getenv("SPINIFEX_LAN_IFACE"); lanIface != "" {
-		lan, err := resolveNIC(lanIface, wanIface)
+	// lan and vpc are optional: an unset _IFACE leaves the role folded, which
+	// is how a single- or two-NIC node collapses onto the planes above it.
+	for _, r := range []struct {
+		plane string
+		dst   *install.NetworkRole
+	}{{"LAN", &cfg.LAN}, {"VPC", &cfg.VPC}} {
+		name := os.Getenv("SPINIFEX_" + r.plane + "_IFACE")
+		if name == "" {
+			continue
+		}
+		iface, err := resolveNIC(name, wanIface)
 		if err != nil {
-			return nil, fmt.Errorf("LAN NIC: %w", err)
+			return nil, fmt.Errorf("%s NIC: %w", r.plane, err)
 		}
-		cfg.LANInterface = lan
-		cfg.LANDHCPMode = strings.ToLower(os.Getenv("SPINIFEX_LAN_MODE")) != "static"
-		if !cfg.LANDHCPMode {
-			cfg.LANAddress = os.Getenv("SPINIFEX_LAN_IP")
-			cfg.LANMask = os.Getenv("SPINIFEX_LAN_MASK")
-			if dns := os.Getenv("SPINIFEX_LAN_DNS"); dns != "" {
-				cfg.LANDNS = strings.Split(dns, ",")
-			}
+		if *r.dst, err = parseRole(r.plane, iface); err != nil {
+			return nil, err
 		}
+		// Only the wan plane installs a default route.
+		r.dst.Gateway = ""
+	}
+
+	if err := cfg.Validate(); err != nil {
+		return nil, fmt.Errorf("network roles: %w", err)
 	}
 
 	if os.Getenv("SPINIFEX_GPU_PASSTHROUGH") == "1" {
 		cfg.GPUPassthrough = true
 	}
 
-	role := strings.ToLower(os.Getenv("SPINIFEX_ROLE"))
-	if role == "" {
-		role = "init"
-	}
-	cfg.ClusterRole = role
-	skipFormation := os.Getenv("SPINIFEX_SKIP_FORMATION") == "1"
-	if skipFormation {
+	if os.Getenv("SPINIFEX_SKIP_FORMATION") == "1" {
 		cfg.SkipFormation = true
 	}
 
-	if role == "join" && !skipFormation {
-		joinAddr := os.Getenv("SPINIFEX_JOIN_ADDR")
-		if joinAddr == "" {
-			return nil, fmt.Errorf("SPINIFEX_JOIN_ADDR required when SPINIFEX_ROLE=join")
-		}
-		cfg.JoinAddr = joinAddr
-	}
-
 	return cfg, nil
 }
 
-// resolveDisk maps the spx_disk value from grub.cfg to a block device path.
+// resolveStorage builds the disk configuration from the kernel cmdline.
+//
+//	SPINIFEX_FS     ext4 (default) or one of the zfs-* topologies
+//	SPINIFEX_DISKS  explicit ordered member list, required for multi-disk pools
+//	SPINIFEX_DISK   single-disk selector, as before
+func resolveStorage() (install.DiskConfig, error) {
+	cfg := install.DiskConfig{FS: install.FSExt4}
+	if v := strings.TrimSpace(os.Getenv("SPINIFEX_FS")); v != "" {
+		fs, err := install.ParseFSType(v)
+		if err != nil {
+			return cfg, err
+		}
+		cfg.FS = fs
+	}
+
+	var err error
+	switch list := strings.TrimSpace(os.Getenv("SPINIFEX_DISKS")); {
+	case list != "":
+		if cfg.Disks, err = disksByName(strings.Split(list, ",")); err != nil {
+			return cfg, err
+		}
+	case cfg.FS.MinDisks() > 1:
+		// No auto-selection for a multi-disk pool. Choosing which disks to
+		// erase is not a decision an unattended install gets to make, and the
+		// member order determines the RAID10 pairing.
+		return cfg, fmt.Errorf("%s needs at least %d disks — set SPINIFEX_DISKS to an explicit comma-separated list",
+			cfg.FS.Label(), cfg.FS.MinDisks())
+	default:
+		d, derr := resolveDisk(os.Getenv("SPINIFEX_DISK"))
+		if derr != nil {
+			return cfg, derr
+		}
+		cfg.Disks = []install.Disk{d}
+	}
+
+	if cfg.ZFS, err = parseZFSOpts(); err != nil {
+		return cfg, err
+	}
+	if err := cfg.Validate(); err != nil {
+		return cfg, err
+	}
+	slog.Info("autoinstall: storage selected", "fs", cfg.FS, "disks", cfg.Paths())
+	return cfg, nil
+}
+
+// parseZFSOpts reads the advanced pool tunables. Every one is optional; unset
+// values are computed from the hardware at install time.
+func parseZFSOpts() (install.ZFSOpts, error) {
+	var o install.ZFSOpts
+	ints := []struct {
+		env string
+		dst *int
+	}{
+		{"SPINIFEX_ZFS_ASHIFT", &o.Ashift},
+		{"SPINIFEX_ZFS_COPIES", &o.Copies},
+		{"SPINIFEX_ZFS_ARC_MAX_MIB", &o.ARCMaxMiB},
+	}
+	for _, f := range ints {
+		v := strings.TrimSpace(os.Getenv(f.env))
+		if v == "" {
+			continue
+		}
+		n, err := strconv.Atoi(v)
+		if err != nil {
+			return o, fmt.Errorf("%s: %w", f.env, err)
+		}
+		*f.dst = n
+	}
+	o.Compress = strings.TrimSpace(os.Getenv("SPINIFEX_ZFS_COMPRESS"))
+	o.Checksum = strings.TrimSpace(os.Getenv("SPINIFEX_ZFS_CHECKSUM"))
+	return o, nil
+}
+
+// disksByName resolves an explicit member list. Names may be kernel names
+// ("sdb"), device paths or by-id paths; the order given is preserved because
+// RAID10 pairs members in it.
+func disksByName(names []string) ([]install.Disk, error) {
+	available, err := listDisks()
+	if err != nil {
+		return nil, err
+	}
+	out := make([]install.Disk, 0, len(names))
+	for _, raw := range names {
+		name := strings.TrimSpace(raw)
+		if name == "" {
+			continue
+		}
+		// A by-id path or a symlink resolves to the kernel device the scan
+		// reported, so both spellings name the same disk.
+		resolved := name
+		if !strings.HasPrefix(resolved, "/dev/") {
+			resolved = "/dev/" + resolved
+		}
+		if target, err := filepath.EvalSymlinks(resolved); err == nil {
+			resolved = target
+		}
+
+		idx := slices.IndexFunc(available, func(d install.Disk) bool { return d.Path == resolved })
+		if idx < 0 {
+			return nil, fmt.Errorf("%q is not an available disk — found:\n%s", name, diskList(available))
+		}
+		out = append(out, available[idx])
+	}
+	return out, nil
+}
+
+// resolveDisk maps the SPINIFEX_DISK value from grub.cfg to a disk.
 //
 // Supported values:
 //
@@ -151,141 +251,78 @@ func buildConfig() (*install.Config, error) {
 //	smallest        — smallest non-removable disk (typical OS-on-SSD pattern)
 //	nvme            — the only NVMe disk; fail if multiple found
 //	/dev/sda (etc.) — exact device path; fail if not present
-func resolveDisk(target string) (string, error) {
-	switch strings.ToLower(target) {
+func resolveDisk(target string) (install.Disk, error) {
+	disks, err := candidateDisks()
+	if err != nil {
+		return install.Disk{}, err
+	}
+	switch strings.ToLower(strings.TrimSpace(target)) {
 	case "", "auto":
-		return singleDisk()
-	case "largest":
-		return diskBySize(true)
-	case "smallest":
-		return diskBySize(false)
-	case "nvme":
-		return nvmeDisk()
-	default:
-		if _, err := os.Stat(target); err != nil {
-			return "", fmt.Errorf("%q not found", target)
+		if len(disks) != 1 {
+			return install.Disk{}, fmt.Errorf(
+				"expected exactly one disk, found %d — set SPINIFEX_DISK to one of:\n%s\n"+
+					"  largest   (largest disk)\n"+
+					"  smallest  (smallest disk)\n"+
+					"  nvme      (NVMe only)\n"+
+					"  /dev/sdX  (exact path)",
+				len(disks), diskList(disks))
 		}
-		return target, nil
+		return disks[0], nil
+	case "largest":
+		return pickBySize(disks, true)
+	case "smallest":
+		return pickBySize(disks, false)
+	case "nvme":
+		nvmes := slices.DeleteFunc(slices.Clone(disks), func(d install.Disk) bool {
+			return !strings.HasPrefix(filepath.Base(d.Path), "nvme")
+		})
+		if len(nvmes) != 1 {
+			return install.Disk{}, fmt.Errorf("expected exactly one NVMe disk, found %d:\n%s",
+				len(nvmes), diskList(nvmes))
+		}
+		return nvmes[0], nil
+	default:
+		found, err := disksByName([]string{target})
+		if err != nil {
+			return install.Disk{}, err
+		}
+		return found[0], nil
 	}
 }
 
-type diskCandidate struct {
-	dev   string
-	bytes int64
-}
-
-// nonRemovableDisks returns all non-removable, non-virtual block devices.
-func nonRemovableDisks() ([]diskCandidate, error) {
-	entries, err := os.ReadDir("/sys/block")
+// candidateDisks lists the disks an unattended install may erase: fixed, and
+// not the installer's own media.
+func candidateDisks() ([]install.Disk, error) {
+	all, err := listDisks()
 	if err != nil {
 		return nil, err
 	}
-	var out []diskCandidate
-	for _, e := range entries {
-		dev := e.Name()
-		switch {
-		case strings.HasPrefix(dev, "loop"),
-			strings.HasPrefix(dev, "ram"),
-			strings.HasPrefix(dev, "dm-"),
-			strings.HasPrefix(dev, "sr"):
-			continue
-		}
-		removable, _ := os.ReadFile("/sys/block/" + dev + "/removable")
-		if strings.TrimSpace(string(removable)) == "1" {
-			continue
-		}
-		sizeRaw, _ := os.ReadFile("/sys/block/" + dev + "/size")
-		sectors, parseErr := strconv.ParseInt(strings.TrimSpace(string(sizeRaw)), 10, 64)
-		if parseErr != nil || sectors == 0 {
-			continue
-		}
-		out = append(out, diskCandidate{dev: dev, bytes: sectors * 512})
-	}
-	return out, nil
+	return slices.DeleteFunc(all, func(d install.Disk) bool {
+		return d.Removable || d.LiveMedia
+	}), nil
 }
 
-// diskList returns a human-readable list of candidates for error messages.
-func diskList(disks []diskCandidate) string {
+func pickBySize(disks []install.Disk, largest bool) (install.Disk, error) {
+	if len(disks) == 0 {
+		return install.Disk{}, fmt.Errorf("no non-removable disks found")
+	}
+	sorted := slices.Clone(disks)
+	slices.SortFunc(sorted, func(a, b install.Disk) int { return cmp.Compare(a.Bytes, b.Bytes) })
+	if largest {
+		return sorted[len(sorted)-1], nil
+	}
+	return sorted[0], nil
+}
+
+// diskList renders candidates for an error message.
+func diskList(disks []install.Disk) string {
 	lines := make([]string, len(disks))
 	for i, d := range disks {
-		lines[i] = fmt.Sprintf("  /dev/%s (%dG)", d.dev, d.bytes>>30)
+		lines[i] = fmt.Sprintf("  %s (%s, %s)", d.Path, d.SizeHuman(), d.Content)
 	}
 	return strings.Join(lines, "\n")
 }
 
-// singleDisk selects the only non-removable disk, or fails listing all found.
-func singleDisk() (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	switch len(disks) {
-	case 0:
-		return "", fmt.Errorf("no non-removable disks found")
-	case 1:
-		slog.Info("autoinstall: single disk selected", "disk", disks[0].dev, "size_gb", disks[0].bytes>>30)
-		return "/dev/" + disks[0].dev, nil
-	default:
-		return "", fmt.Errorf(
-			"multiple disks found — set SPINIFEX_DISK to one of:\n%s\n"+
-				"  largest   (largest disk)\n"+
-				"  smallest  (smallest disk)\n"+
-				"  nvme      (NVMe only)\n"+
-				"  /dev/sdX  (exact path)",
-			diskList(disks),
-		)
-	}
-}
-
-// diskBySize selects the largest (largest=true) or smallest non-removable disk.
-func diskBySize(largest bool) (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	if len(disks) == 0 {
-		return "", fmt.Errorf("no non-removable disks found")
-	}
-	sort.Slice(disks, func(i, j int) bool {
-		if largest {
-			return disks[i].bytes > disks[j].bytes
-		}
-		return disks[i].bytes < disks[j].bytes
-	})
-	label := "smallest"
-	if largest {
-		label = "largest"
-	}
-	slog.Info("autoinstall: disk selected by size", "mode", label, "disk", disks[0].dev, "size_gb", disks[0].bytes>>30)
-	return "/dev/" + disks[0].dev, nil
-}
-
-// nvmeDisk selects the only NVMe disk, or fails listing all NVMe disks found.
-func nvmeDisk() (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	var nvmes []diskCandidate
-	for _, d := range disks {
-		if strings.HasPrefix(d.dev, "nvme") {
-			nvmes = append(nvmes, d)
-		}
-	}
-	switch len(nvmes) {
-	case 0:
-		return "", fmt.Errorf("no NVMe disks found")
-	case 1:
-		slog.Info("autoinstall: NVMe disk selected", "disk", nvmes[0].dev, "size_gb", nvmes[0].bytes>>30)
-		return "/dev/" + nvmes[0].dev, nil
-	default:
-		return "", fmt.Errorf(
-			"multiple NVMe disks found — set SPINIFEX_DISK to one of:\n%s",
-			diskList(nvmes),
-		)
-	}
-}
-
 // virtualNICPrefixes lists interface name prefixes that identify non-physical
 // interfaces (bridges, tunnels, container/OVS veth pairs, etc.). These are
 // skipped when auto-selecting a NIC so we don't configure docker0 or
@@ -297,6 +334,42 @@ var virtualNICPrefixes = []string{
 
 // resolveNIC returns the interface name to use. "auto" or empty picks the
 // first physical (non-loopback, non-virtual) interface that is not exclude.
+// parseRole reads the SPINIFEX_<PLANE>_* kernel-cmdline vars for one plane.
+// iface is the already-resolved interface name; _VLAN and _MTU are optional
+// and default to untagged at the link MTU.
+func parseRole(plane, iface string) (install.NetworkRole, error) {
+	role := install.NetworkRole{
+		Interface: iface,
+		DHCPMode:  strings.ToLower(os.Getenv("SPINIFEX_"+plane+"_MODE")) != "static",
+	}
+	if v := os.Getenv("SPINIFEX_" + plane + "_VLAN"); v != "" {
+		id, err := strconv.Atoi(v)
+		if err != nil {
+			return role, fmt.Errorf("SPINIFEX_%s_VLAN: %w", plane, err)
+		}
+		role.VLAN = id
+	}
+	if v := os.Getenv("SPINIFEX_" + plane + "_MTU"); v != "" {
+		mtu, err := strconv.Atoi(v)
+		if err != nil {
+			return role, fmt.Errorf("SPINIFEX_%s_MTU: %w", plane, err)
+		}
+		role.MTU = mtu
+	}
+	if !role.DHCPMode {
+		role.Address = os.Getenv("SPINIFEX_" + plane + "_IP")
+		role.Mask = os.Getenv("SPINIFEX_" + plane + "_MASK")
+		role.Gateway = os.Getenv("SPINIFEX_" + plane + "_GW")
+		if role.Address == "" || role.Mask == "" {
+			return role, fmt.Errorf("SPINIFEX_%s_IP and SPINIFEX_%s_MASK required for static mode", plane, plane)
+		}
+	}
+	if dns := os.Getenv("SPINIFEX_" + plane + "_DNS"); dns != "" {
+		role.DNS = strings.Split(dns, ",")
+	}
+	return role, nil
+}
+
 func resolveNIC(name, exclude string) (string, error) {
 	if name != "" && name != "auto" {
 		return name, nil

cmd/installer/autoinstall/autoinstall_test.go

@@ -1,12 +1,57 @@
 package autoinstall
 
 import (
+	"errors"
+	"fmt"
+	"maps"
+	"net"
 	"os"
+	"slices"
+	"strings"
 	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
 )
 
+const gib = 1 << 30
+
+// setEnv clears every SPINIFEX_* variable inherited from the test runner before
+// applying want, so one case cannot leak a kernel-cmdline value into the next.
+func setEnv(t *testing.T, want map[string]string) {
+	t.Helper()
+	for _, kv := range os.Environ() {
+		if k, _, ok := strings.Cut(kv, "="); ok && strings.HasPrefix(k, "SPINIFEX_") {
+			t.Setenv(k, "")
+		}
+	}
+	for k, v := range want {
+		t.Setenv(k, v)
+	}
+}
+
+// withDisks replaces the block-device scan. The clone matters: candidateDisks
+// filters with slices.DeleteFunc, which would otherwise mutate the fixture.
+func withDisks(t *testing.T, disks []install.Disk, err error) {
+	t.Helper()
+	prev := listDisks
+	listDisks = func() ([]install.Disk, error) { return slices.Clone(disks), err }
+	t.Cleanup(func() { listDisks = prev })
+}
+
+func mkDisk(path string, bytes int64) install.Disk {
+	return install.Disk{
+		Path:              path,
+		ByID:              "/dev/disk/by-id/test-" + strings.TrimPrefix(path, "/dev/"),
+		Bytes:             bytes,
+		Model:             "TESTDISK",
+		LogicalBlockSize:  512,
+		PhysicalBlockSize: 512,
+		Content:           "empty",
+	}
+}
+
 func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
-	os.Unsetenv("SPINIFEX_AUTO")
+	setEnv(t, nil)
 	cfg, err := Load()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -15,3 +60,586 @@ func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
 		t.Fatal("expected nil config when SPINIFEX_AUTO is not set")
 	}
 }
+
+func TestLoadBuildsConfigFromEnv(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_AUTO":            "1",
+		"SPINIFEX_PASSWORD":        "s3cret",
+		"SPINIFEX_HOSTNAME":        "node-7",
+		"SPINIFEX_EMAIL":           "  ops@example.com  ",
+		"SPINIFEX_WAN_IFACE":       "eno1",
+		"SPINIFEX_WAN_DNS":         "1.1.1.1,8.8.8.8",
+		"SPINIFEX_LAN_IFACE":       "eno2",
+		"SPINIFEX_LAN_VLAN":        "100",
+		"SPINIFEX_LAN_MTU":         "9000",
+		"SPINIFEX_GPU_PASSTHROUGH": "1",
+	})
+
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.Hostname != "node-7" || cfg.RootPassword != "s3cret" {
+		t.Errorf("identity = %q/%q", cfg.Hostname, cfg.RootPassword)
+	}
+	if cfg.Email != "ops@example.com" {
+		t.Errorf("Email = %q, want trimmed", cfg.Email)
+	}
+	if cfg.Storage.FS != install.FSExt4 || len(cfg.Storage.Disks) != 1 {
+		t.Errorf("Storage = %v / %d disks", cfg.Storage.FS, len(cfg.Storage.Disks))
+	}
+	if cfg.WAN.Interface != "eno1" || !cfg.WAN.DHCPMode {
+		t.Errorf("WAN = %+v, want eno1 on DHCP", cfg.WAN)
+	}
+	if !slices.Equal(cfg.WAN.DNS, []string{"1.1.1.1", "8.8.8.8"}) {
+		t.Errorf("WAN.DNS = %v", cfg.WAN.DNS)
+	}
+	if cfg.LAN.Interface != "eno2" || cfg.LAN.VLAN != 100 || cfg.LAN.MTU != 9000 {
+		t.Errorf("LAN = %+v", cfg.LAN)
+	}
+	// Only the wan plane installs a default route.
+	if cfg.LAN.Gateway != "" {
+		t.Errorf("LAN.Gateway = %q, want cleared", cfg.LAN.Gateway)
+	}
+	if !cfg.VPC.Folded() {
+		t.Errorf("VPC = %+v, want folded when no _IFACE is set", cfg.VPC)
+	}
+	if !cfg.GPUPassthrough {
+		t.Error("GPUPassthrough not enabled")
+	}
+}
+
+func TestLoadDefaultsHostname(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_AUTO":      "1",
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+	})
+
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.Hostname != "spinifex-node" {
+		t.Errorf("Hostname = %q, want spinifex-node", cfg.Hostname)
+	}
+}
+
+func TestLoadRequiresPassword(t *testing.T) {
+	setEnv(t, map[string]string{"SPINIFEX_AUTO": "1"})
+
+	cfg, err := Load()
+	if err == nil {
+		t.Fatalf("expected error, got config %+v", cfg)
+	}
+	if !strings.Contains(err.Error(), "SPINIFEX_PASSWORD") {
+		t.Errorf("err = %v, want it to name SPINIFEX_PASSWORD", err)
+	}
+	if !strings.HasPrefix(err.Error(), "autoinstall config:") {
+		t.Errorf("err = %v, want the autoinstall wrapper", err)
+	}
+}
+
+func TestBuildConfigStaticWAN(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     map[string]string
+		wantErr string
+	}{
+		{
+			name: "full static",
+			env: map[string]string{
+				"SPINIFEX_WAN_MODE": "static",
+				"SPINIFEX_WAN_IP":   "10.0.0.5",
+				"SPINIFEX_WAN_MASK": "255.255.255.0",
+				"SPINIFEX_WAN_GW":   "10.0.0.1",
+			},
+		},
+		{
+			name: "missing gateway",
+			env: map[string]string{
+				"SPINIFEX_WAN_MODE": "static",
+				"SPINIFEX_WAN_IP":   "10.0.0.5",
+				"SPINIFEX_WAN_MASK": "255.255.255.0",
+			},
+			wantErr: "SPINIFEX_WAN_GW required",
+		},
+		{
+			name:    "missing address",
+			env:     map[string]string{"SPINIFEX_WAN_MODE": "static"},
+			wantErr: "SPINIFEX_WAN_IP and SPINIFEX_WAN_MASK required",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+			env := map[string]string{
+				"SPINIFEX_PASSWORD":  "s3cret",
+				"SPINIFEX_WAN_IFACE": "eno1",
+			}
+			maps.Copy(env, tt.env)
+			setEnv(t, env)
+
+			cfg, err := buildConfig()
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("buildConfig: %v", err)
+			}
+			if cfg.WAN.DHCPMode {
+				t.Error("WAN.DHCPMode = true, want static")
+			}
+			if cfg.WAN.Address != "10.0.0.5" || cfg.WAN.Gateway != "10.0.0.1" {
+				t.Errorf("WAN = %+v", cfg.WAN)
+			}
+		})
+	}
+}
+
+// SPINIFEX_ROLE and SPINIFEX_JOIN_ADDR no longer exist: the installer cannot
+// form a multi-node cluster, so a stale cmdline from an older ISO has to be
+// inert rather than producing a half-configured node.
+func TestBuildConfigSkipFormation(t *testing.T) {
+	tests := []struct {
+		name         string
+		env          map[string]string
+		wantSkipForm bool
+	}{
+		{
+			name:         "default initializes a single-node cluster",
+			wantSkipForm: false,
+		},
+		{
+			name:         "formation deferred to the provisioner",
+			env:          map[string]string{"SPINIFEX_SKIP_FORMATION": "1"},
+			wantSkipForm: true,
+		},
+		{
+			name:         "stale join cmdline is ignored",
+			env:          map[string]string{"SPINIFEX_ROLE": "join", "SPINIFEX_JOIN_ADDR": "10.0.0.1:9999"},
+			wantSkipForm: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+			env := map[string]string{
+				"SPINIFEX_PASSWORD":  "s3cret",
+				"SPINIFEX_WAN_IFACE": "eno1",
+			}
+			maps.Copy(env, tt.env)
+			setEnv(t, env)
+
+			cfg, err := buildConfig()
+			if err != nil {
+				t.Fatalf("buildConfig: %v", err)
+			}
+			if cfg.SkipFormation != tt.wantSkipForm {
+				t.Errorf("SkipFormation = %v, want %v", cfg.SkipFormation, tt.wantSkipForm)
+			}
+		})
+	}
+}
+
+func TestBuildConfigRejectsDuplicatePlaneBinding(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+		"SPINIFEX_LAN_IFACE": "eno1",
+	})
+
+	if _, err := buildConfig(); err == nil || !strings.Contains(err.Error(), "network roles") {
+		t.Fatalf("err = %v, want a network roles validation failure", err)
+	}
+}
+
+func TestBuildConfigPropagatesStorageError(t *testing.T) {
+	setEnv(t, map[string]string{
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+		"SPINIFEX_FS":        "btrfs",
+	})
+
+	_, err := buildConfig()
+	if err == nil || !strings.HasPrefix(err.Error(), "storage:") {
+		t.Fatalf("err = %v, want a storage: prefix", err)
+	}
+}
+
+func TestParseRole(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     map[string]string
+		want    install.NetworkRole
+		wantErr string
+	}{
+		{
+			name: "dhcp by default",
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true},
+		},
+		{
+			name: "vlan and mtu",
+			env:  map[string]string{"SPINIFEX_LAN_VLAN": "42", "SPINIFEX_LAN_MTU": "9000"},
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true, VLAN: 42, MTU: 9000},
+		},
+		{
+			name: "dns list is comma separated",
+			env:  map[string]string{"SPINIFEX_LAN_DNS": "1.1.1.1,9.9.9.9"},
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true, DNS: []string{"1.1.1.1", "9.9.9.9"}},
+		},
+		{
+			name: "static",
+			env: map[string]string{
+				"SPINIFEX_LAN_MODE": "STATIC",
+				"SPINIFEX_LAN_IP":   "192.168.1.2",
+				"SPINIFEX_LAN_MASK": "255.255.255.0",
+				"SPINIFEX_LAN_GW":   "192.168.1.1",
+			},
+			want: install.NetworkRole{
+				Interface: "eno1",
+				Address:   "192.168.1.2",
+				Mask:      "255.255.255.0",
+				Gateway:   "192.168.1.1",
+			},
+		},
+		{
+			name:    "static without mask",
+			env:     map[string]string{"SPINIFEX_LAN_MODE": "static", "SPINIFEX_LAN_IP": "192.168.1.2"},
+			wantErr: "SPINIFEX_LAN_IP and SPINIFEX_LAN_MASK required",
+		},
+		{
+			name:    "unparsable vlan",
+			env:     map[string]string{"SPINIFEX_LAN_VLAN": "trunk"},
+			wantErr: "SPINIFEX_LAN_VLAN",
+		},
+		{
+			name:    "unparsable mtu",
+			env:     map[string]string{"SPINIFEX_LAN_MTU": "jumbo"},
+			wantErr: "SPINIFEX_LAN_MTU",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			setEnv(t, tt.env)
+
+			got, err := parseRole("LAN", "eno1")
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("parseRole: %v", err)
+			}
+			if fmt.Sprintf("%+v", got) != fmt.Sprintf("%+v", tt.want) {
+				t.Errorf("role = %+v, want %+v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseZFSOpts(t *testing.T) {
+	t.Run("unset leaves the computed defaults", func(t *testing.T) {
+		setEnv(t, nil)
+		got, err := parseZFSOpts()
+		if err != nil {
+			t.Fatalf("parseZFSOpts: %v", err)
+		}
+		if got != (install.ZFSOpts{}) {
+			t.Errorf("opts = %+v, want zero", got)
+		}
+	})
+
+	t.Run("all tunables", func(t *testing.T) {
+		setEnv(t, map[string]string{
+			"SPINIFEX_ZFS_ASHIFT":      " 12 ",
+			"SPINIFEX_ZFS_COPIES":      "2",
+			"SPINIFEX_ZFS_ARC_MAX_MIB": "4096",
+			"SPINIFEX_ZFS_COMPRESS":    " lz4 ",
+			"SPINIFEX_ZFS_CHECKSUM":    "blake3",
+		})
+		got, err := parseZFSOpts()
+		if err != nil {
+			t.Fatalf("parseZFSOpts: %v", err)
+		}
+		want := install.ZFSOpts{Ashift: 12, Copies: 2, ARCMaxMiB: 4096, Compress: "lz4", Checksum: "blake3"}
+		if got != want {
+			t.Errorf("opts = %+v, want %+v", got, want)
+		}
+	})
+
+	for _, env := range []string{"SPINIFEX_ZFS_ASHIFT", "SPINIFEX_ZFS_COPIES", "SPINIFEX_ZFS_ARC_MAX_MIB"} {
+		t.Run("unparsable "+env, func(t *testing.T) {
+			setEnv(t, map[string]string{env: "auto"})
+			if _, err := parseZFSOpts(); err == nil || !strings.Contains(err.Error(), env) {
+				t.Fatalf("err = %v, want it to name %s", err, env)
+			}
+		})
+	}
+}
+
+func TestResolveStorage(t *testing.T) {
+	four := []install.Disk{
+		mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 40*gib),
+		mkDisk("/dev/vdd", 40*gib), mkDisk("/dev/vde", 40*gib),
+	}
+
+	tests := []struct {
+		name      string
+		disks     []install.Disk
+		env       map[string]string
+		wantFS    install.FSType
+		wantPaths []string
+		wantErr   string
+	}{
+		{
+			name:      "ext4 single disk by default",
+			disks:     four[:1],
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb"},
+		},
+		{
+			name:      "explicit member list preserves order",
+			disks:     four,
+			env:       map[string]string{"SPINIFEX_FS": "zfs-raid10", "SPINIFEX_DISKS": "vde,/dev/vdd,vdc, vdb "},
+			wantFS:    install.FSZFSRAID10,
+			wantPaths: []string{"/dev/vde", "/dev/vdd", "/dev/vdc", "/dev/vdb"},
+		},
+		{
+			name:    "unknown filesystem",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_FS": "btrfs"},
+			wantErr: "unknown filesystem",
+		},
+		{
+			// Choosing which disks to erase is not an unattended decision.
+			name:    "multi-disk pool without an explicit list",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_FS": "zfs-raid1"},
+			wantErr: "set SPINIFEX_DISKS to an explicit comma-separated list",
+		},
+		{
+			name:    "member that is not present",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_DISKS": "vdz"},
+			wantErr: `"vdz" is not an available disk`,
+		},
+		{
+			name:    "member list that fails validation",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_DISKS": "vdb,vdc"},
+			wantErr: "supports a single disk only",
+		},
+		{
+			name:    "bad zfs tunable",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_ZFS_ASHIFT": "big"},
+			wantErr: "SPINIFEX_ZFS_ASHIFT",
+		},
+		{
+			name:    "scan failure",
+			disks:   nil,
+			env:     map[string]string{"SPINIFEX_DISKS": "vdb"},
+			wantErr: "sysfs unavailable",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var scanErr error
+			if tt.disks == nil {
+				scanErr = errors.New("sysfs unavailable")
+			}
+			withDisks(t, tt.disks, scanErr)
+			setEnv(t, tt.env)
+
+			got, err := resolveStorage()
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveStorage: %v", err)
+			}
+			if got.FS != tt.wantFS {
+				t.Errorf("FS = %v, want %v", got.FS, tt.wantFS)
+			}
+			if !slices.Equal(got.Paths(), tt.wantPaths) {
+				t.Errorf("Paths = %v, want %v", got.Paths(), tt.wantPaths)
+			}
+		})
+	}
+}
+
+func TestResolveDisk(t *testing.T) {
+	small := mkDisk("/dev/vdb", 40*gib)
+	large := mkDisk("/dev/vdc", 400*gib)
+	nvme := mkDisk("/dev/nvme0n1", 200*gib)
+
+	tests := []struct {
+		name     string
+		disks    []install.Disk
+		target   string
+		wantPath string
+		wantErr  string
+	}{
+		{name: "auto with one candidate", disks: []install.Disk{small}, target: "auto", wantPath: "/dev/vdb"},
+		{name: "empty target is auto", disks: []install.Disk{small}, wantPath: "/dev/vdb"},
+		{
+			name:    "auto with several candidates",
+			disks:   []install.Disk{small, large},
+			target:  "auto",
+			wantErr: "expected exactly one disk, found 2",
+		},
+		{name: "largest", disks: []install.Disk{small, large, nvme}, target: "LARGEST", wantPath: "/dev/vdc"},
+		{name: "smallest", disks: []install.Disk{large, small, nvme}, target: " smallest ", wantPath: "/dev/vdb"},
+		{name: "nvme", disks: []install.Disk{small, large, nvme}, target: "nvme", wantPath: "/dev/nvme0n1"},
+		{
+			name:    "nvme with none present",
+			disks:   []install.Disk{small, large},
+			target:  "nvme",
+			wantErr: "expected exactly one NVMe disk, found 0",
+		},
+		{name: "exact path", disks: []install.Disk{small, large}, target: "/dev/vdc", wantPath: "/dev/vdc"},
+		{
+			name:    "path that is not present",
+			disks:   []install.Disk{small},
+			target:  "/dev/vdz",
+			wantErr: "is not an available disk",
+		},
+		{name: "no candidates at all", disks: []install.Disk{}, target: "largest", wantErr: "no non-removable disks found"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, tt.disks, nil)
+
+			got, err := resolveDisk(tt.target)
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveDisk(%q): %v", tt.target, err)
+			}
+			if got.Path != tt.wantPath {
+				t.Errorf("path = %q, want %q", got.Path, tt.wantPath)
+			}
+		})
+	}
+}
+
+func TestCandidateDisksExcludesRemovableAndLiveMedia(t *testing.T) {
+	fixed := mkDisk("/dev/vdb", 40*gib)
+	usb := mkDisk("/dev/sdz", 16*gib)
+	usb.Removable = true
+	boot := mkDisk("/dev/sdy", 16*gib)
+	boot.LiveMedia = true
+	withDisks(t, []install.Disk{usb, fixed, boot}, nil)
+
+	got, err := candidateDisks()
+	if err != nil {
+		t.Fatalf("candidateDisks: %v", err)
+	}
+	if len(got) != 1 || got[0].Path != "/dev/vdb" {
+		t.Fatalf("candidates = %v, want only /dev/vdb", got)
+	}
+}
+
+func TestCandidateDisksPropagatesScanError(t *testing.T) {
+	withDisks(t, nil, errors.New("read /sys/block: denied"))
+	if _, err := candidateDisks(); err == nil {
+		t.Fatal("expected the scan error to propagate")
+	}
+}
+
+func TestPickBySizeRejectsEmptySet(t *testing.T) {
+	if _, err := pickBySize(nil, true); err == nil {
+		t.Fatal("expected an error for an empty disk set")
+	}
+}
+
+func TestPickBySizeLeavesInputOrderIntact(t *testing.T) {
+	disks := []install.Disk{mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 400*gib)}
+	if _, err := pickBySize(disks, false); err != nil {
+		t.Fatalf("pickBySize: %v", err)
+	}
+	if disks[0].Path != "/dev/vdb" {
+		t.Errorf("input reordered: %v", disks)
+	}
+}
+
+func TestDiskList(t *testing.T) {
+	got := diskList([]install.Disk{mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 2048*gib)})
+	want := "  /dev/vdb (40.0G, empty)\n  /dev/vdc (2.0T, empty)"
+	if got != want {
+		t.Errorf("diskList =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestResolveNICExplicitName(t *testing.T) {
+	got, err := resolveNIC("eno1", "")
+	if err != nil {
+		t.Fatalf("resolveNIC: %v", err)
+	}
+	if got != "eno1" {
+		t.Errorf("nic = %q, want eno1", got)
+	}
+}
+
+// The auto path reads the host's real interfaces, so assert the selection rules
+// rather than a specific name.
+func TestResolveNICAutoSkipsVirtualInterfaces(t *testing.T) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		t.Skipf("no interface list on this host: %v", err)
+	}
+
+	got, err := resolveNIC("auto", "")
+	if err != nil {
+		if len(ifaces) > 0 {
+			t.Logf("no physical NIC among %d interfaces: %v", len(ifaces), err)
+		}
+		return
+	}
+
+	idx := slices.IndexFunc(ifaces, func(i net.Interface) bool { return i.Name == got })
+	if idx < 0 {
+		t.Fatalf("resolveNIC returned %q, which is not a host interface", got)
+	}
+	sel := ifaces[idx]
+	if sel.Flags&net.FlagLoopback != 0 || sel.Flags&net.FlagBroadcast == 0 || len(sel.HardwareAddr) == 0 {
+		t.Errorf("selected %q with flags %v and MAC %q", got, sel.Flags, sel.HardwareAddr)
+	}
+	for _, pfx := range virtualNICPrefixes {
+		if strings.HasPrefix(got, pfx) {
+			t.Errorf("selected virtual interface %q (prefix %q)", got, pfx)
+		}
+	}
+}
+
+func TestResolveNICAutoHonoursExclude(t *testing.T) {
+	first, err := resolveNIC("auto", "")
+	if err != nil {
+		t.Skipf("no physical NIC on this host: %v", err)
+	}
+	got, err := resolveNIC("", first)
+	if err == nil && got == first {
+		t.Errorf("resolveNIC returned the excluded interface %q", first)
+	}
+}

cmd/installer/firstboot/firstboot.go

@@ -14,14 +14,23 @@ import (
 // Config holds the values the firstboot service needs to configure the node.
 type Config struct {
 	Hostname string
-	// EncapIP is the Geneve tunnel IP for OVN. Set to the LAN bridge IP when a
-	// dedicated LAN NIC is present, otherwise the WAN bridge IP. Empty when DHCP
-	// is used — setup-ovn.sh auto-detects the IP from the default route in that case.
+	// EncapIP is the Geneve tunnel IP for OVN, taken from the vpc plane after
+	// collapsing (vpc <- lan <- wan). Empty when that plane uses DHCP —
+	// setup-ovn.sh auto-detects the IP from the default route in that case.
 	EncapIP string
-	// ClusterRole is "init" or "join".
-	ClusterRole string
-	// JoinAddr is host:port of the primary node, only used when ClusterRole is "join".
-	JoinAddr string
+	// LANIP is the internal cluster address, taken from the lan plane after
+	// collapsing. Passed as --bind and --cluster-bind so predastore replication,
+	// the NATS mesh and OVN control traffic stay off the public interface.
+	// Empty leaves the 0.0.0.0 wildcard default, which is correct for a
+	// single-NIC node where lan folds onto wan.
+	LANIP string
+	// WANIP is the public address, taken from the wan plane. Passed as
+	// --advertise so northstar's :53 listener, the awsgw registry host and the
+	// dial target recorded for off-host clients stay on the public interface.
+	// Without it a concrete --bind is echoed back as the advertise address,
+	// which would move all three onto the internal plane. Empty when wan uses
+	// DHCP, leaving spx to auto-detect from the default route.
+	WANIP string
 	// Email is the operator email collected by the TUI or SPINIFEX_EMAIL on
 	// the headless path. Passed to `spx admin init --email=<value>` when set;
 	// omitted entirely when empty.
@@ -197,6 +206,9 @@ fi
 
 # Enable services to start, on reboot
 systemctl enable spinifex.target spinifex-banner.service
+# WantedBy=timers.target only self-activates once enabled — without this the
+# JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart.
+systemctl enable --now spinifex-nats-watchdog.timer
 
 # Mark complete only after every step above has succeeded. Until this point,
 # any failure (set -e) leaves the marker absent and the next reboot retries
@@ -244,14 +256,67 @@ func buildClusterCmd(cfg Config) string {
 	if cfg.GPUPassthrough {
 		gpuFlag = " --gpu-passthrough"
 	}
-	switch cfg.ClusterRole {
-	case "join":
-		return fmt.Sprintf("spx admin join --node %s --host %s%s", cfg.Hostname, cfg.JoinAddr, emailFlag)
-	default:
-		return fmt.Sprintf("spx admin init --node %s --nodes 1%s%s", cfg.Hostname, emailFlag, gpuFlag)
+	// Without these the node takes the 0.0.0.0 wildcard default and every
+	// internal service resolves onto the auto-detected WAN address, which is
+	// exactly what the three-plane model exists to prevent.
+	bindFlags := ""
+	if cfg.LANIP != "" {
+		bindFlags = fmt.Sprintf(" --bind %s --cluster-bind %s", cfg.LANIP, cfg.LANIP)
 	}
+	// --advertise must be explicit whenever --bind is: spx echoes a concrete
+	// bind address straight back as the advertise address and never reaches its
+	// WAN auto-detection, which would silently publish the internal plane as
+	// this node's public dial target.
+	preamble := ""
+	switch {
+	case bindFlags == "":
+		// Nothing pinned, so spx auto-detects both and a guessed advertise
+		// address would only get in the way.
+	case cfg.WANIP != "":
+		bindFlags += " --advertise " + cfg.WANIP
+	default:
+		// A DHCP wan has no address at install time, but it does by the time
+		// firstboot runs, so read it off the bridge instead of shipping the
+		// bind address as the public one.
+		preamble = wanAdvertisePreamble
+		bindFlags += ` $SPX_ADVERTISE`
+	}
+
+	// Always a single-node cluster. The installer cannot form a multi-node one:
+	// cluster membership decides the OVN database topology and the join token
+	// only exists once the primary has booted, so neither is knowable while the
+	// nodes are still being installed. Multi-node is a post-install conversion,
+	// documented in the multi-node install guide.
+	cmd := fmt.Sprintf("spx admin init --node %s --nodes 1%s%s%s", cfg.Hostname, bindFlags, emailFlag, gpuFlag)
+	return preamble + cmd
 }
 
+// wanAdvertisePreamble resolves the wan plane's DHCP lease into $SPX_ADVERTISE
+// just before formation. br-wan is always the wan bridge — that plane is the one
+// role that cannot fold — so its address is the node's public identity.
+//
+// Falling through with SPX_ADVERTISE empty is deliberate: no lease means there
+// is no public address to advertise yet, and spx echoing the bind address is
+// still better than formation failing outright.
+const wanAdvertisePreamble = `# The wan plane leases its address, so it is only knowable at boot. Without this
+# spx would echo --bind back as the advertise address and publish the internal
+# plane as this node's public dial target.
+SPX_ADVERTISE=""
+echo "[firstboot] waiting for the wan plane to acquire an address..."
+for _i in $(seq 1 60); do
+    _wan_ip=$(ip -4 -o addr show br-wan scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1)
+    if [ -n "$_wan_ip" ]; then
+        SPX_ADVERTISE=" --advertise $_wan_ip"
+        echo "[firstboot] wan address $_wan_ip (${_i}s)"
+        break
+    fi
+    sleep 1
+done
+if [ -z "$SPX_ADVERTISE" ]; then
+    echo "[firstboot] warning: br-wan has no address after 60s — this node will advertise its bind address"
+fi
+`
+
 // shellEscapeSingle wraps s in single quotes with any embedded single
 // quotes escaped. Minimal — we only need this because the email value is
 // interpolated into a shell script written by Write().

cmd/installer/firstboot/firstboot_test.go

@@ -2,6 +2,7 @@ package firstboot
 
 import (
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -24,7 +25,7 @@ func TestWriteScriptNoCallbackWhenEmpty(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	cfg := Config{Hostname: "test-node"}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -45,7 +46,6 @@ func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
 	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	cfg := Config{
 		Hostname:        "test-node",
-		ClusterRole:     "init",
 		InstallCallback: callbackURL,
 	}
 	if err := Write(root, cfg); err != nil {
@@ -70,7 +70,7 @@ func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	cfg := Config{Hostname: "test-node"}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -88,7 +88,7 @@ func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
+	cfg := Config{Hostname: "test-node", SkipFormation: true}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -114,6 +114,25 @@ func readScript(t *testing.T, root string) string {
 	return string(b)
 }
 
+// TestWriteScriptEnablesNatsWatchdogTimer pins that firstboot activates the
+// JetStream ENOSPC-latch watchdog timer. The unit file's WantedBy=timers.target
+// only takes effect once enabled — without this line the watchdog is dropped
+// onto disk but never runs, and a full disk needs a manual nats restart forever.
+func TestWriteScriptEnablesNatsWatchdogTimer(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node"}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if !strings.Contains(content, "systemctl enable --now spinifex-nats-watchdog.timer") {
+		t.Error("firstboot script must enable --now spinifex-nats-watchdog.timer")
+	}
+}
+
 func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
@@ -121,7 +140,6 @@ func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	cfg := Config{
 		Hostname:        "node1",
-		ClusterRole:     "init",
 		InstallCallback: callbackURL,
 	}
 	if err := Write(root, cfg); err != nil {
@@ -146,3 +164,121 @@ func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 		t.Error("curl must appear after done marker write")
 	}
 }
+
+// A multi-NIC node must keep cluster traffic on the internal plane while still
+// publishing the public one. --advertise has to be explicit to do that: spx
+// returns a concrete --bind verbatim as the advertise address and never reaches
+// its WAN auto-detection, so omitting it moves northstar's :53 listener and the
+// off-host dial target onto the internal plane.
+func TestBuildClusterCmdSeparatesBindFromAdvertise(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "hydrogen",
+		LANIP:    "10.0.0.3",
+		WANIP:    "216.218.163.99",
+	})
+	for _, want := range []string{
+		"--bind 10.0.0.3",
+		"--cluster-bind 10.0.0.3",
+		"--advertise 216.218.163.99",
+	} {
+		if !strings.Contains(cmd, want) {
+			t.Errorf("missing %q in:\n%s", want, cmd)
+		}
+	}
+}
+
+// Without a bind address there is nothing to correct: spx auto-detects both,
+// and a stray --advertise would pin the node to an address the installer only
+// guessed at. A single-NIC node also collapses wan and lan onto one address,
+// where the split is meaningless.
+func TestBuildClusterCmdOmitsAdvertiseWithoutBind(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "node1",
+		WANIP:    "216.218.163.99",
+	})
+	if strings.Contains(cmd, "--advertise") {
+		t.Errorf("advertise must not be passed without a bind address:\n%s", cmd)
+	}
+}
+
+// A DHCP wan has no address at install time, so the advertise value has to be
+// resolved at boot. Shipping --bind alone here would republish the internal
+// plane as the node's public dial target, which is the case the static path
+// already guards against.
+func TestBuildClusterCmdResolvesDHCPWanAtBoot(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "hydrogen",
+		LANIP:    "10.0.0.3",
+		// WANIP empty — the wan plane is on DHCP.
+	})
+	if !strings.Contains(cmd, "ip -4 -o addr show br-wan") {
+		t.Errorf("must read the wan address off the bridge:\n%s", cmd)
+	}
+	if !strings.Contains(cmd, "$SPX_ADVERTISE") {
+		t.Errorf("resolved address must reach the command:\n%s", cmd)
+	}
+	// The preamble has to run before the command that consumes it.
+	if strings.Index(cmd, "SPX_ADVERTISE=\"\"") > strings.Index(cmd, "spx admin init") {
+		t.Errorf("preamble must precede the spx invocation:\n%s", cmd)
+	}
+	if !strings.Contains(cmd, "--bind 10.0.0.3") {
+		t.Errorf("bind still comes from the lan plane:\n%s", cmd)
+	}
+}
+
+// With no bind address there is nothing to correct, so neither the literal flag
+// nor the boot-time lookup should appear — spx auto-detects both ends.
+func TestBuildClusterCmdSkipsWanLookupWithoutBind(t *testing.T) {
+	cmd := buildClusterCmd(Config{Hostname: "node1"})
+	if strings.Contains(cmd, "SPX_ADVERTISE") || strings.Contains(cmd, "--advertise") {
+		t.Errorf("no bind means no advertise handling at all:\n%s", cmd)
+	}
+}
+
+// The advertise handling injects a shell preamble into the generated script, so
+// the permutations that change its shape are syntax-checked rather than only
+// string-matched — a broken quote here bricks first boot with no installer error.
+func TestGeneratedScriptIsValidShell(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		cfg  Config
+	}{
+		{"dhcp wan resolves at boot", Config{Hostname: "hydrogen", LANIP: "10.0.0.3"}},
+		{"static wan is inlined", Config{Hostname: "hydrogen", LANIP: "10.0.0.3", WANIP: "216.218.163.99"}},
+		{"single nic, nothing pinned", Config{Hostname: "node1"}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			root := t.TempDir()
+			makeRootDirs(t, root)
+			if err := Write(root, tc.cfg); err != nil {
+				t.Fatalf("Write: %v", err)
+			}
+			path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
+			if out, err := exec.Command("bash", "-n", path).CombinedOutput(); err != nil {
+				script, _ := os.ReadFile(path)
+				t.Fatalf("generated script is not valid shell: %v\n%s\n---\n%s", err, out, script)
+			}
+		})
+	}
+}
+
+// The installer cannot form a multi-node cluster: membership decides the OVN
+// database topology, and the join token only exists once the primary has
+// booted. Neither is knowable at install time, so firstboot must only ever
+// initialize a single-node cluster and leave multi-node to a post-install
+// conversion.
+func TestBuildClusterCmdNeverJoins(t *testing.T) {
+	for _, cfg := range []Config{
+		{Hostname: "node1"},
+		{Hostname: "node1", LANIP: "10.0.0.3", WANIP: "216.218.163.99"},
+		{Hostname: "node1", LANIP: "10.0.0.3"},
+	} {
+		cmd := buildClusterCmd(cfg)
+		if strings.Contains(cmd, "spx admin join") {
+			t.Errorf("firstboot must never join a cluster:\n%s", cmd)
+		}
+		if !strings.Contains(cmd, "--nodes 1") {
+			t.Errorf("want a single-node init, got:\n%s", cmd)
+		}
+	}
+}

cmd/installer/install/assets/grub-install-wrapper

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Diversion wrapper for grub-install on a spinifex-boot-tool system.
+#
+# On ZFS root the bootloader lives on per-disk ESPs, not in /boot. A plain
+# grub-install writes a core image that tries to read the pool, which works
+# until the next OpenZFS feature flag lands and then silently does not. Worse,
+# the grub-pc and shim packages run grub-install from their own postinst, so an
+# ordinary `apt upgrade` would undo the layout with no visible error.
+#
+# Mirrors the wrapper Proxmox ships for the same reason.
+
+set -e
+
+REAL="/usr/sbin/grub-install.real"
+TOOL="/usr/sbin/spinifex-boot-tool"
+
+# Not a managed system: behave exactly as the real grub-install.
+if ! "$TOOL" status --quiet 2>/dev/null; then
+	exec "$REAL" "$@"
+fi
+
+# dpkg sets this when a maintainer script is running. The bootloader packages
+# reinstalling themselves is expected and is redirected, not refused.
+case "${DPKG_MAINTSCRIPT_PACKAGE:-}" in
+grub-* | shim-*)
+	echo "This system is managed by spinifex-boot-tool; redirecting grub-install to 'spinifex-boot-tool reinit'." >&2
+	exec "$TOOL" reinit
+	;;
+esac
+
+cat >&2 <<EOF
+E: This system's bootloader is managed by spinifex-boot-tool.
+
+Running grub-install directly would install GRUB somewhere the firmware does
+not read, and would leave the per-disk ESPs holding a stale kernel.
+
+Use instead:
+  spinifex-boot-tool refresh     re-sync kernels and grub.cfg to every ESP
+  spinifex-boot-tool reinit      reinstall GRUB on every ESP
+  spinifex-boot-tool status      list the ESPs under management
+
+To run the real grub-install anyway (you probably do not want this):
+  $REAL $*
+EOF
+exit 1

cmd/installer/install/assets/spinifex-boot-tool

@@ -0,0 +1,412 @@
+#!/bin/bash
+# spinifex-boot-tool — keep a bootloader on every disk of a ZFS root pool.
+#
+# GRUB cannot be trusted to read a ZFS pool: its ZFS reader lags OpenZFS by
+# years, and enabling a new feature flag on the pool can make the machine
+# unbootable with no warning. So /boot on the pool is never read by firmware.
+# Instead each pool member carries a small FAT32 ESP holding its own copy of
+# GRUB, the kernels and the initramfs images, and this tool keeps them in step.
+#
+# Modelled directly on Proxmox's proxmox-boot-tool, which has run this design
+# across their entire installed base for years.
+
+set -u
+set -o pipefail
+
+ESP_LIST="/etc/kernel/spinifex-boot-uuids"
+ESPMOUNTS="/var/tmp/espmounts"
+MARKER="spinifex-boot-uuids"
+
+# Kernels are only ever copied out of the real /boot, which lives on the pool.
+# Once an ESP is bind-mounted over /boot this variable is how we still find it.
+BOOTROOT="/boot"
+
+warn() { echo "W: $*" >&2; }
+die() { echo "E: $*" >&2; exit 1; }
+
+usage() {
+	cat <<EOF
+Usage: $(basename "$0") <command>
+
+  format <partition> [--force]  make <partition> a Spinifex ESP
+  init <partition>              register <partition> and install a bootloader on it
+  clean [--dry-run]             drop ESPs that are no longer present
+  refresh [--hook <name>]       re-sync kernels and grub.cfg to every ESP
+  reinit                        re-run refresh, reinstalling GRUB on every ESP
+  kernel <add|remove> <ver>     pin or unpin a kernel version
+  status [--quiet]              list registered ESPs
+EOF
+}
+
+# reexec_in_mountns re-runs this script inside a private mount namespace. Every
+# code path here bind-mounts an ESP over /boot; if the script is killed at the
+# wrong moment outside a namespace, the running system is left with its real
+# /boot masked and the next kernel upgrade writes into a FAT partition.
+reexec_in_mountns() {
+	if [ "${SPINIFEX_BOOT_TOOL_NS:-0}" = "1" ]; then
+		return 0
+	fi
+	export SPINIFEX_BOOT_TOOL_NS=1
+	# --propagation private stops the bind mounts leaking back to the host.
+	exec unshare --mount --propagation private -- "$0" "$@"
+}
+
+# --- ESP registry -----------------------------------------------------------
+
+# esp_list prints the registered ESP filesystem UUIDs.
+esp_list() {
+	[ -f "$ESP_LIST" ] || return 0
+	grep -v '^[[:space:]]*\(#\|$\)' "$ESP_LIST" || true
+}
+
+# uuid_of prints the FAT filesystem UUID of a partition.
+uuid_of() {
+	blkid "$1" -o value -s UUID 2>/dev/null
+}
+
+# path_for_uuid resolves a registered UUID back to a device, or nothing if the
+# disk has been pulled.
+path_for_uuid() {
+	local dev
+	dev="/dev/disk/by-uuid/$1"
+	[ -e "$dev" ] || return 1
+	readlink -f "$dev"
+}
+
+# mount_esp mounts the ESP for a UUID and echoes its mountpoint.
+mount_esp() {
+	local uuid="$1" dev mp
+	dev=$(path_for_uuid "$uuid") || return 1
+	mp="$ESPMOUNTS/$uuid"
+	mkdir -p "$mp"
+	mountpoint -q "$mp" || mount "$dev" "$mp" || return 1
+	echo "$mp"
+}
+
+umount_esp() {
+	local mp="$ESPMOUNTS/$1"
+	mountpoint -q "$mp" && umount "$mp"
+	rmdir "$mp" 2>/dev/null || true
+}
+
+# --- format / init ----------------------------------------------------------
+
+do_format() {
+	local part="$1" force="${2:-}"
+	[ -b "$part" ] || die "$part is not a block device"
+
+	if [ "$force" != "--force" ]; then
+		local existing
+		existing=$(blkid -o value -s TYPE "$part" 2>/dev/null || true)
+		[ -z "$existing" ] || die "$part already contains a $existing filesystem; pass --force to overwrite"
+	fi
+	mkfs.vfat -F 32 "$part" || die "mkfs.vfat failed on $part"
+}
+
+do_init() {
+	local part="$1"
+	[ -b "$part" ] || die "$part is not a block device"
+
+	local uuid
+	uuid=$(uuid_of "$part") || true
+	[ -n "$uuid" ] || die "$part has no filesystem UUID — run '$(basename "$0") format $part' first"
+
+	mkdir -p "$(dirname "$ESP_LIST")"
+	touch "$ESP_LIST"
+	if ! grep -qx "$uuid" "$ESP_LIST"; then
+		echo "$uuid" >>"$ESP_LIST"
+	fi
+	echo "Registered $part ($uuid)"
+
+	install_hooks
+	# Only the ESP just registered: the others are already current, and
+	# refreshing all of them here makes registering N disks an N-squared
+	# sequence of grub-installs.
+	refresh_one "$uuid" || die "could not make $part bootable"
+}
+
+do_clean() {
+	local dry="${1:-}" uuid keep=()
+	for uuid in $(esp_list); do
+		if path_for_uuid "$uuid" >/dev/null 2>&1; then
+			keep+=("$uuid")
+		else
+			echo "Removing stale ESP $uuid"
+		fi
+	done
+	if [ "$dry" = "--dry-run" ]; then
+		return 0
+	fi
+	printf '%s\n' "${keep[@]+"${keep[@]}"}" >"$ESP_LIST"
+}
+
+# --- refresh ----------------------------------------------------------------
+
+# kernel_versions lists the kernels present in the real /boot, newest first, so
+# the default menu entry is the newest kernel.
+kernel_versions() {
+	local img
+	for img in "$BOOTROOT"/vmlinuz-*; do
+		[ -e "$img" ] || continue
+		basename "$img" | sed 's/^vmlinuz-//'
+	done | sort -Vr
+}
+
+# sync_esp copies the current kernels and initramfs images onto one ESP and
+# removes anything left from a kernel that is no longer installed — an ESP is
+# small and a stale 100MB initramfs is what makes the next upgrade fail.
+sync_esp() {
+	local mp="$1" kver f base
+
+	local -A wanted=()
+	for kver in $(kernel_versions); do
+		wanted["vmlinuz-$kver"]=1
+		wanted["initrd.img-$kver"]=1
+	done
+
+	for f in "$mp"/vmlinuz-* "$mp"/initrd.img-*; do
+		[ -e "$f" ] || continue
+		base=$(basename "$f")
+		[ -n "${wanted[$base]:-}" ] || rm -f "$f"
+	done
+
+	for kver in $(kernel_versions); do
+		for base in "vmlinuz-$kver" "initrd.img-$kver"; do
+			[ -e "$BOOTROOT/$base" ] || continue
+			# Copy to a temp name and rename: a half-written kernel on the ESP
+			# is a machine that does not come back from a reboot.
+			cp -f "$BOOTROOT/$base" "$mp/$base.tmp" || return 1
+			mv -f "$mp/$base.tmp" "$mp/$base" || return 1
+		done
+	done
+
+	# GRUB reads its theme from $prefix, which is the ESP's own grub directory,
+	# so the branding has to be copied out of /boot alongside the kernels.
+	mkdir -p "$mp/grub/fonts"
+	for f in grub/theme.txt grub/splash.png grub/fonts/unicode.pf2; do
+		[ -e "$BOOTROOT/$f" ] && cp -f "$BOOTROOT/$f" "$mp/$f"
+	done
+
+	# Marks the partition as ours, so a later `format` on the wrong disk is
+	# obviously a mistake rather than a silent overwrite.
+	echo "$MARKER" >"$mp/$MARKER"
+	return 0
+}
+
+# grub_install_bin echoes the real grub-install. On a managed system the normal
+# name is a wrapper that refuses to run, and it must not intercept this tool's
+# own calls — that would make the refusal a self-deadlock.
+grub_install_bin() {
+	if [ -x /usr/sbin/grub-install.real ]; then
+		echo /usr/sbin/grub-install.real
+	else
+		echo grub-install
+	fi
+}
+
+# install_grub puts GRUB itself on one ESP, for both firmware types.
+#
+# --boot-directory points at the ESP, not at /boot, so the GRUB core image
+# embeds a FAT reader and a search for the ESP's own UUID. It never needs to
+# read the pool, which is the entire point of this tool.
+install_grub() {
+	local mp="$1" dev="$2" rc=0 grub
+	grub="$(grub_install_bin)"
+
+	if [ ! -d /sys/firmware/efi ]; then
+		"$grub" --target=i386-pc --boot-directory="$mp" --recheck "$dev" || rc=1
+		return $rc
+	fi
+
+	# NVRAM is written at install time only. A refresh runs on every kernel
+	# upgrade, and rewriting boot variables that often has bricked firmware on
+	# more than one vendor's boards.
+	local nvram="--no-nvram"
+	[ "${SPINIFEX_BOOT_TOOL_NVRAM:-0}" = "1" ] && nvram=""
+
+	# Twice: once as a named entry, and once at the removable path, which is all
+	# some firmware will ever look at and is what keeps the disk bootable after
+	# its NVRAM entry is lost or the drive is moved to another machine.
+	# shellcheck disable=SC2086 # $nvram is a single optional flag, not a path
+	"$grub" --target=x86_64-efi \
+		--efi-directory="$mp" --boot-directory="$mp" \
+		--bootloader-id=spinifex $nvram --recheck "$dev" || rc=1
+	"$grub" --target=x86_64-efi \
+		--efi-directory="$mp" --boot-directory="$mp" \
+		--bootloader-id=spinifex --removable --no-nvram --recheck "$dev" || rc=1
+	return $rc
+}
+
+# update_grub_on regenerates grub.cfg for one ESP.
+#
+# The bind mount is the trick that makes this work: grub-mkconfig probes the
+# device behind /boot to decide how to find the kernel at boot. With the ESP
+# mounted there it emits a FAT UUID search and paths relative to the ESP root,
+# so the generated config matches what the embedded core image can actually do.
+update_grub_on() {
+	local mp="$1" rc=0
+	mount --bind "$mp" "$BOOTROOT" || return 1
+	grub-mkconfig -o "$BOOTROOT/grub/grub.cfg" || rc=1
+	umount "$BOOTROOT" || warn "could not unmount the bind mount over $BOOTROOT"
+	return $rc
+}
+
+# refresh_one makes a single registered ESP independently bootable: kernels,
+# bootloader, and a grub.cfg generated against that ESP's own UUID.
+refresh_one() {
+	local uuid="$1" mp dev disk rc=0
+
+	if ! dev=$(path_for_uuid "$uuid"); then
+		warn "ESP $uuid is not present, skipping"
+		return 1
+	fi
+	if ! mp=$(mount_esp "$uuid"); then
+		warn "could not mount ESP $uuid"
+		return 1
+	fi
+
+	# The whole disk, not the partition: BIOS GRUB embeds into the boot
+	# sector and the BIOS boot partition, both of which live on the disk.
+	disk=$(lsblk -no pkname "$dev")
+	disk="/dev/$disk"
+
+	if sync_esp "$mp" && install_grub "$mp" "$disk" && update_grub_on "$mp"; then
+		echo "Synced $dev ($uuid)"
+	else
+		warn "failed to update ESP $uuid on $dev"
+		rc=1
+	fi
+	umount_esp "$uuid"
+	return $rc
+}
+
+do_refresh() {
+	local uuid failed=0 ok=0 total=0
+
+	local uuids
+	uuids=$(esp_list)
+	if [ -z "$uuids" ]; then
+		echo "No ESPs registered, nothing to do."
+		return 0
+	fi
+
+	for uuid in $uuids; do
+		total=$((total + 1))
+		if refresh_one "$uuid"; then
+			ok=$((ok + 1))
+		else
+			failed=$((failed + 1))
+		fi
+	done
+
+	if [ "$ok" -eq 0 ]; then
+		die "no ESP could be updated ($failed of $total failed) — this system may not boot"
+	fi
+	if [ "$failed" -gt 0 ]; then
+		# Strict mode is used by the installer, where a partly-updated set of
+		# ESPs means the install is broken and should fail loudly now rather
+		# than at the first reboot.
+		if [ "${SPINIFEX_BOOT_TOOL_STRICT:-0}" = "1" ]; then
+			die "$failed of $total ESPs failed to update"
+		fi
+		warn "$failed of $total ESPs failed to update; $ok succeeded, so the system will still boot"
+	fi
+	return 0
+}
+
+do_status() {
+	local quiet="${1:-}" uuid dev
+	if [ ! -s "$ESP_LIST" ]; then
+		[ "$quiet" = "--quiet" ] || echo "No ESPs registered — this system does not use spinifex-boot-tool."
+		return 1
+	fi
+	[ "$quiet" = "--quiet" ] && return 0
+	for uuid in $(esp_list); do
+		if dev=$(path_for_uuid "$uuid"); then
+			echo "$uuid  $dev"
+		else
+			echo "$uuid  MISSING"
+		fi
+	done
+}
+
+do_kernel() {
+	local action="$1" kver="${2:-}"
+	local pinfile="/etc/kernel/spinifex-boot-pin"
+	case "$action" in
+	add)
+		[ -n "$kver" ] || die "kernel add needs a version"
+		touch "$pinfile"
+		grep -qx "$kver" "$pinfile" || echo "$kver" >>"$pinfile"
+		;;
+	remove)
+		[ -n "$kver" ] || die "kernel remove needs a version"
+		[ -f "$pinfile" ] && sed -i "\|^$kver\$|d" "$pinfile"
+		;;
+	*) die "unknown kernel action: $action" ;;
+	esac
+	do_refresh
+}
+
+# --- hooks ------------------------------------------------------------------
+
+# install_hooks wires this tool into every package operation that would
+# otherwise write a bootloader to the wrong place.
+install_hooks() {
+	mkdir -p /etc/kernel/postinst.d /etc/kernel/postrm.d /etc/initramfs/post-update.d
+
+	# Debian's own hook runs update-grub against the real /boot, which is on the
+	# pool. Firmware never reads that, so it would produce a config nothing uses
+	# while the ESPs silently kept an old kernel.
+	if [ -e /etc/kernel/postinst.d/zz-update-grub ]; then
+		dpkg-divert --divert /etc/kernel/postinst.d/zz-update-grub.distrib \
+			--rename --add /etc/kernel/postinst.d/zz-update-grub >/dev/null 2>&1 || true
+	fi
+
+	for hook in /etc/kernel/postinst.d/zz-spinifex-boot-tool \
+		/etc/kernel/postrm.d/zz-spinifex-boot-tool \
+		/etc/initramfs/post-update.d/spinifex-boot-tool; do
+		cat >"$hook" <<'HOOK'
+#!/bin/sh
+# Re-sync the ESPs after a kernel or initramfs change.
+set -e
+exec /usr/sbin/spinifex-boot-tool refresh --hook "$(basename "$0")"
+HOOK
+		chmod 0755 "$hook"
+	done
+}
+
+# --- dispatch ---------------------------------------------------------------
+
+cmd="${1:-}"
+[ $# -gt 0 ] && shift
+
+case "$cmd" in
+format) do_format "$@" ;;
+init)
+	reexec_in_mountns init "$@"
+	do_init "$@"
+	;;
+refresh)
+	# --hook <name> is passed by the kernel hooks purely so the reason for a
+	# refresh is visible in the log.
+	if [ "${1:-}" = "--hook" ]; then shift 2 || shift; fi
+	reexec_in_mountns refresh
+	do_refresh
+	;;
+reinit)
+	reexec_in_mountns reinit
+	do_refresh
+	;;
+clean) do_clean "$@" ;;
+kernel)
+	reexec_in_mountns kernel "$@"
+	do_kernel "$@"
+	;;
+status) do_status "$@" ;;
+help | --help | -h | "") usage ;;
+*)
+	usage
+	exit 1
+	;;
+esac

cmd/installer/install/blockdev.go

@@ -0,0 +1,267 @@
+package install
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// Disk is one candidate block device with every attribute the installer needs
+// to validate a selection: sizes for the same-size rule, block sizes for ashift
+// and the 4Kn/BIOS check, rotational for autotrim, and ByID for pool members.
+type Disk struct {
+	Path  string // /dev/sda
+	ByID  string // /dev/disk/by-id/ata-… — stable across controller reordering
+	Bytes int64
+	Model string
+
+	LogicalBlockSize  int
+	PhysicalBlockSize int
+
+	Rotational bool
+	Removable  bool
+
+	// LiveMedia marks the device the installer itself booted from. Selecting it
+	// would erase the running system mid-install.
+	LiveMedia bool
+
+	// Content is a human-readable summary of what is on the disk today
+	// ("ext4 filesystem", "empty"), shown before the operator confirms erasure.
+	Content string
+}
+
+// skipPrefixes are kernel block devices that are never install targets:
+// virtual, mapped, or removable media. Mirrors Proxmox's hd_list filter.
+var skipPrefixes = []string{"ram", "loop", "md", "dm-", "fd", "sr", "zram", "nbd"}
+
+// The kernel-owned trees the scan reads, indirected so tests can point it at a
+// fixture instead of the running machine's hardware.
+var (
+	sysBlockDir      = "/sys/block"
+	sysClassBlockDir = "/sys/class/block"
+	devByIDDir       = "/dev/disk/by-id"
+	procMountsPath   = "/proc/mounts"
+)
+
+// ListDisks enumerates install-candidate disks from /sys/block. It reads sysfs
+// and blkid only — pool detection is separate (ImportablePools) because it is
+// far slower and not every caller needs it.
+func ListDisks() ([]Disk, error) {
+	entries, err := os.ReadDir(sysBlockDir)
+	if err != nil {
+		return nil, fmt.Errorf("read /sys/block: %w", err)
+	}
+
+	byID := buildByIDIndex()
+	live := liveMediaDevices()
+
+	var disks []Disk
+	for _, e := range entries {
+		name := e.Name()
+		if hasAnyPrefix(name, skipPrefixes) {
+			continue
+		}
+		// A zero-sized device is an empty card reader slot, not a disk.
+		sectors := sysfsInt(name, "size")
+		if sectors == 0 {
+			continue
+		}
+		d := Disk{
+			Path:              "/dev/" + name,
+			ByID:              byID[name],
+			Bytes:             sectors * 512,
+			Model:             sysfsString(name, "device/model"),
+			LogicalBlockSize:  sysfsIntDefault(name, "queue/logical_block_size", 512),
+			PhysicalBlockSize: sysfsIntDefault(name, "queue/physical_block_size", 512),
+			Rotational:        sysfsInt(name, "queue/rotational") == 1,
+			Removable:         sysfsInt(name, "removable") == 1,
+			LiveMedia:         live[name],
+		}
+		d.Content = describeContent(d.Path)
+		disks = append(disks, d)
+	}
+	slices.SortFunc(disks, func(a, b Disk) int { return strings.Compare(a.Path, b.Path) })
+	return disks, nil
+}
+
+// SizeHuman renders the disk size for display, e.g. "1.7T".
+func (d Disk) SizeHuman() string {
+	switch {
+	case d.Bytes >= 1<<40:
+		return fmt.Sprintf("%.1fT", float64(d.Bytes)/(1<<40))
+	case d.Bytes >= 1<<30:
+		return fmt.Sprintf("%.1fG", float64(d.Bytes)/(1<<30))
+	case d.Bytes >= 1<<20:
+		return fmt.Sprintf("%.1fM", float64(d.Bytes)/(1<<20))
+	default:
+		return fmt.Sprintf("%dB", d.Bytes)
+	}
+}
+
+// Stable returns the path to use when building a pool: the by-id path when one
+// exists, else the kernel name. Pools referencing /dev/sdX break when a
+// controller enumerates disks in a different order after a reboot.
+func (d Disk) Stable() string {
+	if d.ByID != "" {
+		return d.ByID
+	}
+	return d.Path
+}
+
+// PartitionPath returns the device path for partition n of this disk. NVMe and
+// other devices whose name ends in a digit take a 'p' separator.
+func (d Disk) PartitionPath(n int) string { return partitionPath(d.Path, n) }
+
+// StablePartitionPath is PartitionPath against the by-id path, which udev
+// publishes with a "-partN" suffix rather than the kernel's 'p' convention.
+func (d Disk) StablePartitionPath(n int) string {
+	if d.ByID == "" {
+		return d.PartitionPath(n)
+	}
+	return fmt.Sprintf("%s-part%d", d.ByID, n)
+}
+
+func partitionPath(disk string, n int) string {
+	if len(disk) > 0 && disk[len(disk)-1] >= '0' && disk[len(disk)-1] <= '9' {
+		return fmt.Sprintf("%sp%d", disk, n)
+	}
+	return fmt.Sprintf("%s%d", disk, n)
+}
+
+// byIDPreference orders the /dev/disk/by-id/ namespaces we prefer to name a
+// pool member by. wwn- ids are last: they are stable but opaque, so a degraded
+// pool reports a number nobody can map to a drive bay.
+var byIDPreference = []string{"nvme-", "ata-", "scsi-", "virtio-", "usb-", "wwn-"}
+
+// buildByIDIndex maps kernel device name to its preferred by-id path.
+func buildByIDIndex() map[string]string {
+	entries, err := os.ReadDir(devByIDDir)
+	if err != nil {
+		return map[string]string{}
+	}
+	// Track the chosen preference rank per device so a later, less-preferred
+	// id cannot displace a better one already found.
+	rank := map[string]int{}
+	out := map[string]string{}
+	for _, e := range entries {
+		id := e.Name()
+		// Partition links describe a slice of the disk, not the disk.
+		if strings.Contains(id, "-part") {
+			continue
+		}
+		target, err := filepath.EvalSymlinks(filepath.Join(devByIDDir, id))
+		if err != nil {
+			continue
+		}
+		dev := filepath.Base(target)
+		r := prefixRank(id)
+		if existing, ok := rank[dev]; ok && existing <= r {
+			continue
+		}
+		rank[dev] = r
+		out[dev] = filepath.Join(devByIDDir, id)
+	}
+	return out
+}
+
+func prefixRank(id string) int {
+	for i, p := range byIDPreference {
+		if strings.HasPrefix(id, p) {
+			return i
+		}
+	}
+	return len(byIDPreference)
+}
+
+// liveMediaDevices returns the set of parent disk names backing the live
+// installer's own media, so they can never be selected as install targets.
+func liveMediaDevices() map[string]bool {
+	out := map[string]bool{}
+	data, err := os.ReadFile(procMountsPath)
+	if err != nil {
+		return out
+	}
+	liveMounts := []string{"/cdrom", "/run/live/medium", "/lib/live/mount/medium"}
+	for line := range strings.SplitSeq(string(data), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) < 2 || !strings.HasPrefix(fields[0], "/dev/") {
+			continue
+		}
+		if !slices.Contains(liveMounts, fields[1]) {
+			continue
+		}
+		if parent := parentDisk(filepath.Base(fields[0])); parent != "" {
+			out[parent] = true
+		}
+	}
+	return out
+}
+
+// parentDisk resolves a partition's kernel name to its whole-disk name by
+// walking /sys/class/block/<part>/.. — the sysfs parent of a partition is its
+// disk. Returns the input unchanged when it is already a whole disk.
+func parentDisk(dev string) string {
+	link, err := filepath.EvalSymlinks(filepath.Join(sysClassBlockDir, dev))
+	if err != nil {
+		return dev
+	}
+	// A whole disk lives directly under a bus path; a partition sits one level
+	// deeper, inside its disk's directory.
+	if _, err := os.Stat(filepath.Join(link, "partition")); err != nil {
+		return dev
+	}
+	return filepath.Base(filepath.Dir(link))
+}
+
+// describeContent summarises what is currently on a disk for the confirmation
+// screen. Best-effort: an unreadable disk is reported as unknown, not empty,
+// because "empty" invites an operator to erase something that is not.
+func describeContent(dev string) string {
+	out, err := exec.Command("blkid", "-p", "-o", "value", "-s", "TYPE", dev).Output()
+	if err == nil {
+		if t := strings.TrimSpace(string(out)); t != "" {
+			return t + " filesystem"
+		}
+	}
+	// No filesystem on the whole device: report the partition table instead,
+	// since that is what tells the operator the disk is in use.
+	out, err = exec.Command("blkid", "-p", "-o", "value", "-s", "PTTYPE", dev).Output()
+	if err != nil {
+		return "unknown"
+	}
+	if t := strings.TrimSpace(string(out)); t != "" {
+		return t + " partition table"
+	}
+	return "empty"
+}
+
+func sysfsString(dev, file string) string {
+	data, err := os.ReadFile(filepath.Join(sysBlockDir, dev, file))
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(data))
+}
+
+func sysfsInt(dev, file string) int64 {
+	n, err := strconv.ParseInt(sysfsString(dev, file), 10, 64)
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+func sysfsIntDefault(dev, file string, def int) int {
+	if n := sysfsInt(dev, file); n > 0 {
+		return int(n)
+	}
+	return def
+}
+
+func hasAnyPrefix(s string, prefixes []string) bool {
+	return slices.ContainsFunc(prefixes, func(p string) bool { return strings.HasPrefix(s, p) })
+}

cmd/installer/install/blockdev_test.go

@@ -0,0 +1,354 @@
+package install
+
+import (
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"testing"
+)
+
+// fakeDisk describes one device to lay down in the fixture sysfs tree.
+type fakeDisk struct {
+	name       string
+	sectors    int64
+	model      string
+	logical    int
+	physical   int
+	rotational bool
+	removable  bool
+	partitions []string
+}
+
+// fakeSysfs builds a /sys/block, /sys/class/block, /dev/disk/by-id and
+// /proc/mounts fixture and points the scan at it for the duration of the test.
+type fakeSysfs struct {
+	root string
+	t    *testing.T
+}
+
+func newFakeSysfs(t *testing.T, disks ...fakeDisk) *fakeSysfs {
+	t.Helper()
+	f := &fakeSysfs{root: t.TempDir(), t: t}
+
+	swap := func(dst *string, v string) {
+		prev := *dst
+		*dst = v
+		t.Cleanup(func() { *dst = prev })
+	}
+	swap(&sysBlockDir, filepath.Join(f.root, "sys/block"))
+	swap(&sysClassBlockDir, filepath.Join(f.root, "sys/class/block"))
+	swap(&devByIDDir, filepath.Join(f.root, "dev/disk/by-id"))
+	swap(&procMountsPath, filepath.Join(f.root, "proc/mounts"))
+
+	f.mkdirAll(sysBlockDir)
+	f.mkdirAll(sysClassBlockDir)
+	for _, d := range disks {
+		f.addDisk(d)
+	}
+	return f
+}
+
+func (f *fakeSysfs) mkdirAll(dir string) {
+	f.t.Helper()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		f.t.Fatalf("mkdir %s: %v", dir, err)
+	}
+}
+
+func (f *fakeSysfs) write(path, content string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Dir(path))
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		f.t.Fatalf("write %s: %v", path, err)
+	}
+}
+
+// addDisk lays out one device the way the kernel does: the attribute files
+// under /sys/block/<dev>, and a /sys/class/block symlink per disk and
+// partition, since parentDisk walks the latter.
+func (f *fakeSysfs) addDisk(d fakeDisk) {
+	f.t.Helper()
+	dir := filepath.Join(sysBlockDir, d.name)
+	f.write(filepath.Join(dir, "size"), strconv.FormatInt(d.sectors, 10)+"\n")
+	if d.model != "" {
+		f.write(filepath.Join(dir, "device/model"), d.model+"\n")
+	}
+	if d.logical > 0 {
+		f.write(filepath.Join(dir, "queue/logical_block_size"), strconv.Itoa(d.logical)+"\n")
+	}
+	if d.physical > 0 {
+		f.write(filepath.Join(dir, "queue/physical_block_size"), strconv.Itoa(d.physical)+"\n")
+	}
+	f.write(filepath.Join(dir, "queue/rotational"), boolAttr(d.rotational))
+	f.write(filepath.Join(dir, "removable"), boolAttr(d.removable))
+
+	f.symlink(dir, filepath.Join(sysClassBlockDir, d.name))
+	for _, part := range d.partitions {
+		partDir := filepath.Join(dir, part)
+		f.write(filepath.Join(partDir, "partition"), "1\n")
+		f.symlink(partDir, filepath.Join(sysClassBlockDir, part))
+	}
+}
+
+func (f *fakeSysfs) symlink(target, link string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Dir(link))
+	if err := os.Symlink(target, link); err != nil {
+		f.t.Fatalf("symlink %s -> %s: %v", link, target, err)
+	}
+}
+
+// addByID creates a by-id alias pointing at a kernel device name. The link
+// target must exist for EvalSymlinks, so a placeholder /dev tree is used.
+func (f *fakeSysfs) addByID(id, dev string) {
+	f.t.Helper()
+	devPath := filepath.Join(f.root, "dev", dev)
+	f.write(devPath, "")
+	f.symlink(devPath, filepath.Join(devByIDDir, id))
+}
+
+func (f *fakeSysfs) setMounts(content string) {
+	f.t.Helper()
+	f.write(procMountsPath, content)
+}
+
+func boolAttr(b bool) string {
+	if b {
+		return "1\n"
+	}
+	return "0\n"
+}
+
+func TestListDisksReadsSysfsAttributes(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{
+		name: "sdb", sectors: 2 << 20, model: "SAMSUNG MZ7", logical: 512, physical: 4096, rotational: true,
+	})
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	if len(disks) != 1 {
+		t.Fatalf("got %d disks, want 1: %+v", len(disks), disks)
+	}
+	d := disks[0]
+	if d.Path != "/dev/sdb" || d.Bytes != (2<<20)*512 {
+		t.Errorf("path/size = %s/%d", d.Path, d.Bytes)
+	}
+	if d.Model != "SAMSUNG MZ7" {
+		t.Errorf("Model = %q", d.Model)
+	}
+	if d.LogicalBlockSize != 512 || d.PhysicalBlockSize != 4096 {
+		t.Errorf("block sizes = %d/%d", d.LogicalBlockSize, d.PhysicalBlockSize)
+	}
+	if !d.Rotational || d.Removable || d.LiveMedia {
+		t.Errorf("flags = rot %v, rem %v, live %v", d.Rotational, d.Removable, d.LiveMedia)
+	}
+	if d.Content == "" {
+		t.Error("Content is empty; an unreadable device must report something")
+	}
+}
+
+func TestListDisksSkipsVirtualAndEmptyDevices(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "loop0", sectors: 1 << 20},
+		fakeDisk{name: "dm-0", sectors: 1 << 20},
+		fakeDisk{name: "zram0", sectors: 1 << 20},
+		fakeDisk{name: "sr0", sectors: 1 << 20},
+		fakeDisk{name: "nbd0", sectors: 1 << 20},
+		// An empty card-reader slot reports zero sectors.
+		fakeDisk{name: "mmcblk0", sectors: 0},
+	)
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	var paths []string
+	for _, d := range disks {
+		paths = append(paths, d.Path)
+	}
+	if !slices.Equal(paths, []string{"/dev/sda"}) {
+		t.Errorf("paths = %v, want only /dev/sda", paths)
+	}
+}
+
+func TestListDisksSortsByPath(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sdc", sectors: 1 << 20},
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "sdb", sectors: 1 << 20},
+	)
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	var paths []string
+	for _, d := range disks {
+		paths = append(paths, d.Path)
+	}
+	if !slices.Equal(paths, []string{"/dev/sda", "/dev/sdb", "/dev/sdc"}) {
+		t.Errorf("paths = %v, want sorted", paths)
+	}
+}
+
+func TestListDisksDefaultsBlockSizesWhenSysfsIsSilent(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "vda", sectors: 1 << 20})
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	if disks[0].LogicalBlockSize != 512 || disks[0].PhysicalBlockSize != 512 {
+		t.Errorf("block sizes = %d/%d, want the 512 default",
+			disks[0].LogicalBlockSize, disks[0].PhysicalBlockSize)
+	}
+}
+
+func TestListDisksErrorsWhenSysfsIsUnreadable(t *testing.T) {
+	newFakeSysfs(t)
+	sysBlockDir = filepath.Join(t.TempDir(), "does-not-exist")
+
+	if _, err := ListDisks(); err == nil {
+		t.Fatal("expected an error when /sys/block cannot be read")
+	}
+}
+
+func TestListDisksMarksTheLiveMedium(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "sdz", sectors: 1 << 20, removable: true, partitions: []string{"sdz1"}},
+	)
+	// The installer boots from a partition; the whole disk must be excluded.
+	f.setMounts("/dev/sdz1 /run/live/medium iso9660 ro 0 0\n" +
+		"proc /proc proc rw 0 0\n" +
+		"/dev/sda1 / ext4 rw 0 0\n")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	byPath := map[string]Disk{}
+	for _, d := range disks {
+		byPath[d.Path] = d
+	}
+	if !byPath["/dev/sdz"].LiveMedia {
+		t.Error("/dev/sdz backs the live medium and was not marked")
+	}
+	if byPath["/dev/sda"].LiveMedia {
+		t.Error("/dev/sda is not the live medium but was marked")
+	}
+}
+
+func TestLiveMediaDevicesIsEmptyWithoutProcMounts(t *testing.T) {
+	newFakeSysfs(t)
+	if got := liveMediaDevices(); len(got) != 0 {
+		t.Errorf("liveMediaDevices = %v, want empty when /proc/mounts is missing", got)
+	}
+}
+
+func TestParentDiskLeavesWholeDisksAlone(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20, partitions: []string{"sda1"}})
+
+	if got := parentDisk("sda"); got != "sda" {
+		t.Errorf("parentDisk(sda) = %q, want sda", got)
+	}
+	if got := parentDisk("sda1"); got != "sda" {
+		t.Errorf("parentDisk(sda1) = %q, want sda", got)
+	}
+	// An unknown name cannot be resolved and is returned unchanged.
+	if got := parentDisk("sdq9"); got != "sdq9" {
+		t.Errorf("parentDisk(sdq9) = %q, want it unchanged", got)
+	}
+}
+
+func TestBuildByIDIndexPrefersTheMostLegibleAlias(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20})
+	// wwn- is stable but opaque, so ata- must win regardless of scan order.
+	f.addByID("wwn-0x5000c500", "sda")
+	f.addByID("ata-SAMSUNG_MZ7_S1", "sda")
+	f.addByID("ata-SAMSUNG_MZ7_S1-part1", "sda")
+
+	idx := buildByIDIndex()
+	want := filepath.Join(devByIDDir, "ata-SAMSUNG_MZ7_S1")
+	if idx["sda"] != want {
+		t.Errorf("by-id[sda] = %q, want %q", idx["sda"], want)
+	}
+}
+
+func TestBuildByIDIndexIsEmptyWhenTheDirIsAbsent(t *testing.T) {
+	newFakeSysfs(t)
+	if got := buildByIDIndex(); len(got) != 0 {
+		t.Errorf("buildByIDIndex = %v, want empty", got)
+	}
+}
+
+func TestListDisksAttachesByIDPath(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20})
+	f.addByID("ata-TEST_SERIAL0", "sda")
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	want := filepath.Join(devByIDDir, "ata-TEST_SERIAL0")
+	if disks[0].ByID != want {
+		t.Errorf("ByID = %q, want %q", disks[0].ByID, want)
+	}
+	if disks[0].Stable() != want {
+		t.Errorf("Stable() = %q, want the by-id path", disks[0].Stable())
+	}
+}
+
+func TestPrefixRank(t *testing.T) {
+	if prefixRank("nvme-Foo") >= prefixRank("wwn-0x1") {
+		t.Error("nvme- must outrank wwn-")
+	}
+	if prefixRank("md-uuid-x") != len(byIDPreference) {
+		t.Error("an unknown namespace must rank last")
+	}
+}
+
+func TestSizeHuman(t *testing.T) {
+	tests := []struct {
+		bytes int64
+		want  string
+	}{
+		{2 << 40, "2.0T"},
+		{500 << 30, "500.0G"},
+		{64 << 20, "64.0M"},
+		{512, "512B"},
+	}
+	for _, tt := range tests {
+		if got := (Disk{Bytes: tt.bytes}).SizeHuman(); got != tt.want {
+			t.Errorf("SizeHuman(%d) = %q, want %q", tt.bytes, got, tt.want)
+		}
+	}
+}
+
+func TestStableFallsBackToKernelName(t *testing.T) {
+	d := Disk{Path: "/dev/sda"}
+	if d.Stable() != "/dev/sda" {
+		t.Errorf("Stable() = %q", d.Stable())
+	}
+	if d.StablePartitionPath(2) != "/dev/sda2" {
+		t.Errorf("StablePartitionPath(2) = %q", d.StablePartitionPath(2))
+	}
+}
+
+func TestHasAnyPrefix(t *testing.T) {
+	if !hasAnyPrefix("loop3", skipPrefixes) {
+		t.Error("loop3 should match")
+	}
+	if hasAnyPrefix("sda", skipPrefixes) {
+		t.Error("sda should not match")
+	}
+}

cmd/installer/install/boottool.go

@@ -0,0 +1,202 @@
+package install
+
+import (
+	_ "embed"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+//go:embed assets/spinifex-boot-tool
+var bootToolScript []byte
+
+//go:embed assets/grub-install-wrapper
+var grubInstallWrapper []byte
+
+const (
+	bootToolPath     = "/usr/sbin/spinifex-boot-tool"
+	grubInstallPath  = "/usr/sbin/grub-install"
+	grubInstallReal  = "/usr/sbin/grub-install.real"
+	bootToolUUIDList = "/etc/kernel/spinifex-boot-uuids"
+)
+
+// installBootToolZFS sets up per-disk bootloaders on a ZFS root system.
+//
+// Nothing in the boot path is allowed to read the pool: GRUB's ZFS support
+// trails OpenZFS, so a pool feature flag enabled years from now could make the
+// machine unbootable. Each disk instead carries a self-contained ESP with its
+// own GRUB, kernels and initramfs.
+func installBootToolZFS(cfg DiskConfig) error {
+	if err := writeTargetFile(bootToolPath, bootToolScript, 0o755); err != nil {
+		return err
+	}
+	if err := writeGrubDefaults(cfg); err != nil {
+		return err
+	}
+
+	if err := bindChrootMounts(); err != nil {
+		return err
+	}
+	defer unbindChrootMounts()
+
+	// Diverted before the first init: init runs a refresh, and refresh calls
+	// grub-install. With the wrapper already in place that call has to reach
+	// the real binary, which is what the diversion arranges.
+	if err := divertGrubInstall(); err != nil {
+		return err
+	}
+
+	// A partially-updated set of ESPs is an install that boots today and fails
+	// after the next kernel upgrade, so every disk must succeed here.
+	env := []string{"SPINIFEX_BOOT_TOOL_STRICT=1"}
+	for _, d := range cfg.Disks {
+		esp := d.PartitionPath(espPartNum)
+		if err := runEnv(env, "chroot", mountRoot, bootToolPath, "init", esp); err != nil {
+			return fmt.Errorf("spinifex-boot-tool init %s: %w", esp, err)
+		}
+	}
+
+	// The one moment firmware boot variables are written. Refreshes run on
+	// every kernel upgrade and deliberately leave NVRAM alone: repeatedly
+	// rewriting boot entries has bricked more than one vendor's boards. Losing
+	// the entry is survivable anyway, because each ESP also carries the
+	// removable-path loader that firmware falls back to.
+	if err := runEnv(append(env, "SPINIFEX_BOOT_TOOL_NVRAM=1"),
+		"chroot", mountRoot, bootToolPath, "refresh"); err != nil {
+		return fmt.Errorf("spinifex-boot-tool refresh: %w", err)
+	}
+	return verifyBootConfig(cfg)
+}
+
+// divertGrubInstall moves the real grub-install aside and puts the wrapper in
+// its place. dpkg-divert makes the move survive package upgrades, which is the
+// point: grub-pc's postinst runs grub-install and would otherwise quietly
+// replace a working boot layout with one that reads the pool.
+func divertGrubInstall() error {
+	if err := run("chroot", mountRoot, "dpkg-divert", "--divert", grubInstallReal,
+		"--rename", "--add", grubInstallPath); err != nil {
+		return fmt.Errorf("divert grub-install: %w", err)
+	}
+	return writeTargetFile(grubInstallPath, grubInstallWrapper, 0o755)
+}
+
+// verifyBootConfig asserts that the config the machine will actually boot from
+// names the ZFS root. grub-mkconfig degrades quietly when grub-probe cannot
+// read the pool, emitting a config that gets as far as the GRUB menu and then
+// panics with "unable to mount root fs" — which looks like a hardware fault.
+func verifyBootConfig(cfg DiskConfig) error {
+	uuids, err := os.ReadFile(filepath.Join(mountRoot, strings.TrimPrefix(bootToolUUIDList, "/")))
+	if err != nil {
+		return fmt.Errorf("read ESP list: %w", err)
+	}
+	if len(strings.Fields(string(uuids))) != len(cfg.Disks) {
+		return fmt.Errorf("expected %d registered ESPs, found %d — some disks have no bootloader",
+			len(cfg.Disks), len(strings.Fields(string(uuids))))
+	}
+
+	// One ESP is enough to prove grub-mkconfig produced the right thing; they
+	// are all generated by the same code path in the same run.
+	mp := filepath.Join(mountRoot, "var/tmp/verify-esp")
+	if err := os.MkdirAll(mp, 0o755); err != nil {
+		return err
+	}
+	defer func() { _ = os.Remove(mp) }()
+
+	esp := cfg.Disks[0].PartitionPath(espPartNum)
+	if err := run("mount", esp, mp); err != nil {
+		return fmt.Errorf("mount %s for verification: %w", esp, err)
+	}
+	defer func() { _ = run("umount", mp) }()
+
+	grubCfg, err := os.ReadFile(filepath.Join(mp, "grub/grub.cfg"))
+	if err != nil {
+		return fmt.Errorf("read grub.cfg from %s: %w", esp, err)
+	}
+	want := "root=ZFS=" + ZFSRootDataset
+	if !strings.Contains(string(grubCfg), want) {
+		return fmt.Errorf("grub.cfg on %s does not contain %q — the installed system would not find its root filesystem", esp, want)
+	}
+	if !strings.Contains(string(grubCfg), "vmlinuz-") {
+		return fmt.Errorf("grub.cfg on %s references no kernel", esp)
+	}
+	return nil
+}
+
+// writeTargetFile writes a file into the install target.
+func writeTargetFile(path string, content []byte, mode os.FileMode) error {
+	full := filepath.Join(mountRoot, strings.TrimPrefix(path, "/"))
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		return err
+	}
+	if err := os.WriteFile(full, content, mode); err != nil {
+		return fmt.Errorf("write %s: %w", path, err)
+	}
+	// WriteFile does not apply the mode to a file that already exists, and this
+	// path is re-run on a retried install.
+	return os.Chmod(full, mode)
+}
+
+// writeGrubDefaults writes /etc/default/grub and the branding snippet.
+//
+// Both consoles are listed: tty0 stays primary for a local install while ttyS0
+// mirrors kernel output for headless racks and CI. The last console= on the
+// cmdline becomes the system console, so serial-only hardware still works.
+func writeGrubDefaults(cfg DiskConfig) error {
+	cmdline := "console=tty0 console=ttyS0,115200 systemd.show_status=1"
+	if cfg.FS.IsZFS() {
+		// Stated explicitly rather than relying on grub-mkconfig's probe. It
+		// appears after the root= that 10_linux computes, so it wins if the
+		// probe came back with something wrong.
+		cmdline = fmt.Sprintf("root=ZFS=%s boot=zfs %s", ZFSRootDataset, cmdline)
+	}
+	grubDefault := fmt.Sprintf(`GRUB_DEFAULT=0
+GRUB_TIMEOUT=5
+GRUB_DISTRIBUTOR=Spinifex
+GRUB_CMDLINE_LINUX_DEFAULT=""
+GRUB_CMDLINE_LINUX="%s"
+`, cmdline)
+	if err := os.WriteFile(filepath.Join(mountRoot, "etc/default/grub"), []byte(grubDefault), 0o644); err != nil {
+		return fmt.Errorf("write /etc/default/grub: %w", err)
+	}
+
+	// gfxterm must be activated before serial is appended — background_image
+	// silently does nothing in text mode. $prefix is used rather than a literal
+	// /boot/grub so the same snippet is correct whether GRUB's root is the ext4
+	// filesystem or a per-disk ESP.
+	grubTheme := `#!/bin/sh
+exec tail -n +3 $0
+insmod all_video
+insmod font
+if loadfont $prefix/fonts/unicode.pf2; then
+  set gfxmode=auto
+  insmod gfxterm
+  terminal_output gfxterm
+fi
+insmod serial
+if serial --unit=0 --speed=115200 --timeout=1; then
+  terminal_input  --append serial
+  terminal_output --append serial
+fi
+
+# --- Branding ---
+insmod png
+set theme=$prefix/theme.txt
+export theme
+`
+	if err := os.WriteFile(filepath.Join(mountRoot, "etc/grub.d/06_spinifex"), []byte(grubTheme), 0o755); err != nil {
+		return fmt.Errorf("write /etc/grub.d/06_spinifex: %w", err)
+	}
+	return nil
+}
+
+// refreshBootToolNote logs the command an operator needs after replacing a
+// disk. Nothing in the installer can do this for them later.
+func refreshBootToolNote(cfg DiskConfig) {
+	if cfg.FS.IsZFS() {
+		slog.Info("bootloader installed on every pool member",
+			"disks", len(cfg.Disks),
+			"after_disk_replacement", "spinifex-boot-tool format <esp> && spinifex-boot-tool init <esp>")
+	}
+}

cmd/installer/install/config.go

@@ -2,42 +2,107 @@
 // collected configuration.
 package install
 
+import (
+	"fmt"
+	"net"
+	"strings"
+)
+
+// Plane names the three traffic planes a node presents. Each plane is a role
+// bound to a target, not a fixed NIC: the same three roles collapse onto one,
+// two or three interfaces depending on the hardware.
+type Plane string
+
+const (
+	// PlaneWAN carries public north-south traffic: ingress/egress, EIPs and
+	// SNAT/DNAT. It is the OVN external bridge-mapping.
+	PlaneWAN Plane = "wan"
+
+	// PlaneLAN carries internal cluster traffic: predastore replication, the
+	// NATS cluster mesh, OVN NB/SB control, Raft and S3DB.
+	PlaneLAN Plane = "lan"
+
+	// PlaneVPC carries the tenant overlay: OVN Geneve tunnels between nodes.
+	// It is the ovn-encap-ip endpoint.
+	PlaneVPC Plane = "vpc"
+)
+
+// Bridge returns the Linux bridge that carries this plane.
+func (p Plane) Bridge() string { return "br-" + string(p) }
+
+// NetworkRole binds one plane to a network target. An empty Interface means
+// the role is folded onto the next role up the chain (vpc <- lan <- wan) and
+// no bridge of its own is created.
+//
+// VLAN is what allows two roles to share one physical NIC while staying
+// separate planes: each gets its own NIC.<vlan> subinterface and bridge. Two
+// roles on the same Interface with no VLAN are the same plane, not two.
+type NetworkRole struct {
+	// Interface is the physical NIC this role binds to; empty means folded.
+	Interface string
+
+	// VLAN is the 802.1Q tag, or 0 for untagged.
+	VLAN int
+
+	// DHCPMode leaves addressing to the DHCP client; Address/Mask are then unset.
+	DHCPMode bool
+
+	Address string
+	Mask    string
+
+	// Gateway is only meaningful on the wan role — the other planes are
+	// link-local to the rack and must not install a default route.
+	Gateway string
+
+	DNS []string
+
+	// MTU is written as MTUBytes= on the link; 0 leaves the kernel default.
+	// East-west planes typically run 9000, wan stays at 1500.
+	MTU int
+
+	// WiFiSSID and WiFiPass are set when the selected NIC is wireless.
+	WiFiSSID string
+	WiFiPass string
+}
+
+// Folded reports whether this role has no interface of its own and inherits
+// the plane of the role above it.
+func (r NetworkRole) Folded() bool { return r.Interface == "" }
+
+// Link returns the name of the interface carrying this role's address: the
+// physical NIC, or its VLAN subinterface when tagged. Empty when folded.
+func (r NetworkRole) Link() string {
+	if r.Folded() {
+		return ""
+	}
+	if r.VLAN > 0 {
+		return fmt.Sprintf("%s.%d", r.Interface, r.VLAN)
+	}
+	return r.Interface
+}
+
 // Config holds all values collected by the installer UI.
 type Config struct {
-	// Disk is the block device path to install onto (e.g. /dev/sda).
-	Disk string
+	// Storage is the filesystem choice and the ordered set of disks it is
+	// built from. In ext4 mode it holds exactly one disk.
+	Storage DiskConfig
 
-	// WAN interface — always present. The management IP lives on br-wan (a Linux
-	// bridge over the NIC), not on the physical NIC itself. This allows OVN/OVS
-	// to safely attach to the bridge without disrupting host connectivity.
-	WANInterface string
-	WANDHCPMode  bool
-	WANAddress   string   // empty if WANDHCPMode
-	WANMask      string   // empty if WANDHCPMode
-	WANGateway   string   // empty if WANDHCPMode
-	WANDNS       []string // empty if WANDHCPMode; e.g. ["1.1.1.1", "8.8.8.8"]
-	WANWiFiSSID  string   // non-empty if a WiFi NIC was selected
-	WANWiFiPass  string   // non-empty if a WiFi NIC was selected
+	// WAN is always present and can never be folded — a node needs an uplink.
+	// The address lives on br-wan (a Linux bridge over the NIC), not on the
+	// physical NIC itself, so OVN/OVS can attach to the bridge without
+	// disrupting host connectivity.
+	WAN NetworkRole
 
-	// LAN interface — only set when 2+ NICs are present. The Geneve encap IP
-	// for OVN tunnels lives on br-lan. When empty, the WAN address is used.
-	LANInterface string
-	LANDHCPMode  bool
-	LANAddress   string
-	LANMask      string
-	LANDNS       []string
-	LANWiFiSSID  string
-	LANWiFiPass  string
+	// LAN carries internal cluster traffic. Folded onto WAN on single-NIC nodes.
+	LAN NetworkRole
+
+	// VPC carries Geneve overlay traffic and supplies ovn-encap-ip. Folded onto
+	// LAN unless a node dedicates an interface or VLAN to it.
+	VPC NetworkRole
 
 	// Node identity
 	Hostname string
 
-	// ClusterRole is "init" or "join".
-	ClusterRole string
-
-	// JoinAddr is the primary node address (host:port) when ClusterRole is "join".
-	JoinAddr string
-
 	// SkipFormation skips spx admin init/join in firstboot; the provisioning
 	// controller (e.g. bm-bootstrap.sh) owns cluster formation instead.
 	SkipFormation bool
@@ -60,3 +125,111 @@ type Config struct {
 	// Passes --gpu-passthrough to `spx admin init` during firstboot.
 	GPUPassthrough bool
 }
+
+// Resolve returns the role that actually carries the given plane after
+// collapsing folded roles up the chain vpc <- lan <- wan, along with the plane
+// it landed on. A folded vpc on a single-NIC node resolves to the wan role.
+func (c *Config) Resolve(p Plane) (NetworkRole, Plane) {
+	switch p {
+	case PlaneVPC:
+		if !c.VPC.Folded() {
+			return c.VPC, PlaneVPC
+		}
+		return c.Resolve(PlaneLAN)
+	case PlaneLAN:
+		if !c.LAN.Folded() {
+			return c.LAN, PlaneLAN
+		}
+		return c.WAN, PlaneWAN
+	default:
+		return c.WAN, PlaneWAN
+	}
+}
+
+// PlaneAddress returns the configured address for a plane after collapsing.
+// Empty when the resolved role uses DHCP, since no static address is known
+// until the lease is taken.
+func (c *Config) PlaneAddress(p Plane) string {
+	role, _ := c.Resolve(p)
+	if role.DHCPMode {
+		return ""
+	}
+	return role.Address
+}
+
+// PlaneBridge returns the bridge carrying a plane after collapsing, so callers
+// asking for the vpc bridge on a single-NIC node correctly get br-wan.
+func (c *Config) PlaneBridge(p Plane) string {
+	_, landed := c.Resolve(p)
+	return landed.Bridge()
+}
+
+// RoleBinding pairs a plane with the role that carries it.
+type RoleBinding struct {
+	Plane Plane
+	Role  NetworkRole
+}
+
+// Roles returns the bound roles in bridge-creation order, skipping folded ones.
+// wan comes first because br-lan and br-vpc are activated manually after it.
+func (c *Config) Roles() []RoleBinding {
+	out := []RoleBinding{{PlaneWAN, c.WAN}}
+	if !c.LAN.Folded() {
+		out = append(out, RoleBinding{PlaneLAN, c.LAN})
+	}
+	if !c.VPC.Folded() {
+		out = append(out, RoleBinding{PlaneVPC, c.VPC})
+	}
+	return out
+}
+
+// Validate enforces the sharing rules the UI and the headless path must both
+// obey: wan always needs a target, and two roles may only share an interface
+// when distinct VLAN ids keep them on separate broadcast domains.
+func (c *Config) Validate() error {
+	if c.WAN.Folded() {
+		return fmt.Errorf("wan role must be bound to an interface")
+	}
+
+	seen := map[string]Plane{}
+	for _, r := range c.Roles() {
+		link := r.Role.Link()
+		if prev, dup := seen[link]; dup {
+			return fmt.Errorf("%s and %s both bind %s: give each a distinct VLAN id or fold one onto the other",
+				prev, r.Plane, link)
+		}
+		seen[link] = r.Plane
+		if r.Role.VLAN < 0 || r.Role.VLAN > 4094 {
+			return fmt.Errorf("%s: VLAN id %d out of range (1-4094)", r.Plane, r.Role.VLAN)
+		}
+		if r.Role.MTU != 0 && (r.Role.MTU < 68 || r.Role.MTU > 9216) {
+			return fmt.Errorf("%s: MTU %d out of range (68-9216)", r.Plane, r.Role.MTU)
+		}
+		// Resolvers belong to the plane that holds the default route. Accepting
+		// one here and dropping it during generation would leave an unattended
+		// install believing it had set a resolver that was never written.
+		if r.Plane != PlaneWAN && len(r.Role.DNS) > 0 {
+			return fmt.Errorf("%s: DNS servers belong to the wan plane, which holds the default route", r.Plane)
+		}
+		// Netgen converts the mask with net.ParseIP, so a prefix length reaches
+		// it as garbage and fails the install after the disk is partitioned.
+		// Catch it here, where the operator can still fix it.
+		if m := strings.TrimSpace(r.Role.Mask); m != "" && !validDottedMask(m) {
+			return fmt.Errorf("%s: subnet mask %q must be dotted-decimal, e.g. 255.255.255.0", r.Plane, m)
+		}
+	}
+	return nil
+}
+
+// validDottedMask reports whether s is a contiguous dotted-decimal IPv4 subnet
+// mask. Prefix-length form ("24", "/24") is deliberately not accepted: netgen
+// parses the mask with net.ParseIP, so anything else fails the install itself.
+func validDottedMask(s string) bool {
+	ip := net.ParseIP(strings.TrimSpace(s))
+	if ip == nil || ip.To4() == nil {
+		return false
+	}
+	// Size reports zero bits for a non-contiguous mask, which netgen rejects too.
+	_, bits := net.IPMask(ip.To4()).Size()
+	return bits != 0
+}

cmd/installer/install/config_test.go

@@ -0,0 +1,463 @@
+package install
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// Node layouts drawn from the us-west-1 racks, which between them exercise
+// every collapse permutation the installer has to express.
+func threeNIC() *Config { // hydrogen / radon
+	return &Config{
+		WAN: NetworkRole{Interface: "eno6", Address: "216.218.163.99", Mask: "255.255.255.224", Gateway: "216.218.163.97"},
+		LAN: NetworkRole{Interface: "ens1f0np0", Address: "10.0.0.3", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "ens1f1np1", Address: "10.1.0.3", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func vlanSplit() *Config { // helium — one 100gbe carrying lan and vpc
+	return &Config{
+		WAN: NetworkRole{Interface: "enx6c1ff7c1475a", DHCPMode: true},
+		LAN: NetworkRole{Interface: "enp129s0np0", VLAN: 10, Address: "10.0.0.2", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "enp129s0np0", VLAN: 20, Address: "10.1.0.2", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func dualPortTagged() *Config { // us-west-1-az2 — wan and lan share port 1
+	return &Config{
+		WAN: NetworkRole{Interface: "enp1s0f0", VLAN: 30, Address: "216.218.163.10", Mask: "255.255.255.224", Gateway: "216.218.163.97"},
+		LAN: NetworkRole{Interface: "enp1s0f0", VLAN: 10, Address: "10.0.0.10", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "enp1s0f1", VLAN: 20, Address: "10.1.0.10", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func TestResolveCollapsesUpTheChain(t *testing.T) {
+	singleNIC := &Config{WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"}}
+	twoNIC := &Config{
+		WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"},
+		LAN: NetworkRole{Interface: "eth1", Address: "10.0.0.5", Mask: "255.255.0.0"},
+	}
+
+	tests := []struct {
+		name      string
+		cfg       *Config
+		plane     Plane
+		wantAddr  string
+		wantPlane Plane
+	}{
+		{"1 NIC: lan folds onto wan", singleNIC, PlaneLAN, "192.168.1.10", PlaneWAN},
+		{"1 NIC: vpc folds all the way to wan", singleNIC, PlaneVPC, "192.168.1.10", PlaneWAN},
+		{"2 NIC: vpc folds onto lan", twoNIC, PlaneVPC, "10.0.0.5", PlaneLAN},
+		{"2 NIC: lan stays put", twoNIC, PlaneLAN, "10.0.0.5", PlaneLAN},
+		{"3 NIC: vpc is its own plane", threeNIC(), PlaneVPC, "10.1.0.3", PlaneVPC},
+		{"VLAN split: vpc is its own plane", vlanSplit(), PlaneVPC, "10.1.0.2", PlaneVPC},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.cfg.PlaneAddress(tt.plane); got != tt.wantAddr {
+				t.Errorf("PlaneAddress(%s) = %q, want %q", tt.plane, got, tt.wantAddr)
+			}
+			if _, landed := tt.cfg.Resolve(tt.plane); landed != tt.wantPlane {
+				t.Errorf("Resolve(%s) landed on %s, want %s", tt.plane, landed, tt.wantPlane)
+			}
+		})
+	}
+}
+
+// A DHCP plane has no address known at install time; setup-ovn.sh auto-detects
+// it from the default route at boot instead.
+func TestPlaneAddressEmptyUnderDHCP(t *testing.T) {
+	if got := vlanSplit().PlaneAddress(PlaneWAN); got != "" {
+		t.Errorf("PlaneAddress(wan) = %q, want empty for a DHCP role", got)
+	}
+}
+
+func TestValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     *Config
+		wantErr string
+	}{
+		{"three physical NICs", threeNIC(), ""},
+		{"VLAN split on one NIC", vlanSplit(), ""},
+		{"dual port, all tagged", dualPortTagged(), ""},
+		{
+			"wan cannot be folded",
+			&Config{LAN: NetworkRole{Interface: "eth1"}},
+			"wan role must be bound",
+		},
+		{
+			"two roles share a NIC untagged",
+			&Config{
+				WAN: NetworkRole{Interface: "eth0"},
+				LAN: NetworkRole{Interface: "eth0"},
+			},
+			"both bind eth0",
+		},
+		{
+			"duplicate VLAN id on one NIC",
+			&Config{
+				WAN: NetworkRole{Interface: "eth0", VLAN: 10},
+				LAN: NetworkRole{Interface: "eth0", VLAN: 10},
+			},
+			"both bind eth0.10",
+		},
+		{
+			"VLAN id out of range",
+			&Config{WAN: NetworkRole{Interface: "eth0", VLAN: 5000}},
+			"out of range",
+		},
+		{
+			"MTU out of range",
+			&Config{WAN: NetworkRole{Interface: "eth0", MTU: 70000}},
+			"MTU 70000 out of range",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("Validate() = %v, want nil", err)
+				}
+				return
+			}
+			if err == nil {
+				t.Fatalf("Validate() = nil, want error containing %q", tt.wantErr)
+			}
+			if !strings.Contains(err.Error(), tt.wantErr) {
+				t.Errorf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Two roles sharing one NIC must produce a single parent .network carrying both
+// VLAN declarations — not two files racing to configure the same interface.
+func TestWriteParentNICTaggedSharesOneFile(t *testing.T) {
+	dir := t.TempDir()
+	p := &parentNIC{vlans: []string{"enp129s0np0.10", "enp129s0np0.20"}, mtu: 9000}
+	if err := writeParentNIC(dir, "enp129s0np0", p); err != nil {
+		t.Fatal(err)
+	}
+
+	got := readFile(t, dir, "05-spinifex-nic-enp129s0np0.network")
+	for _, want := range []string{
+		"Name=enp129s0np0",
+		"MTUBytes=9000",
+		"VLAN=enp129s0np0.10",
+		"VLAN=enp129s0np0.20",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("parent .network missing %q:\n%s", want, got)
+		}
+	}
+	// A tagged parent is never itself enslaved — the VLAN devices are.
+	if strings.Contains(got, "Bridge=") {
+		t.Errorf("tagged parent must not be bridged directly:\n%s", got)
+	}
+}
+
+func TestWriteNetworkdBridgeVLAN(t *testing.T) {
+	dir := t.TempDir()
+	cfg := vlanSplit()
+	if err := writeNetworkdBridge(dir, PlaneVPC, cfg.VPC, true); err != nil {
+		t.Fatal(err)
+	}
+
+	netdev := readFile(t, dir, "10-spinifex-vpc-vlan.netdev")
+	if !strings.Contains(netdev, "Kind=vlan") || !strings.Contains(netdev, "Id=20") {
+		t.Errorf("VLAN netdev wrong:\n%s", netdev)
+	}
+	if !strings.Contains(netdev, "Name=enp129s0np0.20") {
+		t.Errorf("VLAN netdev name wrong:\n%s", netdev)
+	}
+
+	vlanNet := readFile(t, dir, "11-spinifex-vpc-vlan.network")
+	if !strings.Contains(vlanNet, "Bridge=br-vpc") {
+		t.Errorf("VLAN device must be enslaved to br-vpc:\n%s", vlanNet)
+	}
+
+	brNet := readFile(t, dir, "20-spinifex-vpc.network")
+	if !strings.Contains(brNet, "Address=10.1.0.2/16") {
+		t.Errorf("bridge address wrong:\n%s", brNet)
+	}
+	// Only wan installs a default route; an east-west plane must not.
+	if strings.Contains(brNet, "Gateway=") {
+		t.Errorf("vpc plane must not install a default route:\n%s", brNet)
+	}
+	if !strings.Contains(brNet, "ActivationPolicy=manual") {
+		t.Errorf("east-west bridge must not auto-activate:\n%s", brNet)
+	}
+}
+
+// The wan plane is the one that carries a default route and auto-activates.
+func TestWriteNetworkdBridgeWANUntagged(t *testing.T) {
+	dir := t.TempDir()
+	cfg := threeNIC()
+	if err := writeNetworkdBridge(dir, PlaneWAN, cfg.WAN, false); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := os.Stat(filepath.Join(dir, "10-spinifex-wan-vlan.netdev")); !os.IsNotExist(err) {
+		t.Error("untagged role must not produce a VLAN netdev")
+	}
+	brNet := readFile(t, dir, "20-spinifex-wan.network")
+	if !strings.Contains(brNet, "Gateway=216.218.163.97") {
+		t.Errorf("wan must install its default route:\n%s", brNet)
+	}
+	if strings.Contains(brNet, "ActivationPolicy=manual") {
+		t.Errorf("wan bridge must auto-activate:\n%s", brNet)
+	}
+}
+
+func TestToFirstbootConfigSourcesPlanes(t *testing.T) {
+	cfg := threeNIC()
+	cfg.Hostname = "hydrogen"
+	fb := cfg.toFirstbootConfig()
+
+	if fb.EncapIP != "10.1.0.3" {
+		t.Errorf("EncapIP = %q, want the vpc plane address", fb.EncapIP)
+	}
+	if fb.LANIP != "10.0.0.3" {
+		t.Errorf("LANIP = %q, want the lan plane address", fb.LANIP)
+	}
+	// The public address has to travel separately: spx echoes a concrete --bind
+	// back as the advertise address, so leaving this empty would publish the
+	// internal plane as the node's public dial target.
+	if fb.WANIP != "216.218.163.99" {
+		t.Errorf("WANIP = %q, want the wan plane address", fb.WANIP)
+	}
+
+	// On a single-NIC node all three collapse onto wan.
+	single := &Config{WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"}}
+	fb = single.toFirstbootConfig()
+	if fb.EncapIP != "192.168.1.10" || fb.LANIP != "192.168.1.10" || fb.WANIP != "192.168.1.10" {
+		t.Errorf("single-NIC collapse wrong: EncapIP=%q LANIP=%q WANIP=%q", fb.EncapIP, fb.LANIP, fb.WANIP)
+	}
+}
+
+func readFile(t *testing.T, dir, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(dir, name))
+	if err != nil {
+		t.Fatalf("reading %s: %v", name, err)
+	}
+	return string(b)
+}
+
+// Resolvers belong to the plane holding the default route. The east-west
+// bridges are link-local and activate manually, so a resolver pinned to one is
+// unreachable while resolved still tries it — a timeout in front of every
+// lookup. The TUI cannot express this; the kernel cmdline can, so it is caught.
+func TestValidateRejectsDNSOffWAN(t *testing.T) {
+	cfg := threeNIC()
+	cfg.LAN.DNS = []string{"10.0.0.1"}
+	if err := cfg.Validate(); err == nil {
+		t.Fatal("lan must not carry DNS servers")
+	}
+
+	cfg = threeNIC()
+	cfg.VPC.DNS = []string{"10.1.0.1"}
+	if err := cfg.Validate(); err == nil {
+		t.Fatal("vpc must not carry DNS servers")
+	}
+
+	cfg = threeNIC()
+	cfg.WAN.DNS = []string{"8.8.8.8", "1.1.1.1"}
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("wan carries the resolvers: %v", err)
+	}
+}
+
+// Even if a role somehow carries DNS, generation must not write it onto an
+// east-west bridge.
+func TestWriteNetworkdBridgeKeepsDNSOnWAN(t *testing.T) {
+	dir := t.TempDir()
+	lan := NetworkRole{Interface: "ens1f0np0", Address: "10.0.0.3", Mask: "255.255.0.0", DNS: []string{"10.0.0.1"}}
+	if err := writeNetworkdBridge(dir, PlaneLAN, lan, true); err != nil {
+		t.Fatalf("writeNetworkdBridge: %v", err)
+	}
+	if got := readFile(t, dir, "20-spinifex-lan.network"); strings.Contains(got, "DNS=") {
+		t.Errorf("lan bridge must carry no resolver:\n%s", got)
+	}
+
+	wan := NetworkRole{Interface: "eno6", Address: "216.218.163.99", Mask: "255.255.255.224", Gateway: "216.218.163.97", DNS: []string{"8.8.8.8"}}
+	if err := writeNetworkdBridge(dir, PlaneWAN, wan, false); err != nil {
+		t.Fatalf("writeNetworkdBridge: %v", err)
+	}
+	if got := readFile(t, dir, "20-spinifex-wan.network"); !strings.Contains(got, "DNS=8.8.8.8") {
+		t.Errorf("wan bridge must carry the resolver:\n%s", got)
+	}
+}
+
+func TestSwapSize(t *testing.T) {
+	dir := t.TempDir()
+
+	// Without an override, RAM drives the size and the disk caps it.
+	ram, err := totalRAM()
+	if err != nil {
+		t.Fatalf("totalRAM: %v", err)
+	}
+	got, err := swapSize(dir)
+	if err != nil {
+		t.Fatalf("swapSize: %v", err)
+	}
+	want := ram / 2
+	if ram >= swapRAMThreshold {
+		want = swapLargeBytes
+	}
+	if got > want {
+		t.Errorf("swapSize = %d, must not exceed what RAM implies (%d)", got, want)
+	}
+	if got != 0 && got < swapMinBytes {
+		t.Errorf("swapSize = %d, must be 0 rather than below the %d floor", got, int64(swapMinBytes))
+	}
+
+	// An explicit size wins, so a large node can be told to take the full 32G
+	// and a constrained one can opt out entirely.
+	for _, tt := range []struct {
+		env  string
+		want int64
+	}{
+		{"32G", 32 << 30},
+		{"512M", 512 << 20},
+		{"0", 0},
+		{"1073741824", 1 << 30},
+	} {
+		t.Setenv("SPINIFEX_SWAP_SIZE", tt.env)
+		got, err := swapSize(dir)
+		if err != nil {
+			t.Fatalf("swapSize(%s): %v", tt.env, err)
+		}
+		if got != tt.want {
+			t.Errorf("swapSize(%s) = %d, want %d", tt.env, got, tt.want)
+		}
+	}
+
+	t.Setenv("SPINIFEX_SWAP_SIZE", "banana")
+	if _, err := swapSize(dir); err == nil {
+		t.Error("an unparseable size must be reported, not silently ignored")
+	}
+}
+
+// The swap line must only appear when the file backing it exists — systemd
+// fails the boot's swap unit on a missing file rather than skipping it.
+func TestFstabSwapLineTracksTheFile(t *testing.T) {
+	if _, err := parseSize("8G"); err != nil {
+		t.Fatalf("parseSize: %v", err)
+	}
+	if n, _ := parseSize("2G"); n != 2<<30 {
+		t.Errorf("parseSize(2G) = %d, want %d", n, int64(2)<<30)
+	}
+	if _, err := parseSize("-5G"); err == nil {
+		t.Error("negative sizes must be rejected")
+	}
+}
+
+// Bridge ports and VLAN trunks never reach the "degraded" state
+// systemd-networkd-wait-online requires — the address sits on the bridge above
+// them. Leaving them required makes that unit fail on every boot and stall
+// until it times out, which is what a three-NIC node was doing.
+func TestSlaveLinksAreNotRequiredForOnline(t *testing.T) {
+	dir := t.TempDir()
+
+	// Untagged bridge port.
+	if err := writeParentNIC(dir, "ens4", &parentNIC{bridge: "br-lan", mtu: 9000}); err != nil {
+		t.Fatal(err)
+	}
+	got := readFile(t, dir, "05-spinifex-nic-ens4.network")
+	if !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("bridge port must not gate network-online:\n%s", got)
+	}
+	if !strings.Contains(got, "MTUBytes=9000") {
+		t.Errorf("MTU must survive alongside it:\n%s", got)
+	}
+
+	// VLAN trunk carrying tagged planes.
+	if err := writeParentNIC(dir, "enp129s0np0", &parentNIC{vlans: []string{"enp129s0np0.10"}}); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "05-spinifex-nic-enp129s0np0.network"); !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("VLAN trunk must not gate network-online:\n%s", got)
+	}
+
+	// The VLAN device itself is enslaved too.
+	cfg := vlanSplit()
+	if err := writeNetworkdBridge(dir, PlaneVPC, cfg.VPC, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "11-spinifex-vpc-vlan.network"); !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("VLAN device must not gate network-online:\n%s", got)
+	}
+
+	// The wan bridge is what should gate it, and still does.
+	three := threeNIC()
+	if err := writeNetworkdBridge(dir, PlaneWAN, three.WAN, false); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "20-spinifex-wan.network"); strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("wan bridge must remain required for network-online:\n%s", got)
+	}
+}
+
+// A prefix length reaches netgen's net.ParseIP as garbage and fails the install
+// after the disk has been partitioned, so it has to be rejected up front. The
+// TUI advertised "255.255.255.0 or 24" while only the first ever worked.
+func TestValidateRejectsPrefixLengthMask(t *testing.T) {
+	for _, bad := range []string{"24", "/24", "16", "255.255.0", "255.0.255.0", "banana"} {
+		cfg := threeNIC()
+		cfg.LAN.Mask = bad
+		if err := cfg.Validate(); err == nil {
+			t.Errorf("mask %q must be rejected", bad)
+		} else if !strings.Contains(err.Error(), "dotted-decimal") {
+			t.Errorf("mask %q: error should name the expected form, got: %v", bad, err)
+		}
+	}
+
+	for _, good := range []string{"255.255.255.0", "255.255.0.0", "255.255.255.252", "255.255.255.255"} {
+		cfg := threeNIC()
+		cfg.LAN.Mask = good
+		if err := cfg.Validate(); err != nil {
+			t.Errorf("mask %q must be accepted: %v", good, err)
+		}
+	}
+
+	// A DHCP role carries no mask, and that must stay valid.
+	cfg := threeNIC()
+	cfg.LAN = NetworkRole{Interface: "ens1f0np0", DHCPMode: true}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("a DHCP role has no mask to check: %v", err)
+	}
+}
+
+// A bridged uplink must present the enslaved NIC's address. Left to its default
+// policy, systemd-networkd generates one for the bridge, so the node appears on
+// the wire as an address belonging to no NIC — DHCP requests carry a source that
+// disagrees with their own chaddr and the unicast reply is never delivered.
+func TestBridgeInheritsPortMAC(t *testing.T) {
+	dir := t.TempDir()
+	cfg := threeNIC()
+
+	for _, tc := range []struct {
+		plane  Plane
+		role   NetworkRole
+		manual bool
+	}{
+		{PlaneWAN, cfg.WAN, false},
+		{PlaneLAN, cfg.LAN, true},
+		{PlaneVPC, cfg.VPC, true},
+	} {
+		if err := writeNetworkdBridge(dir, tc.plane, tc.role, tc.manual); err != nil {
+			t.Fatal(err)
+		}
+		got := readFile(t, dir, "20-spinifex-"+string(tc.plane)+".netdev")
+		if !strings.Contains(got, "MACAddressPolicy=none") {
+			t.Errorf("%s bridge must inherit its port's MAC:\n%s", tc.plane, got)
+		}
+	}
+}

cmd/installer/install/disk.go

@@ -0,0 +1,158 @@
+package install
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"time"
+)
+
+// Partition numbers, identical on every disk the installer touches so that a
+// replacement drive can be cloned with `sgdisk -R` and nothing has to be
+// recomputed at 3am.
+const (
+	biosPartNum = 1
+	espPartNum  = 2
+	rootPartNum = 3
+)
+
+// GPT partition type GUIDs, by their sgdisk short code.
+const (
+	typeBIOSBoot = "EF02"
+	typeESP      = "EF00"
+	typeLinuxFS  = "8300"
+	typeZFS      = "BF00"
+)
+
+// Partition geometry. The ESP is sized as Proxmox does: 512MiB is enough for
+// two kernels, but a 1GiB ESP on a real disk costs nothing and keeps room for
+// the extra kernels an operator pins during an upgrade.
+const (
+	espSmallMiB     = 512
+	espLargeMiB     = 1024
+	espLargeDiskGiB = 100
+
+	// biosBootMiB is the BIOS Boot Partition GRUB's i386-pc core.img is embedded
+	// into. GPT leaves no post-MBR gap, so without it there is nowhere to put it.
+	biosBootMiB = 1
+
+	// tailReserveMiB is left unallocated at the end of every disk. A replacement
+	// drive of the "same" size is routinely a few thousand sectors smaller, and
+	// without slack `sgdisk -R` refuses to clone the table onto it — which is
+	// discovered while standing in front of a degraded pool.
+	tailReserveMiB = 1024
+)
+
+// espSizeMiB picks the ESP size for a disk.
+func espSizeMiB(d Disk) int {
+	if d.Bytes >= espLargeDiskGiB<<30 {
+		return espLargeMiB
+	}
+	return espSmallMiB
+}
+
+// partitionDisks lays down an identical GPT on every selected disk. Each disk
+// gets a BIOS boot partition and an ESP whether or not this machine booted in
+// EFI mode: the cost is 1GiB, and it means a board swap or a firmware change
+// from BIOS to UEFI does not leave an unbootable pool.
+func partitionDisks(cfg DiskConfig) error {
+	ptype := typeLinuxFS
+	if cfg.FS.IsZFS() {
+		ptype = typeZFS
+	}
+	for _, d := range cfg.Disks {
+		if err := partitionOne(d, ptype); err != nil {
+			return fmt.Errorf("partition %s: %w", d.Path, err)
+		}
+	}
+	return waitForPartitions(cfg)
+}
+
+// partitionOne writes the GPT for a single disk.
+func partitionOne(d Disk, rootType string) error {
+	// Stale ZFS labels and filesystem signatures survive a new partition table
+	// and make a retried install fail with errors that read like hardware faults.
+	if err := clearDisk(d); err != nil {
+		return fmt.Errorf("clear: %w", err)
+	}
+
+	esp := espSizeMiB(d)
+	if err := run("sgdisk",
+		"-n", fmt.Sprintf("%d:1M:+%dM", biosPartNum, biosBootMiB),
+		"-t", fmt.Sprintf("%d:%s", biosPartNum, typeBIOSBoot),
+		"-c", fmt.Sprintf("%d:bios_boot", biosPartNum),
+		"-n", fmt.Sprintf("%d:0:+%dM", espPartNum, esp),
+		"-t", fmt.Sprintf("%d:%s", espPartNum, typeESP),
+		"-c", fmt.Sprintf("%d:ESP", espPartNum),
+		"-n", fmt.Sprintf("%d:0:-%dM", rootPartNum, tailReserveMiB),
+		"-t", fmt.Sprintf("%d:%s", rootPartNum, rootType),
+		"-c", fmt.Sprintf("%d:root", rootPartNum),
+		d.Path,
+	); err != nil {
+		return err
+	}
+	// A fresh disk-level GUID: cloning a table with `sgdisk -R` copies it, and
+	// two disks sharing one leaves the pool ambiguous to udev.
+	if err := run("sgdisk", "-G", d.Path); err != nil {
+		slog.Warn("could not randomise GPT GUIDs", "disk", d.Path, "err", err)
+	}
+	return nil
+}
+
+// waitForPartitions ensures every partition device node the installer is about
+// to use exists. Trixie's udev is slow enough after a table rewrite that mkfs
+// races it and fails with ENOENT on a device the kernel has already accepted.
+func waitForPartitions(cfg DiskConfig) error {
+	for _, d := range cfg.Disks {
+		// Best-effort: partprobe failing is not fatal, since udev may still act
+		// on the BLKRRPART ioctl sgdisk issued itself.
+		if err := run("partprobe", d.Path); err != nil {
+			slog.Warn("partprobe failed, continuing", "disk", d.Path, "err", err)
+		}
+	}
+	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
+		slog.Warn("udevadm settle failed, continuing", "err", err)
+	}
+
+	deadline := time.Now().Add(15 * time.Second)
+	for _, d := range cfg.Disks {
+		wanted := []string{d.PartitionPath(espPartNum), d.PartitionPath(rootPartNum)}
+		// The pool is built from by-id paths, so their symlinks have to exist
+		// too — udev publishes them later than the kernel device node.
+		if d.ByID != "" {
+			wanted = append(wanted, d.StablePartitionPath(rootPartNum))
+		}
+		for _, part := range wanted {
+			if err := waitForPath(part, deadline); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func waitForPath(path string, deadline time.Time) error {
+	for {
+		if _, err := os.Stat(path); err == nil {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("device %s did not appear within timeout — the kernel or udev did not pick up the new partition table", path)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// formatESPs makes a FAT32 filesystem on every disk's ESP. Each one is
+// independent — there is no RAID here, they are kept in step by content.
+func formatESPs(cfg DiskConfig) error {
+	for _, d := range cfg.Disks {
+		esp := d.PartitionPath(espPartNum)
+		// A distinct label per member so `lsblk -f` on a degraded machine tells
+		// the operator which disk each ESP belongs to.
+		if err := run("mkfs.fat", "-F", "32", "-n", "EFI", esp); err != nil {
+			return fmt.Errorf("format ESP %s: %w", esp, err)
+		}
+	}
+	return nil
+}

cmd/installer/install/diskconfig.go

@@ -0,0 +1,318 @@
+package install
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"strings"
+)
+
+// Pool and dataset names. rpool matches Proxmox, which is what every ZFS
+// recovery guide and forum answer assumes — the name an operator will be
+// reading off a serial console at 3am.
+const (
+	ZFSPoolName    = "rpool"
+	ZFSRootDataset = ZFSPoolName + "/ROOT/spinifex"
+)
+
+// FSType selects the root filesystem and, for ZFS, the vdev topology.
+type FSType string
+
+const (
+	FSExt4      FSType = "ext4"
+	FSZFSRAID0  FSType = "zfs-raid0"
+	FSZFSRAID1  FSType = "zfs-raid1"
+	FSZFSRAID10 FSType = "zfs-raid10"
+	FSZFSRAIDZ1 FSType = "zfs-raidz1"
+	FSZFSRAIDZ2 FSType = "zfs-raidz2"
+	FSZFSRAIDZ3 FSType = "zfs-raidz3"
+)
+
+// AllFSTypes is the selector order presented by the UI. ext4 is first and is
+// the default, so ZFS — which holds the kernel at its shipped version and
+// reserves RAM for the ARC — is always an explicit choice.
+var AllFSTypes = []FSType{
+	FSExt4, FSZFSRAID0, FSZFSRAID1, FSZFSRAID10, FSZFSRAIDZ1, FSZFSRAIDZ2, FSZFSRAIDZ3,
+}
+
+// fsMeta carries the per-topology facts that drive display and validation.
+var fsMeta = map[FSType]struct {
+	label     string
+	minDisks  int
+	redundant bool   // subject to the same-size rule
+	vdev      string // zpool create keyword; empty means a plain stripe
+}{
+	FSExt4:      {"ext4", 1, false, ""},
+	FSZFSRAID0:  {"zfs (RAID0)", 1, false, ""},
+	FSZFSRAID1:  {"zfs (RAID1)", 2, true, "mirror"},
+	FSZFSRAID10: {"zfs (RAID10)", 4, true, "mirror"},
+	FSZFSRAIDZ1: {"zfs (RAIDZ-1)", 3, true, "raidz1"},
+	FSZFSRAIDZ2: {"zfs (RAIDZ-2)", 4, true, "raidz2"},
+	FSZFSRAIDZ3: {"zfs (RAIDZ-3)", 5, true, "raidz3"},
+}
+
+// IsZFS reports whether this mode creates a pool rather than a plain filesystem.
+func (f FSType) IsZFS() bool { return strings.HasPrefix(string(f), "zfs-") }
+
+// Label is the display name, matching proxinstall's wording so the screen is
+// recognisable to anyone who has installed Proxmox.
+func (f FSType) Label() string { return fsMeta[f].label }
+
+// MinDisks is the smallest member count the topology can be built from.
+func (f FSType) MinDisks() int { return fsMeta[f].minDisks }
+
+// Redundant reports whether the topology stores parity or mirrors, and so
+// requires members of matching size. A RAID0 stripe over unequal disks is
+// legitimate and is deliberately not covered.
+func (f FSType) Redundant() bool { return fsMeta[f].redundant }
+
+// vdevKeyword is the zpool create keyword introducing each vdev.
+func (f FSType) vdevKeyword() string { return fsMeta[f].vdev }
+
+// ParseFSType maps a SPINIFEX_FS value to an FSType.
+func ParseFSType(s string) (FSType, error) {
+	f := FSType(strings.ToLower(strings.TrimSpace(s)))
+	if _, ok := fsMeta[f]; !ok {
+		names := make([]string, 0, len(AllFSTypes))
+		for _, t := range AllFSTypes {
+			names = append(names, string(t))
+		}
+		return "", fmt.Errorf("unknown filesystem %q — must be one of: %s", s, strings.Join(names, ", "))
+	}
+	return f, nil
+}
+
+// ZFSOpts are the advanced tunables. A zero value means "use the computed
+// default", which is why Ashift and ARCMaxMiB are plain ints — 0 is not legal
+// for either, so it is unambiguous as a sentinel.
+type ZFSOpts struct {
+	Ashift    int
+	Compress  string
+	Checksum  string
+	Copies    int
+	ARCMaxMiB int
+}
+
+// DiskConfig is the storage half of the installer's configuration. Disks is
+// ordered and the order is significant: RAID10 pairs members two at a time.
+type DiskConfig struct {
+	FS    FSType
+	Disks []Disk
+	ZFS   ZFSOpts
+}
+
+// Primary is the first selected disk — the whole target in ext4 mode, and the
+// disk shown in single-disk summaries.
+func (d DiskConfig) Primary() Disk {
+	if len(d.Disks) == 0 {
+		return Disk{}
+	}
+	return d.Disks[0]
+}
+
+// Paths lists the selected device paths, for logs and confirmation screens.
+func (d DiskConfig) Paths() []string {
+	out := make([]string, len(d.Disks))
+	for i, disk := range d.Disks {
+		out[i] = disk.Path
+	}
+	return out
+}
+
+// Buildable reports whether the selection has enough members, of the right
+// count, to form the topology at all. Capacity and fault tolerance are
+// meaningless until it does — a 3-disk RAIDZ-3 has no geometry to describe.
+func (d DiskConfig) Buildable() bool {
+	if _, ok := fsMeta[d.FS]; !ok {
+		return false
+	}
+	if len(d.Disks) < d.FS.MinDisks() {
+		return false
+	}
+	return d.FS != FSZFSRAID10 || len(d.Disks)%2 == 0
+}
+
+// Requirement describes what the topology still needs, for the preview line
+// shown in place of a capacity that cannot be computed yet.
+func (d DiskConfig) Requirement() string {
+	req := fmt.Sprintf("needs %d disks", d.FS.MinDisks())
+	if d.FS == FSZFSRAID10 {
+		req = "needs an even number of disks, at least 4"
+	}
+	if d.FS.Redundant() {
+		req += " of the same size"
+	}
+	return req
+}
+
+// UsableBytes estimates post-parity capacity, for the geometry preview. It is
+// an operator-facing approximation and deliberately ignores metadata overhead.
+// Returns 0 for a selection that cannot form the topology, since the parity
+// arithmetic below goes to zero and then negative as members run out.
+func (d DiskConfig) UsableBytes() int64 {
+	n := int64(len(d.Disks))
+	if n == 0 || !d.Buildable() {
+		return 0
+	}
+	smallest := d.Disks[0].Bytes
+	for _, disk := range d.Disks {
+		smallest = min(smallest, disk.Bytes)
+	}
+	switch d.FS {
+	case FSExt4, FSZFSRAID0:
+		var total int64
+		for _, disk := range d.Disks {
+			total += disk.Bytes
+		}
+		return total
+	case FSZFSRAID1:
+		return smallest
+	case FSZFSRAID10:
+		return smallest * (n / 2)
+	case FSZFSRAIDZ1:
+		return smallest * (n - 1)
+	case FSZFSRAIDZ2:
+		return smallest * (n - 2)
+	case FSZFSRAIDZ3:
+		return smallest * (n - 3)
+	}
+	return 0
+}
+
+// Tolerated returns the number of simultaneous disk failures the topology
+// survives, for the geometry preview.
+func (d DiskConfig) Tolerated() int {
+	if !d.Buildable() {
+		return 0
+	}
+	switch d.FS {
+	case FSZFSRAID1:
+		return max(len(d.Disks)-1, 0)
+	case FSZFSRAID10:
+		return 1 // one per mirror pair; the honest worst case is one
+	case FSZFSRAIDZ1:
+		return 1
+	case FSZFSRAIDZ2:
+		return 2
+	case FSZFSRAIDZ3:
+		return 3
+	}
+	return 0
+}
+
+// sizeTolerance is the fraction by which pool members may differ in size.
+// Matches Proxmox's zfs_mirror_size_check: anything beyond 10% and the pool
+// silently truncates every member to the smallest.
+const sizeTolerance = 0.10
+
+// bootedEFI reports whether firmware booted this machine in EFI mode. GRUB's
+// i386-pc target cannot read a 4Kn drive, so legacy BIOS constrains selection.
+func bootedEFI() bool {
+	_, err := os.Stat("/sys/firmware/efi")
+	return err == nil
+}
+
+// Validate checks a disk selection against every rule that must hold before any
+// destructive command runs. The UI calls it on each keystroke so failures block
+// Continue, rather than aborting an install that has already repartitioned.
+func (d DiskConfig) Validate() error {
+	if _, ok := fsMeta[d.FS]; !ok {
+		return fmt.Errorf("no filesystem selected")
+	}
+	if len(d.Disks) == 0 {
+		return fmt.Errorf("%s: select at least one disk", d.FS.Label())
+	}
+	if !d.FS.IsZFS() && len(d.Disks) > 1 {
+		return fmt.Errorf("%s: supports a single disk only, %d selected — choose a zfs mode to use them all",
+			d.FS.Label(), len(d.Disks))
+	}
+	if n := len(d.Disks); n < d.FS.MinDisks() {
+		return fmt.Errorf("%s: needs at least %d disks, %d selected", d.FS.Label(), d.FS.MinDisks(), n)
+	}
+	if d.FS == FSZFSRAID10 && len(d.Disks)%2 != 0 {
+		return fmt.Errorf("%s: needs an even number of disks, %d selected", d.FS.Label(), len(d.Disks))
+	}
+
+	seen := map[string]bool{}
+	for _, disk := range d.Disks {
+		if seen[disk.Path] {
+			return fmt.Errorf("%s is selected more than once", disk.Path)
+		}
+		seen[disk.Path] = true
+
+		if disk.LiveMedia {
+			return fmt.Errorf("%s is the installer's own boot media and cannot be erased", disk.Path)
+		}
+		// GRUB's i386-pc target cannot read a 4Kn drive, so on legacy BIOS the
+		// installed system would have no reachable bootloader.
+		if !bootedEFI() && disk.LogicalBlockSize == 4096 {
+			return fmt.Errorf("%s is a 4Kn drive (4096-byte sectors) and this machine booted in legacy BIOS mode — booting from it is not supported; enable UEFI in firmware",
+				disk.Path)
+		}
+		if disk.Bytes < minRootBytes {
+			return fmt.Errorf("%s is too small (%s, need at least %dGiB)",
+				disk.Path, disk.SizeHuman(), minRootBytes>>30)
+		}
+	}
+
+	return d.validateSizes()
+}
+
+// minRootBytes matches Proxmox's hard floor for a root disk.
+const minRootBytes = 2 << 30
+
+// validateSizes enforces the same-size rule where the topology needs it:
+// across all members for mirrors and raidz, and per-pair for RAID10 — a
+// striped mirror only requires each pair to match, not every disk in the pool.
+func (d DiskConfig) validateSizes() error {
+	if !d.FS.Redundant() {
+		return nil
+	}
+	if d.FS == FSZFSRAID10 {
+		for i := 0; i+1 < len(d.Disks); i += 2 {
+			if err := sizesMatch(d.Disks[i], d.Disks[i+1]); err != nil {
+				return fmt.Errorf("%s mirror pair %d: %w", d.FS.Label(), i/2+1, err)
+			}
+		}
+		return nil
+	}
+	for _, disk := range d.Disks[1:] {
+		if err := sizesMatch(d.Disks[0], disk); err != nil {
+			return fmt.Errorf("%s: %w", d.FS.Label(), err)
+		}
+	}
+	return nil
+}
+
+// SizesWithinTolerance reports whether two disks could be members of the same
+// redundant pool. The UI uses it to decide whether suggesting ZFS is honest.
+func SizesWithinTolerance(a, b Disk) bool { return sizesMatch(a, b) == nil }
+
+// sizesMatch reports whether two members are within the tolerance, naming both
+// disks and their sizes — that message is the operator's only diagnostic.
+func sizesMatch(a, b Disk) error {
+	if math.Abs(float64(a.Bytes-b.Bytes)) > float64(a.Bytes)*sizeTolerance {
+		return fmt.Errorf("disks must be the same size within %.0f%%: %s is %s, %s is %s",
+			sizeTolerance*100, a.Path, a.SizeHuman(), b.Path, b.SizeHuman())
+	}
+	return nil
+}
+
+// Warnings returns non-blocking advice for a valid selection. A RAID0 stripe
+// over mismatched disks is legal but almost always a mistake, so it is
+// surfaced without preventing the install — which is what Proxmox does.
+func (d DiskConfig) Warnings() []string {
+	var out []string
+	if d.FS == FSZFSRAID0 && len(d.Disks) > 1 {
+		for _, disk := range d.Disks[1:] {
+			if sizesMatch(d.Disks[0], disk) != nil {
+				out = append(out, "striping disks of different sizes — capacity is usable but a failure of any disk loses the pool")
+				break
+			}
+		}
+	}
+	if d.FS.IsZFS() && d.FS != FSZFSRAID0 && d.Buildable() && d.Tolerated() == 0 {
+		out = append(out, "this topology stores no redundancy")
+	}
+	return out
+}

cmd/installer/install/diskconfig_test.go

@@ -0,0 +1,267 @@
+package install
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDiskConfigValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     DiskConfig
+		wantErr string
+	}{
+		{"ext4 on one disk", DiskConfig{FS: FSExt4, Disks: disks(1, 40)}, ""},
+		{"raidz1 on three matching disks", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)}, ""},
+		{"raid10 on four matching disks", DiskConfig{FS: FSZFSRAID10, Disks: disks(4, 40)}, ""},
+		// The single-drive ZFS case: no redundancy, but checksums, compression
+		// and snapshots still apply.
+		{"raid0 on one disk", DiskConfig{FS: FSZFSRAID0, Disks: disks(1, 40)}, ""},
+
+		{"no filesystem chosen", DiskConfig{Disks: disks(1, 40)}, "no filesystem selected"},
+		{"no disks", DiskConfig{FS: FSZFSRAIDZ1}, "select at least one disk"},
+		{"ext4 cannot span disks", DiskConfig{FS: FSExt4, Disks: disks(2, 40)}, "single disk only"},
+		{"raidz1 needs three", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(2, 40)}, "at least 3 disks"},
+		{"raidz2 needs four", DiskConfig{FS: FSZFSRAIDZ2, Disks: disks(3, 40)}, "at least 4 disks"},
+		{"raidz3 needs five", DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(4, 40)}, "at least 5 disks"},
+		{"raid1 needs two", DiskConfig{FS: FSZFSRAID1, Disks: disks(1, 40)}, "at least 2 disks"},
+		{"raid10 rejects an odd count", DiskConfig{FS: FSZFSRAID10, Disks: disks(5, 40)}, "even number"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			switch {
+			case tt.wantErr == "" && err != nil:
+				t.Fatalf("Validate() = %v, want nil", err)
+			case tt.wantErr != "" && err == nil:
+				t.Fatalf("Validate() = nil, want error containing %q", tt.wantErr)
+			case tt.wantErr != "" && !strings.Contains(err.Error(), tt.wantErr):
+				t.Fatalf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestValidateRejectsDuplicateDisk(t *testing.T) {
+	d := disk("a", 40)
+	err := DiskConfig{FS: FSZFSRAID1, Disks: []Disk{d, d}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "more than once") {
+		t.Fatalf("Validate() = %v, want a duplicate-selection error", err)
+	}
+}
+
+func TestValidateRejectsLiveMedia(t *testing.T) {
+	d := disk("a", 40)
+	d.LiveMedia = true
+	err := DiskConfig{FS: FSExt4, Disks: []Disk{d}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "boot media") {
+		t.Fatalf("Validate() = %v, want the installer's own media to be refused", err)
+	}
+}
+
+func TestValidateRejectsTooSmallDisk(t *testing.T) {
+	err := DiskConfig{FS: FSExt4, Disks: []Disk{disk("a", 1)}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "too small") {
+		t.Fatalf("Validate() = %v, want a minimum-size error", err)
+	}
+}
+
+// A pool truncates every member to the smallest, so mismatched disks in a
+// redundant topology silently throw capacity away.
+func TestValidateSizeTolerance(t *testing.T) {
+	tests := []struct {
+		name    string
+		fs      FSType
+		sizes   []int64
+		wantErr bool
+	}{
+		{"identical disks", FSZFSRAIDZ1, []int64{40, 40, 40}, false},
+		{"within 10%", FSZFSRAIDZ1, []int64{100, 95, 92}, false},
+		{"beyond 10%", FSZFSRAIDZ1, []int64{100, 100, 80}, true},
+		{"mirror with a mismatched member", FSZFSRAID1, []int64{100, 50}, true},
+		// A stripe has no parity to waste, so mismatched sizes are a warning
+		// rather than a refusal — matching Proxmox.
+		{"raid0 permits mismatched disks", FSZFSRAID0, []int64{100, 40}, false},
+		{"raid10 checks each pair, not the whole pool", FSZFSRAID10, []int64{40, 40, 100, 100}, false},
+		{"raid10 rejects a mismatched pair", FSZFSRAID10, []int64{40, 100, 100, 100}, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			members := make([]Disk, len(tt.sizes))
+			for i, gib := range tt.sizes {
+				members[i] = disk(string(rune('a'+i)), gib)
+			}
+			err := DiskConfig{FS: tt.fs, Disks: members}.Validate()
+			if tt.wantErr && err == nil {
+				t.Fatalf("Validate() = nil, want a size-mismatch error")
+			}
+			if !tt.wantErr && err != nil {
+				t.Fatalf("Validate() = %v, want nil", err)
+			}
+		})
+	}
+}
+
+func TestWarningsFlagMismatchedStripe(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAID0, Disks: []Disk{disk("a", 100), disk("b", 40)}}
+	warnings := cfg.Warnings()
+	if len(warnings) == 0 {
+		t.Fatal("a stripe over mismatched disks should warn")
+	}
+}
+
+func TestUsableBytes(t *testing.T) {
+	tests := []struct {
+		name    string
+		fs      FSType
+		n       int
+		wantGiB int64
+	}{
+		{"stripe adds every disk", FSZFSRAID0, 3, 120},
+		{"mirror gives one disk", FSZFSRAID1, 2, 40},
+		{"raid10 gives half", FSZFSRAID10, 4, 80},
+		{"raidz1 loses one to parity", FSZFSRAIDZ1, 3, 80},
+		{"raidz2 loses two", FSZFSRAIDZ2, 4, 80},
+		{"raidz3 loses three", FSZFSRAIDZ3, 5, 80},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if got := cfg.UsableBytes() >> 30; got != tt.wantGiB {
+				t.Errorf("UsableBytes() = %dG, want %dG", got, tt.wantGiB)
+			}
+		})
+	}
+}
+
+// The parity arithmetic goes to zero and then negative as members run out, and
+// a preview reading "-40GiB usable" is worse than no figure at all.
+func TestUsableBytesIsZeroWhenTopologyCannotBeBuilt(t *testing.T) {
+	tests := []struct {
+		name string
+		fs   FSType
+		n    int
+	}{
+		{"raidz3 with exactly its parity count", FSZFSRAIDZ3, 3},
+		{"raidz3 with fewer disks than parity", FSZFSRAIDZ3, 2},
+		{"raidz2 with two disks", FSZFSRAIDZ2, 2},
+		{"raidz1 with one disk", FSZFSRAIDZ1, 1},
+		{"mirror with one disk", FSZFSRAID1, 1},
+		{"raid10 with an odd count", FSZFSRAID10, 3},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if cfg.Buildable() {
+				t.Fatalf("Buildable() = true for %s across %d disks", tt.fs, tt.n)
+			}
+			if got := cfg.UsableBytes(); got != 0 {
+				t.Errorf("UsableBytes() = %d, want 0", got)
+			}
+			if got := cfg.Tolerated(); got != 0 {
+				t.Errorf("Tolerated() = %d, want 0", got)
+			}
+		})
+	}
+}
+
+// A selection that cannot be built must not also be described as unredundant:
+// the operator's problem is the disk count, not the topology.
+func TestWarningsStaySilentUntilBuildable(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(3, 40)}
+	if got := cfg.Warnings(); len(got) != 0 {
+		t.Errorf("Warnings() = %v, want none", got)
+	}
+}
+
+func TestRequirementNamesTheSameSizeRule(t *testing.T) {
+	if got := (DiskConfig{FS: FSZFSRAIDZ2}).Requirement(); !strings.Contains(got, "4 disks") ||
+		!strings.Contains(got, "same size") {
+		t.Errorf("Requirement() = %q, want the disk count and the same-size rule", got)
+	}
+	if got := (DiskConfig{FS: FSZFSRAID10}).Requirement(); !strings.Contains(got, "even") {
+		t.Errorf("Requirement() = %q, want the even-count rule", got)
+	}
+	// A stripe has no matching requirement, so claiming one would be wrong.
+	if got := (DiskConfig{FS: FSZFSRAID0}).Requirement(); strings.Contains(got, "same size") {
+		t.Errorf("Requirement() = %q, want no same-size rule for a stripe", got)
+	}
+}
+
+func TestTolerated(t *testing.T) {
+	tests := []struct {
+		fs   FSType
+		n    int
+		want int
+	}{
+		{FSExt4, 1, 0},
+		{FSZFSRAID0, 3, 0},
+		{FSZFSRAID1, 2, 1},
+		{FSZFSRAID1, 3, 2},
+		{FSZFSRAID10, 4, 1},
+		{FSZFSRAIDZ1, 3, 1},
+		{FSZFSRAIDZ2, 4, 2},
+		{FSZFSRAIDZ3, 5, 3},
+	}
+	for _, tt := range tests {
+		t.Run(string(tt.fs), func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if got := cfg.Tolerated(); got != tt.want {
+				t.Errorf("Tolerated() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseFSType(t *testing.T) {
+	for _, fs := range AllFSTypes {
+		got, err := ParseFSType(strings.ToUpper(string(fs)))
+		if err != nil {
+			t.Errorf("ParseFSType(%q): %v", fs, err)
+			continue
+		}
+		if got != fs {
+			t.Errorf("ParseFSType(%q) = %q", fs, got)
+		}
+	}
+	// The error has to list the valid values: it is read off a serial console
+	// by someone whose unattended install just stopped.
+	_, err := ParseFSType("btrfs")
+	if err == nil || !strings.Contains(err.Error(), "zfs-raidz1") {
+		t.Fatalf("ParseFSType(btrfs) = %v, want an error listing the accepted values", err)
+	}
+}
+
+func TestPartitionPathSeparator(t *testing.T) {
+	tests := []struct{ dev, want string }{
+		{"/dev/sda", "/dev/sda3"},
+		// A trailing digit needs a 'p' or the name is ambiguous.
+		{"/dev/nvme0n1", "/dev/nvme0n1p3"},
+		{"/dev/vda", "/dev/vda3"},
+	}
+	for _, tt := range tests {
+		if got := (Disk{Path: tt.dev}).PartitionPath(3); got != tt.want {
+			t.Errorf("PartitionPath(%s) = %s, want %s", tt.dev, got, tt.want)
+		}
+	}
+}
+
+// udev names partition links with a -partN suffix regardless of the kernel's
+// separator, so the by-id form must not reuse the 'p' rule.
+func TestStablePartitionPathUsesPartSuffix(t *testing.T) {
+	d := Disk{Path: "/dev/nvme0n1", ByID: "/dev/disk/by-id/nvme-SAMSUNG_1234"}
+	if got := d.StablePartitionPath(3); got != "/dev/disk/by-id/nvme-SAMSUNG_1234-part3" {
+		t.Errorf("StablePartitionPath() = %s", got)
+	}
+}
+
+func TestESPSizeMiB(t *testing.T) {
+	if got := espSizeMiB(disk("a", 40)); got != espSmallMiB {
+		t.Errorf("espSizeMiB(40G) = %d, want %d", got, espSmallMiB)
+	}
+	if got := espSizeMiB(disk("a", 500)); got != espLargeMiB {
+		t.Errorf("espSizeMiB(500G) = %d, want %d", got, espLargeMiB)
+	}
+}

cmd/installer/install/install.go

@@ -11,6 +11,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"slices"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -19,40 +20,55 @@ import (
 	"github.com/mulgadc/spinifex/cmd/installer/systemd"
 )
 
-const (
-	mountRoot = "/mnt/spinifex-install"
-	efiPart   = mountRoot + "/boot/efi"
-)
+// mountRoot is the install target. A variable, not a constant, so tests can
+// drive the file-writing steps against a temp tree rather than a real mount.
+var mountRoot = "/mnt/spinifex-install"
+
+// efiPart is the ESP mount point inside the target. Derived on each call so it
+// cannot go stale against mountRoot.
+func efiPart() string { return filepath.Join(mountRoot, "boot/efi") }
+
+// step is one named unit of work in the install sequence.
+type step struct {
+	name string
+	fn   func() error
+}
 
 // Run executes all installation steps in order. It is intentionally sequential
 // and explicit — each step is visible in logs so failures are easy to diagnose.
 func Run(cfg *Config) error {
+	// Re-checked here even though the UI and the headless path both validate:
+	// this is the last point before anything is erased, and it is the only one
+	// every caller must pass through.
+	if err := cfg.Storage.Validate(); err != nil {
+		return fmt.Errorf("storage: %w", err)
+	}
+
 	// The live environment may not have /sbin or /usr/sbin in PATH. Set it
 	// explicitly so exec.Command's LookPath finds system binaries like grub-install.
 	_ = os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
 
-	// Unmount unconditionally on exit so a failed step never leaves partitions
-	// mounted in the live environment, which would cause a retry to double-mount.
-	defer func() {
-		_ = run("umount", efiPart)
-		_ = run("umount", mountRoot)
-	}()
+	// Unmount unconditionally on exit so a failed step never leaves the target
+	// mounted in the live environment, which would make a retry double-mount or,
+	// on ZFS, fail with "pool is busy".
+	defer cleanupTarget(cfg.Storage)
 
-	steps := []struct {
-		name string
-		fn   func() error
-	}{
-		{"partition disk", func() error { return partitionDisk(cfg.Disk) }},
-		{"format partitions", func() error { return formatPartitions(cfg.Disk) }},
-		{"mount partitions", func() error { return mountPartitions(cfg.Disk) }},
-		{"copy rootfs", copyRootfs},
-		{"write fstab", func() error { return writeFstab(cfg.Disk) }},
-		{"install spinifex", func() error { return installSpinifex(cfg) }},
-		{"write network config", func() error { return writeNetworkConfig(cfg) }},
-		{"write firstboot service", func() error { return firstboot.Write(mountRoot, cfg.toFirstbootConfig()) }},
-		{"install bootloader", func() error { return installBootloader(cfg.Disk) }},
-		{"install CA cert", func() error { return installCACert(cfg) }},
+	steps := []step{{"partition disks", func() error { return partitionDisks(cfg.Storage) }}}
+	if cfg.Storage.FS.IsZFS() {
+		steps = append(steps, zfsRootSteps(cfg)...)
+	} else {
+		steps = append(steps, ext4RootSteps(cfg)...)
 	}
+	steps = append(steps,
+		step{"copy rootfs", func() error { return copyRootfs(cfg.Storage) }},
+		step{"create swap", func() error { return createSwap(cfg.Storage) }},
+		step{"write fstab", func() error { return writeFstab(cfg.Storage) }},
+		step{"install spinifex", func() error { return installSpinifex(cfg) }},
+		step{"write network config", func() error { return writeNetworkConfig(cfg) }},
+		step{"write firstboot service", func() error { return firstboot.Write(mountRoot, cfg.toFirstbootConfig()) }},
+		step{"install bootloader", func() error { return installBootloader(cfg.Storage) }},
+		step{"install CA cert", func() error { return installCACert(cfg) }},
+	)
 
 	for _, s := range steps {
 		slog.Info("installer", "step", s.name)
@@ -62,90 +78,83 @@ func Run(cfg *Config) error {
 	}
 
 	slog.Info("installation complete")
+	refreshBootToolNote(cfg.Storage)
 	fireInstallCallback()
 	promptRemoveUSB()
 	return reboot()
 }
 
-func partitionDisk(disk string) error {
-	// GPT table with three partitions:
-	//   p1: 1MiB BIOS Boot Partition — required for grub-install i386-pc on GPT
-	//   p2: 512MiB EFI System Partition
-	//   p3: remainder as root (ext4)
-	if err := run("parted", "--script", disk,
-		"mklabel", "gpt",
-		"mkpart", "bios_boot", "1MiB", "2MiB",
-		"set", "1", "bios_grub", "on",
-		"mkpart", "ESP", "fat32", "2MiB", "514MiB",
-		"set", "2", "esp", "on",
-		"mkpart", "root", "ext4", "514MiB", "100%",
-	); err != nil {
+// ext4RootSteps formats and mounts a single-disk ext4 root.
+func ext4RootSteps(cfg *Config) []step {
+	return []step{
+		{"format partitions", func() error { return formatPartitions(cfg.Storage) }},
+		{"mount partitions", func() error { return mountPartitions(cfg.Storage) }},
+	}
+}
+
+// zfsRootSteps builds the root pool. The ESPs are formatted here rather than
+// with the bootloader so that a failure to make a filesystem on one of them is
+// reported before the rootfs has been copied.
+func zfsRootSteps(cfg *Config) []step {
+	// Resolved once, before any step runs, so the values written to the pool,
+	// to modprobe.d and to the daemon's reserve are all the same numbers.
+	var opts ZFSOpts
+	return []step{
+		{"format ESPs", func() error { return formatESPs(cfg.Storage) }},
+		{"load zfs module", loadZFSModule},
+		{"create zfs pool", func() error {
+			opts = resolveZFSOpts(cfg.Storage)
+			slog.Info("zfs pool", "topology", cfg.Storage.FS, "disks", len(cfg.Storage.Disks),
+				"ashift", opts.Ashift, "compress", opts.Compress, "arc_max_mib", opts.ARCMaxMiB)
+			return createPool(cfg.Storage, opts)
+		}},
+		{"create zfs datasets", createDatasets},
+		{"configure zfs", func() error { return writeZFSSystemConfig(opts) }},
+	}
+}
+
+// cleanupTarget releases everything the install mounted, in the right order for
+// the filesystem in use.
+func cleanupTarget(cfg DiskConfig) {
+	unbindChrootMounts()
+	if cfg.FS.IsZFS() {
+		// Recursive: the layout is a dozen nested datasets, and leaving one
+		// mounted is enough to block the export.
+		_ = runQuiet("umount", "-R", mountRoot)
+		exportPool()
+		return
+	}
+	_ = runQuiet("umount", efiPart())
+	_ = runQuiet("umount", mountRoot)
+}
+
+func formatPartitions(cfg DiskConfig) error {
+	if err := formatESPs(cfg); err != nil {
 		return err
 	}
-	// Force the kernel to re-read the partition table and wait for udev to
-	// create the partition device nodes. Without this, mkfs.fat in the next
-	// step may race and fail with "No such file or directory" on /dev/sda2 —
-	// the kernel has accepted the new layout but /dev hasn't been populated
-	// yet. Trixie's udev seems slower at this than Bookworm's was.
-	return waitForPartitions(disk)
+	return run("mkfs.ext4", "-F", cfg.Primary().PartitionPath(rootPartNum))
 }
 
-// waitForPartitions ensures the EFI and root partition device nodes exist
-// after parted creates them. It runs partprobe (kernel re-read) and
-// udevadm settle (wait for queued events), then polls /dev for the files.
-func waitForPartitions(disk string) error {
-	// Best-effort: partprobe failure isn't fatal — udev may still pick up
-	// the change from the BLKRRPART ioctl that parted itself issued.
-	if err := run("partprobe", disk); err != nil {
-		slog.Warn("partprobe failed, continuing", "disk", disk, "err", err)
-	}
-	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
-		slog.Warn("udevadm settle failed, continuing", "err", err)
-	}
-	efi, root := partitionPaths(disk)
-	deadline := time.Now().Add(15 * time.Second)
-	for _, part := range []string{efi, root} {
-		for {
-			if _, err := os.Stat(part); err == nil {
-				break
-			}
-			if time.Now().After(deadline) {
-				return fmt.Errorf("partition device %s did not appear within timeout — kernel/udev did not pick up new partition table", part)
-			}
-			time.Sleep(200 * time.Millisecond)
-		}
-	}
-	return nil
-}
-
-func formatPartitions(disk string) error {
-	efi, root := partitionPaths(disk)
-	if err := run("mkfs.fat", "-F32", efi); err != nil {
-		return err
-	}
-	return run("mkfs.ext4", "-F", root)
-}
-
-func mountPartitions(disk string) error {
-	_, root := partitionPaths(disk)
+func mountPartitions(cfg DiskConfig) error {
+	d := cfg.Primary()
 	if err := os.MkdirAll(mountRoot, 0o755); err != nil {
 		return err
 	}
-	if err := run("mount", root, mountRoot); err != nil {
+	if err := run("mount", d.PartitionPath(rootPartNum), mountRoot); err != nil {
 		return err
 	}
-	if err := os.MkdirAll(efiPart, 0o755); err != nil {
+	if err := os.MkdirAll(efiPart(), 0o755); err != nil {
 		return err
 	}
-	efi, _ := partitionPaths(disk)
-	return run("mount", efi, efiPart)
+	return run("mount", d.PartitionPath(espPartNum), efiPart())
 }
 
 // copyRootfs copies the live squashfs environment onto the target disk using
 // rsync. This is the air-gapped alternative to debootstrap — all packages are
 // already embedded in the ISO so no network access is required.
-func copyRootfs() error {
-	if err := run("rsync", "-aHAX", "--delete", "--info=progress2",
+func copyRootfs(cfg DiskConfig) error {
+	args := []string{
+		"-aHAX", "--delete", "--info=progress2",
 		"--exclude=/proc",
 		"--exclude=/sys",
 		"--exclude=/dev",
@@ -161,8 +170,10 @@ func copyRootfs() error {
 		"--exclude=/etc/ssh/ssh_host_*",
 		"--exclude=/lost+found",
 		"--exclude=/boot/efi",
-		"/", mountRoot+"/",
-	); err != nil {
+	}
+	args = append(args, datasetProtectFilters(cfg)...)
+	args = append(args, "/", mountRoot+"/")
+	if err := run("rsync", args...); err != nil {
 		return err
 	}
 
@@ -278,7 +289,7 @@ func installSpinifex(cfg *Config) error {
 	// MANAGEMENT_IFACE is the bridge (br-wan), not the physical NIC.
 	// MANAGEMENT_IP is empty for DHCP — banner's --boot-check fills it in at boot.
 	nodeConf := fmt.Sprintf("MANAGEMENT_IP=%s\nMANAGEMENT_IFACE=br-wan\nNODE_HOSTNAME=%s\n",
-		cfg.WANAddress, cfg.Hostname)
+		cfg.WAN.Address, cfg.Hostname)
 	confDir := filepath.Join(mountRoot, "etc/spinifex")
 	if err := os.MkdirAll(confDir, 0o755); err != nil {
 		return err
@@ -309,23 +320,47 @@ func writeNetworkConfig(cfg *Config) error {
 		return err
 	}
 
-	if err := writeNetworkdBridge(netdDir, cfg.WANInterface, "br-wan", false,
-		cfg.WANDHCPMode, cfg.WANAddress, cfg.WANMask, cfg.WANGateway, cfg.WANDNS,
-		cfg.WANWiFiSSID, cfg.WANWiFiPass); err != nil {
-		return err
+	if err := cfg.Validate(); err != nil {
+		return fmt.Errorf("network roles: %w", err)
 	}
 
-	if cfg.LANInterface != "" {
-		if err := writeNetworkdBridge(netdDir, cfg.LANInterface, "br-lan", true,
-			cfg.LANDHCPMode, cfg.LANAddress, cfg.LANMask, "", cfg.LANDNS,
-			cfg.LANWiFiSSID, cfg.LANWiFiPass); err != nil {
+	// A physical NIC is configured once even when two roles share it via
+	// VLANs, so parent-interface files are keyed by NIC rather than by plane.
+	parents := map[string]*parentNIC{}
+	for _, rb := range cfg.Roles() {
+		p, ok := parents[rb.Role.Interface]
+		if !ok {
+			p = &parentNIC{}
+			parents[rb.Role.Interface] = p
+		}
+		if rb.Role.VLAN > 0 {
+			p.vlans = append(p.vlans, rb.Role.Link())
+		} else {
+			p.bridge = rb.Plane.Bridge()
+		}
+		// The parent must carry at least the largest MTU of any VLAN riding it.
+		if rb.Role.MTU > p.mtu {
+			p.mtu = rb.Role.MTU
+		}
+	}
+	for iface, p := range parents {
+		if err := writeParentNIC(netdDir, iface, p); err != nil {
 			return err
 		}
-		// br-lan timing is controlled by spinifex-lan-bridge.service, not
-		// networkd auto-activation — ActivationPolicy=manual in the .network
-		// file ensures networkd creates the bridge but does not bring it up.
-		if err := systemd.WriteLANBridgeUnit(mountRoot); err != nil {
-			return fmt.Errorf("lan bridge unit: %w", err)
+	}
+
+	for _, rb := range cfg.Roles() {
+		// Only br-wan auto-activates; the east-west bridges are brought up by
+		// their own unit after network-online.target so a missing cable or a
+		// DHCP timeout on them can never stall the management path.
+		manual := rb.Plane != PlaneWAN
+		if err := writeNetworkdBridge(netdDir, rb.Plane, rb.Role, manual); err != nil {
+			return err
+		}
+		if manual {
+			if err := systemd.WriteBridgeUnit(mountRoot, rb.Plane.Bridge()); err != nil {
+				return fmt.Errorf("%s bridge unit: %w", rb.Plane, err)
+			}
 		}
 	}
 
@@ -353,9 +388,9 @@ func writeNetworkConfig(cfg *Config) error {
 
 	// Disable IPv6 via sysctl — belt-and-suspenders alongside IPv6AcceptRA=no
 	// in the networkd .network files.
-	bridges := []string{"br-wan"}
-	if cfg.LANInterface != "" {
-		bridges = append(bridges, "br-lan")
+	var bridges []string
+	for _, rb := range cfg.Roles() {
+		bridges = append(bridges, rb.Plane.Bridge())
 	}
 	var sysctl strings.Builder
 	sysctl.WriteString("# Generated by Spinifex installer — IPv6 disabled on management bridges\n")
@@ -377,10 +412,14 @@ func writeNetworkConfig(cfg *Config) error {
 		return err
 	}
 	var udevRules strings.Builder
-	for _, iface := range []string{cfg.WANInterface, cfg.LANInterface} {
-		if iface == "" {
+	pinned := map[string]bool{}
+	for _, rb := range cfg.Roles() {
+		iface := rb.Role.Interface
+		// Two roles can share one NIC via VLANs; pin the physical name once.
+		if iface == "" || pinned[iface] {
 			continue
 		}
+		pinned[iface] = true
 		mac, err := os.ReadFile("/sys/class/net/" + iface + "/address")
 		if err != nil {
 			slog.Warn("writeNetworkConfig: could not read NIC MAC, skipping udev pin", "iface", iface, "err", err)
@@ -395,35 +434,90 @@ func writeNetworkConfig(cfg *Config) error {
 	return nil
 }
 
-// writeNetworkdBridge writes the three systemd-networkd files that configure a
-// physical NIC enslaved to a Linux bridge:
+// parentNIC accumulates how one physical interface is used across roles. A NIC
+// is either enslaved directly to a bridge (untagged) or carries VLAN
+// subinterfaces — never both for the same plane.
+type parentNIC struct {
+	bridge string
+	vlans  []string
+	mtu    int
+}
+
+// writeParentNIC writes the .network file for a physical interface. An
+// untagged NIC is enslaved straight to its bridge; a tagged one declares its
+// VLAN subinterfaces and stays unbridged, since the VLAN devices are what get
+// enslaved.
+func writeParentNIC(dir, iface string, p *parentNIC) error {
+	var b strings.Builder
+	fmt.Fprintf(&b, "[Match]\nName=%s\n\n", iface)
+	// A physical NIC here is either a bridge port or a VLAN trunk, and neither
+	// carries the address — the bridge above it does. Such a link never reaches
+	// the "degraded" state systemd-networkd-wait-online requires, so leaving it
+	// required makes that unit fail on every boot and stall until it times out.
+	b.WriteString("[Link]\nRequiredForOnline=no\n")
+	if p.mtu > 0 {
+		fmt.Fprintf(&b, "MTUBytes=%d\n", p.mtu)
+	}
+	b.WriteString("\n[Network]\n")
+	if p.bridge != "" {
+		fmt.Fprintf(&b, "Bridge=%s\n", p.bridge)
+	}
+	for _, v := range p.vlans {
+		fmt.Fprintf(&b, "VLAN=%s\n", v)
+	}
+	b.WriteString("IPv6AcceptRA=no\n")
+	return os.WriteFile(filepath.Join(dir, fmt.Sprintf("05-spinifex-nic-%s.network", iface)), []byte(b.String()), 0o644)
+}
+
+// writeNetworkdBridge writes the systemd-networkd files that put a plane's
+// address on its own Linux bridge:
 //
-//   - {prio}-spinifex-{suffix}-nic.network  enslaves the NIC to the bridge
-//   - {prio+1}-spinifex-{suffix}.netdev     declares the bridge device
-//   - {prio+1}-spinifex-{suffix}.network    configures IP on the bridge
+//   - 1{n}-spinifex-{plane}-vlan.netdev   VLAN subinterface (tagged roles only)
+//   - 1{n}-spinifex-{plane}-vlan.network  enslaves the VLAN device to the bridge
+//   - 2{n}-spinifex-{plane}.netdev        declares the bridge device
+//   - 2{n}-spinifex-{plane}.network       configures IP on the bridge
+//
+// The physical NIC itself is written separately by writeParentNIC, because two
+// roles may share one NIC and it must be configured exactly once.
 //
 // manual=true sets ActivationPolicy=manual so networkd creates the bridge but
-// does not auto-activate it — used for br-lan, which spinifex-lan-bridge.service
-// activates after network-online.target.
-func writeNetworkdBridge(dir, nicIface, bridgeName string, manual, dhcp bool,
-	addr, mask, gw string, dns []string, wifiSSID, wifiPass string) error {
-	suffix := strings.TrimPrefix(bridgeName, "br-")
-	nicPrio, brPrio := 10, 11
-	if manual {
-		nicPrio, brPrio = 12, 13
+// does not auto-activate it — used for br-lan and br-vpc, which their own
+// spinifex-{plane}-bridge.service activates after network-online.target.
+func writeNetworkdBridge(dir string, plane Plane, role NetworkRole, manual bool) error {
+	name := string(plane)
+	bridgeName := plane.Bridge()
+
+	if role.VLAN > 0 {
+		link := role.Link()
+		vlanNetdev := fmt.Sprintf("[NetDev]\nName=%s\nKind=vlan\n\n[VLAN]\nId=%d\n", link, role.VLAN)
+		if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("10-spinifex-%s-vlan.netdev", name)), []byte(vlanNetdev), 0o644); err != nil {
+			return err
+		}
+		var v strings.Builder
+		fmt.Fprintf(&v, "[Match]\nName=%s\n\n", link)
+		// Enslaved to the bridge below, so it is not what "online" means here.
+		v.WriteString("[Link]\nRequiredForOnline=no\n")
+		if role.MTU > 0 {
+			fmt.Fprintf(&v, "MTUBytes=%d\n", role.MTU)
+		}
+		fmt.Fprintf(&v, "\n[Network]\nBridge=%s\nIPv6AcceptRA=no\n", bridgeName)
+		if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("11-spinifex-%s-vlan.network", name)), []byte(v.String()), 0o644); err != nil {
+			return err
+		}
 	}
 
-	// NIC .network — enslaves the physical NIC to the bridge.
-	nicNet := fmt.Sprintf("[Match]\nName=%s\n\n[Network]\nBridge=%s\n", nicIface, bridgeName)
-	nicFile := fmt.Sprintf("%02d-spinifex-%s-nic.network", nicPrio, suffix)
-	if err := os.WriteFile(filepath.Join(dir, nicFile), []byte(nicNet), 0o644); err != nil {
-		return err
+	// Without a policy systemd-networkd invents a MAC for the bridge, so the node
+	// appears on the wire as an address that belongs to no NIC — the DHCP server
+	// sees a request whose source does not match its own chaddr and the unicast
+	// reply goes nowhere. "none" leaves the kernel to inherit the enslaved port's
+	// address, which is what a bridged uplink is supposed to present, and keeps
+	// it stable across boots for switch MAC tables and DHCP reservations.
+	brNetdev := fmt.Sprintf("[NetDev]\nName=%s\nKind=bridge\nMACAddressPolicy=none\n", bridgeName)
+	if role.MTU > 0 {
+		brNetdev += fmt.Sprintf("MTUBytes=%d\n", role.MTU)
 	}
-
-	// Bridge .netdev — declares the bridge device.
-	brNetdev := fmt.Sprintf("[NetDev]\nName=%s\nKind=bridge\n\n[Bridge]\nSTP=no\n", bridgeName)
-	brNetdevFile := fmt.Sprintf("%02d-spinifex-%s.netdev", brPrio, suffix)
-	if err := os.WriteFile(filepath.Join(dir, brNetdevFile), []byte(brNetdev), 0o644); err != nil {
+	brNetdev += "\n[Bridge]\nSTP=no\n"
+	if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("20-spinifex-%s.netdev", name)), []byte(brNetdev), 0o644); err != nil {
 		return err
 	}
 
@@ -434,36 +528,43 @@ func writeNetworkdBridge(dir, nicIface, bridgeName string, manual, dhcp bool,
 		b.WriteString("[Link]\nActivationPolicy=manual\nRequiredForOnline=no\n\n")
 	}
 	b.WriteString("[Network]\n")
-	if dhcp {
+	if role.DHCPMode {
 		b.WriteString("DHCP=ipv4\n")
 	} else {
-		cidr, err := addrCIDR(addr, mask)
+		cidr, err := addrCIDR(role.Address, role.Mask)
 		if err != nil {
 			return fmt.Errorf("bridge %s: %w", bridgeName, err)
 		}
 		fmt.Fprintf(&b, "Address=%s\n", cidr)
-		if gw != "" {
-			fmt.Fprintf(&b, "Gateway=%s\n", gw)
+		// Only the wan plane installs a default route; the east-west planes
+		// are link-local to the rack.
+		if role.Gateway != "" && plane == PlaneWAN {
+			fmt.Fprintf(&b, "Gateway=%s\n", role.Gateway)
 		}
-		for _, ns := range dns {
-			if ns = strings.TrimSpace(ns); ns != "" {
-				fmt.Fprintf(&b, "DNS=%s\n", ns)
+		// Resolvers follow the default route for the same reason. A resolver
+		// pinned to an east-west bridge is unreachable — those bridges are
+		// link-local and activate manually — but resolved would still try it,
+		// putting a timeout in front of every lookup on the node.
+		if plane == PlaneWAN {
+			for _, ns := range role.DNS {
+				if ns = strings.TrimSpace(ns); ns != "" {
+					fmt.Fprintf(&b, "DNS=%s\n", ns)
+				}
 			}
 		}
 	}
 	b.WriteString("IPv6AcceptRA=no\nConfigureWithoutCarrier=yes\n")
-	if manual && dhcp {
-		// br-lan is non-critical; fail fast on DHCP so a missing LAN cable
-		// does not stall spinifex-lan-bridge.service indefinitely.
+	if manual && role.DHCPMode {
+		// The east-west bridges are non-critical; fail fast on DHCP so a
+		// missing cable does not stall their activation unit indefinitely.
 		b.WriteString("\n[DHCP]\nTimeout=10\n")
 	}
-	brNetFile := fmt.Sprintf("%02d-spinifex-%s.network", brPrio, suffix)
-	if err := os.WriteFile(filepath.Join(dir, brNetFile), []byte(b.String()), 0o644); err != nil {
+	if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("20-spinifex-%s.network", name)), []byte(b.String()), 0o644); err != nil {
 		return err
 	}
 
-	if wifiSSID != "" {
-		if err := writeWPASupplicant(nicIface, wifiSSID, wifiPass); err != nil {
+	if role.WiFiSSID != "" {
+		if err := writeWPASupplicant(role.Interface, role.WiFiSSID, role.WiFiPass); err != nil {
 			return err
 		}
 	}
@@ -513,7 +614,59 @@ func writeWPASupplicant(nicIface, ssid, psk string) error {
 	return os.Symlink("/lib/systemd/system/wpa_supplicant@.service", link)
 }
 
-func installBootloader(disk string) error {
+func installBootloader(cfg DiskConfig) error {
+	// Branding assets have to be in place before either path regenerates
+	// grub.cfg — the ZFS path copies them out of /boot onto each ESP.
+	copySplashImage(mountRoot)
+	copyGrubFont(mountRoot)
+
+	if cfg.FS.IsZFS() {
+		// The initramfs must be able to import the pool before the ESPs are
+		// synced, since refresh copies whatever initrd exists at that moment.
+		if err := buildZFSInitramfs(); err != nil {
+			return err
+		}
+		return installBootToolZFS(cfg)
+	}
+	return installBootloaderExt4(cfg.Primary())
+}
+
+// buildZFSInitramfs regenerates the target's initramfs so it carries the ZFS
+// module, the pool cache and the hostid. The image copied from the live ISO was
+// built for a machine with an ext4 root and cannot import a pool.
+func buildZFSInitramfs() error {
+	// ZFS refuses to import a pool last touched by a different host without
+	// -f, and the hostid the initramfs sees must match the one recorded on the
+	// pool — otherwise every boot stops in the initramfs shell.
+	if err := run("cp", "-f", "/etc/hostid", filepath.Join(mountRoot, "etc/hostid")); err != nil {
+		slog.Warn("could not copy /etc/hostid, generating one in the target", "err", err)
+		if err := run("zgenhostid", "-f", "-o", filepath.Join(mountRoot, "etc/hostid")); err != nil {
+			return fmt.Errorf("zgenhostid: %w", err)
+		}
+	}
+
+	// The pool was created with cachefile=none so the live environment would
+	// not claim it. The installed system needs the cache to import at boot
+	// without scanning every block device.
+	cacheDir := filepath.Join(mountRoot, "etc/zfs")
+	if err := os.MkdirAll(cacheDir, 0o755); err != nil {
+		return err
+	}
+	if err := run("zpool", "set", "cachefile="+filepath.Join(cacheDir, "zpool.cache"), ZFSPoolName); err != nil {
+		return fmt.Errorf("set zpool cachefile: %w", err)
+	}
+
+	if err := bindChrootMounts(); err != nil {
+		return err
+	}
+	defer unbindChrootMounts()
+	if err := run("chroot", mountRoot, "update-initramfs", "-u", "-k", "all"); err != nil {
+		return fmt.Errorf("update-initramfs: %w", err)
+	}
+	return nil
+}
+
+func installBootloaderExt4(disk Disk) error {
 	// grub-install runs in the live environment (not chroot) using the
 	// grub-pc-bin and grub-efi-amd64-bin packages already present on the ISO.
 	// --boot-directory points at the installed system's /boot.
@@ -535,64 +688,17 @@ func installBootloader(disk string) error {
 		"--target=i386-pc",
 		"--boot-directory="+bootDir,
 		"--recheck",
-		disk,
+		disk.Path,
 	); biosErr != nil {
 		if efiErr != nil {
 			// Both targets failed — the system will not boot.
-			return fmt.Errorf("both bootloader targets failed (EFI: %v; BIOS: %w)", efiErr, biosErr)
+			return fmt.Errorf("both bootloader targets failed (EFI: %w; BIOS: %w)", efiErr, biosErr)
 		}
 		return biosErr
 	}
-	// Copy splash image and unicode font from the ISO (mounted at /cdrom) so the
-	// installed GRUB shows the same branded background as the installer GRUB.
-	// The font must be at /boot/grub/fonts/unicode.pf2 so update-grub finds it
-	// there and emits the same loadfont path as the ISO's grub.cfg — GRUB 2.12
-	// (trixie) needs the font in the boot partition, not just /usr/share/grub/.
-	copySplashImage(mountRoot)
-	copyGrubFont(mountRoot)
 
-	// Both consoles listed: tty0 stays primary for local install, ttyS0 mirrors
-	// kernel output to serial so headless installs (CI, racks with serial-only
-	// IPMI, remote-dev SSH-to-QEMU) see boot messages. The last `console=` on
-	// the cmdline becomes the system console, so ttyS0 wins on serial-only
-	// hardware while tty0 still receives output for local display.
-	grubDefault := `GRUB_DEFAULT=0
-GRUB_TIMEOUT=5
-GRUB_DISTRIBUTOR=Spinifex
-GRUB_CMDLINE_LINUX_DEFAULT=""
-GRUB_CMDLINE_LINUX="console=tty0 console=ttyS0,115200 systemd.show_status=1"
-`
-
-	if err := os.WriteFile(filepath.Join(mountRoot, "etc/default/grub"), []byte(grubDefault), 0o644); err != nil {
-		return fmt.Errorf("write /etc/default/grub: %w", err)
-	}
-
-	// Mirror the ISO grub.cfg graphical block exactly so the installed GRUB menu
-	// looks identical to the installer menu. gfxterm MUST be activated before
-	// serial is appended — background_image silently does nothing in text mode.
-	// Using exec tail so update-grub includes everything from line 3 as raw GRUB config.
-	grubTheme := `#!/bin/sh
-exec tail -n +3 $0
-insmod all_video
-insmod font
-if loadfont /boot/grub/fonts/unicode.pf2; then
-  set gfxmode=auto
-  insmod gfxterm
-  terminal_output gfxterm
-fi
-insmod serial
-if serial --unit=0 --speed=115200 --timeout=1; then
-  terminal_input  --append serial
-  terminal_output --append serial
-fi
-
-# --- Branding ---
-insmod png
-set theme=/boot/grub/theme.txt
-export theme
-`
-	if err := os.WriteFile(filepath.Join(mountRoot, "etc/grub.d/06_spinifex"), []byte(grubTheme), 0o755); err != nil {
-		return fmt.Errorf("write /etc/grub.d/06_spinifex: %w", err)
+	if err := writeGrubDefaults(DiskConfig{FS: FSExt4}); err != nil {
+		return err
 	}
 
 	if err := bindChrootMounts(); err != nil {
@@ -667,42 +773,247 @@ func reboot() error {
 
 // toFirstbootConfig maps installer Config to the firstboot package's Config.
 func (c *Config) toFirstbootConfig() firstboot.Config {
-	// Geneve encap IP: use LAN bridge IP when a dedicated LAN NIC is present,
-	// otherwise fall back to WAN bridge IP. Empty for DHCP — setup-ovn.sh
+	// Both addresses come from their plane after collapsing, so a single-NIC
+	// node resolves them to the wan address and a three-plane rack keeps
+	// Geneve on vpc and cluster traffic on lan. Empty for DHCP — setup-ovn.sh
 	// auto-detects the IP from the default route at boot in that case.
-	encapIP := c.WANAddress
-	if c.LANInterface != "" && c.LANAddress != "" {
-		encapIP = c.LANAddress
-	}
 	return firstboot.Config{
 		Hostname:        c.Hostname,
-		EncapIP:         encapIP,
-		ClusterRole:     c.ClusterRole,
-		JoinAddr:        c.JoinAddr,
+		EncapIP:         c.PlaneAddress(PlaneVPC),
+		LANIP:           c.PlaneAddress(PlaneLAN),
+		WANIP:           c.PlaneAddress(PlaneWAN),
 		Email:           c.Email,
 		InstallCallback: strings.TrimSpace(os.Getenv("SPINIFEX_INSTALL_CALLBACK")),
 		SkipFormation:   c.SkipFormation,
 	}
 }
 
-// writeFstab writes /etc/fstab on the installed system using partition UUIDs so
-// the root filesystem is mounted read-write at boot and the EFI partition is
-// mounted at /boot/efi.
-func writeFstab(disk string) error {
-	efi, root := partitionPaths(disk)
-	rootUUID, err := partUUID(root)
-	if err != nil {
-		return fmt.Errorf("get root UUID: %w", err)
+// writeFstab writes /etc/fstab on the installed system.
+//
+// On ZFS the root and every dataset are mounted by ZFS itself from properties
+// stored on the pool, and the ESPs are mounted on demand by spinifex-boot-tool,
+// so the file holds nothing but swap.
+func writeFstab(cfg DiskConfig) error {
+	fstab := "# /etc/fstab — generated by Spinifex installer\n"
+
+	if !cfg.FS.IsZFS() {
+		d := cfg.Primary()
+		rootUUID, err := partUUID(d.PartitionPath(rootPartNum))
+		if err != nil {
+			return fmt.Errorf("get root UUID: %w", err)
+		}
+		efiUUID, err := partUUID(d.PartitionPath(espPartNum))
+		if err != nil {
+			return fmt.Errorf("get EFI UUID: %w", err)
+		}
+		fstab += fmt.Sprintf("UUID=%s / ext4 errors=remount-ro 0 1\nUUID=%s /boot/efi vfat umask=0077 0 1\n",
+			rootUUID, efiUUID)
 	}
-	efiUUID, err := partUUID(efi)
-	if err != nil {
-		return fmt.Errorf("get EFI UUID: %w", err)
+
+	// Only claim the swap the previous step actually produced; systemd fails the
+	// boot on a missing swap unit rather than ignoring the line.
+	switch {
+	case cfg.FS.IsZFS():
+		if _, err := os.Stat("/dev/zvol/" + swapZvolName); err == nil {
+			fstab += fmt.Sprintf("/dev/zvol/%s none swap discard 0 0\n", swapZvolName)
+		}
+	default:
+		if _, err := os.Stat(filepath.Join(mountRoot, swapFileName)); err == nil {
+			fstab += fmt.Sprintf("/%s none swap sw 0 0\n", swapFileName)
+		}
 	}
-	fstab := fmt.Sprintf("# /etc/fstab — generated by Spinifex installer\nUUID=%s / ext4 errors=remount-ro 0 1\nUUID=%s /boot/efi vfat umask=0077 0 1\n",
-		rootUUID, efiUUID)
 	return os.WriteFile(filepath.Join(mountRoot, "etc/fstab"), []byte(fstab), 0o644)
 }
 
+const (
+	swapFileName = "swapfile"
+	// Nodes below this get half their RAM, which keeps a 4 GiB test VM to a
+	// 2 GiB file instead of eating its disk. At or above it the machine is a
+	// real host with a real disk, so swap is worth sizing for large imports.
+	swapRAMThreshold = 32 << 30
+	swapLargeBytes   = 64 << 30
+	// Below this there is not enough to be worth the space it costs.
+	swapMinBytes = 512 << 20
+	// Hard ceiling as a share of free disk, whatever RAM suggests: a 64 GiB file
+	// must never fill a small root filesystem.
+	swapDiskShare = 4
+)
+
+// createSwapFile lays down /swapfile on the installed root, sized to the disk.
+//
+// Importing an image streams the decompressed disk through viperblock into
+// predastore, which a 2-4 GiB node cannot hold in RAM — the kernel OOM-kills the
+// import mid-flush. Swap trades speed for completing the operation at all. It is
+// a stopgap for the memory profile of that path, not a fix for it.
+//
+// Failure is logged and not fatal: a node that boots without swap is worse at
+// large imports but is otherwise fine, and losing a whole install over it would
+// be the greater harm.
+// swapZvolName is the swap volume on a ZFS root. ZFS cannot host a swap *file*
+// at all, so a zvol is the only option — the same one Proxmox uses.
+const swapZvolName = ZFSPoolName + "/swap"
+
+// createSwap lays down swap in whichever form the root filesystem supports.
+func createSwap(cfg DiskConfig) error {
+	if cfg.FS.IsZFS() {
+		return createSwapZvol()
+	}
+	return createSwapFile()
+}
+
+// createSwapZvol creates the swap volume on a ZFS root.
+//
+// The properties are not tuning preferences. A zvol swapping under memory
+// pressure can deadlock against the ARC, and these are the settings that make
+// it survivable: page-sized blocks so a swap-out is one record, no data
+// caching so swap pages are never copied into the ARC, and sync writes so a
+// page is on disk before the kernel frees it.
+func createSwapZvol() error {
+	size, err := swapSize(mountRoot)
+	if err != nil {
+		slog.Warn("swap: cannot size swap volume, continuing without one", "err", err)
+		return nil
+	}
+	if size == 0 {
+		slog.Warn("swap: pool too small for a useful swap volume, continuing without one")
+		return nil
+	}
+
+	if err := run("zfs", "create",
+		"-V", strconv.FormatInt(size, 10),
+		"-b", strconv.Itoa(os.Getpagesize()),
+		"-o", "logbias=throughput",
+		"-o", "sync=always",
+		"-o", "primarycache=metadata",
+		"-o", "secondarycache=none",
+		"-o", "compression=zle",
+		"-o", "com.sun:auto-snapshot=false",
+		swapZvolName,
+	); err != nil {
+		slog.Warn("swap: could not create swap volume, continuing without swap", "err", err)
+		return nil
+	}
+
+	dev := "/dev/zvol/" + swapZvolName
+	if err := waitForPath(dev, time.Now().Add(10*time.Second)); err != nil {
+		slog.Warn("swap: zvol device node never appeared, continuing without swap", "err", err)
+		return nil
+	}
+	if err := run("mkswap", "-f", dev); err != nil {
+		slog.Warn("swap: mkswap failed, continuing without swap", "err", err)
+		return nil
+	}
+	slog.Info("swap: created", "zvol", swapZvolName, "bytes", size)
+	return nil
+}
+
+func createSwapFile() error {
+	size, err := swapSize(mountRoot)
+	if err != nil {
+		slog.Warn("swap: cannot size swap file, continuing without one", "err", err)
+		return nil
+	}
+	if size == 0 {
+		slog.Warn("swap: disk too small for a useful swap file, continuing without one",
+			"min_bytes", swapMinBytes)
+		return nil
+	}
+
+	path := filepath.Join(mountRoot, swapFileName)
+	// fallocate rather than dd: on ext4 it reserves real extents, so mkswap and
+	// swapon accept it, and a 32 GiB file costs no write time during install.
+	if err := run("fallocate", "-l", strconv.FormatInt(size, 10), path); err != nil {
+		slog.Warn("swap: fallocate failed, continuing without swap", "err", err)
+		_ = os.Remove(path)
+		return nil
+	}
+	// mkswap refuses a world-readable file, and its contents are process memory.
+	if err := os.Chmod(path, 0o600); err != nil {
+		slog.Warn("swap: chmod failed, continuing without swap", "err", err)
+		_ = os.Remove(path)
+		return nil
+	}
+	if err := run("mkswap", path); err != nil {
+		slog.Warn("swap: mkswap failed, continuing without swap", "err", err)
+		_ = os.Remove(path)
+		return nil
+	}
+	slog.Info("swap: created", "path", "/"+swapFileName, "bytes", size)
+	return nil
+}
+
+// swapSize returns the swap file size for the filesystem mounted at root, or 0
+// when the disk cannot spare a useful amount. SPINIFEX_SWAP_SIZE overrides the
+// calculation; 0 disables swap entirely.
+//
+// RAM drives the size and the disk only caps it, so a small test VM stays cheap
+// on disk while a real host gets enough to carry a large import.
+func swapSize(root string) (int64, error) {
+	if v := strings.TrimSpace(os.Getenv("SPINIFEX_SWAP_SIZE")); v != "" {
+		size, err := parseSize(v)
+		if err != nil {
+			return 0, fmt.Errorf("SPINIFEX_SWAP_SIZE: %w", err)
+		}
+		return size, nil
+	}
+
+	ram, err := totalRAM()
+	if err != nil {
+		return 0, err
+	}
+	size := ram / 2
+	if ram >= swapRAMThreshold {
+		size = swapLargeBytes
+	}
+
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(root, &st); err != nil {
+		return 0, fmt.Errorf("statfs %s: %w", root, err)
+	}
+	// Free space, not total: the rootfs is already copied in, so this is what is
+	// genuinely left to give away.
+	free := int64(st.Bavail) * st.Bsize
+	if diskCap := free / swapDiskShare; size > diskCap {
+		size = diskCap
+	}
+	if size < swapMinBytes {
+		return 0, nil
+	}
+	return size, nil
+}
+
+// totalRAM reports installed memory in bytes.
+func totalRAM() (int64, error) {
+	var si syscall.Sysinfo_t
+	if err := syscall.Sysinfo(&si); err != nil {
+		return 0, fmt.Errorf("sysinfo: %w", err)
+	}
+	unit := int64(si.Unit)
+	if unit == 0 {
+		unit = 1
+	}
+	return int64(si.Totalram) * unit, nil
+}
+
+// parseSize reads a plain byte count or a G/M suffixed size ("32G", "512M").
+func parseSize(s string) (int64, error) {
+	mult := int64(1)
+	switch {
+	case strings.HasSuffix(s, "G"), strings.HasSuffix(s, "g"):
+		mult, s = 1<<30, s[:len(s)-1]
+	case strings.HasSuffix(s, "M"), strings.HasSuffix(s, "m"):
+		mult, s = 1<<20, s[:len(s)-1]
+	}
+	n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("invalid size %q", s)
+	}
+	if n < 0 {
+		return 0, fmt.Errorf("negative size %q", s)
+	}
+	return n * mult, nil
+}
+
 func partUUID(dev string) (string, error) {
 	out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", dev).Output()
 	if err != nil {
@@ -715,18 +1026,6 @@ func partUUID(dev string) (string, error) {
 	return uuid, nil
 }
 
-// partitionPaths returns the EFI and root partition device paths for a given
-// disk. p1 is the BIOS Boot Partition (no filesystem), p2 is EFI, p3 is root.
-// Handles both /dev/sdX (→ /dev/sdX2, /dev/sdX3) and /dev/nvmeXnY
-// (→ /dev/nvmeXnYp2, /dev/nvmeXnYp3).
-func partitionPaths(disk string) (efi, root string) {
-	// NVMe devices use a 'p' separator before the partition number.
-	if len(disk) > 0 && disk[len(disk)-1] >= '0' && disk[len(disk)-1] <= '9' {
-		return disk + "p2", disk + "p3"
-	}
-	return disk + "2", disk + "3"
-}
-
 // copyGrubFont copies the unicode.pf2 GRUB font into the installed system's
 // /boot/grub/fonts/ directory. This ensures update-grub finds the font at
 // /boot/grub/fonts/unicode.pf2 — the same path the ISO's grub.cfg uses —
@@ -836,13 +1135,31 @@ func bindChrootMounts() error {
 // Errors are logged but not returned — this is best-effort cleanup.
 func unbindChrootMounts() {
 	for _, v := range slices.Backward(chrootMountPaths) {
-		_ = run("umount", filepath.Join(mountRoot, v))
+		// Quiet: this also runs from the deferred cleanup, where nothing is
+		// mounted and three "no mount point specified" lines above a real error
+		// message are pure distraction.
+		_ = runQuiet("umount", filepath.Join(mountRoot, v))
 	}
 }
 
-func run(name string, args ...string) error {
+var run = func(name string, args ...string) error {
+	return runEnv(nil, name, args...)
+}
+
+// runQuiet is run with output discarded, for best-effort probes whose failure
+// is expected and whose stderr would otherwise look like an install fault.
+var runQuiet = func(name string, args ...string) error {
+	cmd := exec.Command(name, args...)
+	return cmd.Run()
+}
+
+// runEnv is run with extra environment variables appended to the installer's own.
+var runEnv = func(env []string, name string, args ...string) error {
 	cmd := exec.Command(name, args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
+	if len(env) > 0 {
+		cmd.Env = append(os.Environ(), env...)
+	}
 	return cmd.Run()
 }

cmd/installer/install/install_test.go

@@ -9,7 +9,7 @@ func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
 	const url = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	t.Setenv("SPINIFEX_INSTALL_CALLBACK", url)
 
-	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	cfg := &Config{Hostname: "node1"}
 	fb := cfg.toFirstbootConfig()
 	if fb.InstallCallback != url {
 		t.Errorf("InstallCallback = %q, want %q", fb.InstallCallback, url)
@@ -19,7 +19,7 @@ func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
 func TestToFirstbootConfigEmptyCallbackWhenEnvUnset(t *testing.T) {
 	os.Unsetenv("SPINIFEX_INSTALL_CALLBACK")
 
-	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	cfg := &Config{Hostname: "node1"}
 	fb := cfg.toFirstbootConfig()
 	if fb.InstallCallback != "" {
 		t.Errorf("InstallCallback = %q, want empty", fb.InstallCallback)

cmd/installer/install/steps_test.go

@@ -0,0 +1,821 @@
+package install
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// fakeExec records every command the install steps shell out to and lets a
+// test fail a chosen one. Without it these steps can only be exercised by
+// running a real install against real hardware.
+type fakeExec struct {
+	calls []string
+	fail  func(name string, args []string) error
+}
+
+func fakeCommands(t *testing.T) *fakeExec {
+	t.Helper()
+	f := &fakeExec{}
+	prevRun, prevQuiet, prevEnv := run, runQuiet, runEnv
+
+	record := func(name string, args ...string) error {
+		f.calls = append(f.calls, strings.TrimSpace(name+" "+strings.Join(args, " ")))
+		if f.fail != nil {
+			return f.fail(name, args)
+		}
+		return nil
+	}
+	run = record
+	runQuiet = record
+	runEnv = func(env []string, name string, args ...string) error {
+		return record(name, append(env, args...)...)
+	}
+
+	t.Cleanup(func() { run, runQuiet, runEnv = prevRun, prevQuiet, prevEnv })
+	return f
+}
+
+// ran reports whether any recorded command contains every given fragment.
+func (f *fakeExec) ran(fragments ...string) bool {
+	for _, c := range f.calls {
+		all := true
+		for _, frag := range fragments {
+			if !strings.Contains(c, frag) {
+				all = false
+				break
+			}
+		}
+		if all {
+			return true
+		}
+	}
+	return false
+}
+
+func (f *fakeExec) mustRun(t *testing.T, fragments ...string) {
+	t.Helper()
+	if !f.ran(fragments...) {
+		t.Errorf("no command matched %v; recorded:\n  %s", fragments, strings.Join(f.calls, "\n  "))
+	}
+}
+
+func (f *fakeExec) mustNotRun(t *testing.T, fragments ...string) {
+	t.Helper()
+	if f.ran(fragments...) {
+		t.Errorf("unexpected command matching %v; recorded:\n  %s", fragments, strings.Join(f.calls, "\n  "))
+	}
+}
+
+// indexOf returns the position of the first command matching frag, or -1.
+func (f *fakeExec) indexOf(frag string) int {
+	for i, c := range f.calls {
+		if strings.Contains(c, frag) {
+			return i
+		}
+	}
+	return -1
+}
+
+// indexOfExact is indexOf for a whole command, for cases where one command
+// line is a prefix of another.
+func (f *fakeExec) indexOfExact(cmd string) int {
+	for i, c := range f.calls {
+		if c == cmd {
+			return i
+		}
+	}
+	return -1
+}
+
+// withMountRoot points the install target at a temp tree.
+func withMountRoot(t *testing.T) string {
+	t.Helper()
+	prev := mountRoot
+	mountRoot = t.TempDir()
+	t.Cleanup(func() { mountRoot = prev })
+	return mountRoot
+}
+
+// targetSkeleton creates the directories rsync would already have copied in by
+// the time these steps run.
+func targetSkeleton(t *testing.T, root string) {
+	t.Helper()
+	for _, d := range []string{"etc/default", "etc/grub.d", "etc/systemd/system", "etc"} {
+		if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// tempDisks builds disks whose device nodes are real files, so the steps that
+// wait for partition nodes to appear can complete.
+func tempDisks(t *testing.T, n int, gib int64) []Disk {
+	t.Helper()
+	dir := t.TempDir()
+	out := make([]Disk, n)
+	for i := range out {
+		d := Disk{
+			Path:              filepath.Join(dir, fmt.Sprintf("sd%c", 'a'+i)),
+			Bytes:             gib << 30,
+			LogicalBlockSize:  512,
+			PhysicalBlockSize: 512,
+		}
+		for _, part := range []int{biosPartNum, espPartNum, rootPartNum} {
+			if err := os.WriteFile(d.PartitionPath(part), nil, 0o644); err != nil {
+				t.Fatalf("create partition node: %v", err)
+			}
+		}
+		out[i] = d
+	}
+	return out
+}
+
+func TestPartitionDisksWritesTheSameLayoutOnEveryDisk(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 200)}
+
+	if err := partitionDisks(cfg); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+
+	for _, d := range cfg.Disks {
+		// Stale signatures survive a new table, so the wipe comes first.
+		f.mustRun(t, "wipefs -a "+d.Path)
+		f.mustRun(t, "sgdisk -Z "+d.Path)
+		f.mustRun(t,
+			"sgdisk", d.Path,
+			"-n 1:1M:+1M", "-t 1:EF02", "-c 1:bios_boot",
+			// A 200GiB disk is a real host, so it gets the large ESP.
+			"-n 2:0:+1024M", "-t 2:EF00", "-c 2:ESP",
+			"-n 3:0:-1024M", "-t 3:BF00", "-c 3:root",
+		)
+		f.mustRun(t, "sgdisk -G "+d.Path)
+		f.mustRun(t, "partprobe "+d.Path)
+
+		if wipe, part := f.indexOf("wipefs -a "+d.Path), f.indexOf("-c 3:root "+d.Path); wipe > part {
+			t.Errorf("wipefs ran after the table was written (%d > %d)", wipe, part)
+		}
+	}
+	f.mustRun(t, "udevadm settle --timeout=10")
+}
+
+func TestPartitionDisksTypesTheRootPartitionForTheFilesystem(t *testing.T) {
+	f := fakeCommands(t)
+	if err := partitionDisks(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+	f.mustRun(t, "-t 3:8300")
+	f.mustNotRun(t, "-t 3:BF00")
+	// Under 100GiB the ESP stays at the smaller size.
+	f.mustRun(t, "-n 2:0:+512M")
+}
+
+func TestPartitionDisksNamesTheDiskThatFailed(t *testing.T) {
+	f := fakeCommands(t)
+	disks := tempDisks(t, 2, 40)
+	f.fail = func(name string, args []string) error {
+		if name == "sgdisk" && strings.Contains(strings.Join(args, " "), "bios_boot") {
+			return errors.New("device busy")
+		}
+		return nil
+	}
+
+	err := partitionDisks(DiskConfig{FS: FSExt4, Disks: disks})
+	if err == nil || !strings.Contains(err.Error(), disks[0].Path) {
+		t.Fatalf("err = %v, want it to name %s", err, disks[0].Path)
+	}
+}
+
+func TestPartitionDisksSurvivesBestEffortFailures(t *testing.T) {
+	f := fakeCommands(t)
+	// partprobe, udevadm and the GUID randomisation are all advisory: sgdisk
+	// has already issued BLKRRPART and the nodes exist.
+	f.fail = func(name string, args []string) error {
+		switch {
+		case name == "partprobe", name == "udevadm":
+			return errors.New("not available")
+		case name == "sgdisk" && len(args) > 0 && args[0] == "-G":
+			return errors.New("no")
+		}
+		return nil
+	}
+	if err := partitionDisks(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+}
+
+func TestWaitForPathFailsWhenTheNodeNeverAppears(t *testing.T) {
+	missing := filepath.Join(t.TempDir(), "sda3")
+
+	err := waitForPath(missing, time.Now().Add(-time.Second))
+	if err == nil || !strings.Contains(err.Error(), missing) {
+		t.Fatalf("err = %v, want a timeout naming the missing node", err)
+	}
+}
+
+func TestWaitForPathReturnsOnceTheNodeExists(t *testing.T) {
+	f, err := os.CreateTemp(t.TempDir(), "node")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := waitForPath(f.Name(), time.Now().Add(-time.Second)); err != nil {
+		t.Errorf("waitForPath on an existing node: %v", err)
+	}
+}
+
+func TestEspSizeMiB(t *testing.T) {
+	if got := espSizeMiB(Disk{Bytes: 99 << 30}); got != espSmallMiB {
+		t.Errorf("99GiB -> %d, want %d", got, espSmallMiB)
+	}
+	if got := espSizeMiB(Disk{Bytes: 100 << 30}); got != espLargeMiB {
+		t.Errorf("100GiB -> %d, want %d", got, espLargeMiB)
+	}
+}
+
+func TestClearDiskOnlyLabelclearsAnExistingPartition(t *testing.T) {
+	f := fakeCommands(t)
+	disks := tempDisks(t, 1, 40)
+	if err := clearDisk(disks[0]); err != nil {
+		t.Fatalf("clearDisk: %v", err)
+	}
+	f.mustRun(t, "zpool labelclear -f "+disks[0].PartitionPath(rootPartNum))
+
+	f2 := fakeCommands(t)
+	blank := Disk{Path: filepath.Join(t.TempDir(), "sdb")}
+	if err := clearDisk(blank); err != nil {
+		t.Fatalf("clearDisk: %v", err)
+	}
+	f2.mustNotRun(t, "labelclear")
+	f2.mustRun(t, "wipefs -a "+blank.Path)
+}
+
+func TestFormatESPsLabelsEveryMember(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 40)}
+	if err := formatESPs(cfg); err != nil {
+		t.Fatalf("formatESPs: %v", err)
+	}
+	for _, d := range cfg.Disks {
+		f.mustRun(t, "mkfs.fat -F 32 -n EFI "+d.PartitionPath(espPartNum))
+	}
+}
+
+func TestFormatESPsNamesTheFailingESP(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 40)}
+	f.fail = func(name string, _ []string) error {
+		if name == "mkfs.fat" {
+			return errors.New("no such device")
+		}
+		return nil
+	}
+	err := formatESPs(cfg)
+	if err == nil || !strings.Contains(err.Error(), cfg.Disks[0].PartitionPath(espPartNum)) {
+		t.Fatalf("err = %v, want it to name the ESP", err)
+	}
+}
+
+func TestFormatPartitionsMakesTheExt4Root(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}
+	if err := formatPartitions(cfg); err != nil {
+		t.Fatalf("formatPartitions: %v", err)
+	}
+	f.mustRun(t, "mkfs.ext4 -F "+cfg.Disks[0].PartitionPath(rootPartNum))
+}
+
+func TestMountPartitionsMountsRootThenESP(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}
+
+	if err := mountPartitions(cfg); err != nil {
+		t.Fatalf("mountPartitions: %v", err)
+	}
+	d := cfg.Disks[0]
+	f.mustRun(t, "mount "+d.PartitionPath(rootPartNum)+" "+root)
+	f.mustRun(t, "mount "+d.PartitionPath(espPartNum)+" "+efiPart())
+	if _, err := os.Stat(efiPart()); err != nil {
+		t.Errorf("ESP mountpoint was not created: %v", err)
+	}
+	if f.indexOf("mount "+d.PartitionPath(rootPartNum)) > f.indexOf("mount "+d.PartitionPath(espPartNum)) {
+		t.Error("the ESP was mounted before the root it lives under")
+	}
+}
+
+func TestMountPartitionsStopsWhenTheRootWillNotMount(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "mount" {
+			return errors.New("wrong fs type")
+		}
+		return nil
+	}
+	if err := mountPartitions(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err == nil {
+		t.Fatal("expected an error when the root mount fails")
+	}
+	f.mustNotRun(t, "boot/efi")
+}
+
+func TestCleanupTargetExportsThePoolOnZFS(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	cleanupTarget(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
+
+	// Recursive: one dataset left mounted is enough to block the export.
+	f.mustRun(t, "umount -R "+root)
+	f.mustRun(t, "zpool export "+ZFSPoolName)
+	if f.indexOf("umount -R") > f.indexOf("zpool export") {
+		t.Error("the pool was exported before its datasets were unmounted")
+	}
+}
+
+func TestCleanupTargetUnmountsESPBeforeRootOnExt4(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	cleanupTarget(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
+
+	f.mustNotRun(t, "zpool export")
+	esp, rootIdx := f.indexOfExact("umount "+efiPart()), f.indexOfExact("umount "+root)
+	if esp < 0 || rootIdx < 0 {
+		t.Fatalf("missing umounts; recorded:\n  %s", strings.Join(f.calls, "\n  "))
+	}
+	if esp > rootIdx {
+		t.Error("the root was unmounted before the ESP nested inside it")
+	}
+}
+
+func TestCreatePoolBuildsTheZpoolCommand(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	opts := ZFSOpts{Ashift: 12, Compress: "lz4", Checksum: "blake3", Copies: 1, ARCMaxMiB: 4096}
+
+	if err := createPool(cfg, opts); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f.mustRun(t, "zpool create -f",
+		"cachefile=none", "ashift=12", "compression=lz4", "checksum=blake3", "copies=1",
+		// altroot keeps the datasets under the target rather than over the live system.
+		"-R "+root, ZFSPoolName, "mirror",
+		cfg.Disks[0].StablePartitionPath(rootPartNum),
+		cfg.Disks[1].StablePartitionPath(rootPartNum),
+	)
+}
+
+func TestCreatePoolEnablesAutotrimOnFlashOnly(t *testing.T) {
+	withMountRoot(t)
+	flash := disks(2, 40)
+	f := fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: flash}, ZFSOpts{Ashift: 12, Copies: 1}); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f.mustRun(t, "zpool set autotrim=on "+ZFSPoolName)
+
+	spinning := disks(2, 40)
+	spinning[1].Rotational = true
+	f2 := fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: spinning}, ZFSOpts{Ashift: 12, Copies: 1}); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f2.mustNotRun(t, "autotrim")
+}
+
+func TestCreatePoolWrapsTheZpoolFailure(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, args []string) error {
+		if name == "zpool" && len(args) > 0 && args[0] == "create" {
+			return errors.New("device is in use")
+		}
+		return nil
+	}
+	err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}, ZFSOpts{Ashift: 12, Copies: 1})
+	if err == nil || !strings.Contains(err.Error(), "zpool create") {
+		t.Fatalf("err = %v, want a zpool create failure", err)
+	}
+}
+
+func TestCreatePoolRejectsAnUnbuildableTopology(t *testing.T) {
+	withMountRoot(t)
+	fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: disks(1, 40)}, ZFSOpts{Ashift: 12}); err == nil {
+		t.Fatal("expected a vdev error for a one-disk mirror")
+	}
+}
+
+func TestCreateDatasetsCreatesEveryDataset(t *testing.T) {
+	f := fakeCommands(t)
+	if err := createDatasets(); err != nil {
+		t.Fatalf("createDatasets: %v", err)
+	}
+	for _, ds := range datasets {
+		f.mustRun(t, "zfs create", "mountpoint="+ds.mountpoint, ds.name)
+	}
+}
+
+func TestCreateDatasetsNamesTheDatasetThatFailed(t *testing.T) {
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "zfs" {
+			return errors.New("out of space")
+		}
+		return nil
+	}
+	err := createDatasets()
+	if err == nil || !strings.Contains(err.Error(), datasets[0].name) {
+		t.Fatalf("err = %v, want it to name %s", err, datasets[0].name)
+	}
+}
+
+func TestExportPoolToleratesABusyPool(t *testing.T) {
+	f := fakeCommands(t)
+	f.fail = func(string, []string) error { return errors.New("pool is busy") }
+	exportPool()
+	f.mustRun(t, "zpool export "+ZFSPoolName)
+}
+
+func TestWriteZFSSystemConfigDeclaresTheARCCeilingToBothConsumers(t *testing.T) {
+	root := withMountRoot(t)
+
+	if err := writeZFSSystemConfig(ZFSOpts{ARCMaxMiB: 4096}); err != nil {
+		t.Fatalf("writeZFSSystemConfig: %v", err)
+	}
+
+	modprobe := readFile(t, root, "etc/modprobe.d/zfs.conf")
+	if !strings.Contains(modprobe, fmt.Sprintf("zfs_arc_max=%d", int64(4096)<<20)) {
+		t.Errorf("zfs.conf does not cap the ARC in bytes:\n%s", modprobe)
+	}
+
+	// The daemon cannot see the ARC, so the reserve has to account for it or
+	// guests get admitted into memory the ARC already owns.
+	env := readFile(t, root, "etc/spinifex/host.env")
+	want := fmt.Sprintf("SPINIFEX_RESERVED_MEM_GB=%.1f", defaultHostReserveGB+4.0)
+	if !strings.Contains(env, want) {
+		t.Errorf("host.env = %q, want it to contain %q", env, want)
+	}
+}
+
+func TestWriteTargetFileAppliesTheModeOnRewrite(t *testing.T) {
+	root := withMountRoot(t)
+
+	if err := writeTargetFile("/usr/sbin/tool", []byte("#!/bin/sh\n"), 0o600); err != nil {
+		t.Fatalf("writeTargetFile: %v", err)
+	}
+	// A retried install rewrites an existing file, and WriteFile would leave
+	// the original mode in place.
+	if err := writeTargetFile("/usr/sbin/tool", []byte("#!/bin/sh\ntrue\n"), 0o755); err != nil {
+		t.Fatalf("writeTargetFile rewrite: %v", err)
+	}
+
+	st, err := os.Stat(filepath.Join(root, "usr/sbin/tool"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st.Mode().Perm() != 0o755 {
+		t.Errorf("mode = %o, want 755", st.Mode().Perm())
+	}
+	if got := readFile(t, root, "usr/sbin/tool"); !strings.Contains(got, "true") {
+		t.Errorf("content not rewritten: %q", got)
+	}
+}
+
+func TestWriteGrubDefaultsPinsTheZFSRoot(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeGrubDefaults(DiskConfig{FS: FSZFSRAID1}); err != nil {
+		t.Fatalf("writeGrubDefaults: %v", err)
+	}
+	got := readFile(t, root, "etc/default/grub")
+	if !strings.Contains(got, "root=ZFS="+ZFSRootDataset) || !strings.Contains(got, "boot=zfs") {
+		t.Errorf("grub defaults do not name the ZFS root:\n%s", got)
+	}
+	// Serial is listed last so it wins as the system console on headless racks.
+	if !strings.Contains(got, "console=tty0 console=ttyS0,115200") {
+		t.Errorf("grub defaults do not set both consoles:\n%s", got)
+	}
+	if theme := readFile(t, root, "etc/grub.d/06_spinifex"); !strings.Contains(theme, "terminal_output gfxterm") {
+		t.Errorf("branding snippet missing gfxterm:\n%s", theme)
+	}
+}
+
+func TestWriteGrubDefaultsLeavesExt4RootToGrubProbe(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	if err := writeGrubDefaults(DiskConfig{FS: FSExt4}); err != nil {
+		t.Fatalf("writeGrubDefaults: %v", err)
+	}
+	if got := readFile(t, root, "etc/default/grub"); strings.Contains(got, "root=ZFS=") {
+		t.Errorf("ext4 install should not pin a ZFS root:\n%s", got)
+	}
+}
+
+func TestDivertGrubInstallReplacesTheBinary(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	if err := divertGrubInstall(); err != nil {
+		t.Fatalf("divertGrubInstall: %v", err)
+	}
+	// The diversion is what makes the replacement survive a grub-pc upgrade.
+	f.mustRun(t, "dpkg-divert --divert "+grubInstallReal, "--rename --add "+grubInstallPath)
+	if _, err := os.Stat(filepath.Join(root, strings.TrimPrefix(grubInstallPath, "/"))); err != nil {
+		t.Errorf("wrapper not written: %v", err)
+	}
+}
+
+func TestDivertGrubInstallFailsLoudly(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(string, []string) error { return errors.New("dpkg is locked") }
+
+	if err := divertGrubInstall(); err == nil || !strings.Contains(err.Error(), "divert grub-install") {
+		t.Fatalf("err = %v, want a diversion failure", err)
+	}
+}
+
+// espFixture writes the state a successful boot-tool run leaves behind: the
+// registered ESP list, and a grub.cfg on the ESP that verifyBootConfig mounts.
+func espFixture(t *testing.T, root string, uuids string, grubCfg string) {
+	t.Helper()
+	list := filepath.Join(root, strings.TrimPrefix(bootToolUUIDList, "/"))
+	if err := os.MkdirAll(filepath.Dir(list), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(list, []byte(uuids), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if grubCfg == "" {
+		return
+	}
+	dir := filepath.Join(root, "var/tmp/verify-esp/grub")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "grub.cfg"), []byte(grubCfg), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestVerifyBootConfig(t *testing.T) {
+	goodCfg := "linux /vmlinuz-6.12.0 root=ZFS=" + ZFSRootDataset + " boot=zfs\n"
+
+	tests := []struct {
+		name    string
+		uuids   string
+		grubCfg string
+		wantErr string
+	}{
+		{name: "every ESP registered and pointing at the pool", uuids: "AAAA-1111\nBBBB-2222\n", grubCfg: goodCfg},
+		{
+			name:    "a disk got no bootloader",
+			uuids:   "AAAA-1111\n",
+			grubCfg: goodCfg,
+			wantErr: "expected 2 registered ESPs, found 1",
+		},
+		{
+			// grub-mkconfig degrades quietly when grub-probe cannot read the
+			// pool, and the failure only shows up as a panic at boot.
+			name:    "grub.cfg names no root",
+			uuids:   "AAAA-1111\nBBBB-2222\n",
+			grubCfg: "linux /vmlinuz-6.12.0 root=/dev/sda3\n",
+			wantErr: "would not find its root filesystem",
+		},
+		{
+			name:    "grub.cfg names no kernel",
+			uuids:   "AAAA-1111\nBBBB-2222\n",
+			grubCfg: "set root=ZFS=" + ZFSRootDataset + "\n",
+			wantErr: "references no kernel",
+		},
+		{name: "no ESP list at all", wantErr: "read ESP list"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			root := withMountRoot(t)
+			fakeCommands(t)
+			cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+			if tt.uuids != "" {
+				espFixture(t, root, tt.uuids, tt.grubCfg)
+			}
+
+			err := verifyBootConfig(cfg)
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("verifyBootConfig: %v", err)
+				}
+				return
+			}
+			if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+				t.Fatalf("err = %v, want %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestInstallBootToolZFSSetsUpEveryESP(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	targetSkeleton(t, root)
+	espFixture(t, root, "AAAA-1111\nBBBB-2222\n", "linux /vmlinuz-6.12.0 root=ZFS="+ZFSRootDataset+"\n")
+
+	if err := installBootToolZFS(cfg); err != nil {
+		t.Fatalf("installBootToolZFS: %v", err)
+	}
+
+	if _, err := os.Stat(filepath.Join(root, strings.TrimPrefix(bootToolPath, "/"))); err != nil {
+		t.Errorf("boot tool not installed: %v", err)
+	}
+	// A partially-updated set of ESPs boots today and fails on the next kernel
+	// upgrade, so every disk is initialised under strict mode.
+	for _, d := range cfg.Disks {
+		f.mustRun(t, "SPINIFEX_BOOT_TOOL_STRICT=1", "init", d.PartitionPath(espPartNum))
+	}
+	// NVRAM is written exactly once, by the initial refresh.
+	f.mustRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1", "refresh")
+	f.mustRun(t, "dpkg-divert")
+	for _, m := range chrootMountPaths {
+		f.mustRun(t, "mount --bind /"+m)
+	}
+}
+
+func TestInstallBootToolZFSStopsWhenADiskFails(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	targetSkeleton(t, root)
+	espFixture(t, root, "AAAA-1111\nBBBB-2222\n", "linux /vmlinuz root=ZFS="+ZFSRootDataset+"\n")
+	f.fail = func(name string, args []string) error {
+		if strings.Contains(strings.Join(args, " "), "init") {
+			return errors.New("no ESP")
+		}
+		return nil
+	}
+
+	err := installBootToolZFS(cfg)
+	if err == nil || !strings.Contains(err.Error(), "spinifex-boot-tool init") {
+		t.Fatalf("err = %v, want the init failure", err)
+	}
+	f.mustNotRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1")
+}
+
+func TestRefreshBootToolNoteOnlyAppliesToZFS(t *testing.T) {
+	// Log-only, but it is the operator's only record of the disk-replacement
+	// command, so both branches are walked.
+	refreshBootToolNote(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
+	refreshBootToolNote(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
+}
+
+func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
+	root := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(root, "etc/systemd/system"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	for range 2 {
+		if err := maskSystemdUnit(root, "getty@tty1.service"); err != nil {
+			t.Fatalf("maskSystemdUnit: %v", err)
+		}
+	}
+	target, err := os.Readlink(filepath.Join(root, "etc/systemd/system/getty@tty1.service"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if target != "/dev/null" {
+		t.Errorf("mask target = %q, want /dev/null", target)
+	}
+}
+
+func TestCopyAssetsAreBestEffort(t *testing.T) {
+	root := t.TempDir()
+	// Neither asset exists outside the live ISO; a missing one must not fail
+	// the install, only the branding.
+	copyGrubFont(root)
+	copySplashImage(root)
+}
+
+func TestWriteFstabOnZFSHoldsNothingButSwap(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeFstab(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	got := readFile(t, root, "etc/fstab")
+	// Root and every dataset are mounted from pool properties, and the ESPs on
+	// demand by the boot tool.
+	if strings.Contains(got, " / ") || strings.Contains(got, "/boot/efi") {
+		t.Errorf("ZFS fstab should carry no root or ESP line:\n%s", got)
+	}
+}
+
+func TestWriteNetworkConfigWritesTheWholeNetworkdTree(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeNetworkConfig(threeNIC()); err != nil {
+		t.Fatalf("writeNetworkConfig: %v", err)
+	}
+
+	netd := filepath.Join(root, "etc/systemd/network")
+	for _, plane := range []Plane{PlaneWAN, PlaneLAN, PlaneVPC} {
+		if got := readFile(t, netd, fmt.Sprintf("20-spinifex-%s.netdev", plane)); !strings.Contains(got, "Kind=bridge") {
+			t.Errorf("%s netdev is not a bridge:\n%s", plane, got)
+		}
+	}
+	// Only br-wan may block network-online.target; the east-west bridges are
+	// brought up by their own unit afterwards.
+	if got := readFile(t, netd, "20-spinifex-wan.network"); strings.Contains(got, "ActivationPolicy=manual") {
+		t.Errorf("br-wan must auto-activate:\n%s", got)
+	}
+	if got := readFile(t, netd, "20-spinifex-lan.network"); !strings.Contains(got, "ActivationPolicy=manual") {
+		t.Errorf("br-lan must not auto-activate:\n%s", got)
+	}
+
+	// The live ISO masks wait-online; the mask is copied in by rsync and has to
+	// be lifted, then scoped to br-wan so br-lan never blocks the boot.
+	dropin := readFile(t, root, "etc/systemd/system/systemd-networkd-wait-online.service.d/spinifex-wan-only.conf")
+	if !strings.Contains(dropin, "--interface=br-wan") || !strings.Contains(dropin, "--timeout=60") {
+		t.Errorf("wait-online drop-in is not scoped to br-wan:\n%s", dropin)
+	}
+
+	sysctl := readFile(t, root, "etc/sysctl.d/99-spinifex-network.conf")
+	for _, br := range []string{"br-wan", "br-lan", "br-vpc"} {
+		if !strings.Contains(sysctl, "net.ipv6.conf."+br+".disable_ipv6=1") {
+			t.Errorf("IPv6 not disabled on %s:\n%s", br, sysctl)
+		}
+	}
+}
+
+func TestWriteNetworkConfigRejectsAnInvalidRoleSet(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	cfg := threeNIC()
+	cfg.LAN.Interface = cfg.WAN.Interface
+
+	if err := writeNetworkConfig(cfg); err == nil || !strings.Contains(err.Error(), "network roles") {
+		t.Fatalf("err = %v, want the role validation to reject two planes on one link", err)
+	}
+}
+
+func TestWriteNetworkConfigLiftsTheWaitOnlineMask(t *testing.T) {
+	root := withMountRoot(t)
+	mask := filepath.Join(root, "etc/systemd/system/systemd-networkd-wait-online.service")
+	if err := os.MkdirAll(filepath.Dir(mask), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink("/dev/null", mask); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := writeNetworkConfig(threeNIC()); err != nil {
+		t.Fatalf("writeNetworkConfig: %v", err)
+	}
+	if _, err := os.Lstat(mask); err == nil {
+		t.Error("the wait-online mask survived; firstboot would run before br-wan has a lease")
+	}
+}
+
+func TestBindChrootMountsCreatesEveryMountpoint(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	if err := bindChrootMounts(); err != nil {
+		t.Fatalf("bindChrootMounts: %v", err)
+	}
+	for _, m := range chrootMountPaths {
+		if _, err := os.Stat(filepath.Join(root, m)); err != nil {
+			t.Errorf("mountpoint /%s not created: %v", m, err)
+		}
+		f.mustRun(t, "mount --bind /"+m+" "+filepath.Join(root, m))
+	}
+
+	unbindChrootMounts()
+	// Reverse order: /sys and /proc come out before /dev they were nested under.
+	if f.indexOf("umount "+filepath.Join(root, "sys")) > f.indexOf("umount "+filepath.Join(root, "dev")) {
+		t.Error("chroot mounts were unbound in bind order, not reverse")
+	}
+}
+
+func TestBindChrootMountsReportsAFailedBind(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "mount" {
+			return errors.New("permission denied")
+		}
+		return nil
+	}
+	if err := bindChrootMounts(); err == nil || !strings.Contains(err.Error(), "bind-mount") {
+		t.Fatalf("err = %v, want a bind-mount failure", err)
+	}
+}

cmd/installer/install/zfs.go

@@ -0,0 +1,362 @@
+package install
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math"
+	"math/bits"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// ARC bounds. Spinifex RAM is guest RAM: Proxmox defaults non-PVE products to
+// half of system memory, which on a 256GB compute node would remove 128GB of
+// schedulable instance capacity. 10% capped at 16GiB is their PVE behaviour and
+// the right trade for a hypervisor.
+const (
+	arcMinMiB     = 64
+	arcMaxCapMiB  = 16 * 1024
+	arcFraction   = 0.10
+	arcHeadroomMi = 1024 // always leave the OS a GiB above the ARC ceiling
+)
+
+// zfsDataset describes one dataset created at install time. Order matters:
+// parents must precede children so mountpoints nest correctly.
+type zfsDataset struct {
+	name       string
+	mountpoint string
+	props      []string
+}
+
+// datasets is the layout described in the plan. Recordsizes are matched to each
+// service's access pattern; 16K on the predastore index keeps RAIDZ-1 parity
+// overhead at 33% rather than the ~50% a 4-8K record would cost.
+var datasets = []zfsDataset{
+	{ZFSPoolName + "/ROOT", "none", []string{"canmount=off"}},
+	{ZFSRootDataset, "/", []string{"acltype=posix"}},
+	{ZFSPoolName + "/data", "/var/lib/spinifex", []string{"canmount=off"}},
+	{ZFSPoolName + "/data/viperblock", "/var/lib/spinifex/viperblock",
+		[]string{"recordsize=128K", "logbias=throughput"}},
+	{ZFSPoolName + "/data/predastore", "/var/lib/spinifex/predastore",
+		[]string{"recordsize=1M"}},
+	// Broken out so the later move of object data onto dedicated drives is a
+	// zfs send | zfs recv rather than an rsync with downtime.
+	{ZFSPoolName + "/data/predastore-nodes", "/var/lib/spinifex/predastore/distributed/nodes", nil},
+	// BadgerDB index plus the Raft BoltDB: small random, fsync-heavy.
+	{ZFSPoolName + "/data/predastore-db", "/var/lib/spinifex/predastore/distributed/db",
+		[]string{"recordsize=16K", "logbias=latency"}},
+	{ZFSPoolName + "/data/nats", "/var/lib/spinifex/nats", []string{"recordsize=16K"}},
+	{ZFSPoolName + "/log", "/var/log/spinifex", []string{"compression=zstd"}},
+}
+
+// buildVdev renders the vdev specification for zpool create. Pure: it is the
+// function that decides pool shape, so it is exhaustively unit-tested rather
+// than discovered to be wrong on hardware.
+func buildVdev(cfg DiskConfig) (string, error) {
+	if err := cfg.Validate(); err != nil {
+		return "", err
+	}
+	if !cfg.FS.IsZFS() {
+		return "", fmt.Errorf("buildVdev: %s is not a zfs topology", cfg.FS)
+	}
+
+	// Members are named by their partition-3 by-id path: a pool built on
+	// /dev/sdX breaks the first time a controller enumerates in a new order.
+	members := make([]string, len(cfg.Disks))
+	for i, d := range cfg.Disks {
+		members[i] = d.StablePartitionPath(rootPartNum)
+	}
+
+	keyword := cfg.FS.vdevKeyword()
+	if keyword == "" {
+		return strings.Join(members, " "), nil // plain stripe
+	}
+	// RAID10 is a stripe of two-disk mirrors, paired in selection order — which
+	// is why the UI numbers selected disks instead of just ticking them.
+	if cfg.FS == FSZFSRAID10 {
+		var parts []string
+		for i := 0; i+1 < len(members); i += 2 {
+			parts = append(parts, fmt.Sprintf("mirror %s %s", members[i], members[i+1]))
+		}
+		return strings.Join(parts, " "), nil
+	}
+	return keyword + " " + strings.Join(members, " "), nil
+}
+
+// detectAshift derives the pool sector size from the widest sector any member
+// reports. ashift cannot be changed after creation, and one that is too small
+// costs read-modify-write on every block for the life of the pool, so the floor
+// is 12 (4K) even when every disk claims 512.
+func detectAshift(disks []Disk) int {
+	widest := 4096
+	for _, d := range disks {
+		widest = max(widest, d.PhysicalBlockSize, d.LogicalBlockSize)
+	}
+	// ashift is the base-2 exponent of the sector size.
+	return bits.Len(uint(widest)) - 1
+}
+
+// defaultARCMaxMiB computes the ARC ceiling from total system memory.
+func defaultARCMaxMiB(totalMiB int) int {
+	want := int(math.Round(float64(totalMiB) * arcFraction))
+	want = min(want, arcMaxCapMiB)
+	// Never let the ARC crowd the OS out of its last GiB on a small node.
+	if ceiling := totalMiB - arcHeadroomMi; ceiling > 0 {
+		want = min(want, ceiling)
+	}
+	return max(want, arcMinMiB)
+}
+
+// totalMemoryMiB reads MemTotal from /proc/meminfo.
+func totalMemoryMiB() int {
+	data, err := os.ReadFile("/proc/meminfo")
+	if err != nil {
+		return 0
+	}
+	for line := range strings.SplitSeq(string(data), "\n") {
+		if !strings.HasPrefix(line, "MemTotal:") {
+			continue
+		}
+		var kb int
+		if _, err := fmt.Sscanf(line, "MemTotal: %d kB", &kb); err == nil {
+			return kb / 1024
+		}
+	}
+	return 0
+}
+
+// resolveZFSOpts fills unset advanced options with their computed defaults, so
+// every value written to the pool is explicit and auditable afterwards.
+func resolveZFSOpts(cfg DiskConfig) ZFSOpts {
+	o := cfg.ZFS
+	if o.Ashift == 0 {
+		o.Ashift = detectAshift(cfg.Disks)
+	}
+	if o.Compress == "" {
+		// Named explicitly rather than "on" so a future OpenZFS default change
+		// cannot silently alter what this pool does.
+		o.Compress = "lz4"
+	}
+	if o.Checksum == "" {
+		o.Checksum = "on"
+	}
+	if o.Copies == 0 {
+		o.Copies = 1
+	}
+	if o.ARCMaxMiB == 0 {
+		o.ARCMaxMiB = defaultARCMaxMiB(totalMemoryMiB())
+	}
+	return o
+}
+
+// loadZFSModule brings up the ZFS kernel module in the live environment and
+// waits for /dev/zfs. The module is prebuilt against the shipped kernel at ISO
+// build time, so failure here means the ISO is broken, not the hardware.
+func loadZFSModule() error {
+	if err := run("modprobe", "zfs"); err != nil {
+		slog.Warn("modprobe zfs failed, checking for /dev/zfs anyway", "err", err)
+	}
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		if _, err := os.Stat("/dev/zfs"); err == nil {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("/dev/zfs did not appear after modprobe zfs — the ZFS module is missing or does not match the running kernel")
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// ImportablePools returns the names of pools visible to `zpool import`. There
+// is still no machine-readable output in OpenZFS 2.3, so the human format is
+// parsed — the same approach Proxmox::Sys::ZFS takes.
+func ImportablePools() ([]string, error) {
+	out, err := exec.Command("zpool", "import").Output()
+	if err != nil {
+		// zpool exits non-zero when there is nothing to import, which is the
+		// normal case. Anything else — no binary, no /dev/zfs — must surface, or
+		// a machine with an existing pool looks empty to the caller.
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("zpool import: %w", err)
+	}
+	return parseImportablePools(string(out)), nil
+}
+
+// parseImportablePools extracts pool names from `zpool import` output. Split
+// out from the command call so it can be tested against captured fixtures.
+func parseImportablePools(out string) []string {
+	var pools []string
+	scanner := bufio.NewScanner(strings.NewReader(out))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		// Only the "pool:" key introduces a pool; "id:", "state:" and the
+		// config tree that follows are attributes of the one before it.
+		if name, ok := strings.CutPrefix(line, "pool:"); ok {
+			if n := strings.TrimSpace(name); n != "" {
+				pools = append(pools, n)
+			}
+		}
+	}
+	return pools
+}
+
+// createPool builds the root pool across the selected disks. altroot keeps
+// every dataset mounted under the install target instead of over the live
+// filesystem, and cachefile=none stops the live environment from claiming it.
+func createPool(cfg DiskConfig, opts ZFSOpts) error {
+	vdev, err := buildVdev(cfg)
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"create", "-f",
+		"-o", "cachefile=none",
+		"-o", fmt.Sprintf("ashift=%d", opts.Ashift),
+		"-O", "atime=on", "-O", "relatime=on",
+		"-O", "xattr=sa",
+		"-O", fmt.Sprintf("compression=%s", opts.Compress),
+		"-O", fmt.Sprintf("checksum=%s", opts.Checksum),
+		"-O", fmt.Sprintf("copies=%d", opts.Copies),
+		"-O", "mountpoint=none",
+		"-R", mountRoot,
+		ZFSPoolName,
+	}
+	args = append(args, strings.Fields(vdev)...)
+	if err := run("zpool", args...); err != nil {
+		return fmt.Errorf("zpool create: %w", err)
+	}
+
+	// TRIM matters on a pool that churns EBS volumes: without it the drives'
+	// garbage collection degrades write latency over months, and diagnosing
+	// that after the fact is miserable. Detected, never prompted.
+	if allFlash(cfg.Disks) {
+		if err := run("zpool", "set", "autotrim=on", ZFSPoolName); err != nil {
+			slog.Warn("could not enable autotrim", "err", err)
+		}
+	}
+	return nil
+}
+
+// allFlash reports whether every member is non-rotational.
+func allFlash(disks []Disk) bool {
+	for _, d := range disks {
+		if d.Rotational {
+			return false
+		}
+	}
+	return len(disks) > 0
+}
+
+// createDatasets lays out the service datasets. Mountpoints are set relative to
+// the pool's altroot, so they land under the install target now and at their
+// real paths once the installed system imports the pool.
+func createDatasets() error {
+	for _, ds := range datasets {
+		args := []string{"create", "-o", "mountpoint=" + ds.mountpoint}
+		for _, p := range ds.props {
+			args = append(args, "-o", p)
+		}
+		args = append(args, ds.name)
+		if err := run("zfs", args...); err != nil {
+			return fmt.Errorf("create dataset %s: %w", ds.name, err)
+		}
+	}
+	return nil
+}
+
+// datasetProtectFilters keeps rsync --delete away from the dataset mountpoints.
+//
+// The datasets are mounted before the copy so files land on the right one, but
+// the live source has nothing at those paths, so --delete tries to rmdir a busy
+// mountpoint and fails the whole transfer. Protect rules exempt them.
+func datasetProtectFilters(cfg DiskConfig) []string {
+	if !cfg.FS.IsZFS() {
+		return nil
+	}
+	var out []string
+	for _, ds := range datasets {
+		if ds.mountpoint == "/" || ds.mountpoint == "none" {
+			continue
+		}
+		out = append(out, "--filter=P "+ds.mountpoint)
+	}
+	return out
+}
+
+// writeZFSSystemConfig writes the target's ZFS tunables: the ARC ceiling, and
+// the matching host reserve so the daemon's admission gate stops handing out
+// memory the ARC is holding.
+func writeZFSSystemConfig(opts ZFSOpts) error {
+	modprobeDir := filepath.Join(mountRoot, "etc/modprobe.d")
+	if err := os.MkdirAll(modprobeDir, 0o755); err != nil {
+		return err
+	}
+	arcBytes := int64(opts.ARCMaxMiB) << 20
+	conf := fmt.Sprintf("# Generated by the Spinifex installer.\n"+
+		"# Guest RAM is the scarce resource on a hypervisor, so the ARC is\n"+
+		"# capped well below the ZFS default of half of system memory.\n"+
+		"options zfs zfs_arc_max=%d\n", arcBytes)
+	if err := os.WriteFile(filepath.Join(modprobeDir, "zfs.conf"), []byte(conf), 0o644); err != nil {
+		return fmt.Errorf("write zfs.conf: %w", err)
+	}
+
+	// The daemon computes schedulable memory as host total minus this reserve
+	// (defaultHostReserve in daemon/resource.go). ARC is invisible to it, so
+	// the ceiling has to be declared or instances get admitted into memory the
+	// ARC already owns.
+	reserve := defaultHostReserveGB + float64(opts.ARCMaxMiB)/1024.0
+	env := fmt.Sprintf("# Generated by the Spinifex installer.\n"+
+		"# %.1fGB base reserve + %.1fGB ZFS ARC ceiling.\n"+
+		"SPINIFEX_RESERVED_MEM_GB=%.1f\n",
+		defaultHostReserveGB, float64(opts.ARCMaxMiB)/1024.0, reserve)
+	confDir := filepath.Join(mountRoot, "etc/spinifex")
+	if err := os.MkdirAll(confDir, 0o755); err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(confDir, "host.env"), []byte(env), 0o644)
+}
+
+// defaultHostReserveGB mirrors defaultHostReserve.memGB in
+// spinifex/daemon/resource.go. Changing one without the other silently
+// misreports how much memory a ZFS node can schedule.
+const defaultHostReserveGB = 2.0
+
+// exportPool releases the pool so a retry after a failed install is not blocked
+// by "pool is busy". The live environment runs spinifex-init as PID 1, so
+// nothing else will ever clean this up.
+func exportPool() {
+	if err := run("zpool", "export", ZFSPoolName); err != nil {
+		slog.Warn("zpool export failed", "pool", ZFSPoolName, "err", err)
+	}
+}
+
+// clearDisk removes any prior filesystem, partition table and ZFS label from a
+// disk so a retry after a partial install is not rejected for stale metadata —
+// a failure that otherwise reads as mysterious rather than as leftover debris.
+//
+// The probes are quiet because a blank disk makes all of them fail, and their
+// stderr on the install console reads like the install is going wrong.
+func clearDisk(d Disk) error {
+	// ZFS labels live at both ends of the device and wipefs does not know about
+	// them, so an old pool member has to be cleared before its partition goes.
+	if old := d.PartitionPath(rootPartNum); exists(old) {
+		_ = runQuiet("zpool", "labelclear", "-f", old)
+	}
+	_ = runQuiet("wipefs", "-a", d.Path)
+	return runQuiet("sgdisk", "-Z", d.Path)
+}
+
+func exists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}

cmd/installer/install/zfs_test.go

@@ -0,0 +1,311 @@
+package install
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+// disk builds a test disk with a by-id path, since that is what pool members
+// are actually named by.
+func disk(name string, gib int64) Disk {
+	return Disk{
+		Path:              "/dev/" + name,
+		ByID:              "/dev/disk/by-id/virtio-" + name,
+		Bytes:             gib << 30,
+		LogicalBlockSize:  512,
+		PhysicalBlockSize: 512,
+	}
+}
+
+func disks(n int, gib int64) []Disk {
+	out := make([]Disk, n)
+	for i := range out {
+		out[i] = disk(string(rune('a'+i)), gib)
+	}
+	return out
+}
+
+func TestBuildVdev(t *testing.T) {
+	tests := []struct {
+		name  string
+		fs    FSType
+		disks []Disk
+		want  string
+	}{
+		{
+			"raid0 is a bare stripe with no keyword",
+			FSZFSRAID0, disks(2, 40),
+			"/dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3",
+		},
+		{
+			"single-disk raid0 is the one-drive case",
+			FSZFSRAID0, disks(1, 40),
+			"/dev/disk/by-id/virtio-a-part3",
+		},
+		{
+			"raid1 is one mirror vdev",
+			FSZFSRAID1, disks(2, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3",
+		},
+		{
+			"raid1 across three disks is a three-way mirror, not a stripe",
+			FSZFSRAID1, disks(3, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 /dev/disk/by-id/virtio-c-part3",
+		},
+		{
+			"raid10 pairs members in selection order",
+			FSZFSRAID10, disks(4, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 " +
+				"mirror /dev/disk/by-id/virtio-c-part3 /dev/disk/by-id/virtio-d-part3",
+		},
+		{
+			"raidz1 across three disks",
+			FSZFSRAIDZ1, disks(3, 40),
+			"raidz1 /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 /dev/disk/by-id/virtio-c-part3",
+		},
+		{
+			"raidz2 needs four",
+			FSZFSRAIDZ2, disks(4, 40),
+			"raidz2 /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 " +
+				"/dev/disk/by-id/virtio-c-part3 /dev/disk/by-id/virtio-d-part3",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := buildVdev(DiskConfig{FS: tt.fs, Disks: tt.disks})
+			if err != nil {
+				t.Fatalf("buildVdev: %v", err)
+			}
+			if got != tt.want {
+				t.Errorf("buildVdev()\n got: %s\nwant: %s", got, tt.want)
+			}
+		})
+	}
+}
+
+// A pool built on /dev/sdX silently breaks when the controller enumerates in a
+// different order, so the fallback only applies when there is no by-id path.
+func TestBuildVdevFallsBackToKernelNameWithoutByID(t *testing.T) {
+	d := disk("a", 40)
+	d.ByID = ""
+	got, err := buildVdev(DiskConfig{FS: FSZFSRAID0, Disks: []Disk{d}})
+	if err != nil {
+		t.Fatalf("buildVdev: %v", err)
+	}
+	if got != "/dev/a3" {
+		t.Errorf("buildVdev() = %q, want /dev/a3", got)
+	}
+}
+
+func TestBuildVdevRejectsExt4(t *testing.T) {
+	if _, err := buildVdev(DiskConfig{FS: FSExt4, Disks: disks(1, 40)}); err == nil {
+		t.Fatal("buildVdev accepted ext4")
+	}
+}
+
+func TestBuildVdevValidatesFirst(t *testing.T) {
+	// Three disks cannot form striped mirrors, and producing a two-disk mirror
+	// plus a silently dropped disk would be worse than an error.
+	if _, err := buildVdev(DiskConfig{FS: FSZFSRAID10, Disks: disks(3, 40)}); err == nil {
+		t.Fatal("buildVdev accepted an odd disk count for RAID10")
+	}
+}
+
+func TestDetectAshift(t *testing.T) {
+	tests := []struct {
+		name     string
+		logical  int
+		physical int
+		want     int
+	}{
+		// ashift cannot be changed after creation and one that is too small
+		// costs a read-modify-write per block forever, so 512e drives are
+		// deliberately treated as 4K.
+		{"512n drive is floored at 4K", 512, 512, 12},
+		{"512e drive reports 4K physical", 512, 4096, 12},
+		{"4Kn drive", 4096, 4096, 12},
+		{"8K native drive", 8192, 8192, 13},
+		{"16K native drive", 16384, 16384, 14},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			d := Disk{LogicalBlockSize: tt.logical, PhysicalBlockSize: tt.physical}
+			if got := detectAshift([]Disk{d}); got != tt.want {
+				t.Errorf("detectAshift() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+// A pool cannot have two sector sizes, so the widest member wins — the reverse
+// would make every write to the large-sector disk a read-modify-write.
+func TestDetectAshiftTakesWidestMember(t *testing.T) {
+	got := detectAshift([]Disk{
+		{LogicalBlockSize: 512, PhysicalBlockSize: 512},
+		{LogicalBlockSize: 512, PhysicalBlockSize: 8192},
+	})
+	if got != 13 {
+		t.Errorf("detectAshift() = %d, want 13", got)
+	}
+}
+
+func TestDefaultARCMaxMiB(t *testing.T) {
+	tests := []struct {
+		name    string
+		totalMi int
+		want    int
+	}{
+		{"tiny VM gets 10% of very little", 2048, 205},
+		{"8GB node gets 10%", 8192, 819},
+		{"64GB node gets 10%", 65536, 6554},
+		// The point of the cap: guest RAM is what this machine sells, and half
+		// of a 256GB node is 128GB of instance capacity the ARC would swallow.
+		{"256GB node is capped at 16GiB", 262144, 16384},
+		{"1TB node is still capped at 16GiB", 1048576, 16384},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := defaultARCMaxMiB(tt.totalMi); got != tt.want {
+				t.Errorf("defaultARCMaxMiB(%d) = %d, want %d", tt.totalMi, got, tt.want)
+			}
+		})
+	}
+}
+
+// An unreadable /proc/meminfo must not produce a zero or negative ceiling,
+// which ZFS would either reject or read as "no limit".
+func TestDefaultARCMaxMiBNeverGoesBelowFloor(t *testing.T) {
+	for _, total := range []int{0, 128, 512} {
+		if got := defaultARCMaxMiB(total); got < arcMinMiB {
+			t.Errorf("defaultARCMaxMiB(%d) = %d, below the floor of %d", total, got, arcMinMiB)
+		}
+	}
+}
+
+func TestResolveZFSOptsFillsDefaults(t *testing.T) {
+	got := resolveZFSOpts(DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)})
+	if got.Ashift != 12 {
+		t.Errorf("Ashift = %d, want 12", got.Ashift)
+	}
+	if got.Compress != "lz4" {
+		t.Errorf("Compress = %q, want lz4", got.Compress)
+	}
+	if got.Checksum != "on" {
+		t.Errorf("Checksum = %q, want on", got.Checksum)
+	}
+	if got.Copies != 1 {
+		t.Errorf("Copies = %d, want 1", got.Copies)
+	}
+	if got.ARCMaxMiB <= 0 {
+		t.Errorf("ARCMaxMiB = %d, want a positive ceiling", got.ARCMaxMiB)
+	}
+}
+
+func TestResolveZFSOptsKeepsOperatorChoices(t *testing.T) {
+	in := DiskConfig{
+		FS:    FSZFSRAIDZ1,
+		Disks: disks(3, 40),
+		ZFS:   ZFSOpts{Ashift: 13, Compress: "zstd", Checksum: "blake3", Copies: 2, ARCMaxMiB: 2048},
+	}
+	got := resolveZFSOpts(in)
+	if got != in.ZFS {
+		t.Errorf("resolveZFSOpts overrode explicit options: got %+v, want %+v", got, in.ZFS)
+	}
+}
+
+func TestParseImportablePools(t *testing.T) {
+	// Captured from `zpool import` on a machine with a pool from a previous
+	// install: the config tree below each header must not be mistaken for one.
+	const out = `   pool: rpool
+     id: 8543219876543210987
+  state: ONLINE
+ action: The pool can be imported using its name or numeric identifier.
+ config:
+
+	rpool                     ONLINE
+	  raidz1-0                ONLINE
+	    virtio-spinifex-disk1-part3  ONLINE
+	    virtio-spinifex-disk2-part3  ONLINE
+
+   pool: tank
+     id: 1111111111111111111
+  state: ONLINE
+ config:
+
+	tank                      ONLINE
+`
+	got := parseImportablePools(out)
+	want := []string{"rpool", "tank"}
+	if len(got) != len(want) {
+		t.Fatalf("parseImportablePools() = %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("pool %d = %q, want %q", i, got[i], want[i])
+		}
+	}
+}
+
+func TestParseImportablePoolsEmpty(t *testing.T) {
+	if got := parseImportablePools("no pools available to import\n"); len(got) != 0 {
+		t.Errorf("parseImportablePools() = %v, want none", got)
+	}
+}
+
+// Every dataset must be creatable in order: a child cannot be created before
+// its parent, and getting this wrong only shows up on real hardware.
+func TestDatasetsAreOrderedParentFirst(t *testing.T) {
+	seen := map[string]bool{ZFSPoolName: true}
+	for _, ds := range datasets {
+		parent := ds.name[:strings.LastIndex(ds.name, "/")]
+		if !seen[parent] {
+			t.Errorf("dataset %s is created before its parent %s", ds.name, parent)
+		}
+		seen[ds.name] = true
+	}
+}
+
+// The root dataset has to be the one the bootloader names, or the machine
+// mounts an empty filesystem and panics.
+func TestRootDatasetIsCreated(t *testing.T) {
+	for _, ds := range datasets {
+		if ds.name == ZFSRootDataset {
+			if ds.mountpoint != "/" {
+				t.Fatalf("%s mountpoint = %q, want /", ZFSRootDataset, ds.mountpoint)
+			}
+			return
+		}
+	}
+	t.Fatalf("%s is never created", ZFSRootDataset)
+}
+
+// rsync --delete tries to rmdir a dataset mountpoint the live source does not
+// have, which fails on a busy mount and aborts the whole copy. Every dataset
+// with a real mountpoint must be protected.
+func TestDatasetProtectFilters(t *testing.T) {
+	got := datasetProtectFilters(DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)})
+	for _, ds := range datasets {
+		if ds.mountpoint == "/" || ds.mountpoint == "none" {
+			continue
+		}
+		want := "--filter=P " + ds.mountpoint
+		if !slices.Contains(got, want) {
+			t.Errorf("missing protect rule for %s (%s)", ds.name, ds.mountpoint)
+		}
+	}
+	// The root dataset is the destination itself, and "none" datasets are never
+	// mounted, so protecting either would be meaningless.
+	for _, f := range got {
+		if f == "--filter=P /" || f == "--filter=P none" {
+			t.Errorf("unexpected protect rule %q", f)
+		}
+	}
+}
+
+func TestDatasetProtectFiltersSkipsExt4(t *testing.T) {
+	if got := datasetProtectFilters(DiskConfig{FS: FSExt4, Disks: disks(1, 40)}); got != nil {
+		t.Errorf("datasetProtectFilters() = %v on ext4, want none", got)
+	}
+}

cmd/installer/netprobe/netprobe.go

@@ -0,0 +1,426 @@
+// Package netprobe identifies physical network interfaces by hardware rather
+// than kernel name, so an operator choosing between eno1, ens1f0np0 and
+// ens1f1np1 can tell the 1gbe Broadcom from the 25gbe Mellanox.
+package netprobe
+
+import (
+	"cmp"
+	"fmt"
+	"log/slog"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// sysClassNet is the kernel's per-interface attribute tree. A variable, not a
+// constant, so tests can point the probe at a fixture instead of the running
+// machine's hardware.
+var sysClassNet = "/sys/class/net"
+
+// The external helpers the probe leans on, indirected so tests can exercise it
+// without the host's real tooling.
+var (
+	listInterfaces = net.Interfaces
+	commandOutput  = func(name string, args ...string) ([]byte, error) {
+		return exec.Command(name, args...).Output()
+	}
+)
+
+// NIC describes one physical interface as presented in the installer's
+// selection table.
+type NIC struct {
+	Name string
+
+	// Vendor and Model come from the udev hardware database, e.g. "Mellanox
+	// Technologies" and "MT27710 Family [ConnectX-4 Lx]". Empty when the
+	// database carries no entry for the card.
+	Vendor string
+	Model  string
+
+	// Driver is the kernel module that claimed the port, e.g. "mlx5_core". It
+	// comes straight from sysfs, so it still identifies a card the hardware
+	// database misses.
+	Driver string
+
+	// DeviceID is the PCI or USB "vendor:device" pair, the last-resort identity
+	// for hardware neither the database nor a driver name pins down.
+	DeviceID string
+
+	// AltName is the first alternative name systemd assigns, which is what
+	// operators often see in switch and cabling documentation.
+	AltName string
+
+	// Speed is the negotiated link speed rendered for display ("25 Gbps") and
+	// SpeedMbps the raw figure behind it. Both are zero on a port with no
+	// carrier, which is why Probe brings links up before reading.
+	Speed     string
+	SpeedMbps int
+
+	// Carrier reports a cable with link. It orders the picker so the port the
+	// operator has just plugged in is the one offered first.
+	Carrier bool
+
+	// State is the link summary shown in the table: "online", "no cable" or
+	// "down".
+	State string
+
+	MTU    int
+	IsWiFi bool
+}
+
+// Label renders the NIC's identity line: kernel name, negotiated speed and link
+// state. The hardware detail is deliberately absent — it goes on its own line
+// via Hardware, where there is room to read it.
+func (n NIC) Label() string {
+	speed := n.Speed
+	if speed == "" {
+		speed = "—"
+	}
+	state := n.State
+	if state == "" {
+		state = "unknown"
+	}
+	return fmt.Sprintf("%s  %s  %s", n.Name, speed, state)
+}
+
+// Hardware renders the card's identity for the detail line, degrading from the
+// database name to the driver and device id rather than to "unknown". An
+// operator needs something to match against the card in their hand.
+func (n NIC) Hardware() string {
+	if hw := strings.TrimSpace(n.Vendor + " " + n.Model); hw != "" {
+		return hw
+	}
+	switch {
+	case n.Driver != "" && n.DeviceID != "":
+		return n.Driver + " [" + n.DeviceID + "]"
+	case n.Driver != "":
+		return n.Driver
+	case n.DeviceID != "":
+		return n.DeviceID
+	}
+	return "no hardware detail"
+}
+
+// linkUpTimeout bounds how long Probe waits after bringing links up for speed
+// and carrier to settle. A 25gbe port negotiating against a cold switch can
+// take a second or two to report a speed.
+var linkUpTimeout = 2 * time.Second
+
+// Probe enumerates non-loopback physical interfaces and enriches each from
+// sysfs and the udev hardware database, ordered so the ports worth assigning
+// come first. Links are brought up first: a down port reports no speed and no
+// carrier, which would leave every fast port unidentified at exactly the moment
+// the operator needs to pick one.
+//
+// Nothing the probe depends on needs a running systemd. The live installer
+// boots straight into spinifex-init with only udevd started, so networkctl —
+// which has to reach systemd-networkd — enriches gaps and is never the source.
+func Probe() ([]NIC, error) {
+	ifaces, err := listInterfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	var names, physical []string
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagLoopback != 0 {
+			continue
+		}
+		if len(iface.HardwareAddr) == 0 {
+			continue
+		}
+		names = append(names, iface.Name)
+		if hasDevice(iface.Name) {
+			physical = append(physical, iface.Name)
+		}
+	}
+
+	// Bridges, bonds and veth pairs carry a MAC like any port but have no
+	// device behind them, and offering them would let a plane be bound to
+	// something that does not exist until after the install. Keep them only if
+	// the filter found nothing, so an unusual platform still gets a list.
+	if len(physical) > 0 {
+		names = physical
+	}
+
+	brought := false
+	for _, name := range names {
+		if _, err := commandOutput("ip", "link", "set", "dev", name, "up"); err != nil {
+			slog.Debug("netprobe: could not bring link up", "iface", name, "err", err)
+			continue
+		}
+		brought = true
+	}
+	if brought {
+		time.Sleep(linkUpTimeout)
+	}
+
+	nics := make([]NIC, 0, len(names))
+	for _, name := range names {
+		nics = append(nics, describe(name))
+	}
+	slices.SortFunc(nics, byUsefulness)
+	return nics, nil
+}
+
+// describe reads everything about one interface that needs no daemon, then lets
+// networkctl fill the gaps if it happens to be answering.
+func describe(name string) NIC {
+	nic := NIC{
+		Name:     name,
+		Driver:   sysLinkBase(name, "device/driver"),
+		DeviceID: deviceID(name),
+		MTU:      sysInt(name, "mtu"),
+		IsWiFi:   isWiFi(name),
+	}
+
+	// The kernel reports -1 for a port that cannot state its speed; treat that
+	// as unknown rather than letting it sort below every real figure.
+	nic.SpeedMbps = max(sysInt(name, "speed"), 0)
+	nic.Speed = formatSpeed(nic.SpeedMbps)
+	nic.Carrier = sysAttr(name, "carrier") == "1"
+	nic.State = linkState(name, nic.Carrier)
+
+	props := udevProperties(name)
+	nic.Vendor = props["ID_VENDOR_FROM_DATABASE"]
+	nic.Model = props["ID_MODEL_FROM_DATABASE"]
+
+	// udev's predictable names are the ones that appear in cabling notes; the
+	// path-based name is the one networkctl lists first.
+	for _, key := range []string{"ID_NET_NAME_PATH", "ID_NET_NAME_ONBOARD", "ID_NET_NAME_SLOT"} {
+		if alt := props[key]; alt != "" && alt != name {
+			nic.AltName = alt
+			break
+		}
+	}
+
+	if out, err := commandOutput("networkctl", "status", "--no-pager", name); err == nil {
+		fill(&nic, ParseStatus(string(out)))
+	} else {
+		slog.Debug("netprobe: networkctl unavailable, using sysfs only", "iface", name, "err", err)
+	}
+	return nic
+}
+
+// fill copies parsed hardware detail onto a NIC without overwriting anything
+// sysfs already established. State is never taken: the carrier bit is a truer
+// answer to "is this cable live" than networkd's configuration status.
+func fill(dst *NIC, src NIC) {
+	if dst.Vendor == "" {
+		dst.Vendor = src.Vendor
+	}
+	if dst.Model == "" {
+		dst.Model = src.Model
+	}
+	if dst.AltName == "" {
+		dst.AltName = src.AltName
+	}
+	if dst.Speed == "" {
+		dst.Speed = src.Speed
+	}
+	if dst.MTU == 0 {
+		dst.MTU = src.MTU
+	}
+}
+
+// byUsefulness orders the picker so the interfaces an operator would actually
+// assign come first: a live cable, then wired over wireless, then the fastest
+// port. The rest still appear, just below the plausible choices.
+func byUsefulness(a, b NIC) int {
+	if a.Carrier != b.Carrier {
+		return boolOrder(a.Carrier)
+	}
+	if a.IsWiFi != b.IsWiFi {
+		return boolOrder(b.IsWiFi)
+	}
+	if a.SpeedMbps != b.SpeedMbps {
+		return cmp.Compare(b.SpeedMbps, a.SpeedMbps)
+	}
+	return cmp.Compare(a.Name, b.Name)
+}
+
+// boolOrder sorts true ahead of false.
+func boolOrder(first bool) int {
+	if first {
+		return -1
+	}
+	return 1
+}
+
+// formatSpeed renders the kernel's Mbit/s figure the way a datasheet does.
+func formatSpeed(mbps int) string {
+	switch {
+	case mbps <= 0:
+		return ""
+	case mbps%1000 == 0:
+		return fmt.Sprintf("%d Gbps", mbps/1000)
+	default:
+		return fmt.Sprintf("%d Mbps", mbps)
+	}
+}
+
+// linkState collapses the kernel's flags into the three outcomes an operator
+// acts on: the port is live, the port is enabled but unplugged, or the port
+// never came up at all. operstate reports "down" for the latter two alike.
+func linkState(name string, carrier bool) string {
+	switch {
+	case carrier:
+		return "online"
+	case adminUp(name):
+		return "no cable"
+	default:
+		return "down"
+	}
+}
+
+// adminUp reads the IFF_UP bit out of the interface flags.
+func adminUp(name string) bool {
+	flags, err := strconv.ParseUint(strings.TrimPrefix(sysAttr(name, "flags"), "0x"), 16, 64)
+	return err == nil && flags&0x1 != 0
+}
+
+// deviceID reads the PCI or USB "vendor:device" pair. A NIC's device link
+// points at the PCI function directly; on USB it points at the interface, whose
+// parent carries the ids.
+func deviceID(name string) string {
+	dev := filepath.Join(sysClassNet, name, "device")
+	if id := idPair(dev, "vendor", "device"); id != "" {
+		return id
+	}
+	return idPair(filepath.Join(dev, ".."), "idVendor", "idProduct")
+}
+
+func idPair(dir, vendorAttr, productAttr string) string {
+	vendor := readTrimmed(filepath.Join(dir, vendorAttr))
+	product := readTrimmed(filepath.Join(dir, productAttr))
+	if vendor == "" || product == "" {
+		return ""
+	}
+	return strings.TrimPrefix(vendor, "0x") + ":" + strings.TrimPrefix(product, "0x")
+}
+
+// udevProperties reads the interface's udev database entry, which is where the
+// hardware-database vendor and model names live. udevd populates it during the
+// installer's device enumeration, so this works with no systemd running.
+func udevProperties(name string) map[string]string {
+	out, err := commandOutput("udevadm", "info", "--query=property", "--path="+filepath.Join(sysClassNet, name))
+	if err != nil {
+		slog.Debug("netprobe: udevadm info failed", "iface", name, "err", err)
+		return nil
+	}
+
+	props := make(map[string]string)
+	for line := range strings.SplitSeq(string(out), "\n") {
+		if key, value, ok := strings.Cut(strings.TrimSpace(line), "="); ok {
+			props[key] = value
+		}
+	}
+	return props
+}
+
+// ParseStatus extracts the hardware fields from `networkctl status <iface>`
+// output. The format is a set of right-aligned "Key: value" lines, with
+// Alternative Names continuing across subsequent indented lines.
+func ParseStatus(out string) NIC {
+	var nic NIC
+	var inAltNames bool
+
+	for line := range strings.SplitSeq(out, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" {
+			inAltNames = false
+			continue
+		}
+
+		key, value, found := strings.Cut(trimmed, ": ")
+		if !found {
+			// A continuation line under "Alternative Names:" — take the first
+			// only, which is the one operators see in cabling notes.
+			if inAltNames && nic.AltName == "" {
+				nic.AltName = trimmed
+			}
+			continue
+		}
+		inAltNames = false
+
+		key = strings.TrimSpace(key)
+		value = strings.TrimSpace(value)
+		switch key {
+		case "Vendor":
+			nic.Vendor = value
+		case "Model":
+			nic.Model = value
+		case "Speed":
+			nic.Speed = value
+		case "Online state":
+			// Rendered as "online" or "online (configured)".
+			if f := strings.Fields(value); len(f) > 0 {
+				nic.State = f[0]
+			}
+		case "MTU":
+			// Rendered as "9000 (min: 68, max: 9978)".
+			if f := strings.Fields(value); len(f) > 0 {
+				if mtu, err := strconv.Atoi(f[0]); err == nil {
+					nic.MTU = mtu
+				}
+			}
+		case "Alternative Names":
+			inAltNames = true
+			if f := strings.Fields(value); len(f) > 0 && nic.AltName == "" {
+				nic.AltName = f[0]
+			}
+		}
+	}
+	return nic
+}
+
+// hasDevice reports whether the interface is backed by real hardware, which
+// sysfs shows as a device link to the PCI, USB or platform node behind it.
+func hasDevice(name string) bool {
+	_, err := os.Stat(filepath.Join(sysClassNet, name, "device"))
+	return err == nil
+}
+
+// isWiFi reports whether the interface has a wireless subdirectory in sysfs.
+func isWiFi(name string) bool {
+	_, err := os.Stat(filepath.Join(sysClassNet, name, "wireless"))
+	return err == nil
+}
+
+// sysAttr reads one sysfs attribute. A missing or unreadable attribute is not
+// an error worth surfacing: speed and carrier both fail on a port with no link,
+// which the state column already conveys.
+func sysAttr(name, attr string) string {
+	return readTrimmed(filepath.Join(sysClassNet, name, attr))
+}
+
+func sysInt(name, attr string) int {
+	n, err := strconv.Atoi(sysAttr(name, attr))
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+// sysLinkBase resolves a sysfs symlink and returns its final element, which is
+// how the driver name is exposed.
+func sysLinkBase(name, attr string) string {
+	target, err := os.Readlink(filepath.Join(sysClassNet, name, attr))
+	if err != nil {
+		return ""
+	}
+	return filepath.Base(target)
+}
+
+func readTrimmed(path string) string {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(b))
+}