release: update go.mod for v1.15.0

This reverts commit c2b80a9b93.

.github/actions/e2e-analyze/README.md

@@ -70,7 +70,7 @@ When the report format changes intentionally, regenerate the snapshots:
 go test ./.github/actions/e2e-analyze/ -update
 ```
 
-## Per-failure bundles (Stage 2)
+## Per-failure bundles
 
 After parsing, the action materialises a bundle directory per failure
 under `<log-dir>/analysis/NN-<TestName>/`. The report links to it from

.github/actions/e2e-analyze/analyze.go

@@ -1,6 +1,6 @@
 // Package main implements the e2e-analyze GitHub Action.
 //
-// Stage 1: cluster failures from go-junit-report's junit-*.xml by error signature,
+// Cluster failures from go-junit-report's junit-*.xml by error signature,
 // pick the earliest non-cascade failure per suite as the likely root cause,
 // and render a human-readable report.
 //
@@ -73,8 +73,14 @@ type Failure struct {
 	// `test-<suite>.start` file written by the workflow before the suite
 	// runs) can recompute StartAt without re-parsing the XML.
 	OffsetFromSuiteStart time.Duration
+	// SuiteSpan is the suite's own wall-clock span, set only when the suite
+	// ran its tests in parallel. Overlapping tests make the cumulative
+	// offset above pure fiction — it lands minutes past the real failure —
+	// so consumers widen the journal window to the suite instead of
+	// slicing a plausible-looking window around the wrong minute.
+	SuiteSpan time.Duration
 	// BundlePath is the relative path (from log-dir) to the per-failure
-	// bundle written by Stage 2's WriteBundles. Empty when bundles were
+	// bundle written by WriteBundles. Empty when bundles were
 	// not generated (e.g. unit tests that call Render directly).
 	BundlePath string
 }
@@ -88,6 +94,12 @@ type SuiteReport struct {
 	Root      *Failure    // nil if no failures
 	Cascades  []Failure   // everything else that failed in this suite
 	Buckets   [][]Failure // failures grouped by signature (root's bucket first)
+	// EndedAt is go-junit-report's <testsuite timestamp>, which records when
+	// the XML was produced — i.e. when the suite finished, not when it began.
+	EndedAt time.Time
+	// TopLevelTotal is the summed duration of the parentless tests. Longer than
+	// the suite's wall clock means they overlapped.
+	TopLevelTotal time.Duration
 }
 
 // Report is the full analysis across every junit file.
@@ -119,6 +131,28 @@ var cascadeMarkers = []string{
 	"Expected value not to be nil",
 }
 
+// teardownPatterns match the content a test logs from t.Cleanup or from the
+// harness's on-failure diagnostics — i.e. after it has already failed. They are
+// the LAST file:line lines in the failure body, so extractErrorLine's
+// keep-the-last rule picks them over the assertion unless they are skipped.
+// Anchored deliberately: a fragment loose enough to match an assertion would
+// blank out the one line the report exists to show.
+var teardownPatterns = []*regexp.Regexp{
+	regexp.MustCompile(`^DB instance \S+ is gone$`),
+	regexp.MustCompile(`^db diagnostics \S+:`),
+}
+
+// isTeardownLine reports whether a candidate error line was emitted after the
+// failure it would otherwise be mistaken for.
+func isTeardownLine(content string) bool {
+	for _, re := range teardownPatterns {
+		if re.MatchString(content) {
+			return true
+		}
+	}
+	return false
+}
+
 // noiseReplacers normalise dynamic identifiers so that "ten failures, same
 // bug" bucket together. Pattern ordering matters: longer/more-specific
 // patterns must precede the catch-alls.
@@ -206,11 +240,14 @@ func extractErrorLine(body string) string {
 		if m := goFileLineRe.FindStringSubmatch(l); m != nil {
 			// Keep the LAST occurrence: t.Log / harness traces emit
 			// progress lines from the same file:line pattern before the
-			// failing assertion finally fires.
-			goFileLine = strings.TrimSpace(m[1])
+			// failing assertion finally fires. Cleanup runs after it, though,
+			// so its lines are last of all and have to be skipped.
+			if content := strings.TrimSpace(m[1]); !isTeardownLine(content) {
+				goFileLine = content
+			}
 			continue
 		}
-		if fileHintRe.MatchString(l) {
+		if fileHintRe.MatchString(l) && !isTeardownLine(l) {
 			lastTestLine = l
 		}
 	}
@@ -254,11 +291,19 @@ func extractFileHint(body string) string {
 	if m := errorTraceRe.FindStringSubmatch(body); m != nil {
 		return m[1]
 	}
-	all := fileHintRe.FindAllString(body, -1)
-	if len(all) == 0 {
-		return ""
+	// Scanned line by line rather than across the body so a cleanup line's own
+	// file:line does not become the site the report points a reader at.
+	hint := ""
+	for _, raw := range strings.Split(body, "\n") {
+		l := strings.TrimSpace(raw)
+		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isTeardownLine(strings.TrimSpace(m[1])) {
+			continue
+		}
+		if all := fileHintRe.FindAllString(l, -1); len(all) > 0 {
+			hint = all[len(all)-1]
+		}
 	}
-	return all[len(all)-1]
+	return hint
 }
 
 // signature normalises an error line so different runs of the same root
@@ -313,6 +358,47 @@ func computeLeafSet(cases []junitTC) map[string]bool {
 	return leaf
 }
 
+// computeRolledUpSet returns the testcase names whose failure is already
+// reported by a failing descendant. go-junit-report gives such a parent a
+// testcase of its own, and its body is the parent's t.Log output rather than an
+// assertion — a cleanup line like "DB instance … is gone", which passing tests
+// emit too, reads to extractErrorLine exactly like a failure site.
+func computeRolledUpSet(cases []junitTC) map[string]bool {
+	rolledUp := make(map[string]bool, len(cases))
+	for _, tc := range cases {
+		if tc.Failure == nil && tc.Error == nil {
+			continue
+		}
+		// Mark every ancestor: the failure belongs to the deepest case that
+		// carries it, not to the tests it is nested under.
+		name := tc.Name
+		for {
+			i := strings.LastIndex(name, "/")
+			if i <= 0 {
+				break
+			}
+			name = name[:i]
+			rolledUp[name] = true
+		}
+	}
+	return rolledUp
+}
+
+// topLevelTotal sums the durations of the tests with no parent, which are the
+// units t.Parallel overlaps. Compared against the suite's wall clock it says
+// whether the suite ran its tests concurrently — see ApplySuiteStartFiles.
+// suite.Time cannot answer that: go-junit-report sets it to the sum of every
+// testcase, parents included, so it is never less than any subset of itself.
+func topLevelTotal(cases []junitTC) time.Duration {
+	total := 0.0
+	for _, tc := range cases {
+		if !strings.Contains(tc.Name, "/") {
+			total += tc.Time
+		}
+	}
+	return time.Duration(total * float64(time.Second))
+}
+
 // parseStartTime turns a junit timestamp attr (RFC3339) into a time.Time;
 // zero value if unparseable.
 func parseStartTime(s string) time.Time {
@@ -359,6 +445,9 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 		// summing every testcase double-counts. Pre-compute the leaf set so
 		// only leaves contribute to the suite's running offset.
 		isLeaf := computeLeafSet(suite.Cases)
+		rolledUp := computeRolledUpSet(suite.Cases)
+		rep.EndedAt = start
+		rep.TopLevelTotal += topLevelTotal(suite.Cases)
 		cumul := 0.0
 		for _, tc := range suite.Cases {
 			rep.Total++
@@ -376,11 +465,12 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 				continue
 			}
 
-			// Skip the synthetic parent-test "Failed" stub that go-junit-report
-			// emits for any Go test whose subtests failed. It has an empty body
-			// and would otherwise outrank real failures by XML order.
+			// Skip the parent-test entry go-junit-report emits for any Go test
+			// whose subtests failed: it is a rollup of failures already counted
+			// below it, and it would outrank them by XML order. An empty body is
+			// the same stub in the case where the parent logged nothing.
 			trimmed := strings.TrimSpace(stripANSI(body))
-			if trimmed == "" {
+			if trimmed == "" || rolledUp[tc.Name] {
 				if leaf {
 					cumul += tc.Time
 				}
@@ -462,14 +552,24 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 	return rep, nil
 }
 
-// Render writes the markdown report (Stage 1).
+// Render writes the markdown report.
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
 
-	totalFail := 0
+	totalFail, totalTests := 0, 0
 	for _, s := range r.Suites {
 		totalFail += s.FailCount
+		totalTests += s.Total
+	}
+
+	// A cell that dies before the suites run emits junit with no testcases at
+	// all. Reporting that as clean turns "nothing ran" into "nothing wrong", so
+	// name it and point above the suites.
+	if totalTests == 0 {
+		b.WriteString("⚠️ No tests ran — no testcases in any junit file. " +
+			"The failure is upstream of the suites (provisioning, bootstrap, or reimage); check the job log.\n")
+		return b.String()
 	}
 	if totalFail == 0 {
 		b.WriteString("✅ No failures across any suite.\n")
@@ -484,12 +584,22 @@ func Render(r Report) string {
 		fmt.Fprintf(&b, "### Suite `%s`: %d failed, 1 root cause likely\n\n", s.Label, s.FailCount)
 
 		if s.Root != nil {
-			b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			// XML order is completion order, so "earliest" only means earliest
+			// in time for a suite whose tests ran one after another.
+			if s.Root.SuiteSpan > 0 {
+				b.WriteString("**First non-cascade failure** (the suite ran in parallel, so this is report order, not time order)\n\n")
+			} else {
+				b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			}
 			fmt.Fprintf(&b, "- Test: `%s`\n", s.Root.Name)
-			if !s.Root.StartAt.IsZero() {
+			switch {
+			case s.Root.SuiteSpan > 0:
+				fmt.Fprintf(&b, "- Start: unknown, the suite ran in parallel (duration %.1fs)\n",
+					s.Root.Duration)
+			case !s.Root.StartAt.IsZero():
 				fmt.Fprintf(&b, "- Start: %s (duration %.1fs)\n",
 					s.Root.StartAt.Format("15:04:05"), s.Root.Duration)
-			} else {
+			default:
 				fmt.Fprintf(&b, "- Duration: %.1fs\n", s.Root.Duration)
 			}
 			if s.Root.Error != "" {

.github/actions/e2e-analyze/analyze_test.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 )
 
 // updateGolden lets `go test ./.github/actions/e2e-analyze/ -update` rewrite
@@ -124,6 +125,22 @@ func TestExtractErrorLine_FallsBackToLastTestLine(t *testing.T) {
 	}
 }
 
+// A test's cleanup logs after it has failed, so its lines are the last in the
+// body — and the cleanup line here is one passing tests emit too.
+func TestExtractErrorLine_SkipsCleanupLoggedAfterTheFailure(t *testing.T) {
+	body := `    failure_test.go:42: DB instance rds-e2e-failure-1 reached status available
+    failure_test.go:60: EventuallyErr: condition not met within 4m0s: rds-e2e-failure-1 status=available want=failed
+    rds.go:364: db diagnostics rds-e2e-failure-1: no VM to capture a console from (<nil>)
+    main_test.go:325: DB instance rds-e2e-failure-1 is gone`
+	got := extractErrorLine(body)
+	if !strings.HasPrefix(got, "EventuallyErr:") {
+		t.Errorf("extractErrorLine = %q, want the assertion at failure_test.go:60", got)
+	}
+	if hint := extractFileHint(body); hint != "failure_test.go:60" {
+		t.Errorf("extractFileHint = %q, want failure_test.go:60", hint)
+	}
+}
+
 func TestExtractFileHint_PicksLastMatch(t *testing.T) {
 	body := "ec2helpers.go:50: setup\n    vpc_test.go:227: Eventually: condition not met"
 	got := extractFileHint(body)
@@ -157,6 +174,116 @@ func TestParseFile_SkipsEmptyParentFailure(t *testing.T) {
 	}
 }
 
+func TestParseFile_SkipsParentRolledUpBySubtests(t *testing.T) {
+	// A parent that logs during cleanup gets a non-empty body, and a t.Log like
+	// "DB instance … is gone" — which passing tests emit too — reads exactly
+	// like an assertion site. Its subtest holds the real failure.
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="2">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    main_test.go:325: DB instance rds-e2e-backup-1 is gone
+]]></failure></testcase>
+  <testcase name="TestX/RealFailure" time="0.5"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 {
+		t.Fatalf("FailCount = %d, want 1 (the parent rollup should be skipped)", sr.FailCount)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX/RealFailure" {
+		t.Fatalf("expected the subtest as root, got %+v", sr.Root)
+	}
+}
+
+// A parent that fails on its own after its subtests passed is a real failure,
+// and dropping it would lose the only record of it.
+func TestParseFile_KeepsParentWithNoFailingSubtest(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    foo_test.go:42:
+        Messages: the parent asserted after its subtests
+]]></failure></testcase>
+  <testcase name="TestX/Passing" time="0.5"></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX" {
+		t.Fatalf("expected the parent as root, got %+v", sr.Root)
+	}
+}
+
+// suiteFixture writes a junit file plus its .start sidecar into a temp log dir
+// and returns the parsed, start-file-corrected report for one suite.
+func suiteFixture(t *testing.T, start string, xml []byte) SuiteReport {
+	t.Helper()
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "test-x.start"), []byte(start), 0o644); err != nil {
+		t.Fatalf("write start file: %v", err)
+	}
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	rep := Report{Suites: []SuiteReport{sr}}
+	ApplySuiteStartFiles(&rep, dir)
+	return rep.Suites[0]
+}
+
+// Parallel tests overlap, so summing the ones before a failure puts its start
+// minutes past where it really was — and the journal slice cut from that window
+// misses the failure entirely. The tell is the top-level tests taking longer in
+// total than the suite's own wall clock.
+func TestApplySuiteStartFiles_DropsTheOffsetWhenTheSuiteRanInParallel(t *testing.T) {
+	// timestamp is when go-junit-report wrote the XML, i.e. the suite's end.
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestSlow" time="100.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 2*time.Minute {
+		t.Errorf("SuiteSpan = %s, want 2m0s to mark the suite parallel", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 0, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start %s rather than a summed offset", s.Root.StartAt, want)
+	}
+}
+
+// The sequential case must keep its offsets: they are the only thing that makes
+// a per-failure journal slice narrow enough to read.
+func TestApplySuiteStartFiles_KeepsTheOffsetWhenTheSuiteRanSequentially(t *testing.T) {
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestFirst" time="30.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 0 {
+		t.Errorf("SuiteSpan = %s, want 0 for a sequential suite", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 30, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start plus 30s of preceding tests %s", s.Root.StartAt, want)
+	}
+}
+
 func TestParseFile_CascadeFallbackWhenAllCascades(t *testing.T) {
 	// If every failure is a cascade marker, the earliest one becomes root
 	// so the report isn't blank.
@@ -189,3 +316,25 @@ func TestRender_NoFailuresIsClean(t *testing.T) {
 		t.Errorf("expected zero-failure banner, got:\n%s", out)
 	}
 }
+
+// A cell that fails in provisioning writes junit with no testcases. Rendering
+// the clean banner there reports "nothing ran" as "nothing wrong".
+func TestRender_NoTestsRanIsNotClean(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		suites []SuiteReport
+	}{
+		{"empty junit file", []SuiteReport{{File: "junit-baremetal.xml", Label: "baremetal", Total: 0, FailCount: 0}}},
+		{"no junit files at all", nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			out := Render(Report{Title: "E2E failure analysis", Suites: tc.suites})
+			if strings.Contains(out, "No failures across any suite") {
+				t.Errorf("clean banner rendered when no test ran, got:\n%s", out)
+			}
+			if !strings.Contains(out, "No tests ran") {
+				t.Errorf("expected a no-tests-ran warning, got:\n%s", out)
+			}
+		})
+	}
+}

.github/actions/e2e-analyze/bundle.go

@@ -17,9 +17,9 @@ import (
 )
 
 // bundlePad is the per-failure window padding either side of the
-// [StartAt, StartAt+Duration] testcase span. Five seconds matches the
-// plan doc: long enough to catch the daemon's pre-test setup and
-// post-test cleanup chatter, short enough to keep the slice focused.
+// [StartAt, StartAt+Duration] testcase span. Five seconds catches the
+// daemon's pre-test setup and post-test cleanup chatter while keeping
+// the slice focused.
 const bundlePad = 5 * time.Second
 
 // nameSanitiseRe replaces characters unsafe in directory names with `_`.
@@ -58,21 +58,57 @@ func ApplySuiteStartFiles(rep *Report, logDir string) {
 			fmt.Fprintf(os.Stderr, "::warning::e2e-analyze: parse %s: %v\n", startPath, err)
 			continue
 		}
-		if s.Root != nil {
-			s.Root.StartAt = ts.Add(s.Root.OffsetFromSuiteStart)
-		}
-		for ci := range s.Cascades {
-			s.Cascades[ci].StartAt = ts.Add(s.Cascades[ci].OffsetFromSuiteStart)
-		}
-		for bi := range s.Buckets {
-			for fi := range s.Buckets[bi] {
-				f := &s.Buckets[bi][fi]
-				f.StartAt = ts.Add(f.OffsetFromSuiteStart)
+		// The start file is the first wall clock the analyzer has, so it is also
+		// the first chance to tell a parallel suite from a sequential one — and
+		// a parallel one's per-test offsets are fiction, since overlapping tests
+		// never started at the sum of the ones before them.
+		span := suiteSpanIfParallel(s, ts)
+		for _, f := range collectFailures(s) {
+			f.SuiteSpan = span
+			if span > 0 {
+				f.StartAt = ts
+				continue
 			}
+			f.StartAt = ts.Add(f.OffsetFromSuiteStart)
 		}
 	}
 }
 
+// suiteSpanIfParallel returns the suite's wall-clock span when its top-level
+// tests overlapped, and zero when they ran one after another. Consumers read a
+// non-zero span as "no per-test offset is derivable from this suite".
+func suiteSpanIfParallel(s *SuiteReport, start time.Time) time.Duration {
+	if s.EndedAt.IsZero() || s.TopLevelTotal == 0 {
+		return 0
+	}
+	wall := s.EndedAt.Sub(start)
+	// The 5% margin keeps per-testcase rounding from calling a sequential
+	// suite parallel; real overlap runs to several times the wall clock.
+	if wall <= 0 || s.TopLevelTotal <= time.Duration(float64(wall)*1.05) {
+		return 0
+	}
+	return wall
+}
+
+// collectFailures returns every failure in the report by pointer. Root, cascades
+// and buckets hold independent copies of the same failures, so a field that has
+// to reach the bundle writer must be set on all three.
+func collectFailures(s *SuiteReport) []*Failure {
+	var all []*Failure
+	if s.Root != nil {
+		all = append(all, s.Root)
+	}
+	for ci := range s.Cascades {
+		all = append(all, &s.Cascades[ci])
+	}
+	for bi := range s.Buckets {
+		for fi := range s.Buckets[bi] {
+			all = append(all, &s.Buckets[bi][fi])
+		}
+	}
+	return all
+}
+
 // WriteBundles materialises per-failure bundles under <logDir>/analysis/.
 // Mutates rep in-place so each Failure.BundlePath holds the relative
 // path the renderer will link from the summary.
@@ -143,7 +179,11 @@ func writeOneBundle(logDir, base, suiteLabel string, f *Failure) {
 
 	writeFailureSummary(dir, suiteLabel, f)
 
-	if !f.StartAt.IsZero() {
+	// A parallel suite offers no derivable per-test window, and the narrowing is
+	// the whole point of the slice: cutting one from the wrong minute leaves out
+	// the lines that explain the failure, and widening it to the suite just
+	// copies the artifact's own journal into every bundle.
+	if !f.StartAt.IsZero() && f.SuiteSpan == 0 {
 		start := f.StartAt.Add(-bundlePad)
 		end := f.StartAt.Add(time.Duration(f.Duration * float64(time.Second))).Add(bundlePad)
 		journalSrc := filepath.Join(logDir, "spinifex-journal.log")
@@ -188,7 +228,11 @@ func writeFailureSummary(dir, suiteLabel string, f *Failure) {
 	var b strings.Builder
 	fmt.Fprintf(&b, "Suite:    %s\n", suiteLabel)
 	fmt.Fprintf(&b, "Test:     %s\n", f.Name)
-	if !f.StartAt.IsZero() {
+	switch {
+	case f.SuiteSpan > 0:
+		b.WriteString("Start:    unknown, the suite ran in parallel; " +
+			"no journal window is derivable — read ../../spinifex-journal.log\n")
+	case !f.StartAt.IsZero():
 		fmt.Fprintf(&b, "Start:    %s\n", f.StartAt.Format(time.RFC3339))
 	}
 	fmt.Fprintf(&b, "Duration: %.3fs\n", f.Duration)

.github/actions/e2e-analyze/main.go

@@ -45,7 +45,7 @@ func main() {
 		report.Suites = append(report.Suites, sr)
 	}
 
-	// Stage 2: prefer authoritative per-suite start times written by the
+	// Prefer authoritative per-suite start times written by the
 	// workflow over junit-report's conversion-time `timestamp` attribute,
 	// then materialise per-failure bundles so Render can link them.
 	if logDir != "" {

.github/actions/e2e-analyze/slice.go

@@ -1,6 +1,6 @@
 package main
 
-// Stage 2: per-failure log slicers used by bundle.go to materialise time-window /
+// Per-failure log slicers used by bundle.go to materialise time-window /
 // testcase-window log excerpts next to each named failure.
 
 import (

.github/scripts/suite-sets.sh

@@ -20,7 +20,11 @@
 # 8192 halves reverted a generation) and passes against the fixed one (0 of
 # 8192). A suite that has never been shown to go red for the reason it claims
 # does not belong here.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock"
+# rds is single-only for the opposite reason to lb: v1 has no multi-AZ
+# behaviour, so a second node adds no assertion. Its DB VMs are capped at four
+# concurrent by the suite's own semaphore, which is what keeps it inside the
+# job's 50-minute budget alongside the suites above.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
 
 # Suites runnable against a multi-node environment.
 E2E_SUITES_MULTI="multinode lb cert"

.github/workflows/apt-packages.yml

@@ -0,0 +1,48 @@
+name: Apt Packages
+
+# Path-filtered like image-scripts.yml: this resolves setup.sh's runtime package
+# list against each supported suite's real indices, so a renamed, dropped, or
+# ambiguously virtual package fails here rather than on a user's machine.
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  apt_packages:
+    name: Resolve apt list (${{ matrix.image }})
+    runs-on: ubuntu-26.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # The suites the nightly E2E matrix actually installs on. Ubuntu 22.04 is
+        # accepted by detect_os but has never been exercised and does not resolve.
+        image:
+          - ubuntu:24.04
+          - ubuntu:26.04
+          - debian:13
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Resolve runtime package list
+        run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}

.github/workflows/e2e-ddil.yml

@@ -1,8 +1,7 @@
 name: E2E DDIL
 
-# Phase 1 trigger: workflow_dispatch only. Nightly cron + on-main-merge
-# land in mulga-siv-5.8 once ≥4 scenarios are asserting. See
-# docs/development/improvements/ddil-e2e-test-harness.md §4 / §Dependencies.
+# This workflow is manually triggered while the DDIL scenarios are being
+# validated.
 on:
   workflow_dispatch:
     inputs:

.github/workflows/e2e-nightly.yml

@@ -20,7 +20,7 @@ permissions:
 # trigger must stay in sync on main for cron to fire.
 env:
   GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
-  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
+  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
   # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
   # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
@@ -36,11 +36,11 @@ jobs:
       matrix:
         include:
           - { cell: 1,  install: binary, topo: single,     net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
+          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
+          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
           - { cell: 8,  install: binary, topo: real-multi, net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
+          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
+          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
           - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
           - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
           - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }
@@ -128,10 +128,13 @@ jobs:
         run: |
           export TF_VAR_git_branch="${GIT_BRANCH}"
 
+          # Match the family, not a pinned version: an exact match silently stops
+          # overriding the image when the matrix bumps release, dropping the cell
+          # back onto the debian gold.
           TOFU_VAR_ARGS=""
-          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
-            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
-          fi
+          case "$MATRIX_HOST" in
+            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
+          esac
 
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
@@ -370,10 +373,12 @@ jobs:
         env:
           MATRIX_HOST: ${{ matrix.host }}
         run: |
+          # Must mirror the Provision step's override, or destroy runs against a
+          # different image path than apply did.
           TOFU_VAR_ARGS=""
-          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
-            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
-          fi
+          case "$MATRIX_HOST" in
+            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
+          esac
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1

.github/workflows/e2e-suites.yml

@@ -1,7 +1,5 @@
-# E2E (per-suite blocks) — EXPERIMENTAL, workflow_dispatch only.
-#
-# Restructures the looped e2e.yml into env-grouped job blocks so each suite
-# surfaces as its own node in the Actions graph (mulga-siv-283):
+# E2E per-suite blocks. Each suite appears as its own node in the Actions
+# graph:
 #
 #   single_provision ─┬─ single_suite (matrix: the single-node-eligible suites)
 #                     └─ single_teardown (if: always)

.github/workflows/e2e.yml

@@ -15,7 +15,7 @@ on:
   workflow_dispatch:
     inputs:
       suites:
-        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam')"
+        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam e2e-rds')"
         required: false
         default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
       test_run:
@@ -50,7 +50,7 @@ jobs:
     # provision a tofu VM and pay the Spinifex install only to run nothing and
     # tear down. Push triggers keep running via the null-check on
     # inputs.suites.
-    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock')
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
     runs-on: [self-hosted, ci-single]
     timeout-minutes: 50
     env:
@@ -139,7 +139,8 @@ jobs:
           ./build-install-artifacts.sh "${GIT_BRANCH}" \
             "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
             "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar"
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
 
       - name: Provision VM
         working-directory: mulga/scripts/tofu-cluster
@@ -249,6 +250,54 @@ jobs:
           EOF
           ok "kubectl installed on VM"
 
+      # CreateDBInstance boots the DB VM from the rds-postgres AMI, so that image
+      # must be in the VM's image store before the suite runs — same split as the
+      # eks-node pair above: build on the runner (needs source + network), import
+      # on the VM (needs predastore/NATS). The suite's psql client needs no step
+      # here: it runs inside a client VM that installs postgresql-client from its
+      # own user-data, because a private-only endpoint is unreachable from the
+      # runner and from the cluster host alike.
+      - name: Build rds-postgres AMI (libguestfs, runner)
+        if: contains(env.SUITES, 'e2e-rds')
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          banner "build rds-postgres AMI (libguestfs — no host devices)"
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
+          export LIBGUESTFS_BACKEND=direct
+          sudo chmod 0644 /boot/vmlinuz-* || true
+          group_open "make build-rds-postgres-image"
+          make build-rds-postgres-image
+          group_close
+          ok "raw: $(du -h /tmp/rds-postgres-image-build/rds-postgres-alpine.raw | cut -f1)"
+
+      - name: Import rds-postgres AMI on VM
+        if: contains(env.SUITES, 'e2e-rds')
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+            spinifex/scripts/images/rds-postgres/manifest.conf)"
+          # SYSTEM_TAG carries three tags, not one: managed-by marks it a system
+          # image and engine/engine-version are what an EngineVersion request
+          # resolves against. --tag is repeatable, so expand rather than pass the
+          # whole string as a single value.
+          TAG_ARGS=""
+          for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          banner "import rds-postgres AMI to VM image store"
+          scp "${SSH_OPTS[@]}" \
+            /tmp/rds-postgres-image-build/rds-postgres-alpine.raw \
+            "$VM":/tmp/rds-postgres-alpine.raw
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "spx admin images import --file /tmp/rds-postgres-alpine.raw \
+               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+               --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
+          ok "rds-postgres AMI imported"
+
       - name: Build E2E test binaries on runner
         working-directory: spinifex/tests/e2e
         run: |
@@ -568,7 +617,8 @@ jobs:
           ./build-install-artifacts.sh "${GIT_BRANCH}" \
             "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
             "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar"
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
 
       - name: Provision VMs
         working-directory: mulga/scripts/tofu-cluster

.github/workflows/gpu-e2e.yml

@@ -1,6 +1,6 @@
 # GPU E2E — drives wattle's persistent, shared-host Spinifex stack over SSH to
 # exercise GPU passthrough (raw EC2, ECS task GPU exposure, EKS pod GPU
-# exposure). See docs/development/feature/wattle-deploy-and-ci.md (Part B).
+# exposure).
 #
 # ---------------------------------------------------------------------------
 # ONE-TIME HOST-SIDE SETUP — NOT part of this repo, a human must do this:
@@ -32,7 +32,6 @@
 #      to skip cleanly (job succeeds, all work steps skipped) instead of
 #      queuing behind manual work indefinitely.
 #
-#   Runbook: docs/development/feature/wattle-deploy-and-ci.md (Part B).
 # ---------------------------------------------------------------------------
 
 name: GPU E2E
@@ -236,9 +235,8 @@ jobs:
       # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
       # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
       # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
-      # the advertised gateway (mulga-7suds), so the override is required
-      # here (proper fix tracked separately: tests/e2e/harness/env.go should
-      # prefer spinifex.toml's advertised bridge).
+      # the advertised gateway, so the override is required here. The harness
+      # should prefer spinifex.toml's advertised bridge.
       - name: Run GPU e2e suite on wattle
         id: e2e
         if: steps.gate.outputs.paused != 'true'

.github/workflows/image-scripts.yml

@@ -0,0 +1,39 @@
+name: Image Scripts
+
+# Separate from spinifex.yml on purpose: scripts/images/ and images/ are shell
+# and mkosi assets baked into system images, not Go, so they get their own
+# path-filtered gate instead of running on every Go contributor's push (see
+# ui.yml for the same pattern applied to the frontend).
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  image_scripts:
+    name: Image Scripts
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Run image script tests + shellcheck
+        run: make test-images

.github/workflows/segscan-oracle.yml

@@ -0,0 +1,81 @@
+# Segscan storage oracle — push-triggered (main/dev) + manual dispatch only,
+# deliberately NOT wired into spinifex.yml's pull_request path.
+#
+# tests/integration/segscan_oracle_test.go needs the mulga umbrella repo's
+# scripts/segscan checked out alongside spinifex (it is a separate Go module,
+# `package main`, and cannot be imported — see spinifex/testutil/segscanoracle).
+# spinifex.yml's jobs check out only spinifex/viperblock/predastore and run on
+# every PR; adding a second-repo checkout there would make every spinifex PR
+# depend on mulga main being healthy and pay that checkout cost on every run,
+# for one storage-accounting oracle test. Mirrors e2e.yml's push-only trigger
+# This never gates a PR before merge, matching how e2e.yml's own
+# "front gate" job works.
+name: Segscan Storage Oracle
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  segscan_oracle:
+    name: Segscan Oracle
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - name: Checkout Mulga (scripts/segscan only)
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/segscan
+          path: mulga
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore
+
+      - name: Run Segscan Oracle Test
+        env:
+          SEGSCAN_DIR: ${{ github.workspace }}/mulga/scripts/segscan
+        run: make -C spinifex test-segscan-oracle

.github/workflows/ui.yml

@@ -57,3 +57,6 @@ jobs:
 
       - name: Build Frontend
         run: pnpm build
+
+      - name: Check for generated route tree drift
+        run: git diff --exit-code -- src/routeTree.gen.ts

.gitignore

@@ -37,3 +37,6 @@ Thumbs.db
 
 # Test reports
 *-report.txt
+
+# Per-worktree golangci-lint cache
+/.cache/

Makefile

@@ -1,6 +1,11 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
+# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
+# path, not worktree path, so parallel worktrees of this module collide and
+# surface each other's issues. Scope it per worktree.
+export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
+
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
 ifeq ($(ARCH),x86_64)
@@ -100,6 +105,12 @@ build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd +
 import-ecs-node-image: ## Build + register the ecs-node AMI (requires a running cluster)
 	$(MAKE) build-system-image IMAGE=ecs-agent IMPORT=1
 
+build-rds-postgres-image: ## Build the spinifex-rds-postgres AMI (Alpine + PostgreSQL 18 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-postgres
+
+import-rds-postgres-image: ## Build + register the rds-postgres AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-postgres IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -120,15 +131,10 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
 
-# Preflight — the pre-commit gate: manifest checks, lint, vuln, and the unit,
-# race and e2e-harness tiers.
-#
-# The integration tier is deliberately NOT here. CI runs it as its own parallel
-# job, so it is still gated before merge, and keeping it out leaves preflight
-# the fast common-path check — run `make test-integration` directly when a
-# change touches the gateway router or the NATS subjects it drives.
+# the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
+# integration and race tests skipped to keep quick, they run in CI.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-race test-harness
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-harness
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
@@ -139,25 +145,27 @@ test-harness:
 	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)
 
 # In-process integration tier: the real gateway router against embedded NATS
-# JetStream, with only the daemon-side NATS subjects stubbed. Build-tagged
-# `integration` so it's skipped by the default `go test ./spinifex/...` and by
-# `test-cover`/`test-race`. Nothing provisioned — no tofu, no docker, no
-# Spinifex daemons, so the whole package runs in well under a minute. It gets
-# its own PR-blocking CI job rather than the self-hosted, push-triggered live
-# e2e tiers, so a regression here is caught before it can be merged, not just
-# after.
+# JetStream, with only the daemon-side NATS subjects stubbed.
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
 	$(_Q)LOG_IGNORE=1 go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
 
+# Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
+# checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
+# is not the default local or CI layout, so this is a separate target from
+# test-integration rather than folded into it. Skips itself when segscan's
+# source isn't found.
+test-segscan-oracle:
+	@echo -e "\n....Running segscan storage oracle test...."
+	$(_Q)LOG_IGNORE=1 go test -tags=integration,segscanoracle -timeout 120s ./tests/integration/... -run TestSegscanOracle $(_RACEQ)
+
 # Validate docs/service-interfaces.yaml. Schema check + cross-reference
-# of services/suites/fixtures + on-disk path existence. Subject content
-# vs source is enforced separately in Bead 5 drift lint.
+# of services/suites/fixtures + on-disk path existence.
 manifest-check:
 	@echo -e "\n....Checking service-interfaces.yaml...."
 	@go run ./tests/e2e/manifest-check/cmd/manifest-check -repo-root . -manifest docs/service-interfaces.yaml
 
-# Drift guards (Bead 5): direct-create fixture lint + NATS subject lint,
+# Drift guards: direct-create fixture lint + NATS subject lint,
 # ratcheted against tests/e2e/manifest-lint/baseline.txt. Fails only on NEW
 # drift beyond the baseline.
 manifest-lint:
@@ -171,19 +179,19 @@ manifest-lint-update:
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	LOG_IGNORE=1 go test -timeout 120s ./spinifex/...
+	GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -timeout 120s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... $(_COVQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... $(_RACEQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -191,6 +199,20 @@ test-actions:
 	@echo -e "\n....Running action tests...."
 	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
 
+# Shell suites + shellcheck for scripts/images/ helpers baked into system
+# images. Kept out of `preflight` (a dedicated CI job gates it on
+# scripts/images/** changes instead) so image-asset churn doesn't run on
+# every Go contributor's commit.
+test-images:
+	@echo -e "\n....Running scripts/images/**/*_test.sh...."
+	@for t in $$(find scripts/images -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over scripts/images/**/*.sh...."
+	shellcheck -S warning $$(find scripts/images -name '*.sh' | sort)
+	@echo "  test-images ok"
+
 # Check that new/changed code meets coverage threshold (runs tests first)
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
@@ -228,7 +250,7 @@ install-system:
 	sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
 		-o Dpkg::Options::="--force-confdef" \
 		-o Dpkg::Options::="--force-confold" \
-		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils qemu-kvm \
+		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
 		ovmf qemu-efi-aarch64 \
 		libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
 		iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
@@ -260,7 +282,7 @@ quickinstall: install-system install-go install-aws
 
 lint:
 	@echo "Running golangci-lint..."
-	$(_Q)golangci-lint run ./...
+	$(_Q)scripts/run-gate.sh golangci-lint golangci-lint run ./...
 	@echo "  golangci-lint ok"
 
 fix:
@@ -268,14 +290,13 @@ fix:
 
 govulncheck:
 	@echo "Running govulncheck..."
-	$(_Q)go tool govulncheck ./...
+	$(_Q)scripts/run-gate.sh govulncheck go tool govulncheck ./...
 	@echo "  govulncheck ok"
 
-# NilAway — advisory nil-panic analysis. Not in preflight: it has a known
-# false-positive rate, so findings are triaged by hand rather than gating commits.
+# NilAway — advisory nil-panic analysis. Not in preflight due to false positives
 nilaway:
 	@echo "Running nilaway..."
-	$(_Q)go tool nilaway -include-pkgs=github.com/mulgadc/spinifex -exclude-test-files ./...
+	$(_Q)scripts/run-gate.sh nilaway go tool nilaway -include-pkgs=github.com/mulgadc/spinifex -exclude-test-files ./...
 	@echo "  nilaway ok"
 
 # Build release tarballs — use distro-ARCH for single arch, distro for both
@@ -321,7 +342,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness test-integration manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -99,10 +99,10 @@ on-premise, or on a partner Neocloud.
 | **EKS** | Kubernetes | Available |
 | **ECR** | Container registry | Available |
 | **ECS** | Container service | Available |
-| **RDS** | Databases | Q3 2026 |
+| **RDS** | Databases | Available |
+| **Bedrock** | AI Deployment | Q3 2026 |
 
-Roadmap items (Q3 2026) ship under the same AWS API surface. Code written for AWS
-today keeps working the moment they land.
+Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land.
 [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 
 ## Core Components

build/logrotate/spinifex

@@ -1,4 +1,8 @@
 /var/log/spinifex/*.log {
+    # The directory is 0775 root:spinifex so each service user can write its own
+    # log. logrotate refuses to rotate inside a group-writable directory unless
+    # it is told which user to rotate as.
+    su root spinifex
     daily
     rotate 14
     compress

build/scripts/nats-js-watchdog.sh

@@ -1,21 +1,32 @@
 #!/bin/sh
 # JetStream liveness watchdog for spinifex-nats.
 #
-# NATS can disable JetStream at runtime (e.g. a filesystem permission error while
-# flushing stream state) and keep the process alive serving KV as read-only. That
-# is not a process failure, so Restart=on-failure never fires and KV-backed
-# services (awsgw IAM bucket init, etc.) stall until NATS is restarted by hand.
+# NATS can stop accepting JetStream writes at runtime while the process stays
+# alive and keeps serving reads. That is not a process failure, so
+# Restart=on-failure never fires and KV-backed services (awsgw IAM bucket init,
+# etc.) stall until NATS is restarted by hand.
 #
-# This probes the NATS monitoring endpoint's JS health and restarts the service
-# only when JetStream is sustainedly unhealthy while the service is running. The
-# multi-sample window avoids restarting during transient startup/quorum 503s.
+# Two distinct latches produce that symptom, and they need different probes:
+#
+#   1. Server-wide. handleOutOfSpace calls ShutdownJetStream and sets an
+#      internal disabled flag that only a restart clears. healthz reports it.
+#   2. Per message block. filestore records a write error on the block and
+#      returns it on every later write; nothing clears it short of a restart.
+#      healthz does NOT report it, because the server-wide state is untouched —
+#      only an actual write observes it.
+#
+# So a health check alone misses case 2 entirely. Both probes run here, and a
+# sustained failure of either restarts the service. The multi-sample window
+# avoids restarting during transient startup/quorum 503s.
 
 set -eu
 
 HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
 SERVICE="spinifex-nats.service"
+SPX="${SPX_BIN:-/usr/local/bin/spx}"
 SAMPLES=3        # consecutive failing samples required before acting
 SAMPLE_GAP=2     # seconds between samples
+PROBE_TIMEOUT=5  # seconds for one canary round-trip
 
 # Only act on a running service: a stopped/activating unit is systemd's to manage,
 # and try-restart on it would either no-op or fight an in-flight (re)start.
@@ -30,15 +41,29 @@ if ! command -v curl >/dev/null 2>&1; then
     exit 0
 fi
 
-# A single 200 means JetStream is healthy — exit fast on the common case.
 js_healthy() {
     curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
 }
 
+# Round-trips a canary key through the cluster-state bucket. Absent spx this
+# degrades to the health check alone rather than restarting on a missing tool.
+js_writable() {
+    if [ ! -x "$SPX" ]; then
+        return 0
+    fi
+    "$SPX" admin node js-probe --timeout "${PROBE_TIMEOUT}s" >/dev/null 2>&1
+}
+
+failure=""
 i=1
 while [ "$i" -le "$SAMPLES" ]; do
     if js_healthy; then
-        exit 0
+        if js_writable; then
+            exit 0
+        fi
+        failure="JetStream accepts no writes (canary round-trip failed) while $HEALTH_URL reports healthy"
+    else
+        failure="JetStream unhealthy at $HEALTH_URL"
     fi
     if [ "$i" -lt "$SAMPLES" ]; then
         sleep "$SAMPLE_GAP"
@@ -46,5 +71,5 @@ while [ "$i" -le "$SAMPLES" ]; do
     i=$((i + 1))
 done
 
-echo "nats-js-watchdog: JetStream unhealthy on $SAMPLES consecutive probes ($HEALTH_URL); restarting $SERVICE" >&2
+echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2
 systemctl try-restart "$SERVICE"

build/scripts/predastore-start.sh

@@ -1,30 +0,0 @@
-#!/bin/sh
-# Helper script for spinifex-predastore.service
-# Detects host, port, backend, and node-id from spinifex.toml
-# then execs spx. Falls back to single-node defaults if no config found.
-
-CONF=/etc/spinifex/spinifex.toml
-BIND="0.0.0.0:8443"
-# -1 = co-located mode (all [[db]] peers in one process). Single-node spinifex
-# templates list 3 DB peers on 127.0.0.1; only -1 can serve that topology.
-# Multi-node spinifex.toml emits node_id = N (N >= 1) which overrides below.
-# predastore rejects NODE_ID=0 since d3a02f0 (strconv.Atoi-of-garbage guard).
-NODE_ID="-1"
-
-if [ -f "$CONF" ]; then
-    H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
-    if [ -n "$H" ]; then
-        BIND="$H"
-    fi
-
-    N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
-    if [ -n "$N" ]; then
-        NODE_ID="$N"
-    fi
-fi
-
-export SPINIFEX_PREDASTORE_HOST="${BIND%%:*}"
-export SPINIFEX_PREDASTORE_PORT="${BIND##*:}"
-export SPINIFEX_PREDASTORE_NODE_ID="$NODE_ID"
-
-exec /usr/local/bin/spx service predastore start

build/systemd/regenerate-ssh-host-keys.service

@@ -7,6 +7,9 @@ Before=ssh.service
 Type=oneshot
 ExecStart=/usr/bin/ssh-keygen -A
 RemainAfterExit=yes
+# No ExecStop: the keygen process has already exited by the time this unit
+# is stopped, so there is nothing to signal.
+TimeoutStopSec=5
 
 [Install]
 WantedBy=multi-user.target

build/systemd/spinifex-awsgw.service

@@ -14,8 +14,15 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service awsgw start
 Restart=on-failure
 RestartSec=5
+# No SIGTERM handler in this process: it exits immediately on signal, so this
+# only bounds an unexpectedly wedged process before SIGKILL.
+TimeoutStopSec=10
 
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/
 Environment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem

build/systemd/spinifex-daemon.service

@@ -7,6 +7,8 @@ Wants=openvswitch-ipsec.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
+# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see
+EnvironmentFile=-/etc/spinifex/host.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-daemon
@@ -19,13 +21,17 @@ KillMode=process
 TimeoutStopSec=15
 # RG-4: infra tier — reaped only after every guest
 OOMScoreAdjust=-500
+# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead
+MemoryHigh=75%
+MemoryMax=85%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
 # Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)
 Environment=XDG_RUNTIME_DIR=/run/spinifex
 
-# RG-9: daemon tier — privileged by necessity (GPU vfio). NoNewPrivileges omitted (RG-10: sudo ip/ovs-vsctl/dhcpcd)
+# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,
+# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.
 AmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE
 ProtectSystem=full
 PrivateDevices=no

build/systemd/spinifex-nats-watchdog.service

@@ -5,10 +5,16 @@ After=spinifex-nats.service
 
 [Service]
 Type=oneshot
-# Runs as root: the probe is a local curl to the NATS monitoring port, and the
-# remedy is `systemctl try-restart spinifex-nats` when JetStream is disabled while
-# the process stays alive (which Restart=on-failure cannot catch).
+# Runs as root: the probes are a local curl to the NATS monitoring port and a
+# canary KV round-trip via spx, and the remedy is `systemctl try-restart
+# spinifex-nats` when JetStream stops accepting writes while the process stays
+# alive (which Restart=on-failure cannot catch).
 ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
+# Sized for the probe loop's own worst case (3 samples x (5s curl + 5s canary)
+# + 2 gaps x 2s = 34s): a stop mid-cycle lets the script finish rather than
+# cutting it off. Only the failing path runs that long; a healthy node exits on
+# the first sample.
+TimeoutStopSec=45
 
 # Hardening (kept compatible with talking to PID1 over /run/systemd).
 NoNewPrivileges=yes

build/systemd/spinifex-nats.service

@@ -12,7 +12,15 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service nats start
 Restart=on-failure
 RestartSec=5
+# nats-server installs its own SIGTERM handler (JetStream store shutdown, raft
+# stepdown, client drain) with no internal deadline; every dependent service
+# stops before nats does, so size this for nats's own graceful-stop needs.
+TimeoutStopSec=20
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf
 
 # Filesystem access

build/systemd/spinifex-northstar.service

@@ -13,7 +13,14 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service northstar start
 Restart=on-failure
 RestartSec=5
+# server.Shutdown() is called with an internal 5s context; margin covers the
+# NATS unsubscribe/close and process exit that follow it.
+TimeoutStopSec=10
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/northstar/
 

build/systemd/spinifex-predastore.service

@@ -14,10 +14,20 @@ Slice=spinifex-system.slice
 User=spinifex-storage
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
-ExecStart=/var/lib/spinifex/predastore-start.sh
+ExecStart=/usr/local/bin/spx service predastore start
 Restart=on-failure
 RestartSec=5
+# Matches viperblock's drain budget: viperblock's WAL flush writes into
+# predastore, and After= already stops viperblock first, but this is the
+# backstop if that ordering is ever bypassed (direct stop, future edit).
+TimeoutStopSec=30
 OOMScoreAdjust=-500
+# Legitimately the most memory-hungry infra service (S3 buffering/caching);
+# High/Max stay well above idle so normal load isn't throttled, while still
+# bounding a leak short of the slice-wide cap.
+MemoryHigh=40%
+MemoryMax=55%
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml
 Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
 Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem

build/systemd/spinifex-qmp-collector.service

@@ -16,6 +16,13 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service qmp-collector start
 Restart=on-failure
 RestartSec=5
+# Poll loop selects on ctx.Done() directly and closes NATS locally; shutdown
+# is near-instant, so this only bounds an unexpectedly wedged process.
+TimeoutStopSec=10
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 # Same runtime dir as the daemon: telemetry sockets + metadata live there

build/systemd/spinifex-shutdown.service

@@ -9,9 +9,13 @@ After=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.serv
 Type=oneshot
 RemainAfterExit=yes
 ExecStart=/bin/true
-# ExecStop only runs when the target/host goes down, not on a single-service
-# restart — so a daemon restart still leaves guests running (DDIL).
-ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s
+# ExecStop fires on ANY target stop, including the stop half of `systemctl
+# restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless
+# of whether the host is actually going down. --only-if-host-stopping drains
+# only when systemd itself is unwinding into shutdown.target; a plain restart
+# or `systemctl stop spinifex.target` falls through so QEMU/nbdkit survive
+# (KillMode=process) and the daemon reconnects to them on the way back up.
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --only-if-host-stopping
 TimeoutStopSec=180
 
 [Install]

build/systemd/spinifex-ui.service

@@ -12,16 +12,24 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service spinifex-ui start
 Restart=on-failure
 RestartSec=5
+# server.Shutdown() uses an internal 30s context (WriteTimeout=120s covers
+# long streaming proxy responses, e.g. console passthrough); add margin above it.
+TimeoutStopSec=40
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
+Environment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/
 
 # Filesystem access
 NoNewPrivileges=yes
 ProtectSystem=strict
 PrivateDevices=yes
 ReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key
-ReadWritePaths=/var/log/spinifex
+ReadWritePaths=/var/lib/spinifex/spinifex-ui /var/log/spinifex
 
 # Hardening
 ProtectHome=yes

build/systemd/spinifex-viperblock.service

@@ -15,6 +15,16 @@ ExecStart=/usr/local/bin/spx service viperblock start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=15%
+MemoryMax=25%
+# KillMode=process: nbdkit still serving a guest survives a restart here, the
+# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts
+# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.
+KillMode=process
+# Bounds this process's own WAL drain, which the plugin caps at 20s.
+TimeoutStopSec=30
 EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/

build/systemd/spinifex-vpcd.service

@@ -14,13 +14,23 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service vpcd start
 Restart=on-failure
 RestartSec=5
+# Waits for an in-flight OVN drift-reconcile pass to return before exiting;
+# no sub-timeout bounds that call today, so this covers a normal pass.
+TimeoutStopSec=20
 OOMScoreAdjust=-500
+# Per-service ceiling: bounds a leak here instead of the slice-wide cap.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/
 
 # RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no
 # longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem
-# sandbox is restored. NoNewPrivileges stays off (RG-10: sudo ip/ovs-vsctl/dhcpcd).
+# sandbox is restored.
+#
+# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and
+# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR
+# update that retires the last escalation path.
 AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
 CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
 NoNewPrivileges=no

cmd/dhcptest/main.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/insomniacslk/dhcp/dhcpv4"
 	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
 	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
 )
@@ -36,6 +37,7 @@ func main() {
 	timeout := flag.Duration("timeout", 10*time.Second, "Per-attempt DHCP timeout")
 	retries := flag.Int("retries", 3, "Number of DORA attempts before giving up")
 	verbose := flag.Bool("verbose", false, "Print full DHCP packet contents")
+	broadcast := flag.Bool("broadcast", false, "Set the BOOTP broadcast flag, as vpcd's Acquire path does, so replies come back to ff:ff:ff:ff:ff:ff rather than unicast to the bridge MAC")
 	flag.Parse()
 
 	// Resolve the hardware address to use.
@@ -114,6 +116,9 @@ func main() {
 	// Identity modifiers — client-id / hostname / vendor-class for upstream lease
 	// tagging. Built by the vpcd code path so the probe cannot drift from it.
 	mods := dhcp.IdentityModifiers(cid, *hostname, *vendorClass, hwAddr)
+	if *broadcast {
+		mods = append(mods, dhcpv4.WithBroadcast(true))
+	}
 
 	fmt.Println("Sending DHCP Discover...")
 	start := time.Now()

cmd/ecr-credential-provider/main.go

@@ -12,17 +12,16 @@
 package main
 
 import (
-	"bufio"
 	"encoding/json"
 	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/guestenv"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 
 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
@@ -144,13 +143,7 @@ func cacheDurationFrom(expiresAt time.Time) string {
 // loadConfig reads the cloud-init env file then lets real env vars override. The
 // IMDS base is fixed (not in the env file) but overridable for tests.
 func loadConfig(envFile string) config {
-	env := parseEnvFile(envFile)
-	get := func(key string) string {
-		if v := os.Getenv(key); v != "" {
-			return v
-		}
-		return env[key]
-	}
+	get := guestenv.Load(envFile).Get
 
 	cfg := config{
 		GatewayURL: get("EKS_GATEWAY_URL"),
@@ -167,29 +160,6 @@ func loadConfig(envFile string) config {
 	return cfg
 }
 
-// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
-func parseEnvFile(path string) map[string]string {
-	out := map[string]string{}
-	f, err := os.Open(path)
-	if err != nil {
-		return out
-	}
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		key, val, ok := strings.Cut(line, "=")
-		if !ok {
-			continue
-		}
-		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
-	}
-	return out
-}
-
 // emitEmpty writes a valid CredentialProviderResponse with no auth entries.
 func emitEmpty(out io.Writer) {
 	resp := CredentialProviderResponse{

cmd/ecr-credential-provider/main_test.go

@@ -18,12 +18,14 @@ func fakeBackend(t *testing.T) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
 
-	// IMDSv2 token.
+	// IMDSv2 token. Real IMDS echoes the requested TTL back as a response
+	// header; the SDK client parses it from there, not the body.
 	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPut {
 			w.WriteHeader(http.StatusMethodNotAllowed)
 			return
 		}
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("imds-v2-token"))
 	})
 	// IMDS role name.

cmd/ecs-agent/config.go

@@ -1,11 +1,10 @@
 package main
 
 import (
-	"bufio"
-	"os"
 	"strconv"
-	"strings"
 	"time"
+
+	"github.com/mulgadc/spinifex/internal/guestenv"
 )
 
 const (
@@ -40,13 +39,7 @@ type config struct {
 
 // loadConfig reads the cloud-init env file then lets real env vars override.
 func loadConfig(envFile string) config {
-	env := parseEnvFile(envFile)
-	get := func(key string) string {
-		if v := os.Getenv(key); v != "" {
-			return v
-		}
-		return env[key]
-	}
+	get := guestenv.Load(envFile).Get
 
 	cfg := config{
 		GatewayURL:       get("ECS_GATEWAY_URL"),
@@ -94,26 +87,3 @@ func loadConfig(envFile string) config {
 	}
 	return cfg
 }
-
-// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
-func parseEnvFile(path string) map[string]string {
-	out := map[string]string{}
-	f, err := os.Open(path)
-	if err != nil {
-		return out
-	}
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		key, val, ok := strings.Cut(line, "=")
-		if !ok {
-			continue
-		}
-		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
-	}
-	return out
-}

cmd/ecs-agent/config_test.go

@@ -71,18 +71,3 @@ func TestLoadConfig_HeartbeatOverride(t *testing.T) {
 		t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
 	}
 }
-
-func TestParseEnvFile_SkipsBlankAndComments(t *testing.T) {
-	dir := t.TempDir()
-	p := filepath.Join(dir, "f.env")
-	if err := os.WriteFile(p, []byte("\n# c\nA=1\nnokeyval\nB = 2 \n"), 0o600); err != nil {
-		t.Fatal(err)
-	}
-	m := parseEnvFile(p)
-	if m["A"] != "1" || m["B"] != "2" {
-		t.Errorf("parse mismatch: %#v", m)
-	}
-	if _, ok := m["nokeyval"]; ok {
-		t.Errorf("line without = should be skipped")
-	}
-}

cmd/ecs-agent/credentials/imds.go

@@ -3,57 +3,39 @@ package credentials
 import (
 	"context"
 	"net/http"
-	"sync"
-	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 )
 
-// refreshMargin is how long before expiry the cache is considered stale.
-const refreshMargin = 5 * time.Minute
-
-// IMDSProvider fetches instance-role credentials from IMDS and caches them until
-// they near expiry. Safe for concurrent use.
+// IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
+// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache
+// (which refetches once the real expiry passes; ec2rolecreds also caps
+// Expires to at most an hour out). Safe for concurrent use.
 type IMDSProvider struct {
-	client *http.Client
-	base   string
-
-	mu     sync.Mutex
-	cached Credentials
+	cache *aws.CredentialsCache
 }
 
 var _ CredentialsProvider = (*IMDSProvider)(nil)
 
 // NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
-// "http://169.254.169.254/latest"). A nil client uses http.DefaultClient.
+// "http://169.254.169.254/latest"). A nil client uses the SDK IMDS client's
+// default HTTP client.
 func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
-	if client == nil {
-		client = http.DefaultClient
-	}
-	return &IMDSProvider{client: client, base: base}
+	provider := imdscreds.NewProvider(client, base)
+	return &IMDSProvider{cache: aws.NewCredentialsCache(provider)}
 }
 
 // Retrieve returns cached credentials when still valid, else fetches fresh ones.
 func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	if p.cached.Valid(refreshMargin) {
-		return p.cached, nil
-	}
-	if err := ctx.Err(); err != nil {
-		return Credentials{}, err
-	}
-
-	raw, err := imdscreds.Fetch(p.client, p.base)
+	creds, err := p.cache.Retrieve(ctx)
 	if err != nil {
 		return Credentials{}, err
 	}
-	p.cached = Credentials{
-		AccessKeyID:     raw.AccessKeyID,
-		SecretAccessKey: raw.SecretAccessKey,
-		SessionToken:    raw.SessionToken,
-		Expiration:      raw.Expiration,
-	}
-	return p.cached, nil
+	return Credentials{
+		AccessKeyID:     creds.AccessKeyID,
+		SecretAccessKey: creds.SecretAccessKey,
+		SessionToken:    creds.SessionToken,
+		Expiration:      creds.Expires,
+	}, nil
 }

cmd/ecs-agent/credentials/imds_test.go

@@ -10,14 +10,19 @@ import (
 	"time"
 )
 
+// imdsStub serves the IMDSv2 token + role + credentials endpoints under the
+// real /latest prefix the SDK IMDS client always requests.
 func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		// Real IMDS echoes the requested TTL back as a response header; the SDK
+		// client parses it from there, not the body.
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("v2-token"))
 	})
-	mux.HandleFunc("/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/meta-data/iam/security-credentials/" {
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
 			_, _ = w.Write([]byte("node-role"))
 			return
 		}
@@ -33,15 +38,17 @@ func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Serv
 
 func TestRetrieve_FetchesAndCaches(t *testing.T) {
 	var hits int32
-	exp := time.Now().Add(6 * time.Hour).UTC().Round(time.Second)
+	// Kept under an hour: ec2rolecreds caps Expires to now+1h.
+	exp := time.Now().Add(30 * time.Minute).UTC().Round(time.Second)
 	srv := imdsStub(t, &hits, map[string]string{
+		"Code":            "Success",
 		"AccessKeyId":     "AKIA",
 		"SecretAccessKey": "secret",
 		"Token":           "session",
 		"Expiration":      exp.Format(time.RFC3339),
 	})
 
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	got, err := p.Retrieve(context.Background())
 	if err != nil {
 		t.Fatalf("Retrieve: %v", err)
@@ -50,7 +57,7 @@ func TestRetrieve_FetchesAndCaches(t *testing.T) {
 		t.Errorf("creds mismatch: %+v", got)
 	}
 
-	// Second call within validity window must not re-hit IMDS.
+	// Second call well within validity must not re-hit IMDS.
 	if _, err := p.Retrieve(context.Background()); err != nil {
 		t.Fatalf("Retrieve 2: %v", err)
 	}
@@ -59,15 +66,17 @@ func TestRetrieve_FetchesAndCaches(t *testing.T) {
 	}
 }
 
-func TestRetrieve_RefetchesWhenStale(t *testing.T) {
+func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
 	var hits int32
-	// Expiry inside the refresh margin → never cached, always refetched.
+	// Already expired: aws.CredentialsCache refetches once the stored
+	// credentials' real expiry has passed, regardless of ExpiryWindow.
 	srv := imdsStub(t, &hits, map[string]string{
+		"Code":            "Success",
 		"AccessKeyId":     "AKIA",
 		"SecretAccessKey": "secret",
-		"Expiration":      time.Now().Add(time.Minute).UTC().Format(time.RFC3339),
+		"Expiration":      time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
 	})
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	if _, err := p.Retrieve(context.Background()); err != nil {
 		t.Fatal(err)
 	}
@@ -75,13 +84,13 @@ func TestRetrieve_RefetchesWhenStale(t *testing.T) {
 		t.Fatal(err)
 	}
 	if hits != 2 {
-		t.Errorf("hits = %d, want 2 (stale refetch)", hits)
+		t.Errorf("hits = %d, want 2 (expired refetch)", hits)
 	}
 }
 
 func TestRetrieve_CancelledContext(t *testing.T) {
-	srv := imdsStub(t, nil, map[string]string{"AccessKeyId": "A", "SecretAccessKey": "B"})
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 	if _, err := p.Retrieve(ctx); err == nil {
@@ -89,16 +98,10 @@ func TestRetrieve_CancelledContext(t *testing.T) {
 	}
 }
 
-func TestCredentials_Valid(t *testing.T) {
-	if (Credentials{}).Valid(0) {
-		t.Error("empty creds should be invalid")
-	}
-	noExp := Credentials{AccessKeyID: "a", SecretAccessKey: "b"}
-	if !noExp.Valid(time.Hour) {
-		t.Error("zero expiry should be treated valid")
-	}
-	soon := Credentials{AccessKeyID: "a", SecretAccessKey: "b", Expiration: time.Now().Add(time.Minute)}
-	if soon.Valid(5 * time.Minute) {
-		t.Error("expiry inside margin should be invalid")
+func TestRetrieve_CodeNotSuccessErrors(t *testing.T) {
+	srv := imdsStub(t, nil, map[string]string{"Code": "Failure", "AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
+	if _, err := p.Retrieve(context.Background()); err == nil {
+		t.Fatal("expected error when Code != Success")
 	}
 }

cmd/ecs-agent/credentials/provider.go

@@ -17,18 +17,6 @@ type Credentials struct {
 	Expiration      time.Time
 }
 
-// Valid reports whether the credentials are usable now, leaving a margin so a
-// caller does not start a request with a token about to expire mid-flight.
-func (c Credentials) Valid(margin time.Duration) bool {
-	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
-		return false
-	}
-	if c.Expiration.IsZero() {
-		return true
-	}
-	return time.Now().Add(margin).Before(c.Expiration)
-}
-
 // CredentialsProvider yields current credentials, refreshing as needed. Retrieve
 // must be safe for concurrent use.
 type CredentialsProvider interface {

cmd/ecs-agent/metadata.go

@@ -1,11 +1,14 @@
 package main
 
 import (
-	"encoding/json"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
 )
 
 // instanceMetadata is the IMDS-derived identity of the host the agent runs on.
@@ -15,83 +18,44 @@ type instanceMetadata struct {
 	AZ         string
 }
 
-// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2.
-// base is the IMDS root, e.g. "http://169.254.169.254/latest".
+// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2
+// via the SDK IMDS client. base is the IMDS root, e.g.
+// "http://169.254.169.254/latest".
 func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
-	token, err := imdsToken(client, base)
-	if err != nil {
-		return instanceMetadata{}, err
-	}
+	imdsClient := imdscreds.NewClient(client, base)
+	ctx := context.Background()
 
-	instanceID, err := imdsGet(client, base+"/meta-data/instance-id", token)
+	instanceID, err := getMetadata(ctx, imdsClient, "instance-id")
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
 	}
-	az, err := imdsGet(client, base+"/meta-data/placement/availability-zone", token)
+	az, err := getMetadata(ctx, imdsClient, "placement/availability-zone")
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
 	}
 
-	doc, err := imdsGet(client, base+"/dynamic/instance-identity/document", token)
+	doc, err := imdsClient.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
 	}
-	var ident struct {
-		AccountID string `json:"accountId"`
-	}
-	if err := json.Unmarshal([]byte(doc), &ident); err != nil {
-		return instanceMetadata{}, fmt.Errorf("parse identity document: %w", err)
-	}
 
 	return instanceMetadata{
-		AccountID:  ident.AccountID,
+		AccountID:  doc.AccountID,
 		InstanceID: strings.TrimSpace(instanceID),
 		AZ:         strings.TrimSpace(az),
 	}, nil
 }
 
-// imdsToken fetches an IMDSv2 session token.
-func imdsToken(client *http.Client, base string) (string, error) {
-	req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
+// getMetadata reads a single meta-data path and returns its body as a string.
+func getMetadata(ctx context.Context, imdsClient *imds.Client, path string) (string, error) {
+	out, err := imdsClient.GetMetadata(ctx, &imds.GetMetadataInput{Path: path})
 	if err != nil {
 		return "", err
 	}
-	req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("imds token: %w", err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
+	defer out.Content.Close()
+	body, err := io.ReadAll(out.Content)
 	if err != nil {
 		return "", err
 	}
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("imds token: status %d", resp.StatusCode)
-	}
-	return string(body), nil
-}
-
-// imdsGet performs a token-authenticated IMDS GET.
-func imdsGet(client *http.Client, url, token string) (string, error) {
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return "", err
-	}
-	if token != "" {
-		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
-	}
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("status %d", resp.StatusCode)
-	}
 	return string(body), nil
 }

cmd/ecs-agent/metadata_test.go

@@ -9,11 +9,14 @@ import (
 func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPut {
 			w.WriteHeader(http.StatusMethodNotAllowed)
 			return
 		}
+		// Real IMDS echoes the requested TTL back as a response header; the SDK
+		// client parses it from there, not the body.
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("v2-token"))
 	})
 	guard := func(w http.ResponseWriter, r *http.Request) bool {
@@ -23,17 +26,17 @@ func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 		}
 		return true
 	}
-	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte("i-0abc123"))
 		}
 	})
-	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte("us-east-1a"))
 		}
 	})
-	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte(`{"accountId":"123456789012","region":"us-east-1"}`))
 		}
@@ -45,7 +48,7 @@ func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 
 func TestFetchInstanceMetadata_OK(t *testing.T) {
 	srv := metadataStub(t, true)
-	got, err := fetchInstanceMetadata(srv.Client(), srv.URL)
+	got, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest")
 	if err != nil {
 		t.Fatalf("fetch: %v", err)
 	}
@@ -57,20 +60,23 @@ func TestFetchInstanceMetadata_OK(t *testing.T) {
 func TestFetchInstanceMetadata_TokenRequired(t *testing.T) {
 	// Server returns 405 for non-PUT token; a GET-only client would 401 downstream.
 	srv := metadataStub(t, true)
-	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err != nil {
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err != nil {
 		t.Fatalf("v2 path should succeed: %v", err)
 	}
 }
 
 func TestFetchInstanceMetadata_BadDocument(t *testing.T) {
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("t")) })
-	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
-	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
-	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("t"))
+	})
+	mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
+	mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
+	mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
 	srv := httptest.NewServer(mux)
 	t.Cleanup(srv.Close)
-	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err == nil {
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err == nil {
 		t.Fatal("expected parse error on bad identity document")
 	}
 }

cmd/ecs-agent/netns.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"hash/crc32"
+	"os"
 	"os/exec"
 	"strings"
 	"time"
@@ -21,6 +22,12 @@ const (
 	credEndpointCIDR = defaultCredEndpointIP + "/32"
 )
 
+// udhcpcScript is the config script busybox udhcpc execs to apply a lease. It
+// is the reason presence of the udhcpc binary is not enough to use it: Debian
+// and Ubuntu's busybox ship the applet without this script, and udhcpc then
+// takes the lease and exits nonzero having configured nothing.
+const udhcpcScript = "/usr/share/udhcpc/default.script"
+
 // netCmdRunner executes a host networking command. Abstracted so unit tests can
 // assert the ip/netns sequence without touching the kernel.
 type netCmdRunner interface {
@@ -43,10 +50,47 @@ type taskNetns struct {
 	run     netCmdRunner
 	nicWait time.Duration
 	poll    time.Duration
+	// Injected so the DHCP-client choice below is unit-testable on a host that
+	// has neither client installed.
+	lookPath   func(string) (string, error)
+	fileIsExec func(string) bool
 }
 
 func newTaskNetns(run netCmdRunner) *taskNetns {
-	return &taskNetns{run: run, nicWait: 30 * time.Second, poll: time.Second}
+	return &taskNetns{
+		run:        run,
+		nicWait:    30 * time.Second,
+		poll:       time.Second,
+		lookPath:   exec.LookPath,
+		fileIsExec: func(p string) bool { fi, err := os.Stat(p); return err == nil && fi.Mode()&0o111 != 0 },
+	}
+}
+
+// dhcpStep returns the in-netns DHCP command that leases the task ENI's
+// OVN-assigned address.
+//
+// This image ships on two distributions and they do not agree on a DHCP client.
+// Alpine has busybox udhcpc; Ubuntu has no udhcpc at all, and `ip netns exec`
+// exits 1 — not 127 — when it cannot exec the binary, so a hardcoded udhcpc
+// fails as an opaque "network setup failed" with the task stuck in STOPPED.
+// Resolve the client at setup time instead, preferring udhcpc so Alpine keeps
+// its existing behaviour exactly, then falling back to dhcpcd. Same order and
+// same reasoning as dhcp_oneshot() in mulga-mgmt-net.sh.
+//
+// Both invocations are one-shot: udhcpc -q exits once it has the lease and -n
+// gives up rather than backgrounding, and dhcpcd -1 exits after configuring the
+// interface. Neither leaves a daemon behind in the netns.
+func (n *taskNetns) dhcpStep(name, iface string) ([]string, error) {
+	if _, err := n.lookPath("udhcpc"); err == nil && n.fileIsExec(udhcpcScript) {
+		return []string{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"}, nil
+	}
+	// Lease state is keyed by interface name, which is unique per task here:
+	// each task ENI is a separate hot-plug, so it lands on its own PCI slot and
+	// gets its own predictable name before being moved into the netns.
+	if _, err := n.lookPath("dhcpcd"); err == nil {
+		return []string{"ip", "netns", "exec", name, "dhcpcd", "-q", "-1", "-t", "20", iface}, nil
+	}
+	return nil, fmt.Errorf("no DHCP client for %s: need udhcpc (with %s) or dhcpcd", iface, udhcpcScript)
 }
 
 func netnsName(taskID string) string { return "ecs-" + taskID }
@@ -68,11 +112,16 @@ func (n *taskNetns) Setup(taskID, mac string) (string, error) {
 	}
 	hostVeth := credVethName(taskID)
 	hostIP, taskIP := credSubnet(taskID)
+	dhcp, err := n.dhcpStep(name, iface)
+	if err != nil {
+		_ = n.Teardown(taskID)
+		return "", err
+	}
 	steps := [][]string{
 		{"ip", "link", "set", iface, "netns", name},
 		{"ip", "-n", name, "link", "set", "lo", "up"},
 		{"ip", "-n", name, "link", "set", iface, "up"},
-		{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"},
+		dhcp,
 		// Credentials path: a veth into the host netns plus a host-routed /32 to the
 		// credential endpoint, so the container reaches 169.254.170.2 without it
 		// leaking onto the task ENI's VPC route.

cmd/ecs-agent/netns_test.go

@@ -55,10 +55,27 @@ const linkWithENI = `1: lo: <LOOPBACK,UP> mtu 65536 ... link/loopback 00:00:00:0
 2: eth0: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:aa:bb:cc ...
 3: eth1: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:de:ad:01 ...`
 
+// withDHCPClients pins which DHCP clients the netns sees, so a test asserts the
+// branch it names rather than whatever happens to be installed on the machine
+// running it. Without this the DHCP step is host-dependent: an Alpine CI box and
+// an Ubuntu dev box would exercise different code from the same test.
+func withDHCPClients(n *taskNetns, present map[string]bool, scriptOK bool) {
+	n.lookPath = func(f string) (string, error) {
+		if present[f] {
+			return "/sbin/" + f, nil
+		}
+		return "", errors.New("not found")
+	}
+	n.fileIsExec = func(string) bool { return scriptOK }
+}
+
+// newTestNetns defaults to the Alpine arrangement — udhcpc plus its config
+// script — so the sequence assertions below keep describing that platform.
 func newTestNetns(f *fakeNetRunner) *taskNetns {
 	n := newTaskNetns(f)
 	n.nicWait = 50 * time.Millisecond
 	n.poll = 5 * time.Millisecond
+	withDHCPClients(n, map[string]bool{"udhcpc": true}, true)
 	return n
 }
 
@@ -142,6 +159,79 @@ func TestNetns_SetupTearsDownOnStepFailure(t *testing.T) {
 	}
 }
 
+// The image runs on two distributions with different DHCP clients, and picking
+// the wrong one fails as an opaque "network setup failed" with the task stuck
+// in STOPPED — `ip netns exec` exits 1, not 127, when it cannot exec the
+// binary. Each branch is asserted so that misresolution is a test failure.
+func TestNetns_DHCPStepPerPlatform(t *testing.T) {
+	tests := []struct {
+		name     string
+		present  map[string]bool
+		scriptOK bool
+		want     string
+		wantErr  bool
+	}{
+		{
+			name:     "alpine: udhcpc with its config script",
+			present:  map[string]bool{"udhcpc": true, "dhcpcd": true},
+			scriptOK: true,
+			want:     "ip netns exec ecs-t-001 udhcpc -i eth1 -q -n",
+		},
+		{
+			// Ubuntu ships no udhcpc at all — the case that broke awsvpc tasks.
+			name:    "ubuntu: no udhcpc, falls back to dhcpcd",
+			present: map[string]bool{"dhcpcd": true},
+			want:    "ip netns exec ecs-t-001 dhcpcd -q -1 -t 20 eth1",
+		},
+		{
+			// Debian/Ubuntu busybox ships the applet without the script, so
+			// presence of the binary alone must not select it.
+			name:     "udhcpc present but config script missing: still dhcpcd",
+			present:  map[string]bool{"udhcpc": true, "dhcpcd": true},
+			scriptOK: false,
+			want:     "ip netns exec ecs-t-001 dhcpcd -q -1 -t 20 eth1",
+		},
+		{
+			name:    "neither client: explicit error, not a silent bad command",
+			present: map[string]bool{},
+			wantErr: true,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			n := newTestNetns(&fakeNetRunner{linkOut: linkWithENI})
+			withDHCPClients(n, tc.present, tc.scriptOK)
+			got, err := n.dhcpStep("ecs-t-001", "eth1")
+			if tc.wantErr {
+				if err == nil {
+					t.Fatalf("want error with no DHCP client, got %v", got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("dhcpStep: %v", err)
+			}
+			if joined := strings.Join(got, " "); joined != tc.want {
+				t.Errorf("want %q, got %q", tc.want, joined)
+			}
+		})
+	}
+}
+
+// A missing DHCP client must not leave the half-built netns behind for the next
+// attempt to trip over, same as any other failed setup step.
+func TestNetns_SetupTearsDownWhenNoDHCPClient(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	withDHCPClients(n, map[string]bool{}, false)
+	if _, err := n.Setup("t-001", "52:54:00:de:ad:01"); err == nil {
+		t.Fatal("want error when no DHCP client is available")
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected teardown, got %v", f.joined())
+	}
+}
+
 func TestNetns_Teardown(t *testing.T) {
 	f := &fakeNetRunner{}
 	n := newTestNetns(f)

cmd/ecs-agent/reconcile.go

@@ -8,12 +8,18 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
-// reconcile re-adopts containers that survived an agent restart. It lists the
-// runtime's containers and, for each task still running in this cluster,
-// re-registers its IAM credentials, re-attaches the exit-wait, and refreshes its
-// RUNNING state on the gateway. The returned set seeds pollAssignments so the
+// reconcile re-syncs the gateway with what this instance is actually running
+// after an agent restart. It lists the runtime's containers and, for each task
+// in this cluster, either re-adopts it (still running: re-register IAM
+// credentials, re-attach the exit-wait, refresh RUNNING) or reports it STOPPED
+// and clears its leftovers. The returned set seeds pollAssignments so the
 // gateway's re-delivered assignments for these tasks are acked but not re-run.
 //
+// Reporting the dead ones is the load-bearing half. The exit-wait is cancelled
+// when the agent shuts down, so a container that dies while the agent is down
+// has nobody to report its exit; the gateway holds the task RUNNING forever,
+// its service never replaces it, and the target never recovers.
+//
 // It is best-effort: a nil runner or a List error returns an empty set and the
 // agent falls back to its pre-reconcile behaviour (poll + run).
 func (a *Agent) reconcile(ctx context.Context) map[string]bool {
@@ -27,15 +33,17 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 		return adopted
 	}
 
-	// Group this cluster's running containers back into their tasks. The Assign is
-	// reconstructed from labels — enough to re-register creds and tear down netns.
+	// Group this cluster's containers back into their tasks, running or not. The
+	// Assign is reconstructed from labels — enough to re-register creds and tear
+	// down netns.
 	type task struct {
-		as         *bus.Assign
-		containers []ctrruntime.Container
+		as      *bus.Assign
+		running []ctrruntime.Container
+		dead    []ctrruntime.Container
 	}
 	tasks := map[string]*task{}
 	for _, c := range containers {
-		if !c.Running || c.Labels[labelClusterName] != a.id.ClusterName {
+		if c.Labels[labelClusterName] != a.id.ClusterName {
 			continue
 		}
 		taskID := c.Labels[labelTaskID]
@@ -53,20 +61,50 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 			}}
 			tasks[taskID] = t
 		}
-		t.containers = append(t.containers, c)
+		if c.Running {
+			t.running = append(t.running, c)
+		} else {
+			t.dead = append(t.dead, c)
+		}
 	}
 
 	for taskID, t := range tasks {
+		// A task with nothing running is over, whatever the gateway still believes.
+		// Mixed running/dead is left to the running path: the exit-wait re-attached
+		// below reports the individual container when it is reaped.
+		if len(t.running) == 0 {
+			a.reportStoppedTask(ctx, t.as, t.dead)
+			adopted[taskID] = true
+			continue
+		}
+
 		a.registerTaskCreds(t.as)
-		statuses := make([]bus.ContainerStatus, 0, len(t.containers))
-		for _, c := range t.containers {
+		statuses := make([]bus.ContainerStatus, 0, len(t.running))
+		for _, c := range t.running {
 			name := c.Labels[labelContainerName]
 			go a.waitContainer(ctx, t.as, name, c.ID)
 			statuses = append(statuses, bus.ContainerStatus{Name: name, Status: bus.TaskStatusRunning, ContainerID: c.ID})
 		}
 		a.reportTaskState(t.as, bus.TaskStatusRunning, "", statuses)
 		adopted[taskID] = true
-		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.containers))
+		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.running))
 	}
 	return adopted
 }
+
+// reportStoppedTask clears a task's leftover containers and tells the gateway it
+// is STOPPED, so the scheduler stops counting a task this instance is not
+// running and its service can place a replacement.
+func (a *Agent) reportStoppedTask(ctx context.Context, as *bus.Assign, dead []ctrruntime.Container) {
+	statuses := make([]bus.ContainerStatus, 0, len(dead))
+	for _, c := range dead {
+		if err := a.runner.Remove(ctx, c.ID); err != nil {
+			slog.Warn("ecs-agent: reconcile remove failed", "task", as.TaskID, "container", c.ID, "err", err)
+		}
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
+		})
+	}
+	a.reportTaskState(as, bus.TaskStatusStopped, "Container exited while the agent was not running", statuses)
+	slog.Info("ecs-agent: reported stopped task", "task", as.TaskID, "containers", len(dead))
+}

cmd/ecs-agent/reconcile_test.go

@@ -72,6 +72,63 @@ func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
 	waitedFor(t, rt, containerID("t-001", "web"))
 }
 
+// A task whose container died while the agent was down must be reported STOPPED,
+// not silently skipped. The exit-wait is cancelled on agent shutdown, so nothing
+// else ever reports that exit: skipping it leaves the gateway holding the task
+// RUNNING forever, its service never places a replacement, and the load
+// balancer target stays unhealthy with no path back.
+func TestReconcile_ReportsTaskWhoseContainerDied(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-dead", "web", "default", "t-dead", testTaskRole, "52:54:00:de:ad:02", false),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusStopped); n != 1 {
+		t.Errorf("STOPPED reports for t-dead = %d, want 1", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusRunning); n != 0 {
+		t.Errorf("t-dead must not be reported RUNNING, got %d such reports", n)
+	}
+	// Seeded so a re-delivered assignment for the dead task is acked, not re-run.
+	if !adopted["t-dead"] {
+		t.Errorf("t-dead not seeded into the adopted set; got %v", adopted)
+	}
+	if want := containerID("t-dead", "web"); !slices.Contains(rt.Removed, want) {
+		t.Errorf("leftover container %s not removed; removed=%v", want, rt.Removed)
+	}
+}
+
+// A task with one live and one dead container is still running: the live half is
+// re-adopted and the re-attached exit-wait reports the dead one when it is
+// reaped, so reconcile must not declare the whole task STOPPED.
+func TestReconcile_PartiallyDeadTaskStaysRunning(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-mixed", "web", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", true),
+			adoptedContainer("t-mixed", "sidecar", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", false),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	a.reconcile(context.Background())
+
+	if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("RUNNING reports for t-mixed = %d, want 1", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusStopped); n != 0 {
+		t.Errorf("t-mixed must not be reported STOPPED while a container still runs, got %d", n)
+	}
+}
+
 // pollAssignments seeded with an adopted task acks its re-delivered assignment
 // but does not re-run it, while a genuinely new task still dispatches.
 func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
@@ -123,23 +180,35 @@ func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
 	}
 }
 
-// Containers for another cluster or with a stopped task are not adopted.
-func TestReconcile_FiltersOtherClusterAndStopped(t *testing.T) {
+// Containers belonging to another cluster are ignored entirely: this agent
+// speaks for its own cluster only, and must neither adopt nor report them.
+//
+// A stopped container of this cluster IS handled — see
+// TestReconcile_ReportsTaskWhoseContainerDied. It lands in the adopted set so
+// its re-delivered assignment is acked rather than re-run, after reconcile has
+// reported the task STOPPED.
+func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
+	cp := &fakeCP{}
 	rt := &ctrruntime.FakePuller{
 		WaitErr: errors.New("blocked"),
 		Containers: []ctrruntime.Container{
 			adoptedContainer("t-other", "web", "other", "", "", "", true),
-			adoptedContainer("t-stop", "web", "default", "", "", "", false),
+			adoptedContainer("t-other-dead", "web", "other", "", "", "", false),
 			adoptedContainer("t-ok", "web", "default", "", "", "", true),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
 
 	adopted := a.reconcile(context.Background())
 
 	if len(adopted) != 1 || !adopted["t-ok"] {
 		t.Fatalf("adopted = %v, want only t-ok", adopted)
 	}
+	for _, id := range []string{"t-other", "t-other-dead"} {
+		if n := countStatus(cp.taskStates(), id, bus.TaskStatusStopped); n != 0 {
+			t.Errorf("reported %d STOPPED states for another cluster's task %s, want 0", n, id)
+		}
+	}
 }
 
 // A nil runner or a List error degrades to an empty set without panicking.

cmd/ecs-agent/taskrun_test.go

@@ -70,8 +70,8 @@ func TestRunTask_PullRunReportsRunning(t *testing.T) {
 	}
 }
 
-// TestRunTask_GPUCarriedToRunSpec covers mulga-11opz (Epic C task C1): the
-// AssignContainer's GPU count reaches the runtime RunSpec unchanged; a
+// TestRunTask_GPUCarriedToRunSpec verifies that the AssignContainer's GPU count
+// reaches the runtime RunSpec unchanged; a
 // container with no GPU request keeps RunSpec.GPU at zero (regression).
 func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
 	cp := &fakeCP{}
@@ -534,7 +534,7 @@ func (sentinelResolver) Authorize(context.Context, string) (string, string, stri
 	return "", "", "", nil
 }
 
-// TestPullResolver covers siv-459: an assign with no execution role (or no
+// TestPullResolver verifies that an assign with no execution role (or no
 // credential endpoint) uses the instance resolver; with both, a distinct
 // exec-role resolver is built.
 func TestPullResolver(t *testing.T) {

cmd/installer/autoinstall/autoinstall.go

@@ -4,12 +4,14 @@
 package autoinstall
 
 import (
+	"cmp"
 	"fmt"
 	"log/slog"
 	"net"
 	"os"
 	"os/exec"
-	"sort"
+	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"syscall"
@@ -18,6 +20,10 @@ import (
 	"github.com/mulgadc/spinifex/cmd/installer/install"
 )
 
+// listDisks is the block-device scan, indirected so tests can drive selection
+// without real hardware.
+var listDisks = install.ListDisks
+
 // Load returns an install.Config built from SPINIFEX_* environment variables,
 // or nil if SPINIFEX_AUTO is not set to "1" (interactive mode).
 func Load() (*install.Config, error) {
@@ -65,9 +71,9 @@ func buildConfig() (*install.Config, error) {
 		hostname = "spinifex-node"
 	}
 
-	disk, err := resolveDisk(os.Getenv("SPINIFEX_DISK"))
+	storage, err := resolveStorage()
 	if err != nil {
-		return nil, fmt.Errorf("disk: %w", err)
+		return nil, fmt.Errorf("storage: %w", err)
 	}
 
 	wanIface, err := resolveNIC(os.Getenv("SPINIFEX_WAN_IFACE"), "")
@@ -76,73 +82,167 @@ func buildConfig() (*install.Config, error) {
 	}
 
 	cfg := &install.Config{
-		Disk:         disk,
+		Storage:      storage,
 		Hostname:     hostname,
 		RootPassword: password,
 		// Optional on the headless path — empty means no --email passed to
 		// spx admin init, which omits the operator identity from telemetry.
-		Email:        strings.TrimSpace(os.Getenv("SPINIFEX_EMAIL")),
-		WANInterface: wanIface,
-		WANDHCPMode:  strings.ToLower(os.Getenv("SPINIFEX_WAN_MODE")) != "static",
+		Email: strings.TrimSpace(os.Getenv("SPINIFEX_EMAIL")),
 	}
 
-	if !cfg.WANDHCPMode {
-		ip := os.Getenv("SPINIFEX_WAN_IP")
-		mask := os.Getenv("SPINIFEX_WAN_MASK")
-		gw := os.Getenv("SPINIFEX_WAN_GW")
-		if ip == "" || mask == "" || gw == "" {
-			return nil, fmt.Errorf("SPINIFEX_WAN_IP, SPINIFEX_WAN_MASK, SPINIFEX_WAN_GW required for static mode")
-		}
-		cfg.WANAddress = ip
-		cfg.WANMask = mask
-		cfg.WANGateway = gw
-		if dns := os.Getenv("SPINIFEX_WAN_DNS"); dns != "" {
-			cfg.WANDNS = strings.Split(dns, ",")
-		}
+	cfg.WAN, err = parseRole("WAN", wanIface)
+	if err != nil {
+		return nil, err
+	}
+	if cfg.WAN.DHCPMode {
+		cfg.WAN.Gateway = ""
+	} else if cfg.WAN.Gateway == "" {
+		return nil, fmt.Errorf("SPINIFEX_WAN_IP, SPINIFEX_WAN_MASK, SPINIFEX_WAN_GW required for static mode")
 	}
 
-	if lanIface := os.Getenv("SPINIFEX_LAN_IFACE"); lanIface != "" {
-		lan, err := resolveNIC(lanIface, wanIface)
+	// lan and vpc are optional: an unset _IFACE leaves the role folded, which
+	// is how a single- or two-NIC node collapses onto the planes above it.
+	for _, r := range []struct {
+		plane string
+		dst   *install.NetworkRole
+	}{{"LAN", &cfg.LAN}, {"VPC", &cfg.VPC}} {
+		name := os.Getenv("SPINIFEX_" + r.plane + "_IFACE")
+		if name == "" {
+			continue
+		}
+		iface, err := resolveNIC(name, wanIface)
 		if err != nil {
-			return nil, fmt.Errorf("LAN NIC: %w", err)
+			return nil, fmt.Errorf("%s NIC: %w", r.plane, err)
 		}
-		cfg.LANInterface = lan
-		cfg.LANDHCPMode = strings.ToLower(os.Getenv("SPINIFEX_LAN_MODE")) != "static"
-		if !cfg.LANDHCPMode {
-			cfg.LANAddress = os.Getenv("SPINIFEX_LAN_IP")
-			cfg.LANMask = os.Getenv("SPINIFEX_LAN_MASK")
-			if dns := os.Getenv("SPINIFEX_LAN_DNS"); dns != "" {
-				cfg.LANDNS = strings.Split(dns, ",")
-			}
+		if *r.dst, err = parseRole(r.plane, iface); err != nil {
+			return nil, err
 		}
+		// Only the wan plane installs a default route.
+		r.dst.Gateway = ""
+	}
+
+	if err := cfg.Validate(); err != nil {
+		return nil, fmt.Errorf("network roles: %w", err)
 	}
 
 	if os.Getenv("SPINIFEX_GPU_PASSTHROUGH") == "1" {
 		cfg.GPUPassthrough = true
 	}
 
-	role := strings.ToLower(os.Getenv("SPINIFEX_ROLE"))
-	if role == "" {
-		role = "init"
-	}
-	cfg.ClusterRole = role
-	skipFormation := os.Getenv("SPINIFEX_SKIP_FORMATION") == "1"
-	if skipFormation {
+	if os.Getenv("SPINIFEX_SKIP_FORMATION") == "1" {
 		cfg.SkipFormation = true
 	}
 
-	if role == "join" && !skipFormation {
-		joinAddr := os.Getenv("SPINIFEX_JOIN_ADDR")
-		if joinAddr == "" {
-			return nil, fmt.Errorf("SPINIFEX_JOIN_ADDR required when SPINIFEX_ROLE=join")
-		}
-		cfg.JoinAddr = joinAddr
-	}
-
 	return cfg, nil
 }
 
-// resolveDisk maps the spx_disk value from grub.cfg to a block device path.
+// resolveStorage builds the disk configuration from the kernel cmdline.
+//
+//	SPINIFEX_FS     ext4 (default) or one of the zfs-* topologies
+//	SPINIFEX_DISKS  explicit ordered member list, required for multi-disk pools
+//	SPINIFEX_DISK   single-disk selector, as before
+func resolveStorage() (install.DiskConfig, error) {
+	cfg := install.DiskConfig{FS: install.FSExt4}
+	if v := strings.TrimSpace(os.Getenv("SPINIFEX_FS")); v != "" {
+		fs, err := install.ParseFSType(v)
+		if err != nil {
+			return cfg, err
+		}
+		cfg.FS = fs
+	}
+
+	var err error
+	switch list := strings.TrimSpace(os.Getenv("SPINIFEX_DISKS")); {
+	case list != "":
+		if cfg.Disks, err = disksByName(strings.Split(list, ",")); err != nil {
+			return cfg, err
+		}
+	case cfg.FS.MinDisks() > 1:
+		// No auto-selection for a multi-disk pool. Choosing which disks to
+		// erase is not a decision an unattended install gets to make, and the
+		// member order determines the RAID10 pairing.
+		return cfg, fmt.Errorf("%s needs at least %d disks — set SPINIFEX_DISKS to an explicit comma-separated list",
+			cfg.FS.Label(), cfg.FS.MinDisks())
+	default:
+		d, derr := resolveDisk(os.Getenv("SPINIFEX_DISK"))
+		if derr != nil {
+			return cfg, derr
+		}
+		cfg.Disks = []install.Disk{d}
+	}
+
+	if cfg.ZFS, err = parseZFSOpts(); err != nil {
+		return cfg, err
+	}
+	if err := cfg.Validate(); err != nil {
+		return cfg, err
+	}
+	slog.Info("autoinstall: storage selected", "fs", cfg.FS, "disks", cfg.Paths())
+	return cfg, nil
+}
+
+// parseZFSOpts reads the advanced pool tunables. Every one is optional; unset
+// values are computed from the hardware at install time.
+func parseZFSOpts() (install.ZFSOpts, error) {
+	var o install.ZFSOpts
+	ints := []struct {
+		env string
+		dst *int
+	}{
+		{"SPINIFEX_ZFS_ASHIFT", &o.Ashift},
+		{"SPINIFEX_ZFS_COPIES", &o.Copies},
+		{"SPINIFEX_ZFS_ARC_MAX_MIB", &o.ARCMaxMiB},
+	}
+	for _, f := range ints {
+		v := strings.TrimSpace(os.Getenv(f.env))
+		if v == "" {
+			continue
+		}
+		n, err := strconv.Atoi(v)
+		if err != nil {
+			return o, fmt.Errorf("%s: %w", f.env, err)
+		}
+		*f.dst = n
+	}
+	o.Compress = strings.TrimSpace(os.Getenv("SPINIFEX_ZFS_COMPRESS"))
+	o.Checksum = strings.TrimSpace(os.Getenv("SPINIFEX_ZFS_CHECKSUM"))
+	return o, nil
+}
+
+// disksByName resolves an explicit member list. Names may be kernel names
+// ("sdb"), device paths or by-id paths; the order given is preserved because
+// RAID10 pairs members in it.
+func disksByName(names []string) ([]install.Disk, error) {
+	available, err := listDisks()
+	if err != nil {
+		return nil, err
+	}
+	out := make([]install.Disk, 0, len(names))
+	for _, raw := range names {
+		name := strings.TrimSpace(raw)
+		if name == "" {
+			continue
+		}
+		// A by-id path or a symlink resolves to the kernel device the scan
+		// reported, so both spellings name the same disk.
+		resolved := name
+		if !strings.HasPrefix(resolved, "/dev/") {
+			resolved = "/dev/" + resolved
+		}
+		if target, err := filepath.EvalSymlinks(resolved); err == nil {
+			resolved = target
+		}
+
+		idx := slices.IndexFunc(available, func(d install.Disk) bool { return d.Path == resolved })
+		if idx < 0 {
+			return nil, fmt.Errorf("%q is not an available disk — found:\n%s", name, diskList(available))
+		}
+		out = append(out, available[idx])
+	}
+	return out, nil
+}
+
+// resolveDisk maps the SPINIFEX_DISK value from grub.cfg to a disk.
 //
 // Supported values:
 //
@@ -151,141 +251,78 @@ func buildConfig() (*install.Config, error) {
 //	smallest        — smallest non-removable disk (typical OS-on-SSD pattern)
 //	nvme            — the only NVMe disk; fail if multiple found
 //	/dev/sda (etc.) — exact device path; fail if not present
-func resolveDisk(target string) (string, error) {
-	switch strings.ToLower(target) {
+func resolveDisk(target string) (install.Disk, error) {
+	disks, err := candidateDisks()
+	if err != nil {
+		return install.Disk{}, err
+	}
+	switch strings.ToLower(strings.TrimSpace(target)) {
 	case "", "auto":
-		return singleDisk()
-	case "largest":
-		return diskBySize(true)
-	case "smallest":
-		return diskBySize(false)
-	case "nvme":
-		return nvmeDisk()
-	default:
-		if _, err := os.Stat(target); err != nil {
-			return "", fmt.Errorf("%q not found", target)
+		if len(disks) != 1 {
+			return install.Disk{}, fmt.Errorf(
+				"expected exactly one disk, found %d — set SPINIFEX_DISK to one of:\n%s\n"+
+					"  largest   (largest disk)\n"+
+					"  smallest  (smallest disk)\n"+
+					"  nvme      (NVMe only)\n"+
+					"  /dev/sdX  (exact path)",
+				len(disks), diskList(disks))
 		}
-		return target, nil
+		return disks[0], nil
+	case "largest":
+		return pickBySize(disks, true)
+	case "smallest":
+		return pickBySize(disks, false)
+	case "nvme":
+		nvmes := slices.DeleteFunc(slices.Clone(disks), func(d install.Disk) bool {
+			return !strings.HasPrefix(filepath.Base(d.Path), "nvme")
+		})
+		if len(nvmes) != 1 {
+			return install.Disk{}, fmt.Errorf("expected exactly one NVMe disk, found %d:\n%s",
+				len(nvmes), diskList(nvmes))
+		}
+		return nvmes[0], nil
+	default:
+		found, err := disksByName([]string{target})
+		if err != nil {
+			return install.Disk{}, err
+		}
+		return found[0], nil
 	}
 }
 
-type diskCandidate struct {
-	dev   string
-	bytes int64
-}
-
-// nonRemovableDisks returns all non-removable, non-virtual block devices.
-func nonRemovableDisks() ([]diskCandidate, error) {
-	entries, err := os.ReadDir("/sys/block")
+// candidateDisks lists the disks an unattended install may erase: fixed, and
+// not the installer's own media.
+func candidateDisks() ([]install.Disk, error) {
+	all, err := listDisks()
 	if err != nil {
 		return nil, err
 	}
-	var out []diskCandidate
-	for _, e := range entries {
-		dev := e.Name()
-		switch {
-		case strings.HasPrefix(dev, "loop"),
-			strings.HasPrefix(dev, "ram"),
-			strings.HasPrefix(dev, "dm-"),
-			strings.HasPrefix(dev, "sr"):
-			continue
-		}
-		removable, _ := os.ReadFile("/sys/block/" + dev + "/removable")
-		if strings.TrimSpace(string(removable)) == "1" {
-			continue
-		}
-		sizeRaw, _ := os.ReadFile("/sys/block/" + dev + "/size")
-		sectors, parseErr := strconv.ParseInt(strings.TrimSpace(string(sizeRaw)), 10, 64)
-		if parseErr != nil || sectors == 0 {
-			continue
-		}
-		out = append(out, diskCandidate{dev: dev, bytes: sectors * 512})
-	}
-	return out, nil
+	return slices.DeleteFunc(all, func(d install.Disk) bool {
+		return d.Removable || d.LiveMedia
+	}), nil
 }
 
-// diskList returns a human-readable list of candidates for error messages.
-func diskList(disks []diskCandidate) string {
+func pickBySize(disks []install.Disk, largest bool) (install.Disk, error) {
+	if len(disks) == 0 {
+		return install.Disk{}, fmt.Errorf("no non-removable disks found")
+	}
+	sorted := slices.Clone(disks)
+	slices.SortFunc(sorted, func(a, b install.Disk) int { return cmp.Compare(a.Bytes, b.Bytes) })
+	if largest {
+		return sorted[len(sorted)-1], nil
+	}
+	return sorted[0], nil
+}
+
+// diskList renders candidates for an error message.
+func diskList(disks []install.Disk) string {
 	lines := make([]string, len(disks))
 	for i, d := range disks {
-		lines[i] = fmt.Sprintf("  /dev/%s (%dG)", d.dev, d.bytes>>30)
+		lines[i] = fmt.Sprintf("  %s (%s, %s)", d.Path, d.SizeHuman(), d.Content)
 	}
 	return strings.Join(lines, "\n")
 }
 
-// singleDisk selects the only non-removable disk, or fails listing all found.
-func singleDisk() (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	switch len(disks) {
-	case 0:
-		return "", fmt.Errorf("no non-removable disks found")
-	case 1:
-		slog.Info("autoinstall: single disk selected", "disk", disks[0].dev, "size_gb", disks[0].bytes>>30)
-		return "/dev/" + disks[0].dev, nil
-	default:
-		return "", fmt.Errorf(
-			"multiple disks found — set SPINIFEX_DISK to one of:\n%s\n"+
-				"  largest   (largest disk)\n"+
-				"  smallest  (smallest disk)\n"+
-				"  nvme      (NVMe only)\n"+
-				"  /dev/sdX  (exact path)",
-			diskList(disks),
-		)
-	}
-}
-
-// diskBySize selects the largest (largest=true) or smallest non-removable disk.
-func diskBySize(largest bool) (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	if len(disks) == 0 {
-		return "", fmt.Errorf("no non-removable disks found")
-	}
-	sort.Slice(disks, func(i, j int) bool {
-		if largest {
-			return disks[i].bytes > disks[j].bytes
-		}
-		return disks[i].bytes < disks[j].bytes
-	})
-	label := "smallest"
-	if largest {
-		label = "largest"
-	}
-	slog.Info("autoinstall: disk selected by size", "mode", label, "disk", disks[0].dev, "size_gb", disks[0].bytes>>30)
-	return "/dev/" + disks[0].dev, nil
-}
-
-// nvmeDisk selects the only NVMe disk, or fails listing all NVMe disks found.
-func nvmeDisk() (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	var nvmes []diskCandidate
-	for _, d := range disks {
-		if strings.HasPrefix(d.dev, "nvme") {
-			nvmes = append(nvmes, d)
-		}
-	}
-	switch len(nvmes) {
-	case 0:
-		return "", fmt.Errorf("no NVMe disks found")
-	case 1:
-		slog.Info("autoinstall: NVMe disk selected", "disk", nvmes[0].dev, "size_gb", nvmes[0].bytes>>30)
-		return "/dev/" + nvmes[0].dev, nil
-	default:
-		return "", fmt.Errorf(
-			"multiple NVMe disks found — set SPINIFEX_DISK to one of:\n%s",
-			diskList(nvmes),
-		)
-	}
-}
-
 // virtualNICPrefixes lists interface name prefixes that identify non-physical
 // interfaces (bridges, tunnels, container/OVS veth pairs, etc.). These are
 // skipped when auto-selecting a NIC so we don't configure docker0 or
@@ -297,6 +334,42 @@ var virtualNICPrefixes = []string{
 
 // resolveNIC returns the interface name to use. "auto" or empty picks the
 // first physical (non-loopback, non-virtual) interface that is not exclude.
+// parseRole reads the SPINIFEX_<PLANE>_* kernel-cmdline vars for one plane.
+// iface is the already-resolved interface name; _VLAN and _MTU are optional
+// and default to untagged at the link MTU.
+func parseRole(plane, iface string) (install.NetworkRole, error) {
+	role := install.NetworkRole{
+		Interface: iface,
+		DHCPMode:  strings.ToLower(os.Getenv("SPINIFEX_"+plane+"_MODE")) != "static",
+	}
+	if v := os.Getenv("SPINIFEX_" + plane + "_VLAN"); v != "" {
+		id, err := strconv.Atoi(v)
+		if err != nil {
+			return role, fmt.Errorf("SPINIFEX_%s_VLAN: %w", plane, err)
+		}
+		role.VLAN = id
+	}
+	if v := os.Getenv("SPINIFEX_" + plane + "_MTU"); v != "" {
+		mtu, err := strconv.Atoi(v)
+		if err != nil {
+			return role, fmt.Errorf("SPINIFEX_%s_MTU: %w", plane, err)
+		}
+		role.MTU = mtu
+	}
+	if !role.DHCPMode {
+		role.Address = os.Getenv("SPINIFEX_" + plane + "_IP")
+		role.Mask = os.Getenv("SPINIFEX_" + plane + "_MASK")
+		role.Gateway = os.Getenv("SPINIFEX_" + plane + "_GW")
+		if role.Address == "" || role.Mask == "" {
+			return role, fmt.Errorf("SPINIFEX_%s_IP and SPINIFEX_%s_MASK required for static mode", plane, plane)
+		}
+	}
+	if dns := os.Getenv("SPINIFEX_" + plane + "_DNS"); dns != "" {
+		role.DNS = strings.Split(dns, ",")
+	}
+	return role, nil
+}
+
 func resolveNIC(name, exclude string) (string, error) {
 	if name != "" && name != "auto" {
 		return name, nil

cmd/installer/autoinstall/autoinstall_test.go

@@ -1,12 +1,57 @@
 package autoinstall
 
 import (
+	"errors"
+	"fmt"
+	"maps"
+	"net"
 	"os"
+	"slices"
+	"strings"
 	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
 )
 
+const gib = 1 << 30
+
+// setEnv clears every SPINIFEX_* variable inherited from the test runner before
+// applying want, so one case cannot leak a kernel-cmdline value into the next.
+func setEnv(t *testing.T, want map[string]string) {
+	t.Helper()
+	for _, kv := range os.Environ() {
+		if k, _, ok := strings.Cut(kv, "="); ok && strings.HasPrefix(k, "SPINIFEX_") {
+			t.Setenv(k, "")
+		}
+	}
+	for k, v := range want {
+		t.Setenv(k, v)
+	}
+}
+
+// withDisks replaces the block-device scan. The clone matters: candidateDisks
+// filters with slices.DeleteFunc, which would otherwise mutate the fixture.
+func withDisks(t *testing.T, disks []install.Disk, err error) {
+	t.Helper()
+	prev := listDisks
+	listDisks = func() ([]install.Disk, error) { return slices.Clone(disks), err }
+	t.Cleanup(func() { listDisks = prev })
+}
+
+func mkDisk(path string, bytes int64) install.Disk {
+	return install.Disk{
+		Path:              path,
+		ByID:              "/dev/disk/by-id/test-" + strings.TrimPrefix(path, "/dev/"),
+		Bytes:             bytes,
+		Model:             "TESTDISK",
+		LogicalBlockSize:  512,
+		PhysicalBlockSize: 512,
+		Content:           "empty",
+	}
+}
+
 func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
-	os.Unsetenv("SPINIFEX_AUTO")
+	setEnv(t, nil)
 	cfg, err := Load()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -15,3 +60,586 @@ func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
 		t.Fatal("expected nil config when SPINIFEX_AUTO is not set")
 	}
 }
+
+func TestLoadBuildsConfigFromEnv(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_AUTO":            "1",
+		"SPINIFEX_PASSWORD":        "s3cret",
+		"SPINIFEX_HOSTNAME":        "node-7",
+		"SPINIFEX_EMAIL":           "  ops@example.com  ",
+		"SPINIFEX_WAN_IFACE":       "eno1",
+		"SPINIFEX_WAN_DNS":         "1.1.1.1,8.8.8.8",
+		"SPINIFEX_LAN_IFACE":       "eno2",
+		"SPINIFEX_LAN_VLAN":        "100",
+		"SPINIFEX_LAN_MTU":         "9000",
+		"SPINIFEX_GPU_PASSTHROUGH": "1",
+	})
+
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.Hostname != "node-7" || cfg.RootPassword != "s3cret" {
+		t.Errorf("identity = %q/%q", cfg.Hostname, cfg.RootPassword)
+	}
+	if cfg.Email != "ops@example.com" {
+		t.Errorf("Email = %q, want trimmed", cfg.Email)
+	}
+	if cfg.Storage.FS != install.FSExt4 || len(cfg.Storage.Disks) != 1 {
+		t.Errorf("Storage = %v / %d disks", cfg.Storage.FS, len(cfg.Storage.Disks))
+	}
+	if cfg.WAN.Interface != "eno1" || !cfg.WAN.DHCPMode {
+		t.Errorf("WAN = %+v, want eno1 on DHCP", cfg.WAN)
+	}
+	if !slices.Equal(cfg.WAN.DNS, []string{"1.1.1.1", "8.8.8.8"}) {
+		t.Errorf("WAN.DNS = %v", cfg.WAN.DNS)
+	}
+	if cfg.LAN.Interface != "eno2" || cfg.LAN.VLAN != 100 || cfg.LAN.MTU != 9000 {
+		t.Errorf("LAN = %+v", cfg.LAN)
+	}
+	// Only the wan plane installs a default route.
+	if cfg.LAN.Gateway != "" {
+		t.Errorf("LAN.Gateway = %q, want cleared", cfg.LAN.Gateway)
+	}
+	if !cfg.VPC.Folded() {
+		t.Errorf("VPC = %+v, want folded when no _IFACE is set", cfg.VPC)
+	}
+	if !cfg.GPUPassthrough {
+		t.Error("GPUPassthrough not enabled")
+	}
+}
+
+func TestLoadDefaultsHostname(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_AUTO":      "1",
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+	})
+
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.Hostname != "spinifex-node" {
+		t.Errorf("Hostname = %q, want spinifex-node", cfg.Hostname)
+	}
+}
+
+func TestLoadRequiresPassword(t *testing.T) {
+	setEnv(t, map[string]string{"SPINIFEX_AUTO": "1"})
+
+	cfg, err := Load()
+	if err == nil {
+		t.Fatalf("expected error, got config %+v", cfg)
+	}
+	if !strings.Contains(err.Error(), "SPINIFEX_PASSWORD") {
+		t.Errorf("err = %v, want it to name SPINIFEX_PASSWORD", err)
+	}
+	if !strings.HasPrefix(err.Error(), "autoinstall config:") {
+		t.Errorf("err = %v, want the autoinstall wrapper", err)
+	}
+}
+
+func TestBuildConfigStaticWAN(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     map[string]string
+		wantErr string
+	}{
+		{
+			name: "full static",
+			env: map[string]string{
+				"SPINIFEX_WAN_MODE": "static",
+				"SPINIFEX_WAN_IP":   "10.0.0.5",
+				"SPINIFEX_WAN_MASK": "255.255.255.0",
+				"SPINIFEX_WAN_GW":   "10.0.0.1",
+			},
+		},
+		{
+			name: "missing gateway",
+			env: map[string]string{
+				"SPINIFEX_WAN_MODE": "static",
+				"SPINIFEX_WAN_IP":   "10.0.0.5",
+				"SPINIFEX_WAN_MASK": "255.255.255.0",
+			},
+			wantErr: "SPINIFEX_WAN_GW required",
+		},
+		{
+			name:    "missing address",
+			env:     map[string]string{"SPINIFEX_WAN_MODE": "static"},
+			wantErr: "SPINIFEX_WAN_IP and SPINIFEX_WAN_MASK required",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+			env := map[string]string{
+				"SPINIFEX_PASSWORD":  "s3cret",
+				"SPINIFEX_WAN_IFACE": "eno1",
+			}
+			maps.Copy(env, tt.env)
+			setEnv(t, env)
+
+			cfg, err := buildConfig()
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("buildConfig: %v", err)
+			}
+			if cfg.WAN.DHCPMode {
+				t.Error("WAN.DHCPMode = true, want static")
+			}
+			if cfg.WAN.Address != "10.0.0.5" || cfg.WAN.Gateway != "10.0.0.1" {
+				t.Errorf("WAN = %+v", cfg.WAN)
+			}
+		})
+	}
+}
+
+// SPINIFEX_ROLE and SPINIFEX_JOIN_ADDR no longer exist: the installer cannot
+// form a multi-node cluster, so a stale cmdline from an older ISO has to be
+// inert rather than producing a half-configured node.
+func TestBuildConfigSkipFormation(t *testing.T) {
+	tests := []struct {
+		name         string
+		env          map[string]string
+		wantSkipForm bool
+	}{
+		{
+			name:         "default initializes a single-node cluster",
+			wantSkipForm: false,
+		},
+		{
+			name:         "formation deferred to the provisioner",
+			env:          map[string]string{"SPINIFEX_SKIP_FORMATION": "1"},
+			wantSkipForm: true,
+		},
+		{
+			name:         "stale join cmdline is ignored",
+			env:          map[string]string{"SPINIFEX_ROLE": "join", "SPINIFEX_JOIN_ADDR": "10.0.0.1:9999"},
+			wantSkipForm: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+			env := map[string]string{
+				"SPINIFEX_PASSWORD":  "s3cret",
+				"SPINIFEX_WAN_IFACE": "eno1",
+			}
+			maps.Copy(env, tt.env)
+			setEnv(t, env)
+
+			cfg, err := buildConfig()
+			if err != nil {
+				t.Fatalf("buildConfig: %v", err)
+			}
+			if cfg.SkipFormation != tt.wantSkipForm {
+				t.Errorf("SkipFormation = %v, want %v", cfg.SkipFormation, tt.wantSkipForm)
+			}
+		})
+	}
+}
+
+func TestBuildConfigRejectsDuplicatePlaneBinding(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+		"SPINIFEX_LAN_IFACE": "eno1",
+	})
+
+	if _, err := buildConfig(); err == nil || !strings.Contains(err.Error(), "network roles") {
+		t.Fatalf("err = %v, want a network roles validation failure", err)
+	}
+}
+
+func TestBuildConfigPropagatesStorageError(t *testing.T) {
+	setEnv(t, map[string]string{
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+		"SPINIFEX_FS":        "btrfs",
+	})
+
+	_, err := buildConfig()
+	if err == nil || !strings.HasPrefix(err.Error(), "storage:") {
+		t.Fatalf("err = %v, want a storage: prefix", err)
+	}
+}
+
+func TestParseRole(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     map[string]string
+		want    install.NetworkRole
+		wantErr string
+	}{
+		{
+			name: "dhcp by default",
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true},
+		},
+		{
+			name: "vlan and mtu",
+			env:  map[string]string{"SPINIFEX_LAN_VLAN": "42", "SPINIFEX_LAN_MTU": "9000"},
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true, VLAN: 42, MTU: 9000},
+		},
+		{
+			name: "dns list is comma separated",
+			env:  map[string]string{"SPINIFEX_LAN_DNS": "1.1.1.1,9.9.9.9"},
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true, DNS: []string{"1.1.1.1", "9.9.9.9"}},
+		},
+		{
+			name: "static",
+			env: map[string]string{
+				"SPINIFEX_LAN_MODE": "STATIC",
+				"SPINIFEX_LAN_IP":   "192.168.1.2",
+				"SPINIFEX_LAN_MASK": "255.255.255.0",
+				"SPINIFEX_LAN_GW":   "192.168.1.1",
+			},
+			want: install.NetworkRole{
+				Interface: "eno1",
+				Address:   "192.168.1.2",
+				Mask:      "255.255.255.0",
+				Gateway:   "192.168.1.1",
+			},
+		},
+		{
+			name:    "static without mask",
+			env:     map[string]string{"SPINIFEX_LAN_MODE": "static", "SPINIFEX_LAN_IP": "192.168.1.2"},
+			wantErr: "SPINIFEX_LAN_IP and SPINIFEX_LAN_MASK required",
+		},
+		{
+			name:    "unparsable vlan",
+			env:     map[string]string{"SPINIFEX_LAN_VLAN": "trunk"},
+			wantErr: "SPINIFEX_LAN_VLAN",
+		},
+		{
+			name:    "unparsable mtu",
+			env:     map[string]string{"SPINIFEX_LAN_MTU": "jumbo"},
+			wantErr: "SPINIFEX_LAN_MTU",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			setEnv(t, tt.env)
+
+			got, err := parseRole("LAN", "eno1")
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("parseRole: %v", err)
+			}
+			if fmt.Sprintf("%+v", got) != fmt.Sprintf("%+v", tt.want) {
+				t.Errorf("role = %+v, want %+v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseZFSOpts(t *testing.T) {
+	t.Run("unset leaves the computed defaults", func(t *testing.T) {
+		setEnv(t, nil)
+		got, err := parseZFSOpts()
+		if err != nil {
+			t.Fatalf("parseZFSOpts: %v", err)
+		}
+		if got != (install.ZFSOpts{}) {
+			t.Errorf("opts = %+v, want zero", got)
+		}
+	})
+
+	t.Run("all tunables", func(t *testing.T) {
+		setEnv(t, map[string]string{
+			"SPINIFEX_ZFS_ASHIFT":      " 12 ",
+			"SPINIFEX_ZFS_COPIES":      "2",
+			"SPINIFEX_ZFS_ARC_MAX_MIB": "4096",
+			"SPINIFEX_ZFS_COMPRESS":    " lz4 ",
+			"SPINIFEX_ZFS_CHECKSUM":    "blake3",
+		})
+		got, err := parseZFSOpts()
+		if err != nil {
+			t.Fatalf("parseZFSOpts: %v", err)
+		}
+		want := install.ZFSOpts{Ashift: 12, Copies: 2, ARCMaxMiB: 4096, Compress: "lz4", Checksum: "blake3"}
+		if got != want {
+			t.Errorf("opts = %+v, want %+v", got, want)
+		}
+	})
+
+	for _, env := range []string{"SPINIFEX_ZFS_ASHIFT", "SPINIFEX_ZFS_COPIES", "SPINIFEX_ZFS_ARC_MAX_MIB"} {
+		t.Run("unparsable "+env, func(t *testing.T) {
+			setEnv(t, map[string]string{env: "auto"})
+			if _, err := parseZFSOpts(); err == nil || !strings.Contains(err.Error(), env) {
+				t.Fatalf("err = %v, want it to name %s", err, env)
+			}
+		})
+	}
+}
+
+func TestResolveStorage(t *testing.T) {
+	four := []install.Disk{
+		mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 40*gib),
+		mkDisk("/dev/vdd", 40*gib), mkDisk("/dev/vde", 40*gib),
+	}
+
+	tests := []struct {
+		name      string
+		disks     []install.Disk
+		env       map[string]string
+		wantFS    install.FSType
+		wantPaths []string
+		wantErr   string
+	}{
+		{
+			name:      "ext4 single disk by default",
+			disks:     four[:1],
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb"},
+		},
+		{
+			name:      "explicit member list preserves order",
+			disks:     four,
+			env:       map[string]string{"SPINIFEX_FS": "zfs-raid10", "SPINIFEX_DISKS": "vde,/dev/vdd,vdc, vdb "},
+			wantFS:    install.FSZFSRAID10,
+			wantPaths: []string{"/dev/vde", "/dev/vdd", "/dev/vdc", "/dev/vdb"},
+		},
+		{
+			name:    "unknown filesystem",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_FS": "btrfs"},
+			wantErr: "unknown filesystem",
+		},
+		{
+			// Choosing which disks to erase is not an unattended decision.
+			name:    "multi-disk pool without an explicit list",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_FS": "zfs-raid1"},
+			wantErr: "set SPINIFEX_DISKS to an explicit comma-separated list",
+		},
+		{
+			name:    "member that is not present",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_DISKS": "vdz"},
+			wantErr: `"vdz" is not an available disk`,
+		},
+		{
+			name:    "member list that fails validation",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_DISKS": "vdb,vdc"},
+			wantErr: "supports a single disk only",
+		},
+		{
+			name:    "bad zfs tunable",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_ZFS_ASHIFT": "big"},
+			wantErr: "SPINIFEX_ZFS_ASHIFT",
+		},
+		{
+			name:    "scan failure",
+			disks:   nil,
+			env:     map[string]string{"SPINIFEX_DISKS": "vdb"},
+			wantErr: "sysfs unavailable",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var scanErr error
+			if tt.disks == nil {
+				scanErr = errors.New("sysfs unavailable")
+			}
+			withDisks(t, tt.disks, scanErr)
+			setEnv(t, tt.env)
+
+			got, err := resolveStorage()
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveStorage: %v", err)
+			}
+			if got.FS != tt.wantFS {
+				t.Errorf("FS = %v, want %v", got.FS, tt.wantFS)
+			}
+			if !slices.Equal(got.Paths(), tt.wantPaths) {
+				t.Errorf("Paths = %v, want %v", got.Paths(), tt.wantPaths)
+			}
+		})
+	}
+}
+
+func TestResolveDisk(t *testing.T) {
+	small := mkDisk("/dev/vdb", 40*gib)
+	large := mkDisk("/dev/vdc", 400*gib)
+	nvme := mkDisk("/dev/nvme0n1", 200*gib)
+
+	tests := []struct {
+		name     string
+		disks    []install.Disk
+		target   string
+		wantPath string
+		wantErr  string
+	}{
+		{name: "auto with one candidate", disks: []install.Disk{small}, target: "auto", wantPath: "/dev/vdb"},
+		{name: "empty target is auto", disks: []install.Disk{small}, wantPath: "/dev/vdb"},
+		{
+			name:    "auto with several candidates",
+			disks:   []install.Disk{small, large},
+			target:  "auto",
+			wantErr: "expected exactly one disk, found 2",
+		},
+		{name: "largest", disks: []install.Disk{small, large, nvme}, target: "LARGEST", wantPath: "/dev/vdc"},
+		{name: "smallest", disks: []install.Disk{large, small, nvme}, target: " smallest ", wantPath: "/dev/vdb"},
+		{name: "nvme", disks: []install.Disk{small, large, nvme}, target: "nvme", wantPath: "/dev/nvme0n1"},
+		{
+			name:    "nvme with none present",
+			disks:   []install.Disk{small, large},
+			target:  "nvme",
+			wantErr: "expected exactly one NVMe disk, found 0",
+		},
+		{name: "exact path", disks: []install.Disk{small, large}, target: "/dev/vdc", wantPath: "/dev/vdc"},
+		{
+			name:    "path that is not present",
+			disks:   []install.Disk{small},
+			target:  "/dev/vdz",
+			wantErr: "is not an available disk",
+		},
+		{name: "no candidates at all", disks: []install.Disk{}, target: "largest", wantErr: "no non-removable disks found"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, tt.disks, nil)
+
+			got, err := resolveDisk(tt.target)
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveDisk(%q): %v", tt.target, err)
+			}
+			if got.Path != tt.wantPath {
+				t.Errorf("path = %q, want %q", got.Path, tt.wantPath)
+			}
+		})
+	}
+}
+
+func TestCandidateDisksExcludesRemovableAndLiveMedia(t *testing.T) {
+	fixed := mkDisk("/dev/vdb", 40*gib)
+	usb := mkDisk("/dev/sdz", 16*gib)
+	usb.Removable = true
+	boot := mkDisk("/dev/sdy", 16*gib)
+	boot.LiveMedia = true
+	withDisks(t, []install.Disk{usb, fixed, boot}, nil)
+
+	got, err := candidateDisks()
+	if err != nil {
+		t.Fatalf("candidateDisks: %v", err)
+	}
+	if len(got) != 1 || got[0].Path != "/dev/vdb" {
+		t.Fatalf("candidates = %v, want only /dev/vdb", got)
+	}
+}
+
+func TestCandidateDisksPropagatesScanError(t *testing.T) {
+	withDisks(t, nil, errors.New("read /sys/block: denied"))
+	if _, err := candidateDisks(); err == nil {
+		t.Fatal("expected the scan error to propagate")
+	}
+}
+
+func TestPickBySizeRejectsEmptySet(t *testing.T) {
+	if _, err := pickBySize(nil, true); err == nil {
+		t.Fatal("expected an error for an empty disk set")
+	}
+}
+
+func TestPickBySizeLeavesInputOrderIntact(t *testing.T) {
+	disks := []install.Disk{mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 400*gib)}
+	if _, err := pickBySize(disks, false); err != nil {
+		t.Fatalf("pickBySize: %v", err)
+	}
+	if disks[0].Path != "/dev/vdb" {
+		t.Errorf("input reordered: %v", disks)
+	}
+}
+
+func TestDiskList(t *testing.T) {
+	got := diskList([]install.Disk{mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 2048*gib)})
+	want := "  /dev/vdb (40.0G, empty)\n  /dev/vdc (2.0T, empty)"
+	if got != want {
+		t.Errorf("diskList =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestResolveNICExplicitName(t *testing.T) {
+	got, err := resolveNIC("eno1", "")
+	if err != nil {
+		t.Fatalf("resolveNIC: %v", err)
+	}
+	if got != "eno1" {
+		t.Errorf("nic = %q, want eno1", got)
+	}
+}
+
+// The auto path reads the host's real interfaces, so assert the selection rules
+// rather than a specific name.
+func TestResolveNICAutoSkipsVirtualInterfaces(t *testing.T) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		t.Skipf("no interface list on this host: %v", err)
+	}
+
+	got, err := resolveNIC("auto", "")
+	if err != nil {
+		if len(ifaces) > 0 {
+			t.Logf("no physical NIC among %d interfaces: %v", len(ifaces), err)
+		}
+		return
+	}
+
+	idx := slices.IndexFunc(ifaces, func(i net.Interface) bool { return i.Name == got })
+	if idx < 0 {
+		t.Fatalf("resolveNIC returned %q, which is not a host interface", got)
+	}
+	sel := ifaces[idx]
+	if sel.Flags&net.FlagLoopback != 0 || sel.Flags&net.FlagBroadcast == 0 || len(sel.HardwareAddr) == 0 {
+		t.Errorf("selected %q with flags %v and MAC %q", got, sel.Flags, sel.HardwareAddr)
+	}
+	for _, pfx := range virtualNICPrefixes {
+		if strings.HasPrefix(got, pfx) {
+			t.Errorf("selected virtual interface %q (prefix %q)", got, pfx)
+		}
+	}
+}
+
+func TestResolveNICAutoHonoursExclude(t *testing.T) {
+	first, err := resolveNIC("auto", "")
+	if err != nil {
+		t.Skipf("no physical NIC on this host: %v", err)
+	}
+	got, err := resolveNIC("", first)
+	if err == nil && got == first {
+		t.Errorf("resolveNIC returned the excluded interface %q", first)
+	}
+}

cmd/installer/firstboot/firstboot.go

@@ -14,14 +14,23 @@ import (
 // Config holds the values the firstboot service needs to configure the node.
 type Config struct {
 	Hostname string
-	// EncapIP is the Geneve tunnel IP for OVN. Set to the LAN bridge IP when a
-	// dedicated LAN NIC is present, otherwise the WAN bridge IP. Empty when DHCP
-	// is used — setup-ovn.sh auto-detects the IP from the default route in that case.
+	// EncapIP is the Geneve tunnel IP for OVN, taken from the vpc plane after
+	// collapsing (vpc <- lan <- wan). Empty when that plane uses DHCP —
+	// setup-ovn.sh auto-detects the IP from the default route in that case.
 	EncapIP string
-	// ClusterRole is "init" or "join".
-	ClusterRole string
-	// JoinAddr is host:port of the primary node, only used when ClusterRole is "join".
-	JoinAddr string
+	// LANIP is the internal cluster address, taken from the lan plane after
+	// collapsing. Passed as --bind and --cluster-bind so predastore replication,
+	// the NATS mesh and OVN control traffic stay off the public interface.
+	// Empty leaves the 0.0.0.0 wildcard default, which is correct for a
+	// single-NIC node where lan folds onto wan.
+	LANIP string
+	// WANIP is the public address, taken from the wan plane. Passed as
+	// --advertise so northstar's :53 listener, the awsgw registry host and the
+	// dial target recorded for off-host clients stay on the public interface.
+	// Without it a concrete --bind is echoed back as the advertise address,
+	// which would move all three onto the internal plane. Empty when wan uses
+	// DHCP, leaving spx to auto-detect from the default route.
+	WANIP string
 	// Email is the operator email collected by the TUI or SPINIFEX_EMAIL on
 	// the headless path. Passed to `spx admin init --email=<value>` when set;
 	// omitted entirely when empty.
@@ -197,6 +206,9 @@ fi
 
 # Enable services to start, on reboot
 systemctl enable spinifex.target spinifex-banner.service
+# WantedBy=timers.target only self-activates once enabled — without this the
+# JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart.
+systemctl enable --now spinifex-nats-watchdog.timer
 
 # Mark complete only after every step above has succeeded. Until this point,
 # any failure (set -e) leaves the marker absent and the next reboot retries
@@ -244,14 +256,67 @@ func buildClusterCmd(cfg Config) string {
 	if cfg.GPUPassthrough {
 		gpuFlag = " --gpu-passthrough"
 	}
-	switch cfg.ClusterRole {
-	case "join":
-		return fmt.Sprintf("spx admin join --node %s --host %s%s", cfg.Hostname, cfg.JoinAddr, emailFlag)
-	default:
-		return fmt.Sprintf("spx admin init --node %s --nodes 1%s%s", cfg.Hostname, emailFlag, gpuFlag)
+	// Without these the node takes the 0.0.0.0 wildcard default and every
+	// internal service resolves onto the auto-detected WAN address, which is
+	// exactly what the three-plane model exists to prevent.
+	bindFlags := ""
+	if cfg.LANIP != "" {
+		bindFlags = fmt.Sprintf(" --bind %s --cluster-bind %s", cfg.LANIP, cfg.LANIP)
 	}
+	// --advertise must be explicit whenever --bind is: spx echoes a concrete
+	// bind address straight back as the advertise address and never reaches its
+	// WAN auto-detection, which would silently publish the internal plane as
+	// this node's public dial target.
+	preamble := ""
+	switch {
+	case bindFlags == "":
+		// Nothing pinned, so spx auto-detects both and a guessed advertise
+		// address would only get in the way.
+	case cfg.WANIP != "":
+		bindFlags += " --advertise " + cfg.WANIP
+	default:
+		// A DHCP wan has no address at install time, but it does by the time
+		// firstboot runs, so read it off the bridge instead of shipping the
+		// bind address as the public one.
+		preamble = wanAdvertisePreamble
+		bindFlags += ` $SPX_ADVERTISE`
+	}
+
+	// Always a single-node cluster. The installer cannot form a multi-node one:
+	// cluster membership decides the OVN database topology and the join token
+	// only exists once the primary has booted, so neither is knowable while the
+	// nodes are still being installed. Multi-node is a post-install conversion,
+	// documented in the multi-node install guide.
+	cmd := fmt.Sprintf("spx admin init --node %s --nodes 1%s%s%s", cfg.Hostname, bindFlags, emailFlag, gpuFlag)
+	return preamble + cmd
 }
 
+// wanAdvertisePreamble resolves the wan plane's DHCP lease into $SPX_ADVERTISE
+// just before formation. br-wan is always the wan bridge — that plane is the one
+// role that cannot fold — so its address is the node's public identity.
+//
+// Falling through with SPX_ADVERTISE empty is deliberate: no lease means there
+// is no public address to advertise yet, and spx echoing the bind address is
+// still better than formation failing outright.
+const wanAdvertisePreamble = `# The wan plane leases its address, so it is only knowable at boot. Without this
+# spx would echo --bind back as the advertise address and publish the internal
+# plane as this node's public dial target.
+SPX_ADVERTISE=""
+echo "[firstboot] waiting for the wan plane to acquire an address..."
+for _i in $(seq 1 60); do
+    _wan_ip=$(ip -4 -o addr show br-wan scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1)
+    if [ -n "$_wan_ip" ]; then
+        SPX_ADVERTISE=" --advertise $_wan_ip"
+        echo "[firstboot] wan address $_wan_ip (${_i}s)"
+        break
+    fi
+    sleep 1
+done
+if [ -z "$SPX_ADVERTISE" ]; then
+    echo "[firstboot] warning: br-wan has no address after 60s — this node will advertise its bind address"
+fi
+`
+
 // shellEscapeSingle wraps s in single quotes with any embedded single
 // quotes escaped. Minimal — we only need this because the email value is
 // interpolated into a shell script written by Write().

cmd/installer/firstboot/firstboot_test.go

@@ -2,6 +2,7 @@ package firstboot
 
 import (
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -24,7 +25,7 @@ func TestWriteScriptNoCallbackWhenEmpty(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	cfg := Config{Hostname: "test-node"}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -45,7 +46,6 @@ func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
 	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	cfg := Config{
 		Hostname:        "test-node",
-		ClusterRole:     "init",
 		InstallCallback: callbackURL,
 	}
 	if err := Write(root, cfg); err != nil {
@@ -70,7 +70,7 @@ func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	cfg := Config{Hostname: "test-node"}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -88,7 +88,7 @@ func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
+	cfg := Config{Hostname: "test-node", SkipFormation: true}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -114,6 +114,25 @@ func readScript(t *testing.T, root string) string {
 	return string(b)
 }
 
+// TestWriteScriptEnablesNatsWatchdogTimer pins that firstboot activates the
+// JetStream ENOSPC-latch watchdog timer. The unit file's WantedBy=timers.target
+// only takes effect once enabled — without this line the watchdog is dropped
+// onto disk but never runs, and a full disk needs a manual nats restart forever.
+func TestWriteScriptEnablesNatsWatchdogTimer(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node"}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if !strings.Contains(content, "systemctl enable --now spinifex-nats-watchdog.timer") {
+		t.Error("firstboot script must enable --now spinifex-nats-watchdog.timer")
+	}
+}
+
 func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
@@ -121,7 +140,6 @@ func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	cfg := Config{
 		Hostname:        "node1",
-		ClusterRole:     "init",
 		InstallCallback: callbackURL,
 	}
 	if err := Write(root, cfg); err != nil {
@@ -146,3 +164,121 @@ func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 		t.Error("curl must appear after done marker write")
 	}
 }
+
+// A multi-NIC node must keep cluster traffic on the internal plane while still
+// publishing the public one. --advertise has to be explicit to do that: spx
+// returns a concrete --bind verbatim as the advertise address and never reaches
+// its WAN auto-detection, so omitting it moves northstar's :53 listener and the
+// off-host dial target onto the internal plane.
+func TestBuildClusterCmdSeparatesBindFromAdvertise(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "hydrogen",
+		LANIP:    "10.0.0.3",
+		WANIP:    "216.218.163.99",
+	})
+	for _, want := range []string{
+		"--bind 10.0.0.3",
+		"--cluster-bind 10.0.0.3",
+		"--advertise 216.218.163.99",
+	} {
+		if !strings.Contains(cmd, want) {
+			t.Errorf("missing %q in:\n%s", want, cmd)
+		}
+	}
+}
+
+// Without a bind address there is nothing to correct: spx auto-detects both,
+// and a stray --advertise would pin the node to an address the installer only
+// guessed at. A single-NIC node also collapses wan and lan onto one address,
+// where the split is meaningless.
+func TestBuildClusterCmdOmitsAdvertiseWithoutBind(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "node1",
+		WANIP:    "216.218.163.99",
+	})
+	if strings.Contains(cmd, "--advertise") {
+		t.Errorf("advertise must not be passed without a bind address:\n%s", cmd)
+	}
+}
+
+// A DHCP wan has no address at install time, so the advertise value has to be
+// resolved at boot. Shipping --bind alone here would republish the internal
+// plane as the node's public dial target, which is the case the static path
+// already guards against.
+func TestBuildClusterCmdResolvesDHCPWanAtBoot(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "hydrogen",
+		LANIP:    "10.0.0.3",
+		// WANIP empty — the wan plane is on DHCP.
+	})
+	if !strings.Contains(cmd, "ip -4 -o addr show br-wan") {
+		t.Errorf("must read the wan address off the bridge:\n%s", cmd)
+	}
+	if !strings.Contains(cmd, "$SPX_ADVERTISE") {
+		t.Errorf("resolved address must reach the command:\n%s", cmd)
+	}
+	// The preamble has to run before the command that consumes it.
+	if strings.Index(cmd, "SPX_ADVERTISE=\"\"") > strings.Index(cmd, "spx admin init") {
+		t.Errorf("preamble must precede the spx invocation:\n%s", cmd)
+	}
+	if !strings.Contains(cmd, "--bind 10.0.0.3") {
+		t.Errorf("bind still comes from the lan plane:\n%s", cmd)
+	}
+}
+
+// With no bind address there is nothing to correct, so neither the literal flag
+// nor the boot-time lookup should appear — spx auto-detects both ends.
+func TestBuildClusterCmdSkipsWanLookupWithoutBind(t *testing.T) {
+	cmd := buildClusterCmd(Config{Hostname: "node1"})
+	if strings.Contains(cmd, "SPX_ADVERTISE") || strings.Contains(cmd, "--advertise") {
+		t.Errorf("no bind means no advertise handling at all:\n%s", cmd)
+	}
+}
+
+// The advertise handling injects a shell preamble into the generated script, so
+// the permutations that change its shape are syntax-checked rather than only
+// string-matched — a broken quote here bricks first boot with no installer error.
+func TestGeneratedScriptIsValidShell(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		cfg  Config
+	}{
+		{"dhcp wan resolves at boot", Config{Hostname: "hydrogen", LANIP: "10.0.0.3"}},
+		{"static wan is inlined", Config{Hostname: "hydrogen", LANIP: "10.0.0.3", WANIP: "216.218.163.99"}},
+		{"single nic, nothing pinned", Config{Hostname: "node1"}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			root := t.TempDir()
+			makeRootDirs(t, root)
+			if err := Write(root, tc.cfg); err != nil {
+				t.Fatalf("Write: %v", err)
+			}
+			path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
+			if out, err := exec.Command("bash", "-n", path).CombinedOutput(); err != nil {
+				script, _ := os.ReadFile(path)
+				t.Fatalf("generated script is not valid shell: %v\n%s\n---\n%s", err, out, script)
+			}
+		})
+	}
+}
+
+// The installer cannot form a multi-node cluster: membership decides the OVN
+// database topology, and the join token only exists once the primary has
+// booted. Neither is knowable at install time, so firstboot must only ever
+// initialize a single-node cluster and leave multi-node to a post-install
+// conversion.
+func TestBuildClusterCmdNeverJoins(t *testing.T) {
+	for _, cfg := range []Config{
+		{Hostname: "node1"},
+		{Hostname: "node1", LANIP: "10.0.0.3", WANIP: "216.218.163.99"},
+		{Hostname: "node1", LANIP: "10.0.0.3"},
+	} {
+		cmd := buildClusterCmd(cfg)
+		if strings.Contains(cmd, "spx admin join") {
+			t.Errorf("firstboot must never join a cluster:\n%s", cmd)
+		}
+		if !strings.Contains(cmd, "--nodes 1") {
+			t.Errorf("want a single-node init, got:\n%s", cmd)
+		}
+	}
+}

cmd/installer/install/assets/grub-install-wrapper

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Diversion wrapper for grub-install on a spinifex-boot-tool system.
+#
+# On ZFS root the bootloader lives on per-disk ESPs, not in /boot. A plain
+# grub-install writes a core image that tries to read the pool, which works
+# until the next OpenZFS feature flag lands and then silently does not. Worse,
+# the grub-pc and shim packages run grub-install from their own postinst, so an
+# ordinary `apt upgrade` would undo the layout with no visible error.
+#
+# Mirrors the wrapper Proxmox ships for the same reason.
+
+set -e
+
+REAL="/usr/sbin/grub-install.real"
+TOOL="/usr/sbin/spinifex-boot-tool"
+
+# Not a managed system: behave exactly as the real grub-install.
+if ! "$TOOL" status --quiet 2>/dev/null; then
+	exec "$REAL" "$@"
+fi
+
+# dpkg sets this when a maintainer script is running. The bootloader packages
+# reinstalling themselves is expected and is redirected, not refused.
+case "${DPKG_MAINTSCRIPT_PACKAGE:-}" in
+grub-* | shim-*)
+	echo "This system is managed by spinifex-boot-tool; redirecting grub-install to 'spinifex-boot-tool reinit'." >&2
+	exec "$TOOL" reinit
+	;;
+esac
+
+cat >&2 <<EOF
+E: This system's bootloader is managed by spinifex-boot-tool.
+
+Running grub-install directly would install GRUB somewhere the firmware does
+not read, and would leave the per-disk ESPs holding a stale kernel.
+
+Use instead:
+  spinifex-boot-tool refresh     re-sync kernels and grub.cfg to every ESP
+  spinifex-boot-tool reinit      reinstall GRUB on every ESP
+  spinifex-boot-tool status      list the ESPs under management
+
+To run the real grub-install anyway (you probably do not want this):
+  $REAL $*
+EOF
+exit 1

cmd/installer/install/assets/spinifex-boot-tool

@@ -0,0 +1,412 @@
+#!/bin/bash
+# spinifex-boot-tool — keep a bootloader on every disk of a ZFS root pool.
+#
+# GRUB cannot be trusted to read a ZFS pool: its ZFS reader lags OpenZFS by
+# years, and enabling a new feature flag on the pool can make the machine
+# unbootable with no warning. So /boot on the pool is never read by firmware.
+# Instead each pool member carries a small FAT32 ESP holding its own copy of
+# GRUB, the kernels and the initramfs images, and this tool keeps them in step.
+#
+# Modelled directly on Proxmox's proxmox-boot-tool, which has run this design
+# across their entire installed base for years.
+
+set -u
+set -o pipefail
+
+ESP_LIST="/etc/kernel/spinifex-boot-uuids"
+ESPMOUNTS="/var/tmp/espmounts"
+MARKER="spinifex-boot-uuids"
+
+# Kernels are only ever copied out of the real /boot, which lives on the pool.
+# Once an ESP is bind-mounted over /boot this variable is how we still find it.
+BOOTROOT="/boot"
+
+warn() { echo "W: $*" >&2; }
+die() { echo "E: $*" >&2; exit 1; }
+
+usage() {
+	cat <<EOF
+Usage: $(basename "$0") <command>
+
+  format <partition> [--force]  make <partition> a Spinifex ESP
+  init <partition>              register <partition> and install a bootloader on it
+  clean [--dry-run]             drop ESPs that are no longer present
+  refresh [--hook <name>]       re-sync kernels and grub.cfg to every ESP
+  reinit                        re-run refresh, reinstalling GRUB on every ESP
+  kernel <add|remove> <ver>     pin or unpin a kernel version
+  status [--quiet]              list registered ESPs
+EOF
+}
+
+# reexec_in_mountns re-runs this script inside a private mount namespace. Every
+# code path here bind-mounts an ESP over /boot; if the script is killed at the
+# wrong moment outside a namespace, the running system is left with its real
+# /boot masked and the next kernel upgrade writes into a FAT partition.
+reexec_in_mountns() {
+	if [ "${SPINIFEX_BOOT_TOOL_NS:-0}" = "1" ]; then
+		return 0
+	fi
+	export SPINIFEX_BOOT_TOOL_NS=1
+	# --propagation private stops the bind mounts leaking back to the host.
+	exec unshare --mount --propagation private -- "$0" "$@"
+}
+
+# --- ESP registry -----------------------------------------------------------
+
+# esp_list prints the registered ESP filesystem UUIDs.
+esp_list() {
+	[ -f "$ESP_LIST" ] || return 0
+	grep -v '^[[:space:]]*\(#\|$\)' "$ESP_LIST" || true
+}
+
+# uuid_of prints the FAT filesystem UUID of a partition.
+uuid_of() {
+	blkid "$1" -o value -s UUID 2>/dev/null
+}
+
+# path_for_uuid resolves a registered UUID back to a device, or nothing if the
+# disk has been pulled.
+path_for_uuid() {
+	local dev
+	dev="/dev/disk/by-uuid/$1"
+	[ -e "$dev" ] || return 1
+	readlink -f "$dev"
+}
+
+# mount_esp mounts the ESP for a UUID and echoes its mountpoint.
+mount_esp() {
+	local uuid="$1" dev mp
+	dev=$(path_for_uuid "$uuid") || return 1
+	mp="$ESPMOUNTS/$uuid"
+	mkdir -p "$mp"
+	mountpoint -q "$mp" || mount "$dev" "$mp" || return 1
+	echo "$mp"
+}
+
+umount_esp() {
+	local mp="$ESPMOUNTS/$1"
+	mountpoint -q "$mp" && umount "$mp"
+	rmdir "$mp" 2>/dev/null || true
+}
+
+# --- format / init ----------------------------------------------------------
+
+do_format() {
+	local part="$1" force="${2:-}"
+	[ -b "$part" ] || die "$part is not a block device"
+
+	if [ "$force" != "--force" ]; then
+		local existing
+		existing=$(blkid -o value -s TYPE "$part" 2>/dev/null || true)
+		[ -z "$existing" ] || die "$part already contains a $existing filesystem; pass --force to overwrite"
+	fi
+	mkfs.vfat -F 32 "$part" || die "mkfs.vfat failed on $part"
+}
+
+do_init() {
+	local part="$1"
+	[ -b "$part" ] || die "$part is not a block device"
+
+	local uuid
+	uuid=$(uuid_of "$part") || true
+	[ -n "$uuid" ] || die "$part has no filesystem UUID — run '$(basename "$0") format $part' first"
+
+	mkdir -p "$(dirname "$ESP_LIST")"
+	touch "$ESP_LIST"
+	if ! grep -qx "$uuid" "$ESP_LIST"; then
+		echo "$uuid" >>"$ESP_LIST"
+	fi
+	echo "Registered $part ($uuid)"
+
+	install_hooks
+	# Only the ESP just registered: the others are already current, and
+	# refreshing all of them here makes registering N disks an N-squared
+	# sequence of grub-installs.
+	refresh_one "$uuid" || die "could not make $part bootable"
+}
+
+do_clean() {
+	local dry="${1:-}" uuid keep=()
+	for uuid in $(esp_list); do
+		if path_for_uuid "$uuid" >/dev/null 2>&1; then
+			keep+=("$uuid")
+		else
+			echo "Removing stale ESP $uuid"
+		fi
+	done
+	if [ "$dry" = "--dry-run" ]; then
+		return 0
+	fi
+	printf '%s\n' "${keep[@]+"${keep[@]}"}" >"$ESP_LIST"
+}
+
+# --- refresh ----------------------------------------------------------------
+
+# kernel_versions lists the kernels present in the real /boot, newest first, so
+# the default menu entry is the newest kernel.
+kernel_versions() {
+	local img
+	for img in "$BOOTROOT"/vmlinuz-*; do
+		[ -e "$img" ] || continue
+		basename "$img" | sed 's/^vmlinuz-//'
+	done | sort -Vr
+}
+
+# sync_esp copies the current kernels and initramfs images onto one ESP and
+# removes anything left from a kernel that is no longer installed — an ESP is
+# small and a stale 100MB initramfs is what makes the next upgrade fail.
+sync_esp() {
+	local mp="$1" kver f base
+
+	local -A wanted=()
+	for kver in $(kernel_versions); do
+		wanted["vmlinuz-$kver"]=1
+		wanted["initrd.img-$kver"]=1
+	done
+
+	for f in "$mp"/vmlinuz-* "$mp"/initrd.img-*; do
+		[ -e "$f" ] || continue
+		base=$(basename "$f")
+		[ -n "${wanted[$base]:-}" ] || rm -f "$f"
+	done
+
+	for kver in $(kernel_versions); do
+		for base in "vmlinuz-$kver" "initrd.img-$kver"; do
+			[ -e "$BOOTROOT/$base" ] || continue
+			# Copy to a temp name and rename: a half-written kernel on the ESP
+			# is a machine that does not come back from a reboot.
+			cp -f "$BOOTROOT/$base" "$mp/$base.tmp" || return 1
+			mv -f "$mp/$base.tmp" "$mp/$base" || return 1
+		done
+	done
+
+	# GRUB reads its theme from $prefix, which is the ESP's own grub directory,
+	# so the branding has to be copied out of /boot alongside the kernels.
+	mkdir -p "$mp/grub/fonts"
+	for f in grub/theme.txt grub/splash.png grub/fonts/unicode.pf2; do
+		[ -e "$BOOTROOT/$f" ] && cp -f "$BOOTROOT/$f" "$mp/$f"
+	done
+
+	# Marks the partition as ours, so a later `format` on the wrong disk is
+	# obviously a mistake rather than a silent overwrite.
+	echo "$MARKER" >"$mp/$MARKER"
+	return 0
+}
+
+# grub_install_bin echoes the real grub-install. On a managed system the normal
+# name is a wrapper that refuses to run, and it must not intercept this tool's
+# own calls — that would make the refusal a self-deadlock.
+grub_install_bin() {
+	if [ -x /usr/sbin/grub-install.real ]; then
+		echo /usr/sbin/grub-install.real
+	else
+		echo grub-install
+	fi
+}
+
+# install_grub puts GRUB itself on one ESP, for both firmware types.
+#
+# --boot-directory points at the ESP, not at /boot, so the GRUB core image
+# embeds a FAT reader and a search for the ESP's own UUID. It never needs to
+# read the pool, which is the entire point of this tool.
+install_grub() {
+	local mp="$1" dev="$2" rc=0 grub
+	grub="$(grub_install_bin)"
+
+	if [ ! -d /sys/firmware/efi ]; then
+		"$grub" --target=i386-pc --boot-directory="$mp" --recheck "$dev" || rc=1
+		return $rc
+	fi
+
+	# NVRAM is written at install time only. A refresh runs on every kernel
+	# upgrade, and rewriting boot variables that often has bricked firmware on
+	# more than one vendor's boards.
+	local nvram="--no-nvram"
+	[ "${SPINIFEX_BOOT_TOOL_NVRAM:-0}" = "1" ] && nvram=""
+
+	# Twice: once as a named entry, and once at the removable path, which is all
+	# some firmware will ever look at and is what keeps the disk bootable after
+	# its NVRAM entry is lost or the drive is moved to another machine.
+	# shellcheck disable=SC2086 # $nvram is a single optional flag, not a path
+	"$grub" --target=x86_64-efi \
+		--efi-directory="$mp" --boot-directory="$mp" \
+		--bootloader-id=spinifex $nvram --recheck "$dev" || rc=1
+	"$grub" --target=x86_64-efi \
+		--efi-directory="$mp" --boot-directory="$mp" \
+		--bootloader-id=spinifex --removable --no-nvram --recheck "$dev" || rc=1
+	return $rc
+}
+
+# update_grub_on regenerates grub.cfg for one ESP.
+#
+# The bind mount is the trick that makes this work: grub-mkconfig probes the
+# device behind /boot to decide how to find the kernel at boot. With the ESP
+# mounted there it emits a FAT UUID search and paths relative to the ESP root,
+# so the generated config matches what the embedded core image can actually do.
+update_grub_on() {
+	local mp="$1" rc=0
+	mount --bind "$mp" "$BOOTROOT" || return 1
+	grub-mkconfig -o "$BOOTROOT/grub/grub.cfg" || rc=1
+	umount "$BOOTROOT" || warn "could not unmount the bind mount over $BOOTROOT"
+	return $rc
+}
+
+# refresh_one makes a single registered ESP independently bootable: kernels,
+# bootloader, and a grub.cfg generated against that ESP's own UUID.
+refresh_one() {
+	local uuid="$1" mp dev disk rc=0
+
+	if ! dev=$(path_for_uuid "$uuid"); then
+		warn "ESP $uuid is not present, skipping"
+		return 1
+	fi
+	if ! mp=$(mount_esp "$uuid"); then
+		warn "could not mount ESP $uuid"
+		return 1
+	fi
+
+	# The whole disk, not the partition: BIOS GRUB embeds into the boot
+	# sector and the BIOS boot partition, both of which live on the disk.
+	disk=$(lsblk -no pkname "$dev")
+	disk="/dev/$disk"
+
+	if sync_esp "$mp" && install_grub "$mp" "$disk" && update_grub_on "$mp"; then
+		echo "Synced $dev ($uuid)"
+	else
+		warn "failed to update ESP $uuid on $dev"
+		rc=1
+	fi
+	umount_esp "$uuid"
+	return $rc
+}
+
+do_refresh() {
+	local uuid failed=0 ok=0 total=0
+
+	local uuids
+	uuids=$(esp_list)
+	if [ -z "$uuids" ]; then
+		echo "No ESPs registered, nothing to do."
+		return 0
+	fi
+
+	for uuid in $uuids; do
+		total=$((total + 1))
+		if refresh_one "$uuid"; then
+			ok=$((ok + 1))
+		else
+			failed=$((failed + 1))
+		fi
+	done
+
+	if [ "$ok" -eq 0 ]; then
+		die "no ESP could be updated ($failed of $total failed) — this system may not boot"
+	fi
+	if [ "$failed" -gt 0 ]; then
+		# Strict mode is used by the installer, where a partly-updated set of
+		# ESPs means the install is broken and should fail loudly now rather
+		# than at the first reboot.
+		if [ "${SPINIFEX_BOOT_TOOL_STRICT:-0}" = "1" ]; then
+			die "$failed of $total ESPs failed to update"
+		fi
+		warn "$failed of $total ESPs failed to update; $ok succeeded, so the system will still boot"
+	fi
+	return 0
+}
+
+do_status() {
+	local quiet="${1:-}" uuid dev
+	if [ ! -s "$ESP_LIST" ]; then
+		[ "$quiet" = "--quiet" ] || echo "No ESPs registered — this system does not use spinifex-boot-tool."
+		return 1
+	fi
+	[ "$quiet" = "--quiet" ] && return 0
+	for uuid in $(esp_list); do
+		if dev=$(path_for_uuid "$uuid"); then
+			echo "$uuid  $dev"
+		else
+			echo "$uuid  MISSING"
+		fi
+	done
+}
+
+do_kernel() {
+	local action="$1" kver="${2:-}"
+	local pinfile="/etc/kernel/spinifex-boot-pin"
+	case "$action" in
+	add)
+		[ -n "$kver" ] || die "kernel add needs a version"
+		touch "$pinfile"
+		grep -qx "$kver" "$pinfile" || echo "$kver" >>"$pinfile"
+		;;
+	remove)
+		[ -n "$kver" ] || die "kernel remove needs a version"
+		[ -f "$pinfile" ] && sed -i "\|^$kver\$|d" "$pinfile"
+		;;
+	*) die "unknown kernel action: $action" ;;
+	esac
+	do_refresh
+}
+
+# --- hooks ------------------------------------------------------------------
+
+# install_hooks wires this tool into every package operation that would
+# otherwise write a bootloader to the wrong place.
+install_hooks() {
+	mkdir -p /etc/kernel/postinst.d /etc/kernel/postrm.d /etc/initramfs/post-update.d
+
+	# Debian's own hook runs update-grub against the real /boot, which is on the
+	# pool. Firmware never reads that, so it would produce a config nothing uses
+	# while the ESPs silently kept an old kernel.
+	if [ -e /etc/kernel/postinst.d/zz-update-grub ]; then
+		dpkg-divert --divert /etc/kernel/postinst.d/zz-update-grub.distrib \
+			--rename --add /etc/kernel/postinst.d/zz-update-grub >/dev/null 2>&1 || true
+	fi
+
+	for hook in /etc/kernel/postinst.d/zz-spinifex-boot-tool \
+		/etc/kernel/postrm.d/zz-spinifex-boot-tool \
+		/etc/initramfs/post-update.d/spinifex-boot-tool; do
+		cat >"$hook" <<'HOOK'
+#!/bin/sh
+# Re-sync the ESPs after a kernel or initramfs change.
+set -e
+exec /usr/sbin/spinifex-boot-tool refresh --hook "$(basename "$0")"
+HOOK
+		chmod 0755 "$hook"
+	done
+}
+
+# --- dispatch ---------------------------------------------------------------
+
+cmd="${1:-}"
+[ $# -gt 0 ] && shift
+
+case "$cmd" in
+format) do_format "$@" ;;
+init)
+	reexec_in_mountns init "$@"
+	do_init "$@"
+	;;
+refresh)
+	# --hook <name> is passed by the kernel hooks purely so the reason for a
+	# refresh is visible in the log.
+	if [ "${1:-}" = "--hook" ]; then shift 2 || shift; fi
+	reexec_in_mountns refresh
+	do_refresh
+	;;
+reinit)
+	reexec_in_mountns reinit
+	do_refresh
+	;;
+clean) do_clean "$@" ;;
+kernel)
+	reexec_in_mountns kernel "$@"
+	do_kernel "$@"
+	;;
+status) do_status "$@" ;;
+help | --help | -h | "") usage ;;
+*)
+	usage
+	exit 1
+	;;
+esac

cmd/installer/install/blockdev.go

@@ -0,0 +1,267 @@
+package install
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// Disk is one candidate block device with every attribute the installer needs
+// to validate a selection: sizes for the same-size rule, block sizes for ashift
+// and the 4Kn/BIOS check, rotational for autotrim, and ByID for pool members.
+type Disk struct {
+	Path  string // /dev/sda
+	ByID  string // /dev/disk/by-id/ata-… — stable across controller reordering
+	Bytes int64
+	Model string
+
+	LogicalBlockSize  int
+	PhysicalBlockSize int
+
+	Rotational bool
+	Removable  bool
+
+	// LiveMedia marks the device the installer itself booted from. Selecting it
+	// would erase the running system mid-install.
+	LiveMedia bool
+
+	// Content is a human-readable summary of what is on the disk today
+	// ("ext4 filesystem", "empty"), shown before the operator confirms erasure.
+	Content string
+}
+
+// skipPrefixes are kernel block devices that are never install targets:
+// virtual, mapped, or removable media. Mirrors Proxmox's hd_list filter.
+var skipPrefixes = []string{"ram", "loop", "md", "dm-", "fd", "sr", "zram", "nbd"}
+
+// The kernel-owned trees the scan reads, indirected so tests can point it at a
+// fixture instead of the running machine's hardware.
+var (
+	sysBlockDir      = "/sys/block"
+	sysClassBlockDir = "/sys/class/block"
+	devByIDDir       = "/dev/disk/by-id"
+	procMountsPath   = "/proc/mounts"
+)
+
+// ListDisks enumerates install-candidate disks from /sys/block. It reads sysfs
+// and blkid only — pool detection is separate (ImportablePools) because it is
+// far slower and not every caller needs it.
+func ListDisks() ([]Disk, error) {
+	entries, err := os.ReadDir(sysBlockDir)
+	if err != nil {
+		return nil, fmt.Errorf("read /sys/block: %w", err)
+	}
+
+	byID := buildByIDIndex()
+	live := liveMediaDevices()
+
+	var disks []Disk
+	for _, e := range entries {
+		name := e.Name()
+		if hasAnyPrefix(name, skipPrefixes) {
+			continue
+		}
+		// A zero-sized device is an empty card reader slot, not a disk.
+		sectors := sysfsInt(name, "size")
+		if sectors == 0 {
+			continue
+		}
+		d := Disk{
+			Path:              "/dev/" + name,
+			ByID:              byID[name],
+			Bytes:             sectors * 512,
+			Model:             sysfsString(name, "device/model"),
+			LogicalBlockSize:  sysfsIntDefault(name, "queue/logical_block_size", 512),
+			PhysicalBlockSize: sysfsIntDefault(name, "queue/physical_block_size", 512),
+			Rotational:        sysfsInt(name, "queue/rotational") == 1,
+			Removable:         sysfsInt(name, "removable") == 1,
+			LiveMedia:         live[name],
+		}
+		d.Content = describeContent(d.Path)
+		disks = append(disks, d)
+	}
+	slices.SortFunc(disks, func(a, b Disk) int { return strings.Compare(a.Path, b.Path) })
+	return disks, nil
+}
+
+// SizeHuman renders the disk size for display, e.g. "1.7T".
+func (d Disk) SizeHuman() string {
+	switch {
+	case d.Bytes >= 1<<40:
+		return fmt.Sprintf("%.1fT", float64(d.Bytes)/(1<<40))
+	case d.Bytes >= 1<<30:
+		return fmt.Sprintf("%.1fG", float64(d.Bytes)/(1<<30))
+	case d.Bytes >= 1<<20:
+		return fmt.Sprintf("%.1fM", float64(d.Bytes)/(1<<20))
+	default:
+		return fmt.Sprintf("%dB", d.Bytes)
+	}
+}
+
+// Stable returns the path to use when building a pool: the by-id path when one
+// exists, else the kernel name. Pools referencing /dev/sdX break when a
+// controller enumerates disks in a different order after a reboot.
+func (d Disk) Stable() string {
+	if d.ByID != "" {
+		return d.ByID
+	}
+	return d.Path
+}
+
+// PartitionPath returns the device path for partition n of this disk. NVMe and
+// other devices whose name ends in a digit take a 'p' separator.
+func (d Disk) PartitionPath(n int) string { return partitionPath(d.Path, n) }
+
+// StablePartitionPath is PartitionPath against the by-id path, which udev
+// publishes with a "-partN" suffix rather than the kernel's 'p' convention.
+func (d Disk) StablePartitionPath(n int) string {
+	if d.ByID == "" {
+		return d.PartitionPath(n)
+	}
+	return fmt.Sprintf("%s-part%d", d.ByID, n)
+}
+
+func partitionPath(disk string, n int) string {
+	if len(disk) > 0 && disk[len(disk)-1] >= '0' && disk[len(disk)-1] <= '9' {
+		return fmt.Sprintf("%sp%d", disk, n)
+	}
+	return fmt.Sprintf("%s%d", disk, n)
+}
+
+// byIDPreference orders the /dev/disk/by-id/ namespaces we prefer to name a
+// pool member by. wwn- ids are last: they are stable but opaque, so a degraded
+// pool reports a number nobody can map to a drive bay.
+var byIDPreference = []string{"nvme-", "ata-", "scsi-", "virtio-", "usb-", "wwn-"}
+
+// buildByIDIndex maps kernel device name to its preferred by-id path.
+func buildByIDIndex() map[string]string {
+	entries, err := os.ReadDir(devByIDDir)
+	if err != nil {
+		return map[string]string{}
+	}
+	// Track the chosen preference rank per device so a later, less-preferred
+	// id cannot displace a better one already found.
+	rank := map[string]int{}
+	out := map[string]string{}
+	for _, e := range entries {
+		id := e.Name()
+		// Partition links describe a slice of the disk, not the disk.
+		if strings.Contains(id, "-part") {
+			continue
+		}
+		target, err := filepath.EvalSymlinks(filepath.Join(devByIDDir, id))
+		if err != nil {
+			continue
+		}
+		dev := filepath.Base(target)
+		r := prefixRank(id)
+		if existing, ok := rank[dev]; ok && existing <= r {
+			continue
+		}
+		rank[dev] = r
+		out[dev] = filepath.Join(devByIDDir, id)
+	}
+	return out
+}
+
+func prefixRank(id string) int {
+	for i, p := range byIDPreference {
+		if strings.HasPrefix(id, p) {
+			return i
+		}
+	}
+	return len(byIDPreference)
+}
+
+// liveMediaDevices returns the set of parent disk names backing the live
+// installer's own media, so they can never be selected as install targets.
+func liveMediaDevices() map[string]bool {
+	out := map[string]bool{}
+	data, err := os.ReadFile(procMountsPath)
+	if err != nil {
+		return out
+	}
+	liveMounts := []string{"/cdrom", "/run/live/medium", "/lib/live/mount/medium"}
+	for line := range strings.SplitSeq(string(data), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) < 2 || !strings.HasPrefix(fields[0], "/dev/") {
+			continue
+		}
+		if !slices.Contains(liveMounts, fields[1]) {
+			continue
+		}
+		if parent := parentDisk(filepath.Base(fields[0])); parent != "" {
+			out[parent] = true
+		}
+	}
+	return out
+}
+
+// parentDisk resolves a partition's kernel name to its whole-disk name by
+// walking /sys/class/block/<part>/.. — the sysfs parent of a partition is its
+// disk. Returns the input unchanged when it is already a whole disk.
+func parentDisk(dev string) string {
+	link, err := filepath.EvalSymlinks(filepath.Join(sysClassBlockDir, dev))
+	if err != nil {
+		return dev
+	}
+	// A whole disk lives directly under a bus path; a partition sits one level
+	// deeper, inside its disk's directory.
+	if _, err := os.Stat(filepath.Join(link, "partition")); err != nil {
+		return dev
+	}
+	return filepath.Base(filepath.Dir(link))
+}
+
+// describeContent summarises what is currently on a disk for the confirmation
+// screen. Best-effort: an unreadable disk is reported as unknown, not empty,
+// because "empty" invites an operator to erase something that is not.
+func describeContent(dev string) string {
+	out, err := exec.Command("blkid", "-p", "-o", "value", "-s", "TYPE", dev).Output()
+	if err == nil {
+		if t := strings.TrimSpace(string(out)); t != "" {
+			return t + " filesystem"
+		}
+	}
+	// No filesystem on the whole device: report the partition table instead,
+	// since that is what tells the operator the disk is in use.
+	out, err = exec.Command("blkid", "-p", "-o", "value", "-s", "PTTYPE", dev).Output()
+	if err != nil {
+		return "unknown"
+	}
+	if t := strings.TrimSpace(string(out)); t != "" {
+		return t + " partition table"
+	}
+	return "empty"
+}
+
+func sysfsString(dev, file string) string {
+	data, err := os.ReadFile(filepath.Join(sysBlockDir, dev, file))
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(data))
+}
+
+func sysfsInt(dev, file string) int64 {
+	n, err := strconv.ParseInt(sysfsString(dev, file), 10, 64)
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+func sysfsIntDefault(dev, file string, def int) int {
+	if n := sysfsInt(dev, file); n > 0 {
+		return int(n)
+	}
+	return def
+}
+
+func hasAnyPrefix(s string, prefixes []string) bool {
+	return slices.ContainsFunc(prefixes, func(p string) bool { return strings.HasPrefix(s, p) })
+}

cmd/installer/install/blockdev_test.go

@@ -0,0 +1,354 @@
+package install
+
+import (
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"testing"
+)
+
+// fakeDisk describes one device to lay down in the fixture sysfs tree.
+type fakeDisk struct {
+	name       string
+	sectors    int64
+	model      string
+	logical    int
+	physical   int
+	rotational bool
+	removable  bool
+	partitions []string
+}
+
+// fakeSysfs builds a /sys/block, /sys/class/block, /dev/disk/by-id and
+// /proc/mounts fixture and points the scan at it for the duration of the test.
+type fakeSysfs struct {
+	root string
+	t    *testing.T
+}
+
+func newFakeSysfs(t *testing.T, disks ...fakeDisk) *fakeSysfs {
+	t.Helper()
+	f := &fakeSysfs{root: t.TempDir(), t: t}
+
+	swap := func(dst *string, v string) {
+		prev := *dst
+		*dst = v
+		t.Cleanup(func() { *dst = prev })
+	}
+	swap(&sysBlockDir, filepath.Join(f.root, "sys/block"))
+	swap(&sysClassBlockDir, filepath.Join(f.root, "sys/class/block"))
+	swap(&devByIDDir, filepath.Join(f.root, "dev/disk/by-id"))
+	swap(&procMountsPath, filepath.Join(f.root, "proc/mounts"))
+
+	f.mkdirAll(sysBlockDir)
+	f.mkdirAll(sysClassBlockDir)
+	for _, d := range disks {
+		f.addDisk(d)
+	}
+	return f
+}
+
+func (f *fakeSysfs) mkdirAll(dir string) {
+	f.t.Helper()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		f.t.Fatalf("mkdir %s: %v", dir, err)
+	}
+}
+
+func (f *fakeSysfs) write(path, content string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Dir(path))
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		f.t.Fatalf("write %s: %v", path, err)
+	}
+}
+
+// addDisk lays out one device the way the kernel does: the attribute files
+// under /sys/block/<dev>, and a /sys/class/block symlink per disk and
+// partition, since parentDisk walks the latter.
+func (f *fakeSysfs) addDisk(d fakeDisk) {
+	f.t.Helper()
+	dir := filepath.Join(sysBlockDir, d.name)
+	f.write(filepath.Join(dir, "size"), strconv.FormatInt(d.sectors, 10)+"\n")
+	if d.model != "" {
+		f.write(filepath.Join(dir, "device/model"), d.model+"\n")
+	}
+	if d.logical > 0 {
+		f.write(filepath.Join(dir, "queue/logical_block_size"), strconv.Itoa(d.logical)+"\n")
+	}
+	if d.physical > 0 {
+		f.write(filepath.Join(dir, "queue/physical_block_size"), strconv.Itoa(d.physical)+"\n")
+	}
+	f.write(filepath.Join(dir, "queue/rotational"), boolAttr(d.rotational))
+	f.write(filepath.Join(dir, "removable"), boolAttr(d.removable))
+
+	f.symlink(dir, filepath.Join(sysClassBlockDir, d.name))
+	for _, part := range d.partitions {
+		partDir := filepath.Join(dir, part)
+		f.write(filepath.Join(partDir, "partition"), "1\n")
+		f.symlink(partDir, filepath.Join(sysClassBlockDir, part))
+	}
+}
+
+func (f *fakeSysfs) symlink(target, link string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Dir(link))
+	if err := os.Symlink(target, link); err != nil {
+		f.t.Fatalf("symlink %s -> %s: %v", link, target, err)
+	}
+}
+
+// addByID creates a by-id alias pointing at a kernel device name. The link
+// target must exist for EvalSymlinks, so a placeholder /dev tree is used.
+func (f *fakeSysfs) addByID(id, dev string) {
+	f.t.Helper()
+	devPath := filepath.Join(f.root, "dev", dev)
+	f.write(devPath, "")
+	f.symlink(devPath, filepath.Join(devByIDDir, id))
+}
+
+func (f *fakeSysfs) setMounts(content string) {
+	f.t.Helper()
+	f.write(procMountsPath, content)
+}
+
+func boolAttr(b bool) string {
+	if b {
+		return "1\n"
+	}
+	return "0\n"
+}
+
+func TestListDisksReadsSysfsAttributes(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{
+		name: "sdb", sectors: 2 << 20, model: "SAMSUNG MZ7", logical: 512, physical: 4096, rotational: true,
+	})
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	if len(disks) != 1 {
+		t.Fatalf("got %d disks, want 1: %+v", len(disks), disks)
+	}
+	d := disks[0]
+	if d.Path != "/dev/sdb" || d.Bytes != (2<<20)*512 {
+		t.Errorf("path/size = %s/%d", d.Path, d.Bytes)
+	}
+	if d.Model != "SAMSUNG MZ7" {
+		t.Errorf("Model = %q", d.Model)
+	}
+	if d.LogicalBlockSize != 512 || d.PhysicalBlockSize != 4096 {
+		t.Errorf("block sizes = %d/%d", d.LogicalBlockSize, d.PhysicalBlockSize)
+	}
+	if !d.Rotational || d.Removable || d.LiveMedia {
+		t.Errorf("flags = rot %v, rem %v, live %v", d.Rotational, d.Removable, d.LiveMedia)
+	}
+	if d.Content == "" {
+		t.Error("Content is empty; an unreadable device must report something")
+	}
+}
+
+func TestListDisksSkipsVirtualAndEmptyDevices(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "loop0", sectors: 1 << 20},
+		fakeDisk{name: "dm-0", sectors: 1 << 20},
+		fakeDisk{name: "zram0", sectors: 1 << 20},
+		fakeDisk{name: "sr0", sectors: 1 << 20},
+		fakeDisk{name: "nbd0", sectors: 1 << 20},
+		// An empty card-reader slot reports zero sectors.
+		fakeDisk{name: "mmcblk0", sectors: 0},
+	)
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	var paths []string
+	for _, d := range disks {
+		paths = append(paths, d.Path)
+	}
+	if !slices.Equal(paths, []string{"/dev/sda"}) {
+		t.Errorf("paths = %v, want only /dev/sda", paths)
+	}
+}
+
+func TestListDisksSortsByPath(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sdc", sectors: 1 << 20},
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "sdb", sectors: 1 << 20},
+	)
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	var paths []string
+	for _, d := range disks {
+		paths = append(paths, d.Path)
+	}
+	if !slices.Equal(paths, []string{"/dev/sda", "/dev/sdb", "/dev/sdc"}) {
+		t.Errorf("paths = %v, want sorted", paths)
+	}
+}
+
+func TestListDisksDefaultsBlockSizesWhenSysfsIsSilent(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "vda", sectors: 1 << 20})
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	if disks[0].LogicalBlockSize != 512 || disks[0].PhysicalBlockSize != 512 {
+		t.Errorf("block sizes = %d/%d, want the 512 default",
+			disks[0].LogicalBlockSize, disks[0].PhysicalBlockSize)
+	}
+}
+
+func TestListDisksErrorsWhenSysfsIsUnreadable(t *testing.T) {
+	newFakeSysfs(t)
+	sysBlockDir = filepath.Join(t.TempDir(), "does-not-exist")
+
+	if _, err := ListDisks(); err == nil {
+		t.Fatal("expected an error when /sys/block cannot be read")
+	}
+}
+
+func TestListDisksMarksTheLiveMedium(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "sdz", sectors: 1 << 20, removable: true, partitions: []string{"sdz1"}},
+	)
+	// The installer boots from a partition; the whole disk must be excluded.
+	f.setMounts("/dev/sdz1 /run/live/medium iso9660 ro 0 0\n" +
+		"proc /proc proc rw 0 0\n" +
+		"/dev/sda1 / ext4 rw 0 0\n")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	byPath := map[string]Disk{}
+	for _, d := range disks {
+		byPath[d.Path] = d
+	}
+	if !byPath["/dev/sdz"].LiveMedia {
+		t.Error("/dev/sdz backs the live medium and was not marked")
+	}
+	if byPath["/dev/sda"].LiveMedia {
+		t.Error("/dev/sda is not the live medium but was marked")
+	}
+}
+
+func TestLiveMediaDevicesIsEmptyWithoutProcMounts(t *testing.T) {
+	newFakeSysfs(t)
+	if got := liveMediaDevices(); len(got) != 0 {
+		t.Errorf("liveMediaDevices = %v, want empty when /proc/mounts is missing", got)
+	}
+}
+
+func TestParentDiskLeavesWholeDisksAlone(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20, partitions: []string{"sda1"}})
+
+	if got := parentDisk("sda"); got != "sda" {
+		t.Errorf("parentDisk(sda) = %q, want sda", got)
+	}
+	if got := parentDisk("sda1"); got != "sda" {
+		t.Errorf("parentDisk(sda1) = %q, want sda", got)
+	}
+	// An unknown name cannot be resolved and is returned unchanged.
+	if got := parentDisk("sdq9"); got != "sdq9" {
+		t.Errorf("parentDisk(sdq9) = %q, want it unchanged", got)
+	}
+}
+
+func TestBuildByIDIndexPrefersTheMostLegibleAlias(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20})
+	// wwn- is stable but opaque, so ata- must win regardless of scan order.
+	f.addByID("wwn-0x5000c500", "sda")
+	f.addByID("ata-SAMSUNG_MZ7_S1", "sda")
+	f.addByID("ata-SAMSUNG_MZ7_S1-part1", "sda")
+
+	idx := buildByIDIndex()
+	want := filepath.Join(devByIDDir, "ata-SAMSUNG_MZ7_S1")
+	if idx["sda"] != want {
+		t.Errorf("by-id[sda] = %q, want %q", idx["sda"], want)
+	}
+}
+
+func TestBuildByIDIndexIsEmptyWhenTheDirIsAbsent(t *testing.T) {
+	newFakeSysfs(t)
+	if got := buildByIDIndex(); len(got) != 0 {
+		t.Errorf("buildByIDIndex = %v, want empty", got)
+	}
+}
+
+func TestListDisksAttachesByIDPath(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20})
+	f.addByID("ata-TEST_SERIAL0", "sda")
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	want := filepath.Join(devByIDDir, "ata-TEST_SERIAL0")
+	if disks[0].ByID != want {
+		t.Errorf("ByID = %q, want %q", disks[0].ByID, want)
+	}
+	if disks[0].Stable() != want {
+		t.Errorf("Stable() = %q, want the by-id path", disks[0].Stable())
+	}
+}
+
+func TestPrefixRank(t *testing.T) {
+	if prefixRank("nvme-Foo") >= prefixRank("wwn-0x1") {
+		t.Error("nvme- must outrank wwn-")
+	}
+	if prefixRank("md-uuid-x") != len(byIDPreference) {
+		t.Error("an unknown namespace must rank last")
+	}
+}
+
+func TestSizeHuman(t *testing.T) {
+	tests := []struct {
+		bytes int64
+		want  string
+	}{
+		{2 << 40, "2.0T"},
+		{500 << 30, "500.0G"},
+		{64 << 20, "64.0M"},
+		{512, "512B"},
+	}
+	for _, tt := range tests {
+		if got := (Disk{Bytes: tt.bytes}).SizeHuman(); got != tt.want {
+			t.Errorf("SizeHuman(%d) = %q, want %q", tt.bytes, got, tt.want)
+		}
+	}
+}
+
+func TestStableFallsBackToKernelName(t *testing.T) {
+	d := Disk{Path: "/dev/sda"}
+	if d.Stable() != "/dev/sda" {
+		t.Errorf("Stable() = %q", d.Stable())
+	}
+	if d.StablePartitionPath(2) != "/dev/sda2" {
+		t.Errorf("StablePartitionPath(2) = %q", d.StablePartitionPath(2))
+	}
+}
+
+func TestHasAnyPrefix(t *testing.T) {
+	if !hasAnyPrefix("loop3", skipPrefixes) {
+		t.Error("loop3 should match")
+	}
+	if hasAnyPrefix("sda", skipPrefixes) {
+		t.Error("sda should not match")
+	}
+}

cmd/installer/install/boottool.go

@@ -0,0 +1,202 @@
+package install
+
+import (
+	_ "embed"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+//go:embed assets/spinifex-boot-tool
+var bootToolScript []byte
+
+//go:embed assets/grub-install-wrapper
+var grubInstallWrapper []byte
+
+const (
+	bootToolPath     = "/usr/sbin/spinifex-boot-tool"
+	grubInstallPath  = "/usr/sbin/grub-install"
+	grubInstallReal  = "/usr/sbin/grub-install.real"
+	bootToolUUIDList = "/etc/kernel/spinifex-boot-uuids"
+)
+
+// installBootToolZFS sets up per-disk bootloaders on a ZFS root system.
+//
+// Nothing in the boot path is allowed to read the pool: GRUB's ZFS support
+// trails OpenZFS, so a pool feature flag enabled years from now could make the
+// machine unbootable. Each disk instead carries a self-contained ESP with its
+// own GRUB, kernels and initramfs.
+func installBootToolZFS(cfg DiskConfig) error {
+	if err := writeTargetFile(bootToolPath, bootToolScript, 0o755); err != nil {
+		return err
+	}
+	if err := writeGrubDefaults(cfg); err != nil {
+		return err
+	}
+
+	if err := bindChrootMounts(); err != nil {
+		return err
+	}
+	defer unbindChrootMounts()
+
+	// Diverted before the first init: init runs a refresh, and refresh calls
+	// grub-install. With the wrapper already in place that call has to reach
+	// the real binary, which is what the diversion arranges.
+	if err := divertGrubInstall(); err != nil {
+		return err
+	}
+
+	// A partially-updated set of ESPs is an install that boots today and fails
+	// after the next kernel upgrade, so every disk must succeed here.
+	env := []string{"SPINIFEX_BOOT_TOOL_STRICT=1"}
+	for _, d := range cfg.Disks {
+		esp := d.PartitionPath(espPartNum)
+		if err := runEnv(env, "chroot", mountRoot, bootToolPath, "init", esp); err != nil {
+			return fmt.Errorf("spinifex-boot-tool init %s: %w", esp, err)
+		}
+	}
+
+	// The one moment firmware boot variables are written. Refreshes run on
+	// every kernel upgrade and deliberately leave NVRAM alone: repeatedly
+	// rewriting boot entries has bricked more than one vendor's boards. Losing
+	// the entry is survivable anyway, because each ESP also carries the
+	// removable-path loader that firmware falls back to.
+	if err := runEnv(append(env, "SPINIFEX_BOOT_TOOL_NVRAM=1"),
+		"chroot", mountRoot, bootToolPath, "refresh"); err != nil {
+		return fmt.Errorf("spinifex-boot-tool refresh: %w", err)
+	}
+	return verifyBootConfig(cfg)
+}
+
+// divertGrubInstall moves the real grub-install aside and puts the wrapper in
+// its place. dpkg-divert makes the move survive package upgrades, which is the
+// point: grub-pc's postinst runs grub-install and would otherwise quietly
+// replace a working boot layout with one that reads the pool.
+func divertGrubInstall() error {
+	if err := run("chroot", mountRoot, "dpkg-divert", "--divert", grubInstallReal,
+		"--rename", "--add", grubInstallPath); err != nil {
+		return fmt.Errorf("divert grub-install: %w", err)
+	}
+	return writeTargetFile(grubInstallPath, grubInstallWrapper, 0o755)
+}
+
+// verifyBootConfig asserts that the config the machine will actually boot from
+// names the ZFS root. grub-mkconfig degrades quietly when grub-probe cannot
+// read the pool, emitting a config that gets as far as the GRUB menu and then
+// panics with "unable to mount root fs" — which looks like a hardware fault.
+func verifyBootConfig(cfg DiskConfig) error {
+	uuids, err := os.ReadFile(filepath.Join(mountRoot, strings.TrimPrefix(bootToolUUIDList, "/")))
+	if err != nil {
+		return fmt.Errorf("read ESP list: %w", err)
+	}
+	if len(strings.Fields(string(uuids))) != len(cfg.Disks) {
+		return fmt.Errorf("expected %d registered ESPs, found %d — some disks have no bootloader",
+			len(cfg.Disks), len(strings.Fields(string(uuids))))
+	}
+
+	// One ESP is enough to prove grub-mkconfig produced the right thing; they
+	// are all generated by the same code path in the same run.
+	mp := filepath.Join(mountRoot, "var/tmp/verify-esp")
+	if err := os.MkdirAll(mp, 0o755); err != nil {
+		return err
+	}
+	defer func() { _ = os.Remove(mp) }()
+
+	esp := cfg.Disks[0].PartitionPath(espPartNum)
+	if err := run("mount", esp, mp); err != nil {
+		return fmt.Errorf("mount %s for verification: %w", esp, err)
+	}
+	defer func() { _ = run("umount", mp) }()
+
+	grubCfg, err := os.ReadFile(filepath.Join(mp, "grub/grub.cfg"))
+	if err != nil {
+		return fmt.Errorf("read grub.cfg from %s: %w", esp, err)
+	}
+	want := "root=ZFS=" + ZFSRootDataset
+	if !strings.Contains(string(grubCfg), want) {
+		return fmt.Errorf("grub.cfg on %s does not contain %q — the installed system would not find its root filesystem", esp, want)
+	}
+	if !strings.Contains(string(grubCfg), "vmlinuz-") {
+		return fmt.Errorf("grub.cfg on %s references no kernel", esp)
+	}
+	return nil
+}
+
+// writeTargetFile writes a file into the install target.
+func writeTargetFile(path string, content []byte, mode os.FileMode) error {
+	full := filepath.Join(mountRoot, strings.TrimPrefix(path, "/"))
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		return err
+	}
+	if err := os.WriteFile(full, content, mode); err != nil {
+		return fmt.Errorf("write %s: %w", path, err)
+	}
+	// WriteFile does not apply the mode to a file that already exists, and this
+	// path is re-run on a retried install.
+	return os.Chmod(full, mode)
+}
+
+// writeGrubDefaults writes /etc/default/grub and the branding snippet.
+//
+// Both consoles are listed: tty0 stays primary for a local install while ttyS0
+// mirrors kernel output for headless racks and CI. The last console= on the
+// cmdline becomes the system console, so serial-only hardware still works.
+func writeGrubDefaults(cfg DiskConfig) error {
+	cmdline := "console=tty0 console=ttyS0,115200 systemd.show_status=1"
+	if cfg.FS.IsZFS() {
+		// Stated explicitly rather than relying on grub-mkconfig's probe. It
+		// appears after the root= that 10_linux computes, so it wins if the
+		// probe came back with something wrong.
+		cmdline = fmt.Sprintf("root=ZFS=%s boot=zfs %s", ZFSRootDataset, cmdline)
+	}
+	grubDefault := fmt.Sprintf(`GRUB_DEFAULT=0
+GRUB_TIMEOUT=5
+GRUB_DISTRIBUTOR=Spinifex
+GRUB_CMDLINE_LINUX_DEFAULT=""
+GRUB_CMDLINE_LINUX="%s"
+`, cmdline)
+	if err := os.WriteFile(filepath.Join(mountRoot, "etc/default/grub"), []byte(grubDefault), 0o644); err != nil {
+		return fmt.Errorf("write /etc/default/grub: %w", err)
+	}
+
+	// gfxterm must be activated before serial is appended — background_image
+	// silently does nothing in text mode. $prefix is used rather than a literal
+	// /boot/grub so the same snippet is correct whether GRUB's root is the ext4
+	// filesystem or a per-disk ESP.
+	grubTheme := `#!/bin/sh
+exec tail -n +3 $0
+insmod all_video
+insmod font
+if loadfont $prefix/fonts/unicode.pf2; then
+  set gfxmode=auto
+  insmod gfxterm
+  terminal_output gfxterm
+fi
+insmod serial
+if serial --unit=0 --speed=115200 --timeout=1; then
+  terminal_input  --append serial
+  terminal_output --append serial
+fi
+
+# --- Branding ---
+insmod png
+set theme=$prefix/theme.txt
+export theme
+`
+	if err := os.WriteFile(filepath.Join(mountRoot, "etc/grub.d/06_spinifex"), []byte(grubTheme), 0o755); err != nil {
+		return fmt.Errorf("write /etc/grub.d/06_spinifex: %w", err)
+	}
+	return nil
+}
+
+// refreshBootToolNote logs the command an operator needs after replacing a
+// disk. Nothing in the installer can do this for them later.
+func refreshBootToolNote(cfg DiskConfig) {
+	if cfg.FS.IsZFS() {
+		slog.Info("bootloader installed on every pool member",
+			"disks", len(cfg.Disks),
+			"after_disk_replacement", "spinifex-boot-tool format <esp> && spinifex-boot-tool init <esp>")
+	}
+}

cmd/installer/install/config.go

@@ -2,42 +2,107 @@
 // collected configuration.
 package install
 
+import (
+	"fmt"
+	"net"
+	"strings"
+)
+
+// Plane names the three traffic planes a node presents. Each plane is a role
+// bound to a target, not a fixed NIC: the same three roles collapse onto one,
+// two or three interfaces depending on the hardware.
+type Plane string
+
+const (
+	// PlaneWAN carries public north-south traffic: ingress/egress, EIPs and
+	// SNAT/DNAT. It is the OVN external bridge-mapping.
+	PlaneWAN Plane = "wan"
+
+	// PlaneLAN carries internal cluster traffic: predastore replication, the
+	// NATS cluster mesh, OVN NB/SB control, Raft and S3DB.
+	PlaneLAN Plane = "lan"
+
+	// PlaneVPC carries the tenant overlay: OVN Geneve tunnels between nodes.
+	// It is the ovn-encap-ip endpoint.
+	PlaneVPC Plane = "vpc"
+)
+
+// Bridge returns the Linux bridge that carries this plane.
+func (p Plane) Bridge() string { return "br-" + string(p) }
+
+// NetworkRole binds one plane to a network target. An empty Interface means
+// the role is folded onto the next role up the chain (vpc <- lan <- wan) and
+// no bridge of its own is created.
+//
+// VLAN is what allows two roles to share one physical NIC while staying
+// separate planes: each gets its own NIC.<vlan> subinterface and bridge. Two
+// roles on the same Interface with no VLAN are the same plane, not two.
+type NetworkRole struct {
+	// Interface is the physical NIC this role binds to; empty means folded.
+	Interface string
+
+	// VLAN is the 802.1Q tag, or 0 for untagged.
+	VLAN int
+
+	// DHCPMode leaves addressing to the DHCP client; Address/Mask are then unset.
+	DHCPMode bool
+
+	Address string
+	Mask    string
+
+	// Gateway is only meaningful on the wan role — the other planes are
+	// link-local to the rack and must not install a default route.
+	Gateway string
+
+	DNS []string
+
+	// MTU is written as MTUBytes= on the link; 0 leaves the kernel default.
+	// East-west planes typically run 9000, wan stays at 1500.
+	MTU int
+
+	// WiFiSSID and WiFiPass are set when the selected NIC is wireless.
+	WiFiSSID string
+	WiFiPass string
+}
+
+// Folded reports whether this role has no interface of its own and inherits
+// the plane of the role above it.
+func (r NetworkRole) Folded() bool { return r.Interface == "" }
+
+// Link returns the name of the interface carrying this role's address: the
+// physical NIC, or its VLAN subinterface when tagged. Empty when folded.
+func (r NetworkRole) Link() string {
+	if r.Folded() {
+		return ""
+	}
+	if r.VLAN > 0 {
+		return fmt.Sprintf("%s.%d", r.Interface, r.VLAN)
+	}
+	return r.Interface
+}
+
 // Config holds all values collected by the installer UI.
 type Config struct {
-	// Disk is the block device path to install onto (e.g. /dev/sda).
-	Disk string
+	// Storage is the filesystem choice and the ordered set of disks it is
+	// built from. In ext4 mode it holds exactly one disk.
+	Storage DiskConfig
 
-	// WAN interface — always present. The management IP lives on br-wan (a Linux
-	// bridge over the NIC), not on the physical NIC itself. This allows OVN/OVS
-	// to safely attach to the bridge without disrupting host connectivity.
-	WANInterface string
-	WANDHCPMode  bool
-	WANAddress   string   // empty if WANDHCPMode
-	WANMask      string   // empty if WANDHCPMode
-	WANGateway   string   // empty if WANDHCPMode
-	WANDNS       []string // empty if WANDHCPMode; e.g. ["1.1.1.1", "8.8.8.8"]
-	WANWiFiSSID  string   // non-empty if a WiFi NIC was selected
-	WANWiFiPass  string   // non-empty if a WiFi NIC was selected
+	// WAN is always present and can never be folded — a node needs an uplink.
+	// The address lives on br-wan (a Linux bridge over the NIC), not on the
+	// physical NIC itself, so OVN/OVS can attach to the bridge without
+	// disrupting host connectivity.
+	WAN NetworkRole
 
-	// LAN interface — only set when 2+ NICs are present. The Geneve encap IP
-	// for OVN tunnels lives on br-lan. When empty, the WAN address is used.
-	LANInterface string
-	LANDHCPMode  bool
-	LANAddress   string
-	LANMask      string
-	LANDNS       []string
-	LANWiFiSSID  string
-	LANWiFiPass  string
+	// LAN carries internal cluster traffic. Folded onto WAN on single-NIC nodes.
+	LAN NetworkRole
+
+	// VPC carries Geneve overlay traffic and supplies ovn-encap-ip. Folded onto
+	// LAN unless a node dedicates an interface or VLAN to it.
+	VPC NetworkRole
 
 	// Node identity
 	Hostname string
 
-	// ClusterRole is "init" or "join".
-	ClusterRole string
-
-	// JoinAddr is the primary node address (host:port) when ClusterRole is "join".
-	JoinAddr string
-
 	// SkipFormation skips spx admin init/join in firstboot; the provisioning
 	// controller (e.g. bm-bootstrap.sh) owns cluster formation instead.
 	SkipFormation bool
@@ -60,3 +125,111 @@ type Config struct {
 	// Passes --gpu-passthrough to `spx admin init` during firstboot.
 	GPUPassthrough bool
 }
+
+// Resolve returns the role that actually carries the given plane after
+// collapsing folded roles up the chain vpc <- lan <- wan, along with the plane
+// it landed on. A folded vpc on a single-NIC node resolves to the wan role.
+func (c *Config) Resolve(p Plane) (NetworkRole, Plane) {
+	switch p {
+	case PlaneVPC:
+		if !c.VPC.Folded() {
+			return c.VPC, PlaneVPC
+		}
+		return c.Resolve(PlaneLAN)
+	case PlaneLAN:
+		if !c.LAN.Folded() {
+			return c.LAN, PlaneLAN
+		}
+		return c.WAN, PlaneWAN
+	default:
+		return c.WAN, PlaneWAN
+	}
+}
+
+// PlaneAddress returns the configured address for a plane after collapsing.
+// Empty when the resolved role uses DHCP, since no static address is known
+// until the lease is taken.
+func (c *Config) PlaneAddress(p Plane) string {
+	role, _ := c.Resolve(p)
+	if role.DHCPMode {
+		return ""
+	}
+	return role.Address
+}
+
+// PlaneBridge returns the bridge carrying a plane after collapsing, so callers
+// asking for the vpc bridge on a single-NIC node correctly get br-wan.
+func (c *Config) PlaneBridge(p Plane) string {
+	_, landed := c.Resolve(p)
+	return landed.Bridge()
+}
+
+// RoleBinding pairs a plane with the role that carries it.
+type RoleBinding struct {
+	Plane Plane
+	Role  NetworkRole
+}
+
+// Roles returns the bound roles in bridge-creation order, skipping folded ones.
+// wan comes first because br-lan and br-vpc are activated manually after it.
+func (c *Config) Roles() []RoleBinding {
+	out := []RoleBinding{{PlaneWAN, c.WAN}}
+	if !c.LAN.Folded() {
+		out = append(out, RoleBinding{PlaneLAN, c.LAN})
+	}
+	if !c.VPC.Folded() {
+		out = append(out, RoleBinding{PlaneVPC, c.VPC})
+	}
+	return out
+}
+
+// Validate enforces the sharing rules the UI and the headless path must both
+// obey: wan always needs a target, and two roles may only share an interface
+// when distinct VLAN ids keep them on separate broadcast domains.
+func (c *Config) Validate() error {
+	if c.WAN.Folded() {
+		return fmt.Errorf("wan role must be bound to an interface")
+	}
+
+	seen := map[string]Plane{}
+	for _, r := range c.Roles() {
+		link := r.Role.Link()
+		if prev, dup := seen[link]; dup {
+			return fmt.Errorf("%s and %s both bind %s: give each a distinct VLAN id or fold one onto the other",
+				prev, r.Plane, link)
+		}
+		seen[link] = r.Plane
+		if r.Role.VLAN < 0 || r.Role.VLAN > 4094 {
+			return fmt.Errorf("%s: VLAN id %d out of range (1-4094)", r.Plane, r.Role.VLAN)
+		}
+		if r.Role.MTU != 0 && (r.Role.MTU < 68 || r.Role.MTU > 9216) {
+			return fmt.Errorf("%s: MTU %d out of range (68-9216)", r.Plane, r.Role.MTU)
+		}
+		// Resolvers belong to the plane that holds the default route. Accepting
+		// one here and dropping it during generation would leave an unattended
+		// install believing it had set a resolver that was never written.
+		if r.Plane != PlaneWAN && len(r.Role.DNS) > 0 {
+			return fmt.Errorf("%s: DNS servers belong to the wan plane, which holds the default route", r.Plane)
+		}
+		// Netgen converts the mask with net.ParseIP, so a prefix length reaches
+		// it as garbage and fails the install after the disk is partitioned.
+		// Catch it here, where the operator can still fix it.
+		if m := strings.TrimSpace(r.Role.Mask); m != "" && !validDottedMask(m) {
+			return fmt.Errorf("%s: subnet mask %q must be dotted-decimal, e.g. 255.255.255.0", r.Plane, m)
+		}
+	}
+	return nil
+}
+
+// validDottedMask reports whether s is a contiguous dotted-decimal IPv4 subnet
+// mask. Prefix-length form ("24", "/24") is deliberately not accepted: netgen
+// parses the mask with net.ParseIP, so anything else fails the install itself.
+func validDottedMask(s string) bool {
+	ip := net.ParseIP(strings.TrimSpace(s))
+	if ip == nil || ip.To4() == nil {
+		return false
+	}
+	// Size reports zero bits for a non-contiguous mask, which netgen rejects too.
+	_, bits := net.IPMask(ip.To4()).Size()
+	return bits != 0
+}

cmd/installer/install/config_test.go

@@ -0,0 +1,463 @@
+package install
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// Node layouts drawn from the us-west-1 racks, which between them exercise
+// every collapse permutation the installer has to express.
+func threeNIC() *Config { // hydrogen / radon
+	return &Config{
+		WAN: NetworkRole{Interface: "eno6", Address: "216.218.163.99", Mask: "255.255.255.224", Gateway: "216.218.163.97"},
+		LAN: NetworkRole{Interface: "ens1f0np0", Address: "10.0.0.3", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "ens1f1np1", Address: "10.1.0.3", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func vlanSplit() *Config { // helium — one 100gbe carrying lan and vpc
+	return &Config{
+		WAN: NetworkRole{Interface: "enx6c1ff7c1475a", DHCPMode: true},
+		LAN: NetworkRole{Interface: "enp129s0np0", VLAN: 10, Address: "10.0.0.2", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "enp129s0np0", VLAN: 20, Address: "10.1.0.2", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func dualPortTagged() *Config { // us-west-1-az2 — wan and lan share port 1
+	return &Config{
+		WAN: NetworkRole{Interface: "enp1s0f0", VLAN: 30, Address: "216.218.163.10", Mask: "255.255.255.224", Gateway: "216.218.163.97"},
+		LAN: NetworkRole{Interface: "enp1s0f0", VLAN: 10, Address: "10.0.0.10", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "enp1s0f1", VLAN: 20, Address: "10.1.0.10", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func TestResolveCollapsesUpTheChain(t *testing.T) {
+	singleNIC := &Config{WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"}}
+	twoNIC := &Config{
+		WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"},
+		LAN: NetworkRole{Interface: "eth1", Address: "10.0.0.5", Mask: "255.255.0.0"},
+	}
+
+	tests := []struct {
+		name      string
+		cfg       *Config
+		plane     Plane
+		wantAddr  string
+		wantPlane Plane
+	}{
+		{"1 NIC: lan folds onto wan", singleNIC, PlaneLAN, "192.168.1.10", PlaneWAN},
+		{"1 NIC: vpc folds all the way to wan", singleNIC, PlaneVPC, "192.168.1.10", PlaneWAN},
+		{"2 NIC: vpc folds onto lan", twoNIC, PlaneVPC, "10.0.0.5", PlaneLAN},
+		{"2 NIC: lan stays put", twoNIC, PlaneLAN, "10.0.0.5", PlaneLAN},
+		{"3 NIC: vpc is its own plane", threeNIC(), PlaneVPC, "10.1.0.3", PlaneVPC},
+		{"VLAN split: vpc is its own plane", vlanSplit(), PlaneVPC, "10.1.0.2", PlaneVPC},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.cfg.PlaneAddress(tt.plane); got != tt.wantAddr {
+				t.Errorf("PlaneAddress(%s) = %q, want %q", tt.plane, got, tt.wantAddr)
+			}
+			if _, landed := tt.cfg.Resolve(tt.plane); landed != tt.wantPlane {
+				t.Errorf("Resolve(%s) landed on %s, want %s", tt.plane, landed, tt.wantPlane)
+			}
+		})
+	}
+}
+
+// A DHCP plane has no address known at install time; setup-ovn.sh auto-detects
+// it from the default route at boot instead.
+func TestPlaneAddressEmptyUnderDHCP(t *testing.T) {
+	if got := vlanSplit().PlaneAddress(PlaneWAN); got != "" {
+		t.Errorf("PlaneAddress(wan) = %q, want empty for a DHCP role", got)
+	}
+}
+
+func TestValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     *Config
+		wantErr string
+	}{
+		{"three physical NICs", threeNIC(), ""},
+		{"VLAN split on one NIC", vlanSplit(), ""},
+		{"dual port, all tagged", dualPortTagged(), ""},
+		{
+			"wan cannot be folded",
+			&Config{LAN: NetworkRole{Interface: "eth1"}},
+			"wan role must be bound",
+		},
+		{
+			"two roles share a NIC untagged",
+			&Config{
+				WAN: NetworkRole{Interface: "eth0"},
+				LAN: NetworkRole{Interface: "eth0"},
+			},
+			"both bind eth0",
+		},
+		{
+			"duplicate VLAN id on one NIC",
+			&Config{
+				WAN: NetworkRole{Interface: "eth0", VLAN: 10},
+				LAN: NetworkRole{Interface: "eth0", VLAN: 10},
+			},
+			"both bind eth0.10",
+		},
+		{
+			"VLAN id out of range",
+			&Config{WAN: NetworkRole{Interface: "eth0", VLAN: 5000}},
+			"out of range",
+		},
+		{
+			"MTU out of range",
+			&Config{WAN: NetworkRole{Interface: "eth0", MTU: 70000}},
+			"MTU 70000 out of range",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("Validate() = %v, want nil", err)
+				}
+				return
+			}
+			if err == nil {
+				t.Fatalf("Validate() = nil, want error containing %q", tt.wantErr)
+			}
+			if !strings.Contains(err.Error(), tt.wantErr) {
+				t.Errorf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Two roles sharing one NIC must produce a single parent .network carrying both
+// VLAN declarations — not two files racing to configure the same interface.
+func TestWriteParentNICTaggedSharesOneFile(t *testing.T) {
+	dir := t.TempDir()
+	p := &parentNIC{vlans: []string{"enp129s0np0.10", "enp129s0np0.20"}, mtu: 9000}
+	if err := writeParentNIC(dir, "enp129s0np0", p); err != nil {
+		t.Fatal(err)
+	}
+
+	got := readFile(t, dir, "05-spinifex-nic-enp129s0np0.network")
+	for _, want := range []string{
+		"Name=enp129s0np0",
+		"MTUBytes=9000",
+		"VLAN=enp129s0np0.10",
+		"VLAN=enp129s0np0.20",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("parent .network missing %q:\n%s", want, got)
+		}
+	}
+	// A tagged parent is never itself enslaved — the VLAN devices are.
+	if strings.Contains(got, "Bridge=") {
+		t.Errorf("tagged parent must not be bridged directly:\n%s", got)
+	}
+}
+
+func TestWriteNetworkdBridgeVLAN(t *testing.T) {
+	dir := t.TempDir()
+	cfg := vlanSplit()
+	if err := writeNetworkdBridge(dir, PlaneVPC, cfg.VPC, true); err != nil {
+		t.Fatal(err)
+	}
+
+	netdev := readFile(t, dir, "10-spinifex-vpc-vlan.netdev")
+	if !strings.Contains(netdev, "Kind=vlan") || !strings.Contains(netdev, "Id=20") {
+		t.Errorf("VLAN netdev wrong:\n%s", netdev)
+	}
+	if !strings.Contains(netdev, "Name=enp129s0np0.20") {
+		t.Errorf("VLAN netdev name wrong:\n%s", netdev)
+	}
+
+	vlanNet := readFile(t, dir, "11-spinifex-vpc-vlan.network")
+	if !strings.Contains(vlanNet, "Bridge=br-vpc") {
+		t.Errorf("VLAN device must be enslaved to br-vpc:\n%s", vlanNet)
+	}
+
+	brNet := readFile(t, dir, "20-spinifex-vpc.network")
+	if !strings.Contains(brNet, "Address=10.1.0.2/16") {
+		t.Errorf("bridge address wrong:\n%s", brNet)
+	}
+	// Only wan installs a default route; an east-west plane must not.
+	if strings.Contains(brNet, "Gateway=") {
+		t.Errorf("vpc plane must not install a default route:\n%s", brNet)
+	}
+	if !strings.Contains(brNet, "ActivationPolicy=manual") {
+		t.Errorf("east-west bridge must not auto-activate:\n%s", brNet)
+	}
+}
+
+// The wan plane is the one that carries a default route and auto-activates.
+func TestWriteNetworkdBridgeWANUntagged(t *testing.T) {
+	dir := t.TempDir()
+	cfg := threeNIC()
+	if err := writeNetworkdBridge(dir, PlaneWAN, cfg.WAN, false); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := os.Stat(filepath.Join(dir, "10-spinifex-wan-vlan.netdev")); !os.IsNotExist(err) {
+		t.Error("untagged role must not produce a VLAN netdev")
+	}
+	brNet := readFile(t, dir, "20-spinifex-wan.network")
+	if !strings.Contains(brNet, "Gateway=216.218.163.97") {
+		t.Errorf("wan must install its default route:\n%s", brNet)
+	}
+	if strings.Contains(brNet, "ActivationPolicy=manual") {
+		t.Errorf("wan bridge must auto-activate:\n%s", brNet)
+	}
+}
+
+func TestToFirstbootConfigSourcesPlanes(t *testing.T) {
+	cfg := threeNIC()
+	cfg.Hostname = "hydrogen"
+	fb := cfg.toFirstbootConfig()
+
+	if fb.EncapIP != "10.1.0.3" {
+		t.Errorf("EncapIP = %q, want the vpc plane address", fb.EncapIP)
+	}
+	if fb.LANIP != "10.0.0.3" {
+		t.Errorf("LANIP = %q, want the lan plane address", fb.LANIP)
+	}
+	// The public address has to travel separately: spx echoes a concrete --bind
+	// back as the advertise address, so leaving this empty would publish the
+	// internal plane as the node's public dial target.
+	if fb.WANIP != "216.218.163.99" {
+		t.Errorf("WANIP = %q, want the wan plane address", fb.WANIP)
+	}
+
+	// On a single-NIC node all three collapse onto wan.
+	single := &Config{WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"}}
+	fb = single.toFirstbootConfig()
+	if fb.EncapIP != "192.168.1.10" || fb.LANIP != "192.168.1.10" || fb.WANIP != "192.168.1.10" {
+		t.Errorf("single-NIC collapse wrong: EncapIP=%q LANIP=%q WANIP=%q", fb.EncapIP, fb.LANIP, fb.WANIP)
+	}
+}
+
+func readFile(t *testing.T, dir, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(dir, name))
+	if err != nil {
+		t.Fatalf("reading %s: %v", name, err)
+	}
+	return string(b)
+}
+
+// Resolvers belong to the plane holding the default route. The east-west
+// bridges are link-local and activate manually, so a resolver pinned to one is
+// unreachable while resolved still tries it — a timeout in front of every
+// lookup. The TUI cannot express this; the kernel cmdline can, so it is caught.
+func TestValidateRejectsDNSOffWAN(t *testing.T) {
+	cfg := threeNIC()
+	cfg.LAN.DNS = []string{"10.0.0.1"}
+	if err := cfg.Validate(); err == nil {
+		t.Fatal("lan must not carry DNS servers")
+	}
+
+	cfg = threeNIC()
+	cfg.VPC.DNS = []string{"10.1.0.1"}
+	if err := cfg.Validate(); err == nil {
+		t.Fatal("vpc must not carry DNS servers")
+	}
+
+	cfg = threeNIC()
+	cfg.WAN.DNS = []string{"8.8.8.8", "1.1.1.1"}
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("wan carries the resolvers: %v", err)
+	}
+}
+
+// Even if a role somehow carries DNS, generation must not write it onto an
+// east-west bridge.
+func TestWriteNetworkdBridgeKeepsDNSOnWAN(t *testing.T) {
+	dir := t.TempDir()
+	lan := NetworkRole{Interface: "ens1f0np0", Address: "10.0.0.3", Mask: "255.255.0.0", DNS: []string{"10.0.0.1"}}
+	if err := writeNetworkdBridge(dir, PlaneLAN, lan, true); err != nil {
+		t.Fatalf("writeNetworkdBridge: %v", err)
+	}
+	if got := readFile(t, dir, "20-spinifex-lan.network"); strings.Contains(got, "DNS=") {
+		t.Errorf("lan bridge must carry no resolver:\n%s", got)
+	}
+
+	wan := NetworkRole{Interface: "eno6", Address: "216.218.163.99", Mask: "255.255.255.224", Gateway: "216.218.163.97", DNS: []string{"8.8.8.8"}}
+	if err := writeNetworkdBridge(dir, PlaneWAN, wan, false); err != nil {
+		t.Fatalf("writeNetworkdBridge: %v", err)
+	}
+	if got := readFile(t, dir, "20-spinifex-wan.network"); !strings.Contains(got, "DNS=8.8.8.8") {
+		t.Errorf("wan bridge must carry the resolver:\n%s", got)
+	}
+}
+
+func TestSwapSize(t *testing.T) {
+	dir := t.TempDir()
+
+	// Without an override, RAM drives the size and the disk caps it.
+	ram, err := totalRAM()
+	if err != nil {
+		t.Fatalf("totalRAM: %v", err)
+	}
+	got, err := swapSize(dir)
+	if err != nil {
+		t.Fatalf("swapSize: %v", err)
+	}
+	want := ram / 2
+	if ram >= swapRAMThreshold {
+		want = swapLargeBytes
+	}
+	if got > want {
+		t.Errorf("swapSize = %d, must not exceed what RAM implies (%d)", got, want)
+	}
+	if got != 0 && got < swapMinBytes {
+		t.Errorf("swapSize = %d, must be 0 rather than below the %d floor", got, int64(swapMinBytes))
+	}
+
+	// An explicit size wins, so a large node can be told to take the full 32G
+	// and a constrained one can opt out entirely.
+	for _, tt := range []struct {
+		env  string
+		want int64
+	}{
+		{"32G", 32 << 30},
+		{"512M", 512 << 20},
+		{"0", 0},
+		{"1073741824", 1 << 30},
+	} {
+		t.Setenv("SPINIFEX_SWAP_SIZE", tt.env)
+		got, err := swapSize(dir)
+		if err != nil {
+			t.Fatalf("swapSize(%s): %v", tt.env, err)
+		}
+		if got != tt.want {
+			t.Errorf("swapSize(%s) = %d, want %d", tt.env, got, tt.want)
+		}
+	}
+
+	t.Setenv("SPINIFEX_SWAP_SIZE", "banana")
+	if _, err := swapSize(dir); err == nil {
+		t.Error("an unparseable size must be reported, not silently ignored")
+	}
+}
+
+// The swap line must only appear when the file backing it exists — systemd
+// fails the boot's swap unit on a missing file rather than skipping it.
+func TestFstabSwapLineTracksTheFile(t *testing.T) {
+	if _, err := parseSize("8G"); err != nil {
+		t.Fatalf("parseSize: %v", err)
+	}
+	if n, _ := parseSize("2G"); n != 2<<30 {
+		t.Errorf("parseSize(2G) = %d, want %d", n, int64(2)<<30)
+	}
+	if _, err := parseSize("-5G"); err == nil {
+		t.Error("negative sizes must be rejected")
+	}
+}
+
+// Bridge ports and VLAN trunks never reach the "degraded" state
+// systemd-networkd-wait-online requires — the address sits on the bridge above
+// them. Leaving them required makes that unit fail on every boot and stall
+// until it times out, which is what a three-NIC node was doing.
+func TestSlaveLinksAreNotRequiredForOnline(t *testing.T) {
+	dir := t.TempDir()
+
+	// Untagged bridge port.
+	if err := writeParentNIC(dir, "ens4", &parentNIC{bridge: "br-lan", mtu: 9000}); err != nil {
+		t.Fatal(err)
+	}
+	got := readFile(t, dir, "05-spinifex-nic-ens4.network")
+	if !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("bridge port must not gate network-online:\n%s", got)
+	}
+	if !strings.Contains(got, "MTUBytes=9000") {
+		t.Errorf("MTU must survive alongside it:\n%s", got)
+	}
+
+	// VLAN trunk carrying tagged planes.
+	if err := writeParentNIC(dir, "enp129s0np0", &parentNIC{vlans: []string{"enp129s0np0.10"}}); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "05-spinifex-nic-enp129s0np0.network"); !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("VLAN trunk must not gate network-online:\n%s", got)
+	}
+
+	// The VLAN device itself is enslaved too.
+	cfg := vlanSplit()
+	if err := writeNetworkdBridge(dir, PlaneVPC, cfg.VPC, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "11-spinifex-vpc-vlan.network"); !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("VLAN device must not gate network-online:\n%s", got)
+	}
+
+	// The wan bridge is what should gate it, and still does.
+	three := threeNIC()
+	if err := writeNetworkdBridge(dir, PlaneWAN, three.WAN, false); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "20-spinifex-wan.network"); strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("wan bridge must remain required for network-online:\n%s", got)
+	}
+}
+
+// A prefix length reaches netgen's net.ParseIP as garbage and fails the install
+// after the disk has been partitioned, so it has to be rejected up front. The
+// TUI advertised "255.255.255.0 or 24" while only the first ever worked.
+func TestValidateRejectsPrefixLengthMask(t *testing.T) {
+	for _, bad := range []string{"24", "/24", "16", "255.255.0", "255.0.255.0", "banana"} {
+		cfg := threeNIC()
+		cfg.LAN.Mask = bad
+		if err := cfg.Validate(); err == nil {
+			t.Errorf("mask %q must be rejected", bad)
+		} else if !strings.Contains(err.Error(), "dotted-decimal") {
+			t.Errorf("mask %q: error should name the expected form, got: %v", bad, err)
+		}
+	}
+
+	for _, good := range []string{"255.255.255.0", "255.255.0.0", "255.255.255.252", "255.255.255.255"} {
+		cfg := threeNIC()
+		cfg.LAN.Mask = good
+		if err := cfg.Validate(); err != nil {
+			t.Errorf("mask %q must be accepted: %v", good, err)
+		}
+	}
+
+	// A DHCP role carries no mask, and that must stay valid.
+	cfg := threeNIC()
+	cfg.LAN = NetworkRole{Interface: "ens1f0np0", DHCPMode: true}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("a DHCP role has no mask to check: %v", err)
+	}
+}
+
+// A bridged uplink must present the enslaved NIC's address. Left to its default
+// policy, systemd-networkd generates one for the bridge, so the node appears on
+// the wire as an address belonging to no NIC — DHCP requests carry a source that
+// disagrees with their own chaddr and the unicast reply is never delivered.
+func TestBridgeInheritsPortMAC(t *testing.T) {
+	dir := t.TempDir()
+	cfg := threeNIC()
+
+	for _, tc := range []struct {
+		plane  Plane
+		role   NetworkRole
+		manual bool
+	}{
+		{PlaneWAN, cfg.WAN, false},
+		{PlaneLAN, cfg.LAN, true},
+		{PlaneVPC, cfg.VPC, true},
+	} {
+		if err := writeNetworkdBridge(dir, tc.plane, tc.role, tc.manual); err != nil {
+			t.Fatal(err)
+		}
+		got := readFile(t, dir, "20-spinifex-"+string(tc.plane)+".netdev")
+		if !strings.Contains(got, "MACAddressPolicy=none") {
+			t.Errorf("%s bridge must inherit its port's MAC:\n%s", tc.plane, got)
+		}
+	}
+}

cmd/installer/install/disk.go

@@ -0,0 +1,158 @@
+package install
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"time"
+)
+
+// Partition numbers, identical on every disk the installer touches so that a
+// replacement drive can be cloned with `sgdisk -R` and nothing has to be
+// recomputed at 3am.
+const (
+	biosPartNum = 1
+	espPartNum  = 2
+	rootPartNum = 3
+)
+
+// GPT partition type GUIDs, by their sgdisk short code.
+const (
+	typeBIOSBoot = "EF02"
+	typeESP      = "EF00"
+	typeLinuxFS  = "8300"
+	typeZFS      = "BF00"
+)
+
+// Partition geometry. The ESP is sized as Proxmox does: 512MiB is enough for
+// two kernels, but a 1GiB ESP on a real disk costs nothing and keeps room for
+// the extra kernels an operator pins during an upgrade.
+const (
+	espSmallMiB     = 512
+	espLargeMiB     = 1024
+	espLargeDiskGiB = 100
+
+	// biosBootMiB is the BIOS Boot Partition GRUB's i386-pc core.img is embedded
+	// into. GPT leaves no post-MBR gap, so without it there is nowhere to put it.
+	biosBootMiB = 1
+
+	// tailReserveMiB is left unallocated at the end of every disk. A replacement
+	// drive of the "same" size is routinely a few thousand sectors smaller, and
+	// without slack `sgdisk -R` refuses to clone the table onto it — which is
+	// discovered while standing in front of a degraded pool.
+	tailReserveMiB = 1024
+)
+
+// espSizeMiB picks the ESP size for a disk.
+func espSizeMiB(d Disk) int {
+	if d.Bytes >= espLargeDiskGiB<<30 {
+		return espLargeMiB
+	}
+	return espSmallMiB
+}
+
+// partitionDisks lays down an identical GPT on every selected disk. Each disk
+// gets a BIOS boot partition and an ESP whether or not this machine booted in
+// EFI mode: the cost is 1GiB, and it means a board swap or a firmware change
+// from BIOS to UEFI does not leave an unbootable pool.
+func partitionDisks(cfg DiskConfig) error {
+	ptype := typeLinuxFS
+	if cfg.FS.IsZFS() {
+		ptype = typeZFS
+	}
+	for _, d := range cfg.Disks {
+		if err := partitionOne(d, ptype); err != nil {
+			return fmt.Errorf("partition %s: %w", d.Path, err)
+		}
+	}
+	return waitForPartitions(cfg)
+}
+
+// partitionOne writes the GPT for a single disk.
+func partitionOne(d Disk, rootType string) error {
+	// Stale ZFS labels and filesystem signatures survive a new partition table
+	// and make a retried install fail with errors that read like hardware faults.
+	if err := clearDisk(d); err != nil {
+		return fmt.Errorf("clear: %w", err)
+	}
+
+	esp := espSizeMiB(d)
+	if err := run("sgdisk",
+		"-n", fmt.Sprintf("%d:1M:+%dM", biosPartNum, biosBootMiB),
+		"-t", fmt.Sprintf("%d:%s", biosPartNum, typeBIOSBoot),
+		"-c", fmt.Sprintf("%d:bios_boot", biosPartNum),
+		"-n", fmt.Sprintf("%d:0:+%dM", espPartNum, esp),
+		"-t", fmt.Sprintf("%d:%s", espPartNum, typeESP),
+		"-c", fmt.Sprintf("%d:ESP", espPartNum),
+		"-n", fmt.Sprintf("%d:0:-%dM", rootPartNum, tailReserveMiB),
+		"-t", fmt.Sprintf("%d:%s", rootPartNum, rootType),
+		"-c", fmt.Sprintf("%d:root", rootPartNum),
+		d.Path,
+	); err != nil {
+		return err
+	}
+	// A fresh disk-level GUID: cloning a table with `sgdisk -R` copies it, and
+	// two disks sharing one leaves the pool ambiguous to udev.
+	if err := run("sgdisk", "-G", d.Path); err != nil {
+		slog.Warn("could not randomise GPT GUIDs", "disk", d.Path, "err", err)
+	}
+	return nil
+}
+
+// waitForPartitions ensures every partition device node the installer is about
+// to use exists. Trixie's udev is slow enough after a table rewrite that mkfs
+// races it and fails with ENOENT on a device the kernel has already accepted.
+func waitForPartitions(cfg DiskConfig) error {
+	for _, d := range cfg.Disks {
+		// Best-effort: partprobe failing is not fatal, since udev may still act
+		// on the BLKRRPART ioctl sgdisk issued itself.
+		if err := run("partprobe", d.Path); err != nil {
+			slog.Warn("partprobe failed, continuing", "disk", d.Path, "err", err)
+		}
+	}
+	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
+		slog.Warn("udevadm settle failed, continuing", "err", err)
+	}
+
+	deadline := time.Now().Add(15 * time.Second)
+	for _, d := range cfg.Disks {
+		wanted := []string{d.PartitionPath(espPartNum), d.PartitionPath(rootPartNum)}
+		// The pool is built from by-id paths, so their symlinks have to exist
+		// too — udev publishes them later than the kernel device node.
+		if d.ByID != "" {
+			wanted = append(wanted, d.StablePartitionPath(rootPartNum))
+		}
+		for _, part := range wanted {
+			if err := waitForPath(part, deadline); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func waitForPath(path string, deadline time.Time) error {
+	for {
+		if _, err := os.Stat(path); err == nil {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("device %s did not appear within timeout — the kernel or udev did not pick up the new partition table", path)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// formatESPs makes a FAT32 filesystem on every disk's ESP. Each one is
+// independent — there is no RAID here, they are kept in step by content.
+func formatESPs(cfg DiskConfig) error {
+	for _, d := range cfg.Disks {
+		esp := d.PartitionPath(espPartNum)
+		// A distinct label per member so `lsblk -f` on a degraded machine tells
+		// the operator which disk each ESP belongs to.
+		if err := run("mkfs.fat", "-F", "32", "-n", "EFI", esp); err != nil {
+			return fmt.Errorf("format ESP %s: %w", esp, err)
+		}
+	}
+	return nil
+}

cmd/installer/install/diskconfig.go

@@ -0,0 +1,318 @@
+package install
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"strings"
+)
+
+// Pool and dataset names. rpool matches Proxmox, which is what every ZFS
+// recovery guide and forum answer assumes — the name an operator will be
+// reading off a serial console at 3am.
+const (
+	ZFSPoolName    = "rpool"
+	ZFSRootDataset = ZFSPoolName + "/ROOT/spinifex"
+)
+
+// FSType selects the root filesystem and, for ZFS, the vdev topology.
+type FSType string
+
+const (
+	FSExt4      FSType = "ext4"
+	FSZFSRAID0  FSType = "zfs-raid0"
+	FSZFSRAID1  FSType = "zfs-raid1"
+	FSZFSRAID10 FSType = "zfs-raid10"
+	FSZFSRAIDZ1 FSType = "zfs-raidz1"
+	FSZFSRAIDZ2 FSType = "zfs-raidz2"
+	FSZFSRAIDZ3 FSType = "zfs-raidz3"
+)
+
+// AllFSTypes is the selector order presented by the UI. ext4 is first and is
+// the default, so ZFS — which holds the kernel at its shipped version and
+// reserves RAM for the ARC — is always an explicit choice.
+var AllFSTypes = []FSType{
+	FSExt4, FSZFSRAID0, FSZFSRAID1, FSZFSRAID10, FSZFSRAIDZ1, FSZFSRAIDZ2, FSZFSRAIDZ3,
+}
+
+// fsMeta carries the per-topology facts that drive display and validation.
+var fsMeta = map[FSType]struct {
+	label     string
+	minDisks  int
+	redundant bool   // subject to the same-size rule
+	vdev      string // zpool create keyword; empty means a plain stripe
+}{
+	FSExt4:      {"ext4", 1, false, ""},
+	FSZFSRAID0:  {"zfs (RAID0)", 1, false, ""},
+	FSZFSRAID1:  {"zfs (RAID1)", 2, true, "mirror"},
+	FSZFSRAID10: {"zfs (RAID10)", 4, true, "mirror"},
+	FSZFSRAIDZ1: {"zfs (RAIDZ-1)", 3, true, "raidz1"},
+	FSZFSRAIDZ2: {"zfs (RAIDZ-2)", 4, true, "raidz2"},
+	FSZFSRAIDZ3: {"zfs (RAIDZ-3)", 5, true, "raidz3"},
+}
+
+// IsZFS reports whether this mode creates a pool rather than a plain filesystem.
+func (f FSType) IsZFS() bool { return strings.HasPrefix(string(f), "zfs-") }
+
+// Label is the display name, matching proxinstall's wording so the screen is
+// recognisable to anyone who has installed Proxmox.
+func (f FSType) Label() string { return fsMeta[f].label }
+
+// MinDisks is the smallest member count the topology can be built from.
+func (f FSType) MinDisks() int { return fsMeta[f].minDisks }
+
+// Redundant reports whether the topology stores parity or mirrors, and so
+// requires members of matching size. A RAID0 stripe over unequal disks is
+// legitimate and is deliberately not covered.
+func (f FSType) Redundant() bool { return fsMeta[f].redundant }
+
+// vdevKeyword is the zpool create keyword introducing each vdev.
+func (f FSType) vdevKeyword() string { return fsMeta[f].vdev }
+
+// ParseFSType maps a SPINIFEX_FS value to an FSType.
+func ParseFSType(s string) (FSType, error) {
+	f := FSType(strings.ToLower(strings.TrimSpace(s)))
+	if _, ok := fsMeta[f]; !ok {
+		names := make([]string, 0, len(AllFSTypes))
+		for _, t := range AllFSTypes {
+			names = append(names, string(t))
+		}
+		return "", fmt.Errorf("unknown filesystem %q — must be one of: %s", s, strings.Join(names, ", "))
+	}
+	return f, nil
+}
+
+// ZFSOpts are the advanced tunables. A zero value means "use the computed
+// default", which is why Ashift and ARCMaxMiB are plain ints — 0 is not legal
+// for either, so it is unambiguous as a sentinel.
+type ZFSOpts struct {
+	Ashift    int
+	Compress  string
+	Checksum  string
+	Copies    int
+	ARCMaxMiB int
+}
+
+// DiskConfig is the storage half of the installer's configuration. Disks is
+// ordered and the order is significant: RAID10 pairs members two at a time.
+type DiskConfig struct {
+	FS    FSType
+	Disks []Disk
+	ZFS   ZFSOpts
+}
+
+// Primary is the first selected disk — the whole target in ext4 mode, and the
+// disk shown in single-disk summaries.
+func (d DiskConfig) Primary() Disk {
+	if len(d.Disks) == 0 {
+		return Disk{}
+	}
+	return d.Disks[0]
+}
+
+// Paths lists the selected device paths, for logs and confirmation screens.
+func (d DiskConfig) Paths() []string {
+	out := make([]string, len(d.Disks))
+	for i, disk := range d.Disks {
+		out[i] = disk.Path
+	}
+	return out
+}
+
+// Buildable reports whether the selection has enough members, of the right
+// count, to form the topology at all. Capacity and fault tolerance are
+// meaningless until it does — a 3-disk RAIDZ-3 has no geometry to describe.
+func (d DiskConfig) Buildable() bool {
+	if _, ok := fsMeta[d.FS]; !ok {
+		return false
+	}
+	if len(d.Disks) < d.FS.MinDisks() {
+		return false
+	}
+	return d.FS != FSZFSRAID10 || len(d.Disks)%2 == 0
+}
+
+// Requirement describes what the topology still needs, for the preview line
+// shown in place of a capacity that cannot be computed yet.
+func (d DiskConfig) Requirement() string {
+	req := fmt.Sprintf("needs %d disks", d.FS.MinDisks())
+	if d.FS == FSZFSRAID10 {
+		req = "needs an even number of disks, at least 4"
+	}
+	if d.FS.Redundant() {
+		req += " of the same size"
+	}
+	return req
+}
+
+// UsableBytes estimates post-parity capacity, for the geometry preview. It is
+// an operator-facing approximation and deliberately ignores metadata overhead.
+// Returns 0 for a selection that cannot form the topology, since the parity
+// arithmetic below goes to zero and then negative as members run out.
+func (d DiskConfig) UsableBytes() int64 {
+	n := int64(len(d.Disks))
+	if n == 0 || !d.Buildable() {
+		return 0
+	}
+	smallest := d.Disks[0].Bytes
+	for _, disk := range d.Disks {
+		smallest = min(smallest, disk.Bytes)
+	}
+	switch d.FS {
+	case FSExt4, FSZFSRAID0:
+		var total int64
+		for _, disk := range d.Disks {
+			total += disk.Bytes
+		}
+		return total
+	case FSZFSRAID1:
+		return smallest
+	case FSZFSRAID10:
+		return smallest * (n / 2)
+	case FSZFSRAIDZ1:
+		return smallest * (n - 1)
+	case FSZFSRAIDZ2:
+		return smallest * (n - 2)
+	case FSZFSRAIDZ3:
+		return smallest * (n - 3)
+	}
+	return 0
+}
+
+// Tolerated returns the number of simultaneous disk failures the topology
+// survives, for the geometry preview.
+func (d DiskConfig) Tolerated() int {
+	if !d.Buildable() {
+		return 0
+	}
+	switch d.FS {
+	case FSZFSRAID1:
+		return max(len(d.Disks)-1, 0)
+	case FSZFSRAID10:
+		return 1 // one per mirror pair; the honest worst case is one
+	case FSZFSRAIDZ1:
+		return 1
+	case FSZFSRAIDZ2:
+		return 2
+	case FSZFSRAIDZ3:
+		return 3
+	}
+	return 0
+}
+
+// sizeTolerance is the fraction by which pool members may differ in size.
+// Matches Proxmox's zfs_mirror_size_check: anything beyond 10% and the pool
+// silently truncates every member to the smallest.
+const sizeTolerance = 0.10
+
+// bootedEFI reports whether firmware booted this machine in EFI mode. GRUB's
+// i386-pc target cannot read a 4Kn drive, so legacy BIOS constrains selection.
+func bootedEFI() bool {
+	_, err := os.Stat("/sys/firmware/efi")
+	return err == nil
+}
+
+// Validate checks a disk selection against every rule that must hold before any
+// destructive command runs. The UI calls it on each keystroke so failures block
+// Continue, rather than aborting an install that has already repartitioned.
+func (d DiskConfig) Validate() error {
+	if _, ok := fsMeta[d.FS]; !ok {
+		return fmt.Errorf("no filesystem selected")
+	}
+	if len(d.Disks) == 0 {
+		return fmt.Errorf("%s: select at least one disk", d.FS.Label())
+	}
+	if !d.FS.IsZFS() && len(d.Disks) > 1 {
+		return fmt.Errorf("%s: supports a single disk only, %d selected — choose a zfs mode to use them all",
+			d.FS.Label(), len(d.Disks))
+	}
+	if n := len(d.Disks); n < d.FS.MinDisks() {
+		return fmt.Errorf("%s: needs at least %d disks, %d selected", d.FS.Label(), d.FS.MinDisks(), n)
+	}
+	if d.FS == FSZFSRAID10 && len(d.Disks)%2 != 0 {
+		return fmt.Errorf("%s: needs an even number of disks, %d selected", d.FS.Label(), len(d.Disks))
+	}
+
+	seen := map[string]bool{}
+	for _, disk := range d.Disks {
+		if seen[disk.Path] {
+			return fmt.Errorf("%s is selected more than once", disk.Path)
+		}
+		seen[disk.Path] = true
+
+		if disk.LiveMedia {
+			return fmt.Errorf("%s is the installer's own boot media and cannot be erased", disk.Path)
+		}
+		// GRUB's i386-pc target cannot read a 4Kn drive, so on legacy BIOS the
+		// installed system would have no reachable bootloader.
+		if !bootedEFI() && disk.LogicalBlockSize == 4096 {
+			return fmt.Errorf("%s is a 4Kn drive (4096-byte sectors) and this machine booted in legacy BIOS mode — booting from it is not supported; enable UEFI in firmware",
+				disk.Path)
+		}
+		if disk.Bytes < minRootBytes {
+			return fmt.Errorf("%s is too small (%s, need at least %dGiB)",
+				disk.Path, disk.SizeHuman(), minRootBytes>>30)
+		}
+	}
+
+	return d.validateSizes()
+}
+
+// minRootBytes matches Proxmox's hard floor for a root disk.
+const minRootBytes = 2 << 30
+
+// validateSizes enforces the same-size rule where the topology needs it:
+// across all members for mirrors and raidz, and per-pair for RAID10 — a
+// striped mirror only requires each pair to match, not every disk in the pool.
+func (d DiskConfig) validateSizes() error {
+	if !d.FS.Redundant() {
+		return nil
+	}
+	if d.FS == FSZFSRAID10 {
+		for i := 0; i+1 < len(d.Disks); i += 2 {
+			if err := sizesMatch(d.Disks[i], d.Disks[i+1]); err != nil {
+				return fmt.Errorf("%s mirror pair %d: %w", d.FS.Label(), i/2+1, err)
+			}
+		}
+		return nil
+	}
+	for _, disk := range d.Disks[1:] {
+		if err := sizesMatch(d.Disks[0], disk); err != nil {
+			return fmt.Errorf("%s: %w", d.FS.Label(), err)
+		}
+	}
+	return nil
+}
+
+// SizesWithinTolerance reports whether two disks could be members of the same
+// redundant pool. The UI uses it to decide whether suggesting ZFS is honest.
+func SizesWithinTolerance(a, b Disk) bool { return sizesMatch(a, b) == nil }
+
+// sizesMatch reports whether two members are within the tolerance, naming both
+// disks and their sizes — that message is the operator's only diagnostic.
+func sizesMatch(a, b Disk) error {
+	if math.Abs(float64(a.Bytes-b.Bytes)) > float64(a.Bytes)*sizeTolerance {
+		return fmt.Errorf("disks must be the same size within %.0f%%: %s is %s, %s is %s",
+			sizeTolerance*100, a.Path, a.SizeHuman(), b.Path, b.SizeHuman())
+	}
+	return nil
+}
+
+// Warnings returns non-blocking advice for a valid selection. A RAID0 stripe
+// over mismatched disks is legal but almost always a mistake, so it is
+// surfaced without preventing the install — which is what Proxmox does.
+func (d DiskConfig) Warnings() []string {
+	var out []string
+	if d.FS == FSZFSRAID0 && len(d.Disks) > 1 {
+		for _, disk := range d.Disks[1:] {
+			if sizesMatch(d.Disks[0], disk) != nil {
+				out = append(out, "striping disks of different sizes — capacity is usable but a failure of any disk loses the pool")
+				break
+			}
+		}
+	}
+	if d.FS.IsZFS() && d.FS != FSZFSRAID0 && d.Buildable() && d.Tolerated() == 0 {
+		out = append(out, "this topology stores no redundancy")
+	}
+	return out
+}

cmd/installer/install/diskconfig_test.go

@@ -0,0 +1,267 @@
+package install
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDiskConfigValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     DiskConfig
+		wantErr string
+	}{
+		{"ext4 on one disk", DiskConfig{FS: FSExt4, Disks: disks(1, 40)}, ""},
+		{"raidz1 on three matching disks", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)}, ""},
+		{"raid10 on four matching disks", DiskConfig{FS: FSZFSRAID10, Disks: disks(4, 40)}, ""},
+		// The single-drive ZFS case: no redundancy, but checksums, compression
+		// and snapshots still apply.
+		{"raid0 on one disk", DiskConfig{FS: FSZFSRAID0, Disks: disks(1, 40)}, ""},
+
+		{"no filesystem chosen", DiskConfig{Disks: disks(1, 40)}, "no filesystem selected"},
+		{"no disks", DiskConfig{FS: FSZFSRAIDZ1}, "select at least one disk"},
+		{"ext4 cannot span disks", DiskConfig{FS: FSExt4, Disks: disks(2, 40)}, "single disk only"},
+		{"raidz1 needs three", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(2, 40)}, "at least 3 disks"},
+		{"raidz2 needs four", DiskConfig{FS: FSZFSRAIDZ2, Disks: disks(3, 40)}, "at least 4 disks"},
+		{"raidz3 needs five", DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(4, 40)}, "at least 5 disks"},
+		{"raid1 needs two", DiskConfig{FS: FSZFSRAID1, Disks: disks(1, 40)}, "at least 2 disks"},
+		{"raid10 rejects an odd count", DiskConfig{FS: FSZFSRAID10, Disks: disks(5, 40)}, "even number"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			switch {
+			case tt.wantErr == "" && err != nil:
+				t.Fatalf("Validate() = %v, want nil", err)
+			case tt.wantErr != "" && err == nil:
+				t.Fatalf("Validate() = nil, want error containing %q", tt.wantErr)
+			case tt.wantErr != "" && !strings.Contains(err.Error(), tt.wantErr):
+				t.Fatalf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestValidateRejectsDuplicateDisk(t *testing.T) {
+	d := disk("a", 40)
+	err := DiskConfig{FS: FSZFSRAID1, Disks: []Disk{d, d}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "more than once") {
+		t.Fatalf("Validate() = %v, want a duplicate-selection error", err)
+	}
+}
+
+func TestValidateRejectsLiveMedia(t *testing.T) {
+	d := disk("a", 40)
+	d.LiveMedia = true
+	err := DiskConfig{FS: FSExt4, Disks: []Disk{d}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "boot media") {
+		t.Fatalf("Validate() = %v, want the installer's own media to be refused", err)
+	}
+}
+
+func TestValidateRejectsTooSmallDisk(t *testing.T) {
+	err := DiskConfig{FS: FSExt4, Disks: []Disk{disk("a", 1)}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "too small") {
+		t.Fatalf("Validate() = %v, want a minimum-size error", err)
+	}
+}
+
+// A pool truncates every member to the smallest, so mismatched disks in a
+// redundant topology silently throw capacity away.
+func TestValidateSizeTolerance(t *testing.T) {
+	tests := []struct {
+		name    string
+		fs      FSType
+		sizes   []int64
+		wantErr bool
+	}{
+		{"identical disks", FSZFSRAIDZ1, []int64{40, 40, 40}, false},
+		{"within 10%", FSZFSRAIDZ1, []int64{100, 95, 92}, false},
+		{"beyond 10%", FSZFSRAIDZ1, []int64{100, 100, 80}, true},
+		{"mirror with a mismatched member", FSZFSRAID1, []int64{100, 50}, true},
+		// A stripe has no parity to waste, so mismatched sizes are a warning
+		// rather than a refusal — matching Proxmox.
+		{"raid0 permits mismatched disks", FSZFSRAID0, []int64{100, 40}, false},
+		{"raid10 checks each pair, not the whole pool", FSZFSRAID10, []int64{40, 40, 100, 100}, false},
+		{"raid10 rejects a mismatched pair", FSZFSRAID10, []int64{40, 100, 100, 100}, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			members := make([]Disk, len(tt.sizes))
+			for i, gib := range tt.sizes {
+				members[i] = disk(string(rune('a'+i)), gib)
+			}
+			err := DiskConfig{FS: tt.fs, Disks: members}.Validate()
+			if tt.wantErr && err == nil {
+				t.Fatalf("Validate() = nil, want a size-mismatch error")
+			}
+			if !tt.wantErr && err != nil {
+				t.Fatalf("Validate() = %v, want nil", err)
+			}
+		})
+	}
+}
+
+func TestWarningsFlagMismatchedStripe(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAID0, Disks: []Disk{disk("a", 100), disk("b", 40)}}
+	warnings := cfg.Warnings()
+	if len(warnings) == 0 {
+		t.Fatal("a stripe over mismatched disks should warn")
+	}
+}
+
+func TestUsableBytes(t *testing.T) {
+	tests := []struct {
+		name    string
+		fs      FSType
+		n       int
+		wantGiB int64
+	}{
+		{"stripe adds every disk", FSZFSRAID0, 3, 120},
+		{"mirror gives one disk", FSZFSRAID1, 2, 40},
+		{"raid10 gives half", FSZFSRAID10, 4, 80},
+		{"raidz1 loses one to parity", FSZFSRAIDZ1, 3, 80},
+		{"raidz2 loses two", FSZFSRAIDZ2, 4, 80},
+		{"raidz3 loses three", FSZFSRAIDZ3, 5, 80},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if got := cfg.UsableBytes() >> 30; got != tt.wantGiB {
+				t.Errorf("UsableBytes() = %dG, want %dG", got, tt.wantGiB)
+			}
+		})
+	}
+}
+
+// The parity arithmetic goes to zero and then negative as members run out, and
+// a preview reading "-40GiB usable" is worse than no figure at all.
+func TestUsableBytesIsZeroWhenTopologyCannotBeBuilt(t *testing.T) {
+	tests := []struct {
+		name string
+		fs   FSType
+		n    int
+	}{
+		{"raidz3 with exactly its parity count", FSZFSRAIDZ3, 3},
+		{"raidz3 with fewer disks than parity", FSZFSRAIDZ3, 2},
+		{"raidz2 with two disks", FSZFSRAIDZ2, 2},
+		{"raidz1 with one disk", FSZFSRAIDZ1, 1},
+		{"mirror with one disk", FSZFSRAID1, 1},
+		{"raid10 with an odd count", FSZFSRAID10, 3},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if cfg.Buildable() {
+				t.Fatalf("Buildable() = true for %s across %d disks", tt.fs, tt.n)
+			}
+			if got := cfg.UsableBytes(); got != 0 {
+				t.Errorf("UsableBytes() = %d, want 0", got)
+			}
+			if got := cfg.Tolerated(); got != 0 {
+				t.Errorf("Tolerated() = %d, want 0", got)
+			}
+		})
+	}
+}
+
+// A selection that cannot be built must not also be described as unredundant:
+// the operator's problem is the disk count, not the topology.
+func TestWarningsStaySilentUntilBuildable(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(3, 40)}
+	if got := cfg.Warnings(); len(got) != 0 {
+		t.Errorf("Warnings() = %v, want none", got)
+	}
+}
+
+func TestRequirementNamesTheSameSizeRule(t *testing.T) {
+	if got := (DiskConfig{FS: FSZFSRAIDZ2}).Requirement(); !strings.Contains(got, "4 disks") ||
+		!strings.Contains(got, "same size") {
+		t.Errorf("Requirement() = %q, want the disk count and the same-size rule", got)
+	}
+	if got := (DiskConfig{FS: FSZFSRAID10}).Requirement(); !strings.Contains(got, "even") {
+		t.Errorf("Requirement() = %q, want the even-count rule", got)
+	}
+	// A stripe has no matching requirement, so claiming one would be wrong.
+	if got := (DiskConfig{FS: FSZFSRAID0}).Requirement(); strings.Contains(got, "same size") {
+		t.Errorf("Requirement() = %q, want no same-size rule for a stripe", got)
+	}
+}
+
+func TestTolerated(t *testing.T) {
+	tests := []struct {
+		fs   FSType
+		n    int
+		want int
+	}{
+		{FSExt4, 1, 0},
+		{FSZFSRAID0, 3, 0},
+		{FSZFSRAID1, 2, 1},
+		{FSZFSRAID1, 3, 2},
+		{FSZFSRAID10, 4, 1},
+		{FSZFSRAIDZ1, 3, 1},
+		{FSZFSRAIDZ2, 4, 2},
+		{FSZFSRAIDZ3, 5, 3},
+	}
+	for _, tt := range tests {
+		t.Run(string(tt.fs), func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if got := cfg.Tolerated(); got != tt.want {
+				t.Errorf("Tolerated() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseFSType(t *testing.T) {
+	for _, fs := range AllFSTypes {
+		got, err := ParseFSType(strings.ToUpper(string(fs)))
+		if err != nil {
+			t.Errorf("ParseFSType(%q): %v", fs, err)
+			continue
+		}
+		if got != fs {
+			t.Errorf("ParseFSType(%q) = %q", fs, got)
+		}
+	}
+	// The error has to list the valid values: it is read off a serial console
+	// by someone whose unattended install just stopped.
+	_, err := ParseFSType("btrfs")
+	if err == nil || !strings.Contains(err.Error(), "zfs-raidz1") {
+		t.Fatalf("ParseFSType(btrfs) = %v, want an error listing the accepted values", err)
+	}
+}
+
+func TestPartitionPathSeparator(t *testing.T) {
+	tests := []struct{ dev, want string }{
+		{"/dev/sda", "/dev/sda3"},
+		// A trailing digit needs a 'p' or the name is ambiguous.
+		{"/dev/nvme0n1", "/dev/nvme0n1p3"},
+		{"/dev/vda", "/dev/vda3"},
+	}
+	for _, tt := range tests {
+		if got := (Disk{Path: tt.dev}).PartitionPath(3); got != tt.want {
+			t.Errorf("PartitionPath(%s) = %s, want %s", tt.dev, got, tt.want)
+		}
+	}
+}
+
+// udev names partition links with a -partN suffix regardless of the kernel's
+// separator, so the by-id form must not reuse the 'p' rule.
+func TestStablePartitionPathUsesPartSuffix(t *testing.T) {
+	d := Disk{Path: "/dev/nvme0n1", ByID: "/dev/disk/by-id/nvme-SAMSUNG_1234"}
+	if got := d.StablePartitionPath(3); got != "/dev/disk/by-id/nvme-SAMSUNG_1234-part3" {
+		t.Errorf("StablePartitionPath() = %s", got)
+	}
+}
+
+func TestESPSizeMiB(t *testing.T) {
+	if got := espSizeMiB(disk("a", 40)); got != espSmallMiB {
+		t.Errorf("espSizeMiB(40G) = %d, want %d", got, espSmallMiB)
+	}
+	if got := espSizeMiB(disk("a", 500)); got != espLargeMiB {
+		t.Errorf("espSizeMiB(500G) = %d, want %d", got, espLargeMiB)
+	}
+}

cmd/installer/install/install.go

@@ -11,6 +11,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"slices"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -19,40 +20,55 @@ import (
 	"github.com/mulgadc/spinifex/cmd/installer/systemd"
 )
 
-const (
-	mountRoot = "/mnt/spinifex-install"
-	efiPart   = mountRoot + "/boot/efi"
-)
+// mountRoot is the install target. A variable, not a constant, so tests can
+// drive the file-writing steps against a temp tree rather than a real mount.
+var mountRoot = "/mnt/spinifex-install"
+
+// efiPart is the ESP mount point inside the target. Derived on each call so it
+// cannot go stale against mountRoot.
+func efiPart() string { return filepath.Join(mountRoot, "boot/efi") }
+
+// step is one named unit of work in the install sequence.
+type step struct {
+	name string
+	fn   func() error
+}
 
 // Run executes all installation steps in order. It is intentionally sequential
 // and explicit — each step is visible in logs so failures are easy to diagnose.
 func Run(cfg *Config) error {
+	// Re-checked here even though the UI and the headless path both validate:
+	// this is the last point before anything is erased, and it is the only one
+	// every caller must pass through.
+	if err := cfg.Storage.Validate(); err != nil {
+		return fmt.Errorf("storage: %w", err)
+	}
+
 	// The live environment may not have /sbin or /usr/sbin in PATH. Set it
 	// explicitly so exec.Command's LookPath finds system binaries like grub-install.
 	_ = os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
 
-	// Unmount unconditionally on exit so a failed step never leaves partitions
-	// mounted in the live environment, which would cause a retry to double-mount.
-	defer func() {
-		_ = run("umount", efiPart)
-		_ = run("umount", mountRoot)
-	}()
+	// Unmount unconditionally on exit so a failed step never leaves the target
+	// mounted in the live environment, which would make a retry double-mount or,
+	// on ZFS, fail with "pool is busy".
+	defer cleanupTarget(cfg.Storage)
 
-	steps := []struct {
-		name string
-		fn   func() error
-	}{
-		{"partition disk", func() error { return partitionDisk(cfg.Disk) }},
-		{"format partitions", func() error { return formatPartitions(cfg.Disk) }},
-		{"mount partitions", func() error { return mountPartitions(cfg.Disk) }},
-		{"copy rootfs", copyRootfs},
-		{"write fstab", func() error { return writeFstab(cfg.Disk) }},
-		{"install spinifex", func() error { return installSpinifex(cfg) }},
-		{"write network config", func() error { return writeNetworkConfig(cfg) }},
-		{"write firstboot service", func() error { return firstboot.Write(mountRoot, cfg.toFirstbootConfig()) }},
-		{"install bootloader", func() error { return installBootloader(cfg.Disk) }},
-		{"install CA cert", func() error { return installCACert(cfg) }},
+	steps := []step{{"partition disks", func() error { return partitionDisks(cfg.Storage) }}}
+	if cfg.Storage.FS.IsZFS() {
+		steps = append(steps, zfsRootSteps(cfg)...)
+	} else {
+		steps = append(steps, ext4RootSteps(cfg)...)
 	}
+	steps = append(steps,
+		step{"copy rootfs", func() error { return copyRootfs(cfg.Storage) }},
+		step{"create swap", func() error { return createSwap(cfg.Storage) }},
+		step{"write fstab", func() error { return writeFstab(cfg.Storage) }},
+		step{"install spinifex", func() error { return installSpinifex(cfg) }},
+		step{"write network config", func() error { return writeNetworkConfig(cfg) }},
+		step{"write firstboot service", func() error { return firstboot.Write(mountRoot, cfg.toFirstbootConfig()) }},
+		step{"install bootloader", func() error { return installBootloader(cfg.Storage) }},
+		step{"install CA cert", func() error { return installCACert(cfg) }},
+	)
 
 	for _, s := range steps {
 		slog.Info("installer", "step", s.name)
@@ -62,90 +78,83 @@ func Run(cfg *Config) error {
 	}
 
 	slog.Info("installation complete")
+	refreshBootToolNote(cfg.Storage)
 	fireInstallCallback()
 	promptRemoveUSB()
 	return reboot()
 }
 
-func partitionDisk(disk string) error {
-	// GPT table with three partitions:
-	//   p1: 1MiB BIOS Boot Partition — required for grub-install i386-pc on GPT
-	//   p2: 512MiB EFI System Partition
-	//   p3: remainder as root (ext4)
-	if err := run("parted", "--script", disk,
-		"mklabel", "gpt",
-		"mkpart", "bios_boot", "1MiB", "2MiB",
-		"set", "1", "bios_grub", "on",
-		"mkpart", "ESP", "fat32", "2MiB", "514MiB",
-		"set", "2", "esp", "on",
-		"mkpart", "root", "ext4", "514MiB", "100%",
-	); err != nil {
+// ext4RootSteps formats and mounts a single-disk ext4 root.
+func ext4RootSteps(cfg *Config) []step {
+	return []step{
+		{"format partitions", func() error { return formatPartitions(cfg.Storage) }},
+		{"mount partitions", func() error { return mountPartitions(cfg.Storage) }},
+	}
+}
+
+// zfsRootSteps builds the root pool. The ESPs are formatted here rather than
+// with the bootloader so that a failure to make a filesystem on one of them is
+// reported before the rootfs has been copied.
+func zfsRootSteps(cfg *Config) []step {
+	// Resolved once, before any step runs, so the values written to the pool,
+	// to modprobe.d and to the daemon's reserve are all the same numbers.
+	var opts ZFSOpts
+	return []step{
+		{"format ESPs", func() error { return formatESPs(cfg.Storage) }},
+		{"load zfs module", loadZFSModule},
+		{"create zfs pool", func() error {
+			opts = resolveZFSOpts(cfg.Storage)
+			slog.Info("zfs pool", "topology", cfg.Storage.FS, "disks", len(cfg.Storage.Disks),
+				"ashift", opts.Ashift, "compress", opts.Compress, "arc_max_mib", opts.ARCMaxMiB)
+			return createPool(cfg.Storage, opts)
+		}},
+		{"create zfs datasets", createDatasets},
+		{"configure zfs", func() error { return writeZFSSystemConfig(opts) }},
+	}
+}
+
+// cleanupTarget releases everything the install mounted, in the right order for
+// the filesystem in use.
+func cleanupTarget(cfg DiskConfig) {
+	unbindChrootMounts()
+	if cfg.FS.IsZFS() {
+		// Recursive: the layout is a dozen nested datasets, and leaving one
+		// mounted is enough to block the export.
+		_ = runQuiet("umount", "-R", mountRoot)
+		exportPool()
+		return
+	}
+	_ = runQuiet("umount", efiPart())
+	_ = runQuiet("umount", mountRoot)
+}
+
+func formatPartitions(cfg DiskConfig) error {
+	if err := formatESPs(cfg); err != nil {
 		return err
 	}
-	// Force the kernel to re-read the partition table and wait for udev to
-	// create the partition device nodes. Without this, mkfs.fat in the next
-	// step may race and fail with "No such file or directory" on /dev/sda2 —
-	// the kernel has accepted the new layout but /dev hasn't been populated
-	// yet. Trixie's udev seems slower at this than Bookworm's was.
-	return waitForPartitions(disk)
+	return run("mkfs.ext4", "-F", cfg.Primary().PartitionPath(rootPartNum))
 }
 
-// waitForPartitions ensures the EFI and root partition device nodes exist
-// after parted creates them. It runs partprobe (kernel re-read) and
-// udevadm settle (wait for queued events), then polls /dev for the files.
-func waitForPartitions(disk string) error {
-	// Best-effort: partprobe failure isn't fatal — udev may still pick up
-	// the change from the BLKRRPART ioctl that parted itself issued.
-	if err := run("partprobe", disk); err != nil {
-		slog.Warn("partprobe failed, continuing", "disk", disk, "err", err)
-	}
-	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
-		slog.Warn("udevadm settle failed, continuing", "err", err)
-	}
-	efi, root := partitionPaths(disk)
-	deadline := time.Now().Add(15 * time.Second)
-	for _, part := range []string{efi, root} {
-		for {
-			if _, err := os.Stat(part); err == nil {
-				break
-			}
-			if time.Now().After(deadline) {
-				return fmt.Errorf("partition device %s did not appear within timeout — kernel/udev did not pick up new partition table", part)
-			}
-			time.Sleep(200 * time.Millisecond)
-		}
-	}
-	return nil
-}
-
-func formatPartitions(disk string) error {
-	efi, root := partitionPaths(disk)
-	if err := run("mkfs.fat", "-F32", efi); err != nil {
-		return err
-	}
-	return run("mkfs.ext4", "-F", root)
-}
-
-func mountPartitions(disk string) error {
-	_, root := partitionPaths(disk)
+func mountPartitions(cfg DiskConfig) error {
+	d := cfg.Primary()
 	if err := os.MkdirAll(mountRoot, 0o755); err != nil {
 		return err
 	}
-	if err := run("mount", root, mountRoot); err != nil {
+	if err := run("mount", d.PartitionPath(rootPartNum), mountRoot); err != nil {
 		return err
 	}
-	if err := os.MkdirAll(efiPart, 0o755); err != nil {
+	if err := os.MkdirAll(efiPart(), 0o755); err != nil {
 		return err
 	}
-	efi, _ := partitionPaths(disk)
-	return run("mount", efi, efiPart)
+	return run("mount", d.PartitionPath(espPartNum), efiPart())
 }
 
 // copyRootfs copies the live squashfs environment onto the target disk using
 // rsync. This is the air-gapped alternative to debootstrap — all packages are
 // already embedded in the ISO so no network access is required.
-func copyRootfs() error {
-	if err := run("rsync", "-aHAX", "--delete", "--info=progress2",
+func copyRootfs(cfg DiskConfig) error {
+	args := []string{
+		"-aHAX", "--delete", "--info=progress2",
 		"--exclude=/proc",
 		"--exclude=/sys",
 		"--exclude=/dev",
@@ -161,8 +170,10 @@ func copyRootfs() error {
 		"--exclude=/etc/ssh/ssh_host_*",
 		"--exclude=/lost+found",
 		"--exclude=/boot/efi",
-		"/", mountRoot+"/",
-	); err != nil {
+	}
+	args = append(args, datasetProtectFilters(cfg)...)
+	args = append(args, "/", mountRoot+"/")
+	if err := run("rsync", args...); err != nil {
 		return err
 	}
 
@@ -278,7 +289,7 @@ func installSpinifex(cfg *Config) error {
 	// MANAGEMENT_IFACE is the bridge (br-wan), not the physical NIC.
 	// MANAGEMENT_IP is empty for DHCP — banner's --boot-check fills it in at boot.
 	nodeConf := fmt.Sprintf("MANAGEMENT_IP=%s\nMANAGEMENT_IFACE=br-wan\nNODE_HOSTNAME=%s\n",
-		cfg.WANAddress, cfg.Hostname)
+		cfg.WAN.Address, cfg.Hostname)
 	confDir := filepath.Join(mountRoot, "etc/spinifex")
 	if err := os.MkdirAll(confDir, 0o755); err != nil {
 		return err
@@ -309,23 +320,47 @@ func writeNetworkConfig(cfg *Config) error {
 		return err
 	}
 
-	if err := writeNetworkdBridge(netdDir, cfg.WANInterface, "br-wan", false,
-		cfg.WANDHCPMode, cfg.WANAddress, cfg.WANMask, cfg.WANGateway, cfg.WANDNS,
-		cfg.WANWiFiSSID, cfg.WANWiFiPass); err != nil {
-		return err
+	if err := cfg.Validate(); err != nil {
+		return fmt.Errorf("network roles: %w", err)
 	}
 
-	if cfg.LANInterface != "" {
-		if err := writeNetworkdBridge(netdDir, cfg.LANInterface, "br-lan", true,
-			cfg.LANDHCPMode, cfg.LANAddress, cfg.LANMask, "", cfg.LANDNS,
-			cfg.LANWiFiSSID, cfg.LANWiFiPass); err != nil {
+	// A physical NIC is configured once even when two roles share it via
+	// VLANs, so parent-interface files are keyed by NIC rather than by plane.
+	parents := map[string]*parentNIC{}
+	for _, rb := range cfg.Roles() {
+		p, ok := parents[rb.Role.Interface]
+		if !ok {
+			p = &parentNIC{}
+			parents[rb.Role.Interface] = p
+		}
+		if rb.Role.VLAN > 0 {
+			p.vlans = append(p.vlans, rb.Role.Link())
+		} else {
+			p.bridge = rb.Plane.Bridge()
+		}
+		// The parent must carry at least the largest MTU of any VLAN riding it.
+		if rb.Role.MTU > p.mtu {
+			p.mtu = rb.Role.MTU
+		}
+	}
+	for iface, p := range parents {
+		if err := writeParentNIC(netdDir, iface, p); err != nil {
 			return err
 		}
-		// br-lan timing is controlled by spinifex-lan-bridge.service, not
-		// networkd auto-activation — ActivationPolicy=manual in the .network
-		// file ensures networkd creates the bridge but does not bring it up.
-		if err := systemd.WriteLANBridgeUnit(mountRoot); err != nil {
-			return fmt.Errorf("lan bridge unit: %w", err)
+	}
+
+	for _, rb := range cfg.Roles() {
+		// Only br-wan auto-activates; the east-west bridges are brought up by
+		// their own unit after network-online.target so a missing cable or a
+		// DHCP timeout on them can never stall the management path.
+		manual := rb.Plane != PlaneWAN
+		if err := writeNetworkdBridge(netdDir, rb.Plane, rb.Role, manual); err != nil {
+			return err
+		}
+		if manual {
+			if err := systemd.WriteBridgeUnit(mountRoot, rb.Plane.Bridge()); err != nil {
+				return fmt.Errorf("%s bridge unit: %w", rb.Plane, err)
+			}
 		}
 	}
 
@@ -353,9 +388,9 @@ func writeNetworkConfig(cfg *Config) error {
 
 	// Disable IPv6 via sysctl — belt-and-suspenders alongside IPv6AcceptRA=no
 	// in the networkd .network files.
-	bridges := []string{"br-wan"}
-	if cfg.LANInterface != "" {
-		bridges = append(bridges, "br-lan")
+	var bridges []string
+	for _, rb := range cfg.Roles() {
+		bridges = append(bridges, rb.Plane.Bridge())
 	}
 	var sysctl strings.Builder
 	sysctl.WriteString("# Generated by Spinifex installer — IPv6 disabled on management bridges\n")
@@ -377,10 +412,14 @@ func writeNetworkConfig(cfg *Config) error {
 		return err
 	}
 	var udevRules strings.Builder
-	for _, iface := range []string{cfg.WANInterface, cfg.LANInterface} {
-		if iface == "" {
+	pinned := map[string]bool{}
+	for _, rb := range cfg.Roles() {
+		iface := rb.Role.Interface
+		// Two roles can share one NIC via VLANs; pin the physical name once.
+		if iface == "" || pinned[iface] {
 			continue
 		}
+		pinned[iface] = true
 		mac, err := os.ReadFile("/sys/class/net/" + iface + "/address")
 		if err != nil {
 			slog.Warn("writeNetworkConfig: could not read NIC MAC, skipping udev pin", "iface", iface, "err", err)
@@ -395,35 +434,90 @@ func writeNetworkConfig(cfg *Config) error {
 	return nil
 }
 
-// writeNetworkdBridge writes the three systemd-networkd files that configure a
-// physical NIC enslaved to a Linux bridge:
+// parentNIC accumulates how one physical interface is used across roles. A NIC
+// is either enslaved directly to a bridge (untagged) or carries VLAN
+// subinterfaces — never both for the same plane.
+type parentNIC struct {
+	bridge string
+	vlans  []string
+	mtu    int
+}
+
+// writeParentNIC writes the .network file for a physical interface. An
+// untagged NIC is enslaved straight to its bridge; a tagged one declares its
+// VLAN subinterfaces and stays unbridged, since the VLAN devices are what get
+// enslaved.
+func writeParentNIC(dir, iface string, p *parentNIC) error {
+	var b strings.Builder
+	fmt.Fprintf(&b, "[Match]\nName=%s\n\n", iface)
+	// A physical NIC here is either a bridge port or a VLAN trunk, and neither
+	// carries the address — the bridge above it does. Such a link never reaches
+	// the "degraded" state systemd-networkd-wait-online requires, so leaving it
+	// required makes that unit fail on every boot and stall until it times out.
+	b.WriteString("[Link]\nRequiredForOnline=no\n")
+	if p.mtu > 0 {
+		fmt.Fprintf(&b, "MTUBytes=%d\n", p.mtu)
+	}
+	b.WriteString("\n[Network]\n")
+	if p.bridge != "" {
+		fmt.Fprintf(&b, "Bridge=%s\n", p.bridge)
+	}
+	for _, v := range p.vlans {
+		fmt.Fprintf(&b, "VLAN=%s\n", v)
+	}
+	b.WriteString("IPv6AcceptRA=no\n")
+	return os.WriteFile(filepath.Join(dir, fmt.Sprintf("05-spinifex-nic-%s.network", iface)), []byte(b.String()), 0o644)
+}
+
+// writeNetworkdBridge writes the systemd-networkd files that put a plane's
+// address on its own Linux bridge:
 //
-//   - {prio}-spinifex-{suffix}-nic.network  enslaves the NIC to the bridge
-//   - {prio+1}-spinifex-{suffix}.netdev     declares the bridge device
-//   - {prio+1}-spinifex-{suffix}.network    configures IP on the bridge
+//   - 1{n}-spinifex-{plane}-vlan.netdev   VLAN subinterface (tagged roles only)
+//   - 1{n}-spinifex-{plane}-vlan.network  enslaves the VLAN device to the bridge
+//   - 2{n}-spinifex-{plane}.netdev        declares the bridge device
+//   - 2{n}-spinifex-{plane}.network       configures IP on the bridge
+//
+// The physical NIC itself is written separately by writeParentNIC, because two
+// roles may share one NIC and it must be configured exactly once.
 //
 // manual=true sets ActivationPolicy=manual so networkd creates the bridge but
-// does not auto-activate it — used for br-lan, which spinifex-lan-bridge.service
-// activates after network-online.target.
-func writeNetworkdBridge(dir, nicIface, bridgeName string, manual, dhcp bool,
-	addr, mask, gw string, dns []string, wifiSSID, wifiPass string) error {
-	suffix := strings.TrimPrefix(bridgeName, "br-")
-	nicPrio, brPrio := 10, 11
-	if manual {
-		nicPrio, brPrio = 12, 13
+// does not auto-activate it — used for br-lan and br-vpc, which their own
+// spinifex-{plane}-bridge.service activates after network-online.target.
+func writeNetworkdBridge(dir string, plane Plane, role NetworkRole, manual bool) error {
+	name := string(plane)
+	bridgeName := plane.Bridge()
+
+	if role.VLAN > 0 {
+		link := role.Link()
+		vlanNetdev := fmt.Sprintf("[NetDev]\nName=%s\nKind=vlan\n\n[VLAN]\nId=%d\n", link, role.VLAN)
+		if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("10-spinifex-%s-vlan.netdev", name)), []byte(vlanNetdev), 0o644); err != nil {
+			return err
+		}
+		var v strings.Builder
+		fmt.Fprintf(&v, "[Match]\nName=%s\n\n", link)
+		// Enslaved to the bridge below, so it is not what "online" means here.
+		v.WriteString("[Link]\nRequiredForOnline=no\n")
+		if role.MTU > 0 {
+			fmt.Fprintf(&v, "MTUBytes=%d\n", role.MTU)
+		}
+		fmt.Fprintf(&v, "\n[Network]\nBridge=%s\nIPv6AcceptRA=no\n", bridgeName)
+		if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("11-spinifex-%s-vlan.network", name)), []byte(v.String()), 0o644); err != nil {
+			return err
+		}
 	}
 
-	// NIC .network — enslaves the physical NIC to the bridge.
-	nicNet := fmt.Sprintf("[Match]\nName=%s\n\n[Network]\nBridge=%s\n", nicIface, bridgeName)
-	nicFile := fmt.Sprintf("%02d-spinifex-%s-nic.network", nicPrio, suffix)
-	if err := os.WriteFile(filepath.Join(dir, nicFile), []byte(nicNet), 0o644); err != nil {
-		return err
+	// Without a policy systemd-networkd invents a MAC for the bridge, so the node
+	// appears on the wire as an address that belongs to no NIC — the DHCP server
+	// sees a request whose source does not match its own chaddr and the unicast
+	// reply goes nowhere. "none" leaves the kernel to inherit the enslaved port's
+	// address, which is what a bridged uplink is supposed to present, and keeps
+	// it stable across boots for switch MAC tables and DHCP reservations.
+	brNetdev := fmt.Sprintf("[NetDev]\nName=%s\nKind=bridge\nMACAddressPolicy=none\n", bridgeName)
+	if role.MTU > 0 {
+		brNetdev += fmt.Sprintf("MTUBytes=%d\n", role.MTU)
 	}
-
-	// Bridge .netdev — declares the bridge device.
-	brNetdev := fmt.Sprintf("[NetDev]\nName=%s\nKind=bridge\n\n[Bridge]\nSTP=no\n", bridgeName)
-	brNetdevFile := fmt.Sprintf("%02d-spinifex-%s.netdev", brPrio, suffix)
-	if err := os.WriteFile(filepath.Join(dir, brNetdevFile), []byte(brNetdev), 0o644); err != nil {
+	brNetdev += "\n[Bridge]\nSTP=no\n"
+	if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("20-spinifex-%s.netdev", name)), []byte(brNetdev), 0o644); err != nil {
 		return err
 	}
 
@@ -434,36 +528,43 @@ func writeNetworkdBridge(dir, nicIface, bridgeName string, manual, dhcp bool,
 		b.WriteString("[Link]\nActivationPolicy=manual\nRequiredForOnline=no\n\n")
 	}
 	b.WriteString("[Network]\n")
-	if dhcp {
+	if role.DHCPMode {
 		b.WriteString("DHCP=ipv4\n")
 	} else {
-		cidr, err := addrCIDR(addr, mask)
+		cidr, err := addrCIDR(role.Address, role.Mask)
 		if err != nil {
 			return fmt.Errorf("bridge %s: %w", bridgeName, err)
 		}
 		fmt.Fprintf(&b, "Address=%s\n", cidr)
-		if gw != "" {
-			fmt.Fprintf(&b, "Gateway=%s\n", gw)
+		// Only the wan plane installs a default route; the east-west planes
+		// are link-local to the rack.
+		if role.Gateway != "" && plane == PlaneWAN {
+			fmt.Fprintf(&b, "Gateway=%s\n", role.Gateway)
 		}
-		for _, ns := range dns {
-			if ns = strings.TrimSpace(ns); ns != "" {
-				fmt.Fprintf(&b, "DNS=%s\n", ns)
+		// Resolvers follow the default route for the same reason. A resolver
+		// pinned to an east-west bridge is unreachable — those bridges are
+		// link-local and activate manually — but resolved would still try it,
+		// putting a timeout in front of every lookup on the node.
+		if plane == PlaneWAN {
+			for _, ns := range role.DNS {
+				if ns = strings.TrimSpace(ns); ns != "" {
+					fmt.Fprintf(&b, "DNS=%s\n", ns)
+				}
 			}
 		}
 	}
 	b.WriteString("IPv6AcceptRA=no\nConfigureWithoutCarrier=yes\n")
-	if manual && dhcp {
-		// br-lan is non-critical; fail fast on DHCP so a missing LAN cable
-		// does not stall spinifex-lan-bridge.service indefinitely.
+	if manual && role.DHCPMode {
+		// The east-west bridges are non-critical; fail fast on DHCP so a
+		// missing cable does not stall their activation unit indefinitely.
 		b.WriteString("\n[DHCP]\nTimeout=10\n")
 	}
-	brNetFile := fmt.Sprintf("%02d-spinifex-%s.network", brPrio, suffix)
-	if err := os.WriteFile(filepath.Join(dir, brNetFile), []byte(b.String()), 0o644); err != nil {
+	if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("20-spinifex-%s.network", name)), []byte(b.String()), 0o644); err != nil {
 		return err
 	}
 
-	if wifiSSID != "" {
-		if err := writeWPASupplicant(nicIface, wifiSSID, wifiPass); err != nil {
+	if role.WiFiSSID != "" {
+		if err := writeWPASupplicant(role.Interface, role.WiFiSSID, role.WiFiPass); err != nil {
 			return err
 		}
 	}
@@ -513,7 +614,59 @@ func writeWPASupplicant(nicIface, ssid, psk string) error {
 	return os.Symlink("/lib/systemd/system/wpa_supplicant@.service", link)
 }
 
-func installBootloader(disk string) error {
+func installBootloader(cfg DiskConfig) error {
+	// Branding assets have to be in place before either path regenerates
+	// grub.cfg — the ZFS path copies them out of /boot onto each ESP.
+	copySplashImage(mountRoot)
+	copyGrubFont(mountRoot)
+
+	if cfg.FS.IsZFS() {
+		// The initramfs must be able to import the pool before the ESPs are
+		// synced, since refresh copies whatever initrd exists at that moment.
+		if err := buildZFSInitramfs(); err != nil {
+			return err
+		}
+		return installBootToolZFS(cfg)
+	}
+	return installBootloaderExt4(cfg.Primary())
+}
+
+// buildZFSInitramfs regenerates the target's initramfs so it carries the ZFS
+// module, the pool cache and the hostid. The image copied from the live ISO was
+// built for a machine with an ext4 root and cannot import a pool.
+func buildZFSInitramfs() error {
+	// ZFS refuses to import a pool last touched by a different host without
+	// -f, and the hostid the initramfs sees must match the one recorded on the
+	// pool — otherwise every boot stops in the initramfs shell.
+	if err := run("cp", "-f", "/etc/hostid", filepath.Join(mountRoot, "etc/hostid")); err != nil {
+		slog.Warn("could not copy /etc/hostid, generating one in the target", "err", err)
+		if err := run("zgenhostid", "-f", "-o", filepath.Join(mountRoot, "etc/hostid")); err != nil {
+			return fmt.Errorf("zgenhostid: %w", err)
+		}
+	}
+
+	// The pool was created with cachefile=none so the live environment would
+	// not claim it. The installed system needs the cache to import at boot
+	// without scanning every block device.
+	cacheDir := filepath.Join(mountRoot, "etc/zfs")
+	if err := os.MkdirAll(cacheDir, 0o755); err != nil {
+		return err
+	}
+	if err := run("zpool", "set", "cachefile="+filepath.Join(cacheDir, "zpool.cache"), ZFSPoolName); err != nil {
+		return fmt.Errorf("set zpool cachefile: %w", err)
+	}
+
+	if err := bindChrootMounts(); err != nil {
+		return err
+	}
+	defer unbindChrootMounts()
+	if err := run("chroot", mountRoot, "update-initramfs", "-u", "-k", "all"); err != nil {
+		return fmt.Errorf("update-initramfs: %w", err)
+	}
+	return nil
+}
+
+func installBootloaderExt4(disk Disk) error {
 	// grub-install runs in the live environment (not chroot) using the
 	// grub-pc-bin and grub-efi-amd64-bin packages already present on the ISO.
 	// --boot-directory points at the installed system's /boot.
@@ -535,7 +688,7 @@ func installBootloader(disk string) error {
 		"--target=i386-pc",
 		"--boot-directory="+bootDir,
 		"--recheck",
-		disk,
+		disk.Path,
 	); biosErr != nil {
 		if efiErr != nil {
 			// Both targets failed — the system will not boot.
@@ -543,56 +696,9 @@ func installBootloader(disk string) error {
 		}
 		return biosErr
 	}
-	// Copy splash image and unicode font from the ISO (mounted at /cdrom) so the
-	// installed GRUB shows the same branded background as the installer GRUB.
-	// The font must be at /boot/grub/fonts/unicode.pf2 so update-grub finds it
-	// there and emits the same loadfont path as the ISO's grub.cfg — GRUB 2.12
-	// (trixie) needs the font in the boot partition, not just /usr/share/grub/.
-	copySplashImage(mountRoot)
-	copyGrubFont(mountRoot)
 
-	// Both consoles listed: tty0 stays primary for local install, ttyS0 mirrors
-	// kernel output to serial so headless installs (CI, racks with serial-only
-	// IPMI, remote-dev SSH-to-QEMU) see boot messages. The last `console=` on
-	// the cmdline becomes the system console, so ttyS0 wins on serial-only
-	// hardware while tty0 still receives output for local display.
-	grubDefault := `GRUB_DEFAULT=0
-GRUB_TIMEOUT=5
-GRUB_DISTRIBUTOR=Spinifex
-GRUB_CMDLINE_LINUX_DEFAULT=""
-GRUB_CMDLINE_LINUX="console=tty0 console=ttyS0,115200 systemd.show_status=1"
-`
-
-	if err := os.WriteFile(filepath.Join(mountRoot, "etc/default/grub"), []byte(grubDefault), 0o644); err != nil {
-		return fmt.Errorf("write /etc/default/grub: %w", err)
-	}
-
-	// Mirror the ISO grub.cfg graphical block exactly so the installed GRUB menu
-	// looks identical to the installer menu. gfxterm MUST be activated before
-	// serial is appended — background_image silently does nothing in text mode.
-	// Using exec tail so update-grub includes everything from line 3 as raw GRUB config.
-	grubTheme := `#!/bin/sh
-exec tail -n +3 $0
-insmod all_video
-insmod font
-if loadfont /boot/grub/fonts/unicode.pf2; then
-  set gfxmode=auto
-  insmod gfxterm
-  terminal_output gfxterm
-fi
-insmod serial
-if serial --unit=0 --speed=115200 --timeout=1; then
-  terminal_input  --append serial
-  terminal_output --append serial
-fi
-
-# --- Branding ---
-insmod png
-set theme=/boot/grub/theme.txt
-export theme
-`
-	if err := os.WriteFile(filepath.Join(mountRoot, "etc/grub.d/06_spinifex"), []byte(grubTheme), 0o755); err != nil {
-		return fmt.Errorf("write /etc/grub.d/06_spinifex: %w", err)
+	if err := writeGrubDefaults(DiskConfig{FS: FSExt4}); err != nil {
+		return err
 	}
 
 	if err := bindChrootMounts(); err != nil {
@@ -667,42 +773,247 @@ func reboot() error {
 
 // toFirstbootConfig maps installer Config to the firstboot package's Config.
 func (c *Config) toFirstbootConfig() firstboot.Config {
-	// Geneve encap IP: use LAN bridge IP when a dedicated LAN NIC is present,
-	// otherwise fall back to WAN bridge IP. Empty for DHCP — setup-ovn.sh
+	// Both addresses come from their plane after collapsing, so a single-NIC
+	// node resolves them to the wan address and a three-plane rack keeps
+	// Geneve on vpc and cluster traffic on lan. Empty for DHCP — setup-ovn.sh
 	// auto-detects the IP from the default route at boot in that case.
-	encapIP := c.WANAddress
-	if c.LANInterface != "" && c.LANAddress != "" {
-		encapIP = c.LANAddress
-	}
 	return firstboot.Config{
 		Hostname:        c.Hostname,
-		EncapIP:         encapIP,
-		ClusterRole:     c.ClusterRole,
-		JoinAddr:        c.JoinAddr,
+		EncapIP:         c.PlaneAddress(PlaneVPC),
+		LANIP:           c.PlaneAddress(PlaneLAN),
+		WANIP:           c.PlaneAddress(PlaneWAN),
 		Email:           c.Email,
 		InstallCallback: strings.TrimSpace(os.Getenv("SPINIFEX_INSTALL_CALLBACK")),
 		SkipFormation:   c.SkipFormation,
 	}
 }
 
-// writeFstab writes /etc/fstab on the installed system using partition UUIDs so
-// the root filesystem is mounted read-write at boot and the EFI partition is
-// mounted at /boot/efi.
-func writeFstab(disk string) error {
-	efi, root := partitionPaths(disk)
-	rootUUID, err := partUUID(root)
-	if err != nil {
-		return fmt.Errorf("get root UUID: %w", err)
+// writeFstab writes /etc/fstab on the installed system.
+//
+// On ZFS the root and every dataset are mounted by ZFS itself from properties
+// stored on the pool, and the ESPs are mounted on demand by spinifex-boot-tool,
+// so the file holds nothing but swap.
+func writeFstab(cfg DiskConfig) error {
+	fstab := "# /etc/fstab — generated by Spinifex installer\n"
+
+	if !cfg.FS.IsZFS() {
+		d := cfg.Primary()
+		rootUUID, err := partUUID(d.PartitionPath(rootPartNum))
+		if err != nil {
+			return fmt.Errorf("get root UUID: %w", err)
+		}
+		efiUUID, err := partUUID(d.PartitionPath(espPartNum))
+		if err != nil {
+			return fmt.Errorf("get EFI UUID: %w", err)
+		}
+		fstab += fmt.Sprintf("UUID=%s / ext4 errors=remount-ro 0 1\nUUID=%s /boot/efi vfat umask=0077 0 1\n",
+			rootUUID, efiUUID)
 	}
-	efiUUID, err := partUUID(efi)
-	if err != nil {
-		return fmt.Errorf("get EFI UUID: %w", err)
+
+	// Only claim the swap the previous step actually produced; systemd fails the
+	// boot on a missing swap unit rather than ignoring the line.
+	switch {
+	case cfg.FS.IsZFS():
+		if _, err := os.Stat("/dev/zvol/" + swapZvolName); err == nil {
+			fstab += fmt.Sprintf("/dev/zvol/%s none swap discard 0 0\n", swapZvolName)
+		}
+	default:
+		if _, err := os.Stat(filepath.Join(mountRoot, swapFileName)); err == nil {
+			fstab += fmt.Sprintf("/%s none swap sw 0 0\n", swapFileName)
+		}
 	}
-	fstab := fmt.Sprintf("# /etc/fstab — generated by Spinifex installer\nUUID=%s / ext4 errors=remount-ro 0 1\nUUID=%s /boot/efi vfat umask=0077 0 1\n",
-		rootUUID, efiUUID)
 	return os.WriteFile(filepath.Join(mountRoot, "etc/fstab"), []byte(fstab), 0o644)
 }
 
+const (
+	swapFileName = "swapfile"
+	// Nodes below this get half their RAM, which keeps a 4 GiB test VM to a
+	// 2 GiB file instead of eating its disk. At or above it the machine is a
+	// real host with a real disk, so swap is worth sizing for large imports.
+	swapRAMThreshold = 32 << 30
+	swapLargeBytes   = 64 << 30
+	// Below this there is not enough to be worth the space it costs.
+	swapMinBytes = 512 << 20
+	// Hard ceiling as a share of free disk, whatever RAM suggests: a 64 GiB file
+	// must never fill a small root filesystem.
+	swapDiskShare = 4
+)
+
+// createSwapFile lays down /swapfile on the installed root, sized to the disk.
+//
+// Importing an image streams the decompressed disk through viperblock into
+// predastore, which a 2-4 GiB node cannot hold in RAM — the kernel OOM-kills the
+// import mid-flush. Swap trades speed for completing the operation at all. It is
+// a stopgap for the memory profile of that path, not a fix for it.
+//
+// Failure is logged and not fatal: a node that boots without swap is worse at
+// large imports but is otherwise fine, and losing a whole install over it would
+// be the greater harm.
+// swapZvolName is the swap volume on a ZFS root. ZFS cannot host a swap *file*
+// at all, so a zvol is the only option — the same one Proxmox uses.
+const swapZvolName = ZFSPoolName + "/swap"
+
+// createSwap lays down swap in whichever form the root filesystem supports.
+func createSwap(cfg DiskConfig) error {
+	if cfg.FS.IsZFS() {
+		return createSwapZvol()
+	}
+	return createSwapFile()
+}
+
+// createSwapZvol creates the swap volume on a ZFS root.
+//
+// The properties are not tuning preferences. A zvol swapping under memory
+// pressure can deadlock against the ARC, and these are the settings that make
+// it survivable: page-sized blocks so a swap-out is one record, no data
+// caching so swap pages are never copied into the ARC, and sync writes so a
+// page is on disk before the kernel frees it.
+func createSwapZvol() error {
+	size, err := swapSize(mountRoot)
+	if err != nil {
+		slog.Warn("swap: cannot size swap volume, continuing without one", "err", err)
+		return nil
+	}
+	if size == 0 {
+		slog.Warn("swap: pool too small for a useful swap volume, continuing without one")
+		return nil
+	}
+
+	if err := run("zfs", "create",
+		"-V", strconv.FormatInt(size, 10),
+		"-b", strconv.Itoa(os.Getpagesize()),
+		"-o", "logbias=throughput",
+		"-o", "sync=always",
+		"-o", "primarycache=metadata",
+		"-o", "secondarycache=none",
+		"-o", "compression=zle",
+		"-o", "com.sun:auto-snapshot=false",
+		swapZvolName,
+	); err != nil {
+		slog.Warn("swap: could not create swap volume, continuing without swap", "err", err)
+		return nil
+	}
+
+	dev := "/dev/zvol/" + swapZvolName
+	if err := waitForPath(dev, time.Now().Add(10*time.Second)); err != nil {
+		slog.Warn("swap: zvol device node never appeared, continuing without swap", "err", err)
+		return nil
+	}
+	if err := run("mkswap", "-f", dev); err != nil {
+		slog.Warn("swap: mkswap failed, continuing without swap", "err", err)
+		return nil
+	}
+	slog.Info("swap: created", "zvol", swapZvolName, "bytes", size)
+	return nil
+}
+
+func createSwapFile() error {
+	size, err := swapSize(mountRoot)
+	if err != nil {
+		slog.Warn("swap: cannot size swap file, continuing without one", "err", err)
+		return nil
+	}
+	if size == 0 {
+		slog.Warn("swap: disk too small for a useful swap file, continuing without one",
+			"min_bytes", swapMinBytes)
+		return nil
+	}
+
+	path := filepath.Join(mountRoot, swapFileName)
+	// fallocate rather than dd: on ext4 it reserves real extents, so mkswap and
+	// swapon accept it, and a 32 GiB file costs no write time during install.
+	if err := run("fallocate", "-l", strconv.FormatInt(size, 10), path); err != nil {
+		slog.Warn("swap: fallocate failed, continuing without swap", "err", err)
+		_ = os.Remove(path)
+		return nil
+	}
+	// mkswap refuses a world-readable file, and its contents are process memory.
+	if err := os.Chmod(path, 0o600); err != nil {
+		slog.Warn("swap: chmod failed, continuing without swap", "err", err)
+		_ = os.Remove(path)
+		return nil
+	}
+	if err := run("mkswap", path); err != nil {
+		slog.Warn("swap: mkswap failed, continuing without swap", "err", err)
+		_ = os.Remove(path)
+		return nil
+	}
+	slog.Info("swap: created", "path", "/"+swapFileName, "bytes", size)
+	return nil
+}
+
+// swapSize returns the swap file size for the filesystem mounted at root, or 0
+// when the disk cannot spare a useful amount. SPINIFEX_SWAP_SIZE overrides the
+// calculation; 0 disables swap entirely.
+//
+// RAM drives the size and the disk only caps it, so a small test VM stays cheap
+// on disk while a real host gets enough to carry a large import.
+func swapSize(root string) (int64, error) {
+	if v := strings.TrimSpace(os.Getenv("SPINIFEX_SWAP_SIZE")); v != "" {
+		size, err := parseSize(v)
+		if err != nil {
+			return 0, fmt.Errorf("SPINIFEX_SWAP_SIZE: %w", err)
+		}
+		return size, nil
+	}
+
+	ram, err := totalRAM()
+	if err != nil {
+		return 0, err
+	}
+	size := ram / 2
+	if ram >= swapRAMThreshold {
+		size = swapLargeBytes
+	}
+
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(root, &st); err != nil {
+		return 0, fmt.Errorf("statfs %s: %w", root, err)
+	}
+	// Free space, not total: the rootfs is already copied in, so this is what is
+	// genuinely left to give away.
+	free := int64(st.Bavail) * st.Bsize
+	if diskCap := free / swapDiskShare; size > diskCap {
+		size = diskCap
+	}
+	if size < swapMinBytes {
+		return 0, nil
+	}
+	return size, nil
+}
+
+// totalRAM reports installed memory in bytes.
+func totalRAM() (int64, error) {
+	var si syscall.Sysinfo_t
+	if err := syscall.Sysinfo(&si); err != nil {
+		return 0, fmt.Errorf("sysinfo: %w", err)
+	}
+	unit := int64(si.Unit)
+	if unit == 0 {
+		unit = 1
+	}
+	return int64(si.Totalram) * unit, nil
+}
+
+// parseSize reads a plain byte count or a G/M suffixed size ("32G", "512M").
+func parseSize(s string) (int64, error) {
+	mult := int64(1)
+	switch {
+	case strings.HasSuffix(s, "G"), strings.HasSuffix(s, "g"):
+		mult, s = 1<<30, s[:len(s)-1]
+	case strings.HasSuffix(s, "M"), strings.HasSuffix(s, "m"):
+		mult, s = 1<<20, s[:len(s)-1]
+	}
+	n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("invalid size %q", s)
+	}
+	if n < 0 {
+		return 0, fmt.Errorf("negative size %q", s)
+	}
+	return n * mult, nil
+}
+
 func partUUID(dev string) (string, error) {
 	out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", dev).Output()
 	if err != nil {
@@ -715,18 +1026,6 @@ func partUUID(dev string) (string, error) {
 	return uuid, nil
 }
 
-// partitionPaths returns the EFI and root partition device paths for a given
-// disk. p1 is the BIOS Boot Partition (no filesystem), p2 is EFI, p3 is root.
-// Handles both /dev/sdX (→ /dev/sdX2, /dev/sdX3) and /dev/nvmeXnY
-// (→ /dev/nvmeXnYp2, /dev/nvmeXnYp3).
-func partitionPaths(disk string) (efi, root string) {
-	// NVMe devices use a 'p' separator before the partition number.
-	if len(disk) > 0 && disk[len(disk)-1] >= '0' && disk[len(disk)-1] <= '9' {
-		return disk + "p2", disk + "p3"
-	}
-	return disk + "2", disk + "3"
-}
-
 // copyGrubFont copies the unicode.pf2 GRUB font into the installed system's
 // /boot/grub/fonts/ directory. This ensures update-grub finds the font at
 // /boot/grub/fonts/unicode.pf2 — the same path the ISO's grub.cfg uses —
@@ -836,13 +1135,31 @@ func bindChrootMounts() error {
 // Errors are logged but not returned — this is best-effort cleanup.
 func unbindChrootMounts() {
 	for _, v := range slices.Backward(chrootMountPaths) {
-		_ = run("umount", filepath.Join(mountRoot, v))
+		// Quiet: this also runs from the deferred cleanup, where nothing is
+		// mounted and three "no mount point specified" lines above a real error
+		// message are pure distraction.
+		_ = runQuiet("umount", filepath.Join(mountRoot, v))
 	}
 }
 
-func run(name string, args ...string) error {
+var run = func(name string, args ...string) error {
+	return runEnv(nil, name, args...)
+}
+
+// runQuiet is run with output discarded, for best-effort probes whose failure
+// is expected and whose stderr would otherwise look like an install fault.
+var runQuiet = func(name string, args ...string) error {
+	cmd := exec.Command(name, args...)
+	return cmd.Run()
+}
+
+// runEnv is run with extra environment variables appended to the installer's own.
+var runEnv = func(env []string, name string, args ...string) error {
 	cmd := exec.Command(name, args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
+	if len(env) > 0 {
+		cmd.Env = append(os.Environ(), env...)
+	}
 	return cmd.Run()
 }

cmd/installer/install/install_test.go

@@ -9,7 +9,7 @@ func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
 	const url = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	t.Setenv("SPINIFEX_INSTALL_CALLBACK", url)
 
-	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	cfg := &Config{Hostname: "node1"}
 	fb := cfg.toFirstbootConfig()
 	if fb.InstallCallback != url {
 		t.Errorf("InstallCallback = %q, want %q", fb.InstallCallback, url)
@@ -19,7 +19,7 @@ func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
 func TestToFirstbootConfigEmptyCallbackWhenEnvUnset(t *testing.T) {
 	os.Unsetenv("SPINIFEX_INSTALL_CALLBACK")
 
-	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	cfg := &Config{Hostname: "node1"}
 	fb := cfg.toFirstbootConfig()
 	if fb.InstallCallback != "" {
 		t.Errorf("InstallCallback = %q, want empty", fb.InstallCallback)

cmd/installer/install/steps_test.go

@@ -0,0 +1,821 @@
+package install
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// fakeExec records every command the install steps shell out to and lets a
+// test fail a chosen one. Without it these steps can only be exercised by
+// running a real install against real hardware.
+type fakeExec struct {
+	calls []string
+	fail  func(name string, args []string) error
+}
+
+func fakeCommands(t *testing.T) *fakeExec {
+	t.Helper()
+	f := &fakeExec{}
+	prevRun, prevQuiet, prevEnv := run, runQuiet, runEnv
+
+	record := func(name string, args ...string) error {
+		f.calls = append(f.calls, strings.TrimSpace(name+" "+strings.Join(args, " ")))
+		if f.fail != nil {
+			return f.fail(name, args)
+		}
+		return nil
+	}
+	run = record
+	runQuiet = record
+	runEnv = func(env []string, name string, args ...string) error {
+		return record(name, append(env, args...)...)
+	}
+
+	t.Cleanup(func() { run, runQuiet, runEnv = prevRun, prevQuiet, prevEnv })
+	return f
+}
+
+// ran reports whether any recorded command contains every given fragment.
+func (f *fakeExec) ran(fragments ...string) bool {
+	for _, c := range f.calls {
+		all := true
+		for _, frag := range fragments {
+			if !strings.Contains(c, frag) {
+				all = false
+				break
+			}
+		}
+		if all {
+			return true
+		}
+	}
+	return false
+}
+
+func (f *fakeExec) mustRun(t *testing.T, fragments ...string) {
+	t.Helper()
+	if !f.ran(fragments...) {
+		t.Errorf("no command matched %v; recorded:\n  %s", fragments, strings.Join(f.calls, "\n  "))
+	}
+}
+
+func (f *fakeExec) mustNotRun(t *testing.T, fragments ...string) {
+	t.Helper()
+	if f.ran(fragments...) {
+		t.Errorf("unexpected command matching %v; recorded:\n  %s", fragments, strings.Join(f.calls, "\n  "))
+	}
+}
+
+// indexOf returns the position of the first command matching frag, or -1.
+func (f *fakeExec) indexOf(frag string) int {
+	for i, c := range f.calls {
+		if strings.Contains(c, frag) {
+			return i
+		}
+	}
+	return -1
+}
+
+// indexOfExact is indexOf for a whole command, for cases where one command
+// line is a prefix of another.
+func (f *fakeExec) indexOfExact(cmd string) int {
+	for i, c := range f.calls {
+		if c == cmd {
+			return i
+		}
+	}
+	return -1
+}
+
+// withMountRoot points the install target at a temp tree.
+func withMountRoot(t *testing.T) string {
+	t.Helper()
+	prev := mountRoot
+	mountRoot = t.TempDir()
+	t.Cleanup(func() { mountRoot = prev })
+	return mountRoot
+}
+
+// targetSkeleton creates the directories rsync would already have copied in by
+// the time these steps run.
+func targetSkeleton(t *testing.T, root string) {
+	t.Helper()
+	for _, d := range []string{"etc/default", "etc/grub.d", "etc/systemd/system", "etc"} {
+		if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// tempDisks builds disks whose device nodes are real files, so the steps that
+// wait for partition nodes to appear can complete.
+func tempDisks(t *testing.T, n int, gib int64) []Disk {
+	t.Helper()
+	dir := t.TempDir()
+	out := make([]Disk, n)
+	for i := range out {
+		d := Disk{
+			Path:              filepath.Join(dir, fmt.Sprintf("sd%c", 'a'+i)),
+			Bytes:             gib << 30,
+			LogicalBlockSize:  512,
+			PhysicalBlockSize: 512,
+		}
+		for _, part := range []int{biosPartNum, espPartNum, rootPartNum} {
+			if err := os.WriteFile(d.PartitionPath(part), nil, 0o644); err != nil {
+				t.Fatalf("create partition node: %v", err)
+			}
+		}
+		out[i] = d
+	}
+	return out
+}
+
+func TestPartitionDisksWritesTheSameLayoutOnEveryDisk(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 200)}
+
+	if err := partitionDisks(cfg); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+
+	for _, d := range cfg.Disks {
+		// Stale signatures survive a new table, so the wipe comes first.
+		f.mustRun(t, "wipefs -a "+d.Path)
+		f.mustRun(t, "sgdisk -Z "+d.Path)
+		f.mustRun(t,
+			"sgdisk", d.Path,
+			"-n 1:1M:+1M", "-t 1:EF02", "-c 1:bios_boot",
+			// A 200GiB disk is a real host, so it gets the large ESP.
+			"-n 2:0:+1024M", "-t 2:EF00", "-c 2:ESP",
+			"-n 3:0:-1024M", "-t 3:BF00", "-c 3:root",
+		)
+		f.mustRun(t, "sgdisk -G "+d.Path)
+		f.mustRun(t, "partprobe "+d.Path)
+
+		if wipe, part := f.indexOf("wipefs -a "+d.Path), f.indexOf("-c 3:root "+d.Path); wipe > part {
+			t.Errorf("wipefs ran after the table was written (%d > %d)", wipe, part)
+		}
+	}
+	f.mustRun(t, "udevadm settle --timeout=10")
+}
+
+func TestPartitionDisksTypesTheRootPartitionForTheFilesystem(t *testing.T) {
+	f := fakeCommands(t)
+	if err := partitionDisks(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+	f.mustRun(t, "-t 3:8300")
+	f.mustNotRun(t, "-t 3:BF00")
+	// Under 100GiB the ESP stays at the smaller size.
+	f.mustRun(t, "-n 2:0:+512M")
+}
+
+func TestPartitionDisksNamesTheDiskThatFailed(t *testing.T) {
+	f := fakeCommands(t)
+	disks := tempDisks(t, 2, 40)
+	f.fail = func(name string, args []string) error {
+		if name == "sgdisk" && strings.Contains(strings.Join(args, " "), "bios_boot") {
+			return errors.New("device busy")
+		}
+		return nil
+	}
+
+	err := partitionDisks(DiskConfig{FS: FSExt4, Disks: disks})
+	if err == nil || !strings.Contains(err.Error(), disks[0].Path) {
+		t.Fatalf("err = %v, want it to name %s", err, disks[0].Path)
+	}
+}
+
+func TestPartitionDisksSurvivesBestEffortFailures(t *testing.T) {
+	f := fakeCommands(t)
+	// partprobe, udevadm and the GUID randomisation are all advisory: sgdisk
+	// has already issued BLKRRPART and the nodes exist.
+	f.fail = func(name string, args []string) error {
+		switch {
+		case name == "partprobe", name == "udevadm":
+			return errors.New("not available")
+		case name == "sgdisk" && len(args) > 0 && args[0] == "-G":
+			return errors.New("no")
+		}
+		return nil
+	}
+	if err := partitionDisks(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+}
+
+func TestWaitForPathFailsWhenTheNodeNeverAppears(t *testing.T) {
+	missing := filepath.Join(t.TempDir(), "sda3")
+
+	err := waitForPath(missing, time.Now().Add(-time.Second))
+	if err == nil || !strings.Contains(err.Error(), missing) {
+		t.Fatalf("err = %v, want a timeout naming the missing node", err)
+	}
+}
+
+func TestWaitForPathReturnsOnceTheNodeExists(t *testing.T) {
+	f, err := os.CreateTemp(t.TempDir(), "node")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := waitForPath(f.Name(), time.Now().Add(-time.Second)); err != nil {
+		t.Errorf("waitForPath on an existing node: %v", err)
+	}
+}
+
+func TestEspSizeMiB(t *testing.T) {
+	if got := espSizeMiB(Disk{Bytes: 99 << 30}); got != espSmallMiB {
+		t.Errorf("99GiB -> %d, want %d", got, espSmallMiB)
+	}
+	if got := espSizeMiB(Disk{Bytes: 100 << 30}); got != espLargeMiB {
+		t.Errorf("100GiB -> %d, want %d", got, espLargeMiB)
+	}
+}
+
+func TestClearDiskOnlyLabelclearsAnExistingPartition(t *testing.T) {
+	f := fakeCommands(t)
+	disks := tempDisks(t, 1, 40)
+	if err := clearDisk(disks[0]); err != nil {
+		t.Fatalf("clearDisk: %v", err)
+	}
+	f.mustRun(t, "zpool labelclear -f "+disks[0].PartitionPath(rootPartNum))
+
+	f2 := fakeCommands(t)
+	blank := Disk{Path: filepath.Join(t.TempDir(), "sdb")}
+	if err := clearDisk(blank); err != nil {
+		t.Fatalf("clearDisk: %v", err)
+	}
+	f2.mustNotRun(t, "labelclear")
+	f2.mustRun(t, "wipefs -a "+blank.Path)
+}
+
+func TestFormatESPsLabelsEveryMember(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 40)}
+	if err := formatESPs(cfg); err != nil {
+		t.Fatalf("formatESPs: %v", err)
+	}
+	for _, d := range cfg.Disks {
+		f.mustRun(t, "mkfs.fat -F 32 -n EFI "+d.PartitionPath(espPartNum))
+	}
+}
+
+func TestFormatESPsNamesTheFailingESP(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 40)}
+	f.fail = func(name string, _ []string) error {
+		if name == "mkfs.fat" {
+			return errors.New("no such device")
+		}
+		return nil
+	}
+	err := formatESPs(cfg)
+	if err == nil || !strings.Contains(err.Error(), cfg.Disks[0].PartitionPath(espPartNum)) {
+		t.Fatalf("err = %v, want it to name the ESP", err)
+	}
+}
+
+func TestFormatPartitionsMakesTheExt4Root(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}
+	if err := formatPartitions(cfg); err != nil {
+		t.Fatalf("formatPartitions: %v", err)
+	}
+	f.mustRun(t, "mkfs.ext4 -F "+cfg.Disks[0].PartitionPath(rootPartNum))
+}
+
+func TestMountPartitionsMountsRootThenESP(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}
+
+	if err := mountPartitions(cfg); err != nil {
+		t.Fatalf("mountPartitions: %v", err)
+	}
+	d := cfg.Disks[0]
+	f.mustRun(t, "mount "+d.PartitionPath(rootPartNum)+" "+root)
+	f.mustRun(t, "mount "+d.PartitionPath(espPartNum)+" "+efiPart())
+	if _, err := os.Stat(efiPart()); err != nil {
+		t.Errorf("ESP mountpoint was not created: %v", err)
+	}
+	if f.indexOf("mount "+d.PartitionPath(rootPartNum)) > f.indexOf("mount "+d.PartitionPath(espPartNum)) {
+		t.Error("the ESP was mounted before the root it lives under")
+	}
+}
+
+func TestMountPartitionsStopsWhenTheRootWillNotMount(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "mount" {
+			return errors.New("wrong fs type")
+		}
+		return nil
+	}
+	if err := mountPartitions(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err == nil {
+		t.Fatal("expected an error when the root mount fails")
+	}
+	f.mustNotRun(t, "boot/efi")
+}
+
+func TestCleanupTargetExportsThePoolOnZFS(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	cleanupTarget(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
+
+	// Recursive: one dataset left mounted is enough to block the export.
+	f.mustRun(t, "umount -R "+root)
+	f.mustRun(t, "zpool export "+ZFSPoolName)
+	if f.indexOf("umount -R") > f.indexOf("zpool export") {
+		t.Error("the pool was exported before its datasets were unmounted")
+	}
+}
+
+func TestCleanupTargetUnmountsESPBeforeRootOnExt4(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	cleanupTarget(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
+
+	f.mustNotRun(t, "zpool export")
+	esp, rootIdx := f.indexOfExact("umount "+efiPart()), f.indexOfExact("umount "+root)
+	if esp < 0 || rootIdx < 0 {
+		t.Fatalf("missing umounts; recorded:\n  %s", strings.Join(f.calls, "\n  "))
+	}
+	if esp > rootIdx {
+		t.Error("the root was unmounted before the ESP nested inside it")
+	}
+}
+
+func TestCreatePoolBuildsTheZpoolCommand(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	opts := ZFSOpts{Ashift: 12, Compress: "lz4", Checksum: "blake3", Copies: 1, ARCMaxMiB: 4096}
+
+	if err := createPool(cfg, opts); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f.mustRun(t, "zpool create -f",
+		"cachefile=none", "ashift=12", "compression=lz4", "checksum=blake3", "copies=1",
+		// altroot keeps the datasets under the target rather than over the live system.
+		"-R "+root, ZFSPoolName, "mirror",
+		cfg.Disks[0].StablePartitionPath(rootPartNum),
+		cfg.Disks[1].StablePartitionPath(rootPartNum),
+	)
+}
+
+func TestCreatePoolEnablesAutotrimOnFlashOnly(t *testing.T) {
+	withMountRoot(t)
+	flash := disks(2, 40)
+	f := fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: flash}, ZFSOpts{Ashift: 12, Copies: 1}); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f.mustRun(t, "zpool set autotrim=on "+ZFSPoolName)
+
+	spinning := disks(2, 40)
+	spinning[1].Rotational = true
+	f2 := fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: spinning}, ZFSOpts{Ashift: 12, Copies: 1}); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f2.mustNotRun(t, "autotrim")
+}
+
+func TestCreatePoolWrapsTheZpoolFailure(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, args []string) error {
+		if name == "zpool" && len(args) > 0 && args[0] == "create" {
+			return errors.New("device is in use")
+		}
+		return nil
+	}
+	err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}, ZFSOpts{Ashift: 12, Copies: 1})
+	if err == nil || !strings.Contains(err.Error(), "zpool create") {
+		t.Fatalf("err = %v, want a zpool create failure", err)
+	}
+}
+
+func TestCreatePoolRejectsAnUnbuildableTopology(t *testing.T) {
+	withMountRoot(t)
+	fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: disks(1, 40)}, ZFSOpts{Ashift: 12}); err == nil {
+		t.Fatal("expected a vdev error for a one-disk mirror")
+	}
+}
+
+func TestCreateDatasetsCreatesEveryDataset(t *testing.T) {
+	f := fakeCommands(t)
+	if err := createDatasets(); err != nil {
+		t.Fatalf("createDatasets: %v", err)
+	}
+	for _, ds := range datasets {
+		f.mustRun(t, "zfs create", "mountpoint="+ds.mountpoint, ds.name)
+	}
+}
+
+func TestCreateDatasetsNamesTheDatasetThatFailed(t *testing.T) {
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "zfs" {
+			return errors.New("out of space")
+		}
+		return nil
+	}
+	err := createDatasets()
+	if err == nil || !strings.Contains(err.Error(), datasets[0].name) {
+		t.Fatalf("err = %v, want it to name %s", err, datasets[0].name)
+	}
+}
+
+func TestExportPoolToleratesABusyPool(t *testing.T) {
+	f := fakeCommands(t)
+	f.fail = func(string, []string) error { return errors.New("pool is busy") }
+	exportPool()
+	f.mustRun(t, "zpool export "+ZFSPoolName)
+}
+
+func TestWriteZFSSystemConfigDeclaresTheARCCeilingToBothConsumers(t *testing.T) {
+	root := withMountRoot(t)
+
+	if err := writeZFSSystemConfig(ZFSOpts{ARCMaxMiB: 4096}); err != nil {
+		t.Fatalf("writeZFSSystemConfig: %v", err)
+	}
+
+	modprobe := readFile(t, root, "etc/modprobe.d/zfs.conf")
+	if !strings.Contains(modprobe, fmt.Sprintf("zfs_arc_max=%d", int64(4096)<<20)) {
+		t.Errorf("zfs.conf does not cap the ARC in bytes:\n%s", modprobe)
+	}
+
+	// The daemon cannot see the ARC, so the reserve has to account for it or
+	// guests get admitted into memory the ARC already owns.
+	env := readFile(t, root, "etc/spinifex/host.env")
+	want := fmt.Sprintf("SPINIFEX_RESERVED_MEM_GB=%.1f", defaultHostReserveGB+4.0)
+	if !strings.Contains(env, want) {
+		t.Errorf("host.env = %q, want it to contain %q", env, want)
+	}
+}
+
+func TestWriteTargetFileAppliesTheModeOnRewrite(t *testing.T) {
+	root := withMountRoot(t)
+
+	if err := writeTargetFile("/usr/sbin/tool", []byte("#!/bin/sh\n"), 0o600); err != nil {
+		t.Fatalf("writeTargetFile: %v", err)
+	}
+	// A retried install rewrites an existing file, and WriteFile would leave
+	// the original mode in place.
+	if err := writeTargetFile("/usr/sbin/tool", []byte("#!/bin/sh\ntrue\n"), 0o755); err != nil {
+		t.Fatalf("writeTargetFile rewrite: %v", err)
+	}
+
+	st, err := os.Stat(filepath.Join(root, "usr/sbin/tool"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st.Mode().Perm() != 0o755 {
+		t.Errorf("mode = %o, want 755", st.Mode().Perm())
+	}
+	if got := readFile(t, root, "usr/sbin/tool"); !strings.Contains(got, "true") {
+		t.Errorf("content not rewritten: %q", got)
+	}
+}
+
+func TestWriteGrubDefaultsPinsTheZFSRoot(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeGrubDefaults(DiskConfig{FS: FSZFSRAID1}); err != nil {
+		t.Fatalf("writeGrubDefaults: %v", err)
+	}
+	got := readFile(t, root, "etc/default/grub")
+	if !strings.Contains(got, "root=ZFS="+ZFSRootDataset) || !strings.Contains(got, "boot=zfs") {
+		t.Errorf("grub defaults do not name the ZFS root:\n%s", got)
+	}
+	// Serial is listed last so it wins as the system console on headless racks.
+	if !strings.Contains(got, "console=tty0 console=ttyS0,115200") {
+		t.Errorf("grub defaults do not set both consoles:\n%s", got)
+	}
+	if theme := readFile(t, root, "etc/grub.d/06_spinifex"); !strings.Contains(theme, "terminal_output gfxterm") {
+		t.Errorf("branding snippet missing gfxterm:\n%s", theme)
+	}
+}
+
+func TestWriteGrubDefaultsLeavesExt4RootToGrubProbe(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	if err := writeGrubDefaults(DiskConfig{FS: FSExt4}); err != nil {
+		t.Fatalf("writeGrubDefaults: %v", err)
+	}
+	if got := readFile(t, root, "etc/default/grub"); strings.Contains(got, "root=ZFS=") {
+		t.Errorf("ext4 install should not pin a ZFS root:\n%s", got)
+	}
+}
+
+func TestDivertGrubInstallReplacesTheBinary(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	if err := divertGrubInstall(); err != nil {
+		t.Fatalf("divertGrubInstall: %v", err)
+	}
+	// The diversion is what makes the replacement survive a grub-pc upgrade.
+	f.mustRun(t, "dpkg-divert --divert "+grubInstallReal, "--rename --add "+grubInstallPath)
+	if _, err := os.Stat(filepath.Join(root, strings.TrimPrefix(grubInstallPath, "/"))); err != nil {
+		t.Errorf("wrapper not written: %v", err)
+	}
+}
+
+func TestDivertGrubInstallFailsLoudly(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(string, []string) error { return errors.New("dpkg is locked") }
+
+	if err := divertGrubInstall(); err == nil || !strings.Contains(err.Error(), "divert grub-install") {
+		t.Fatalf("err = %v, want a diversion failure", err)
+	}
+}
+
+// espFixture writes the state a successful boot-tool run leaves behind: the
+// registered ESP list, and a grub.cfg on the ESP that verifyBootConfig mounts.
+func espFixture(t *testing.T, root string, uuids string, grubCfg string) {
+	t.Helper()
+	list := filepath.Join(root, strings.TrimPrefix(bootToolUUIDList, "/"))
+	if err := os.MkdirAll(filepath.Dir(list), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(list, []byte(uuids), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if grubCfg == "" {
+		return
+	}
+	dir := filepath.Join(root, "var/tmp/verify-esp/grub")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "grub.cfg"), []byte(grubCfg), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestVerifyBootConfig(t *testing.T) {
+	goodCfg := "linux /vmlinuz-6.12.0 root=ZFS=" + ZFSRootDataset + " boot=zfs\n"
+
+	tests := []struct {
+		name    string
+		uuids   string
+		grubCfg string
+		wantErr string
+	}{
+		{name: "every ESP registered and pointing at the pool", uuids: "AAAA-1111\nBBBB-2222\n", grubCfg: goodCfg},
+		{
+			name:    "a disk got no bootloader",
+			uuids:   "AAAA-1111\n",
+			grubCfg: goodCfg,
+			wantErr: "expected 2 registered ESPs, found 1",
+		},
+		{
+			// grub-mkconfig degrades quietly when grub-probe cannot read the
+			// pool, and the failure only shows up as a panic at boot.
+			name:    "grub.cfg names no root",
+			uuids:   "AAAA-1111\nBBBB-2222\n",
+			grubCfg: "linux /vmlinuz-6.12.0 root=/dev/sda3\n",
+			wantErr: "would not find its root filesystem",
+		},
+		{
+			name:    "grub.cfg names no kernel",
+			uuids:   "AAAA-1111\nBBBB-2222\n",
+			grubCfg: "set root=ZFS=" + ZFSRootDataset + "\n",
+			wantErr: "references no kernel",
+		},
+		{name: "no ESP list at all", wantErr: "read ESP list"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			root := withMountRoot(t)
+			fakeCommands(t)
+			cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+			if tt.uuids != "" {
+				espFixture(t, root, tt.uuids, tt.grubCfg)
+			}
+
+			err := verifyBootConfig(cfg)
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("verifyBootConfig: %v", err)
+				}
+				return
+			}
+			if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+				t.Fatalf("err = %v, want %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestInstallBootToolZFSSetsUpEveryESP(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	targetSkeleton(t, root)
+	espFixture(t, root, "AAAA-1111\nBBBB-2222\n", "linux /vmlinuz-6.12.0 root=ZFS="+ZFSRootDataset+"\n")
+
+	if err := installBootToolZFS(cfg); err != nil {
+		t.Fatalf("installBootToolZFS: %v", err)
+	}
+
+	if _, err := os.Stat(filepath.Join(root, strings.TrimPrefix(bootToolPath, "/"))); err != nil {
+		t.Errorf("boot tool not installed: %v", err)
+	}
+	// A partially-updated set of ESPs boots today and fails on the next kernel
+	// upgrade, so every disk is initialised under strict mode.
+	for _, d := range cfg.Disks {
+		f.mustRun(t, "SPINIFEX_BOOT_TOOL_STRICT=1", "init", d.PartitionPath(espPartNum))
+	}
+	// NVRAM is written exactly once, by the initial refresh.
+	f.mustRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1", "refresh")
+	f.mustRun(t, "dpkg-divert")
+	for _, m := range chrootMountPaths {
+		f.mustRun(t, "mount --bind /"+m)
+	}
+}
+
+func TestInstallBootToolZFSStopsWhenADiskFails(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	targetSkeleton(t, root)
+	espFixture(t, root, "AAAA-1111\nBBBB-2222\n", "linux /vmlinuz root=ZFS="+ZFSRootDataset+"\n")
+	f.fail = func(name string, args []string) error {
+		if strings.Contains(strings.Join(args, " "), "init") {
+			return errors.New("no ESP")
+		}
+		return nil
+	}
+
+	err := installBootToolZFS(cfg)
+	if err == nil || !strings.Contains(err.Error(), "spinifex-boot-tool init") {
+		t.Fatalf("err = %v, want the init failure", err)
+	}
+	f.mustNotRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1")
+}
+
+func TestRefreshBootToolNoteOnlyAppliesToZFS(t *testing.T) {
+	// Log-only, but it is the operator's only record of the disk-replacement
+	// command, so both branches are walked.
+	refreshBootToolNote(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
+	refreshBootToolNote(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
+}
+
+func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
+	root := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(root, "etc/systemd/system"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	for range 2 {
+		if err := maskSystemdUnit(root, "getty@tty1.service"); err != nil {
+			t.Fatalf("maskSystemdUnit: %v", err)
+		}
+	}
+	target, err := os.Readlink(filepath.Join(root, "etc/systemd/system/getty@tty1.service"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if target != "/dev/null" {
+		t.Errorf("mask target = %q, want /dev/null", target)
+	}
+}
+
+func TestCopyAssetsAreBestEffort(t *testing.T) {
+	root := t.TempDir()
+	// Neither asset exists outside the live ISO; a missing one must not fail
+	// the install, only the branding.
+	copyGrubFont(root)
+	copySplashImage(root)
+}
+
+func TestWriteFstabOnZFSHoldsNothingButSwap(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeFstab(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	got := readFile(t, root, "etc/fstab")
+	// Root and every dataset are mounted from pool properties, and the ESPs on
+	// demand by the boot tool.
+	if strings.Contains(got, " / ") || strings.Contains(got, "/boot/efi") {
+		t.Errorf("ZFS fstab should carry no root or ESP line:\n%s", got)
+	}
+}
+
+func TestWriteNetworkConfigWritesTheWholeNetworkdTree(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeNetworkConfig(threeNIC()); err != nil {
+		t.Fatalf("writeNetworkConfig: %v", err)
+	}
+
+	netd := filepath.Join(root, "etc/systemd/network")
+	for _, plane := range []Plane{PlaneWAN, PlaneLAN, PlaneVPC} {
+		if got := readFile(t, netd, fmt.Sprintf("20-spinifex-%s.netdev", plane)); !strings.Contains(got, "Kind=bridge") {
+			t.Errorf("%s netdev is not a bridge:\n%s", plane, got)
+		}
+	}
+	// Only br-wan may block network-online.target; the east-west bridges are
+	// brought up by their own unit afterwards.
+	if got := readFile(t, netd, "20-spinifex-wan.network"); strings.Contains(got, "ActivationPolicy=manual") {
+		t.Errorf("br-wan must auto-activate:\n%s", got)
+	}
+	if got := readFile(t, netd, "20-spinifex-lan.network"); !strings.Contains(got, "ActivationPolicy=manual") {
+		t.Errorf("br-lan must not auto-activate:\n%s", got)
+	}
+
+	// The live ISO masks wait-online; the mask is copied in by rsync and has to
+	// be lifted, then scoped to br-wan so br-lan never blocks the boot.
+	dropin := readFile(t, root, "etc/systemd/system/systemd-networkd-wait-online.service.d/spinifex-wan-only.conf")
+	if !strings.Contains(dropin, "--interface=br-wan") || !strings.Contains(dropin, "--timeout=60") {
+		t.Errorf("wait-online drop-in is not scoped to br-wan:\n%s", dropin)
+	}
+
+	sysctl := readFile(t, root, "etc/sysctl.d/99-spinifex-network.conf")
+	for _, br := range []string{"br-wan", "br-lan", "br-vpc"} {
+		if !strings.Contains(sysctl, "net.ipv6.conf."+br+".disable_ipv6=1") {
+			t.Errorf("IPv6 not disabled on %s:\n%s", br, sysctl)
+		}
+	}
+}
+
+func TestWriteNetworkConfigRejectsAnInvalidRoleSet(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	cfg := threeNIC()
+	cfg.LAN.Interface = cfg.WAN.Interface
+
+	if err := writeNetworkConfig(cfg); err == nil || !strings.Contains(err.Error(), "network roles") {
+		t.Fatalf("err = %v, want the role validation to reject two planes on one link", err)
+	}
+}
+
+func TestWriteNetworkConfigLiftsTheWaitOnlineMask(t *testing.T) {
+	root := withMountRoot(t)
+	mask := filepath.Join(root, "etc/systemd/system/systemd-networkd-wait-online.service")
+	if err := os.MkdirAll(filepath.Dir(mask), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink("/dev/null", mask); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := writeNetworkConfig(threeNIC()); err != nil {
+		t.Fatalf("writeNetworkConfig: %v", err)
+	}
+	if _, err := os.Lstat(mask); err == nil {
+		t.Error("the wait-online mask survived; firstboot would run before br-wan has a lease")
+	}
+}
+
+func TestBindChrootMountsCreatesEveryMountpoint(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	if err := bindChrootMounts(); err != nil {
+		t.Fatalf("bindChrootMounts: %v", err)
+	}
+	for _, m := range chrootMountPaths {
+		if _, err := os.Stat(filepath.Join(root, m)); err != nil {
+			t.Errorf("mountpoint /%s not created: %v", m, err)
+		}
+		f.mustRun(t, "mount --bind /"+m+" "+filepath.Join(root, m))
+	}
+
+	unbindChrootMounts()
+	// Reverse order: /sys and /proc come out before /dev they were nested under.
+	if f.indexOf("umount "+filepath.Join(root, "sys")) > f.indexOf("umount "+filepath.Join(root, "dev")) {
+		t.Error("chroot mounts were unbound in bind order, not reverse")
+	}
+}
+
+func TestBindChrootMountsReportsAFailedBind(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "mount" {
+			return errors.New("permission denied")
+		}
+		return nil
+	}
+	if err := bindChrootMounts(); err == nil || !strings.Contains(err.Error(), "bind-mount") {
+		t.Fatalf("err = %v, want a bind-mount failure", err)
+	}
+}

cmd/installer/install/zfs.go

@@ -0,0 +1,362 @@
+package install
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math"
+	"math/bits"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// ARC bounds. Spinifex RAM is guest RAM: Proxmox defaults non-PVE products to
+// half of system memory, which on a 256GB compute node would remove 128GB of
+// schedulable instance capacity. 10% capped at 16GiB is their PVE behaviour and
+// the right trade for a hypervisor.
+const (
+	arcMinMiB     = 64
+	arcMaxCapMiB  = 16 * 1024
+	arcFraction   = 0.10
+	arcHeadroomMi = 1024 // always leave the OS a GiB above the ARC ceiling
+)
+
+// zfsDataset describes one dataset created at install time. Order matters:
+// parents must precede children so mountpoints nest correctly.
+type zfsDataset struct {
+	name       string
+	mountpoint string
+	props      []string
+}
+
+// datasets is the layout described in the plan. Recordsizes are matched to each
+// service's access pattern; 16K on the predastore index keeps RAIDZ-1 parity
+// overhead at 33% rather than the ~50% a 4-8K record would cost.
+var datasets = []zfsDataset{
+	{ZFSPoolName + "/ROOT", "none", []string{"canmount=off"}},
+	{ZFSRootDataset, "/", []string{"acltype=posix"}},
+	{ZFSPoolName + "/data", "/var/lib/spinifex", []string{"canmount=off"}},
+	{ZFSPoolName + "/data/viperblock", "/var/lib/spinifex/viperblock",
+		[]string{"recordsize=128K", "logbias=throughput"}},
+	{ZFSPoolName + "/data/predastore", "/var/lib/spinifex/predastore",
+		[]string{"recordsize=1M"}},
+	// Broken out so the later move of object data onto dedicated drives is a
+	// zfs send | zfs recv rather than an rsync with downtime.
+	{ZFSPoolName + "/data/predastore-nodes", "/var/lib/spinifex/predastore/distributed/nodes", nil},
+	// BadgerDB index plus the Raft BoltDB: small random, fsync-heavy.
+	{ZFSPoolName + "/data/predastore-db", "/var/lib/spinifex/predastore/distributed/db",
+		[]string{"recordsize=16K", "logbias=latency"}},
+	{ZFSPoolName + "/data/nats", "/var/lib/spinifex/nats", []string{"recordsize=16K"}},
+	{ZFSPoolName + "/log", "/var/log/spinifex", []string{"compression=zstd"}},
+}
+
+// buildVdev renders the vdev specification for zpool create. Pure: it is the
+// function that decides pool shape, so it is exhaustively unit-tested rather
+// than discovered to be wrong on hardware.
+func buildVdev(cfg DiskConfig) (string, error) {
+	if err := cfg.Validate(); err != nil {
+		return "", err
+	}
+	if !cfg.FS.IsZFS() {
+		return "", fmt.Errorf("buildVdev: %s is not a zfs topology", cfg.FS)
+	}
+
+	// Members are named by their partition-3 by-id path: a pool built on
+	// /dev/sdX breaks the first time a controller enumerates in a new order.
+	members := make([]string, len(cfg.Disks))
+	for i, d := range cfg.Disks {
+		members[i] = d.StablePartitionPath(rootPartNum)
+	}
+
+	keyword := cfg.FS.vdevKeyword()
+	if keyword == "" {
+		return strings.Join(members, " "), nil // plain stripe
+	}
+	// RAID10 is a stripe of two-disk mirrors, paired in selection order — which
+	// is why the UI numbers selected disks instead of just ticking them.
+	if cfg.FS == FSZFSRAID10 {
+		var parts []string
+		for i := 0; i+1 < len(members); i += 2 {
+			parts = append(parts, fmt.Sprintf("mirror %s %s", members[i], members[i+1]))
+		}
+		return strings.Join(parts, " "), nil
+	}
+	return keyword + " " + strings.Join(members, " "), nil
+}
+
+// detectAshift derives the pool sector size from the widest sector any member
+// reports. ashift cannot be changed after creation, and one that is too small
+// costs read-modify-write on every block for the life of the pool, so the floor
+// is 12 (4K) even when every disk claims 512.
+func detectAshift(disks []Disk) int {
+	widest := 4096
+	for _, d := range disks {
+		widest = max(widest, d.PhysicalBlockSize, d.LogicalBlockSize)
+	}
+	// ashift is the base-2 exponent of the sector size.
+	return bits.Len(uint(widest)) - 1
+}
+
+// defaultARCMaxMiB computes the ARC ceiling from total system memory.
+func defaultARCMaxMiB(totalMiB int) int {
+	want := int(math.Round(float64(totalMiB) * arcFraction))
+	want = min(want, arcMaxCapMiB)
+	// Never let the ARC crowd the OS out of its last GiB on a small node.
+	if ceiling := totalMiB - arcHeadroomMi; ceiling > 0 {
+		want = min(want, ceiling)
+	}
+	return max(want, arcMinMiB)
+}
+
+// totalMemoryMiB reads MemTotal from /proc/meminfo.
+func totalMemoryMiB() int {
+	data, err := os.ReadFile("/proc/meminfo")
+	if err != nil {
+		return 0
+	}
+	for line := range strings.SplitSeq(string(data), "\n") {
+		if !strings.HasPrefix(line, "MemTotal:") {
+			continue
+		}
+		var kb int
+		if _, err := fmt.Sscanf(line, "MemTotal: %d kB", &kb); err == nil {
+			return kb / 1024
+		}
+	}
+	return 0
+}
+
+// resolveZFSOpts fills unset advanced options with their computed defaults, so
+// every value written to the pool is explicit and auditable afterwards.
+func resolveZFSOpts(cfg DiskConfig) ZFSOpts {
+	o := cfg.ZFS
+	if o.Ashift == 0 {
+		o.Ashift = detectAshift(cfg.Disks)
+	}
+	if o.Compress == "" {
+		// Named explicitly rather than "on" so a future OpenZFS default change
+		// cannot silently alter what this pool does.
+		o.Compress = "lz4"
+	}
+	if o.Checksum == "" {
+		o.Checksum = "on"
+	}
+	if o.Copies == 0 {
+		o.Copies = 1
+	}
+	if o.ARCMaxMiB == 0 {
+		o.ARCMaxMiB = defaultARCMaxMiB(totalMemoryMiB())
+	}
+	return o
+}
+
+// loadZFSModule brings up the ZFS kernel module in the live environment and
+// waits for /dev/zfs. The module is prebuilt against the shipped kernel at ISO
+// build time, so failure here means the ISO is broken, not the hardware.
+func loadZFSModule() error {
+	if err := run("modprobe", "zfs"); err != nil {
+		slog.Warn("modprobe zfs failed, checking for /dev/zfs anyway", "err", err)
+	}
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		if _, err := os.Stat("/dev/zfs"); err == nil {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("/dev/zfs did not appear after modprobe zfs — the ZFS module is missing or does not match the running kernel")
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// ImportablePools returns the names of pools visible to `zpool import`. There
+// is still no machine-readable output in OpenZFS 2.3, so the human format is
+// parsed — the same approach Proxmox::Sys::ZFS takes.
+func ImportablePools() ([]string, error) {
+	out, err := exec.Command("zpool", "import").Output()
+	if err != nil {
+		// zpool exits non-zero when there is nothing to import, which is the
+		// normal case. Anything else — no binary, no /dev/zfs — must surface, or
+		// a machine with an existing pool looks empty to the caller.
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("zpool import: %w", err)
+	}
+	return parseImportablePools(string(out)), nil
+}
+
+// parseImportablePools extracts pool names from `zpool import` output. Split
+// out from the command call so it can be tested against captured fixtures.
+func parseImportablePools(out string) []string {
+	var pools []string
+	scanner := bufio.NewScanner(strings.NewReader(out))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		// Only the "pool:" key introduces a pool; "id:", "state:" and the
+		// config tree that follows are attributes of the one before it.
+		if name, ok := strings.CutPrefix(line, "pool:"); ok {
+			if n := strings.TrimSpace(name); n != "" {
+				pools = append(pools, n)
+			}
+		}
+	}
+	return pools
+}
+
+// createPool builds the root pool across the selected disks. altroot keeps
+// every dataset mounted under the install target instead of over the live
+// filesystem, and cachefile=none stops the live environment from claiming it.
+func createPool(cfg DiskConfig, opts ZFSOpts) error {
+	vdev, err := buildVdev(cfg)
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"create", "-f",
+		"-o", "cachefile=none",
+		"-o", fmt.Sprintf("ashift=%d", opts.Ashift),
+		"-O", "atime=on", "-O", "relatime=on",
+		"-O", "xattr=sa",
+		"-O", fmt.Sprintf("compression=%s", opts.Compress),
+		"-O", fmt.Sprintf("checksum=%s", opts.Checksum),
+		"-O", fmt.Sprintf("copies=%d", opts.Copies),
+		"-O", "mountpoint=none",
+		"-R", mountRoot,
+		ZFSPoolName,
+	}
+	args = append(args, strings.Fields(vdev)...)
+	if err := run("zpool", args...); err != nil {
+		return fmt.Errorf("zpool create: %w", err)
+	}
+
+	// TRIM matters on a pool that churns EBS volumes: without it the drives'
+	// garbage collection degrades write latency over months, and diagnosing
+	// that after the fact is miserable. Detected, never prompted.
+	if allFlash(cfg.Disks) {
+		if err := run("zpool", "set", "autotrim=on", ZFSPoolName); err != nil {
+			slog.Warn("could not enable autotrim", "err", err)
+		}
+	}
+	return nil
+}
+
+// allFlash reports whether every member is non-rotational.
+func allFlash(disks []Disk) bool {
+	for _, d := range disks {
+		if d.Rotational {
+			return false
+		}
+	}
+	return len(disks) > 0
+}
+
+// createDatasets lays out the service datasets. Mountpoints are set relative to
+// the pool's altroot, so they land under the install target now and at their
+// real paths once the installed system imports the pool.
+func createDatasets() error {
+	for _, ds := range datasets {
+		args := []string{"create", "-o", "mountpoint=" + ds.mountpoint}
+		for _, p := range ds.props {
+			args = append(args, "-o", p)
+		}
+		args = append(args, ds.name)
+		if err := run("zfs", args...); err != nil {
+			return fmt.Errorf("create dataset %s: %w", ds.name, err)
+		}
+	}
+	return nil
+}
+
+// datasetProtectFilters keeps rsync --delete away from the dataset mountpoints.
+//
+// The datasets are mounted before the copy so files land on the right one, but
+// the live source has nothing at those paths, so --delete tries to rmdir a busy
+// mountpoint and fails the whole transfer. Protect rules exempt them.
+func datasetProtectFilters(cfg DiskConfig) []string {
+	if !cfg.FS.IsZFS() {
+		return nil
+	}
+	var out []string
+	for _, ds := range datasets {
+		if ds.mountpoint == "/" || ds.mountpoint == "none" {
+			continue
+		}
+		out = append(out, "--filter=P "+ds.mountpoint)
+	}
+	return out
+}
+
+// writeZFSSystemConfig writes the target's ZFS tunables: the ARC ceiling, and
+// the matching host reserve so the daemon's admission gate stops handing out
+// memory the ARC is holding.
+func writeZFSSystemConfig(opts ZFSOpts) error {
+	modprobeDir := filepath.Join(mountRoot, "etc/modprobe.d")
+	if err := os.MkdirAll(modprobeDir, 0o755); err != nil {
+		return err
+	}
+	arcBytes := int64(opts.ARCMaxMiB) << 20
+	conf := fmt.Sprintf("# Generated by the Spinifex installer.\n"+
+		"# Guest RAM is the scarce resource on a hypervisor, so the ARC is\n"+
+		"# capped well below the ZFS default of half of system memory.\n"+
+		"options zfs zfs_arc_max=%d\n", arcBytes)
+	if err := os.WriteFile(filepath.Join(modprobeDir, "zfs.conf"), []byte(conf), 0o644); err != nil {
+		return fmt.Errorf("write zfs.conf: %w", err)
+	}
+
+	// The daemon computes schedulable memory as host total minus this reserve
+	// (defaultHostReserve in daemon/resource.go). ARC is invisible to it, so
+	// the ceiling has to be declared or instances get admitted into memory the
+	// ARC already owns.
+	reserve := defaultHostReserveGB + float64(opts.ARCMaxMiB)/1024.0
+	env := fmt.Sprintf("# Generated by the Spinifex installer.\n"+
+		"# %.1fGB base reserve + %.1fGB ZFS ARC ceiling.\n"+
+		"SPINIFEX_RESERVED_MEM_GB=%.1f\n",
+		defaultHostReserveGB, float64(opts.ARCMaxMiB)/1024.0, reserve)
+	confDir := filepath.Join(mountRoot, "etc/spinifex")
+	if err := os.MkdirAll(confDir, 0o755); err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(confDir, "host.env"), []byte(env), 0o644)
+}
+
+// defaultHostReserveGB mirrors defaultHostReserve.memGB in
+// spinifex/daemon/resource.go. Changing one without the other silently
+// misreports how much memory a ZFS node can schedule.
+const defaultHostReserveGB = 2.0
+
+// exportPool releases the pool so a retry after a failed install is not blocked
+// by "pool is busy". The live environment runs spinifex-init as PID 1, so
+// nothing else will ever clean this up.
+func exportPool() {
+	if err := run("zpool", "export", ZFSPoolName); err != nil {
+		slog.Warn("zpool export failed", "pool", ZFSPoolName, "err", err)
+	}
+}
+
+// clearDisk removes any prior filesystem, partition table and ZFS label from a
+// disk so a retry after a partial install is not rejected for stale metadata —
+// a failure that otherwise reads as mysterious rather than as leftover debris.
+//
+// The probes are quiet because a blank disk makes all of them fail, and their
+// stderr on the install console reads like the install is going wrong.
+func clearDisk(d Disk) error {
+	// ZFS labels live at both ends of the device and wipefs does not know about
+	// them, so an old pool member has to be cleared before its partition goes.
+	if old := d.PartitionPath(rootPartNum); exists(old) {
+		_ = runQuiet("zpool", "labelclear", "-f", old)
+	}
+	_ = runQuiet("wipefs", "-a", d.Path)
+	return runQuiet("sgdisk", "-Z", d.Path)
+}
+
+func exists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}

cmd/installer/install/zfs_test.go

@@ -0,0 +1,311 @@
+package install
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+// disk builds a test disk with a by-id path, since that is what pool members
+// are actually named by.
+func disk(name string, gib int64) Disk {
+	return Disk{
+		Path:              "/dev/" + name,
+		ByID:              "/dev/disk/by-id/virtio-" + name,
+		Bytes:             gib << 30,
+		LogicalBlockSize:  512,
+		PhysicalBlockSize: 512,
+	}
+}
+
+func disks(n int, gib int64) []Disk {
+	out := make([]Disk, n)
+	for i := range out {
+		out[i] = disk(string(rune('a'+i)), gib)
+	}
+	return out
+}
+
+func TestBuildVdev(t *testing.T) {
+	tests := []struct {
+		name  string
+		fs    FSType
+		disks []Disk
+		want  string
+	}{
+		{
+			"raid0 is a bare stripe with no keyword",
+			FSZFSRAID0, disks(2, 40),
+			"/dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3",
+		},
+		{
+			"single-disk raid0 is the one-drive case",
+			FSZFSRAID0, disks(1, 40),
+			"/dev/disk/by-id/virtio-a-part3",
+		},
+		{
+			"raid1 is one mirror vdev",
+			FSZFSRAID1, disks(2, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3",
+		},
+		{
+			"raid1 across three disks is a three-way mirror, not a stripe",
+			FSZFSRAID1, disks(3, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 /dev/disk/by-id/virtio-c-part3",
+		},
+		{
+			"raid10 pairs members in selection order",
+			FSZFSRAID10, disks(4, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 " +
+				"mirror /dev/disk/by-id/virtio-c-part3 /dev/disk/by-id/virtio-d-part3",
+		},
+		{
+			"raidz1 across three disks",
+			FSZFSRAIDZ1, disks(3, 40),
+			"raidz1 /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 /dev/disk/by-id/virtio-c-part3",
+		},
+		{
+			"raidz2 needs four",
+			FSZFSRAIDZ2, disks(4, 40),
+			"raidz2 /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 " +
+				"/dev/disk/by-id/virtio-c-part3 /dev/disk/by-id/virtio-d-part3",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := buildVdev(DiskConfig{FS: tt.fs, Disks: tt.disks})
+			if err != nil {
+				t.Fatalf("buildVdev: %v", err)
+			}
+			if got != tt.want {
+				t.Errorf("buildVdev()\n got: %s\nwant: %s", got, tt.want)
+			}
+		})
+	}
+}
+
+// A pool built on /dev/sdX silently breaks when the controller enumerates in a
+// different order, so the fallback only applies when there is no by-id path.
+func TestBuildVdevFallsBackToKernelNameWithoutByID(t *testing.T) {
+	d := disk("a", 40)
+	d.ByID = ""
+	got, err := buildVdev(DiskConfig{FS: FSZFSRAID0, Disks: []Disk{d}})
+	if err != nil {
+		t.Fatalf("buildVdev: %v", err)
+	}
+	if got != "/dev/a3" {
+		t.Errorf("buildVdev() = %q, want /dev/a3", got)
+	}
+}
+
+func TestBuildVdevRejectsExt4(t *testing.T) {
+	if _, err := buildVdev(DiskConfig{FS: FSExt4, Disks: disks(1, 40)}); err == nil {
+		t.Fatal("buildVdev accepted ext4")
+	}
+}
+
+func TestBuildVdevValidatesFirst(t *testing.T) {
+	// Three disks cannot form striped mirrors, and producing a two-disk mirror
+	// plus a silently dropped disk would be worse than an error.
+	if _, err := buildVdev(DiskConfig{FS: FSZFSRAID10, Disks: disks(3, 40)}); err == nil {
+		t.Fatal("buildVdev accepted an odd disk count for RAID10")
+	}
+}
+
+func TestDetectAshift(t *testing.T) {
+	tests := []struct {
+		name     string
+		logical  int
+		physical int
+		want     int
+	}{
+		// ashift cannot be changed after creation and one that is too small
+		// costs a read-modify-write per block forever, so 512e drives are
+		// deliberately treated as 4K.
+		{"512n drive is floored at 4K", 512, 512, 12},
+		{"512e drive reports 4K physical", 512, 4096, 12},
+		{"4Kn drive", 4096, 4096, 12},
+		{"8K native drive", 8192, 8192, 13},
+		{"16K native drive", 16384, 16384, 14},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			d := Disk{LogicalBlockSize: tt.logical, PhysicalBlockSize: tt.physical}
+			if got := detectAshift([]Disk{d}); got != tt.want {
+				t.Errorf("detectAshift() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+// A pool cannot have two sector sizes, so the widest member wins — the reverse
+// would make every write to the large-sector disk a read-modify-write.
+func TestDetectAshiftTakesWidestMember(t *testing.T) {
+	got := detectAshift([]Disk{
+		{LogicalBlockSize: 512, PhysicalBlockSize: 512},
+		{LogicalBlockSize: 512, PhysicalBlockSize: 8192},
+	})
+	if got != 13 {
+		t.Errorf("detectAshift() = %d, want 13", got)
+	}
+}
+
+func TestDefaultARCMaxMiB(t *testing.T) {
+	tests := []struct {
+		name    string
+		totalMi int
+		want    int
+	}{
+		{"tiny VM gets 10% of very little", 2048, 205},
+		{"8GB node gets 10%", 8192, 819},
+		{"64GB node gets 10%", 65536, 6554},
+		// The point of the cap: guest RAM is what this machine sells, and half
+		// of a 256GB node is 128GB of instance capacity the ARC would swallow.
+		{"256GB node is capped at 16GiB", 262144, 16384},
+		{"1TB node is still capped at 16GiB", 1048576, 16384},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := defaultARCMaxMiB(tt.totalMi); got != tt.want {
+				t.Errorf("defaultARCMaxMiB(%d) = %d, want %d", tt.totalMi, got, tt.want)
+			}
+		})
+	}
+}
+
+// An unreadable /proc/meminfo must not produce a zero or negative ceiling,
+// which ZFS would either reject or read as "no limit".
+func TestDefaultARCMaxMiBNeverGoesBelowFloor(t *testing.T) {
+	for _, total := range []int{0, 128, 512} {
+		if got := defaultARCMaxMiB(total); got < arcMinMiB {
+			t.Errorf("defaultARCMaxMiB(%d) = %d, below the floor of %d", total, got, arcMinMiB)
+		}
+	}
+}
+
+func TestResolveZFSOptsFillsDefaults(t *testing.T) {
+	got := resolveZFSOpts(DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)})
+	if got.Ashift != 12 {
+		t.Errorf("Ashift = %d, want 12", got.Ashift)
+	}
+	if got.Compress != "lz4" {
+		t.Errorf("Compress = %q, want lz4", got.Compress)
+	}
+	if got.Checksum != "on" {
+		t.Errorf("Checksum = %q, want on", got.Checksum)
+	}
+	if got.Copies != 1 {
+		t.Errorf("Copies = %d, want 1", got.Copies)
+	}
+	if got.ARCMaxMiB <= 0 {
+		t.Errorf("ARCMaxMiB = %d, want a positive ceiling", got.ARCMaxMiB)
+	}
+}
+
+func TestResolveZFSOptsKeepsOperatorChoices(t *testing.T) {
+	in := DiskConfig{
+		FS:    FSZFSRAIDZ1,
+		Disks: disks(3, 40),
+		ZFS:   ZFSOpts{Ashift: 13, Compress: "zstd", Checksum: "blake3", Copies: 2, ARCMaxMiB: 2048},
+	}
+	got := resolveZFSOpts(in)
+	if got != in.ZFS {
+		t.Errorf("resolveZFSOpts overrode explicit options: got %+v, want %+v", got, in.ZFS)
+	}
+}
+
+func TestParseImportablePools(t *testing.T) {
+	// Captured from `zpool import` on a machine with a pool from a previous
+	// install: the config tree below each header must not be mistaken for one.
+	const out = `   pool: rpool
+     id: 8543219876543210987
+  state: ONLINE
+ action: The pool can be imported using its name or numeric identifier.
+ config:
+
+	rpool                     ONLINE
+	  raidz1-0                ONLINE
+	    virtio-spinifex-disk1-part3  ONLINE
+	    virtio-spinifex-disk2-part3  ONLINE
+
+   pool: tank
+     id: 1111111111111111111
+  state: ONLINE
+ config:
+
+	tank                      ONLINE
+`
+	got := parseImportablePools(out)
+	want := []string{"rpool", "tank"}
+	if len(got) != len(want) {
+		t.Fatalf("parseImportablePools() = %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("pool %d = %q, want %q", i, got[i], want[i])
+		}
+	}
+}
+
+func TestParseImportablePoolsEmpty(t *testing.T) {
+	if got := parseImportablePools("no pools available to import\n"); len(got) != 0 {
+		t.Errorf("parseImportablePools() = %v, want none", got)
+	}
+}
+
+// Every dataset must be creatable in order: a child cannot be created before
+// its parent, and getting this wrong only shows up on real hardware.
+func TestDatasetsAreOrderedParentFirst(t *testing.T) {
+	seen := map[string]bool{ZFSPoolName: true}
+	for _, ds := range datasets {
+		parent := ds.name[:strings.LastIndex(ds.name, "/")]
+		if !seen[parent] {
+			t.Errorf("dataset %s is created before its parent %s", ds.name, parent)
+		}
+		seen[ds.name] = true
+	}
+}
+
+// The root dataset has to be the one the bootloader names, or the machine
+// mounts an empty filesystem and panics.
+func TestRootDatasetIsCreated(t *testing.T) {
+	for _, ds := range datasets {
+		if ds.name == ZFSRootDataset {
+			if ds.mountpoint != "/" {
+				t.Fatalf("%s mountpoint = %q, want /", ZFSRootDataset, ds.mountpoint)
+			}
+			return
+		}
+	}
+	t.Fatalf("%s is never created", ZFSRootDataset)
+}
+
+// rsync --delete tries to rmdir a dataset mountpoint the live source does not
+// have, which fails on a busy mount and aborts the whole copy. Every dataset
+// with a real mountpoint must be protected.
+func TestDatasetProtectFilters(t *testing.T) {
+	got := datasetProtectFilters(DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)})
+	for _, ds := range datasets {
+		if ds.mountpoint == "/" || ds.mountpoint == "none" {
+			continue
+		}
+		want := "--filter=P " + ds.mountpoint
+		if !slices.Contains(got, want) {
+			t.Errorf("missing protect rule for %s (%s)", ds.name, ds.mountpoint)
+		}
+	}
+	// The root dataset is the destination itself, and "none" datasets are never
+	// mounted, so protecting either would be meaningless.
+	for _, f := range got {
+		if f == "--filter=P /" || f == "--filter=P none" {
+			t.Errorf("unexpected protect rule %q", f)
+		}
+	}
+}
+
+func TestDatasetProtectFiltersSkipsExt4(t *testing.T) {
+	if got := datasetProtectFilters(DiskConfig{FS: FSExt4, Disks: disks(1, 40)}); got != nil {
+		t.Errorf("datasetProtectFilters() = %v on ext4, want none", got)
+	}
+}

cmd/installer/netprobe/netprobe.go

@@ -0,0 +1,426 @@
+// Package netprobe identifies physical network interfaces by hardware rather
+// than kernel name, so an operator choosing between eno1, ens1f0np0 and
+// ens1f1np1 can tell the 1gbe Broadcom from the 25gbe Mellanox.
+package netprobe
+
+import (
+	"cmp"
+	"fmt"
+	"log/slog"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// sysClassNet is the kernel's per-interface attribute tree. A variable, not a
+// constant, so tests can point the probe at a fixture instead of the running
+// machine's hardware.
+var sysClassNet = "/sys/class/net"
+
+// The external helpers the probe leans on, indirected so tests can exercise it
+// without the host's real tooling.
+var (
+	listInterfaces = net.Interfaces
+	commandOutput  = func(name string, args ...string) ([]byte, error) {
+		return exec.Command(name, args...).Output()
+	}
+)
+
+// NIC describes one physical interface as presented in the installer's
+// selection table.
+type NIC struct {
+	Name string
+
+	// Vendor and Model come from the udev hardware database, e.g. "Mellanox
+	// Technologies" and "MT27710 Family [ConnectX-4 Lx]". Empty when the
+	// database carries no entry for the card.
+	Vendor string
+	Model  string
+
+	// Driver is the kernel module that claimed the port, e.g. "mlx5_core". It
+	// comes straight from sysfs, so it still identifies a card the hardware
+	// database misses.
+	Driver string
+
+	// DeviceID is the PCI or USB "vendor:device" pair, the last-resort identity
+	// for hardware neither the database nor a driver name pins down.
+	DeviceID string
+
+	// AltName is the first alternative name systemd assigns, which is what
+	// operators often see in switch and cabling documentation.
+	AltName string
+
+	// Speed is the negotiated link speed rendered for display ("25 Gbps") and
+	// SpeedMbps the raw figure behind it. Both are zero on a port with no
+	// carrier, which is why Probe brings links up before reading.
+	Speed     string
+	SpeedMbps int
+
+	// Carrier reports a cable with link. It orders the picker so the port the
+	// operator has just plugged in is the one offered first.
+	Carrier bool
+
+	// State is the link summary shown in the table: "online", "no cable" or
+	// "down".
+	State string
+
+	MTU    int
+	IsWiFi bool
+}
+
+// Label renders the NIC's identity line: kernel name, negotiated speed and link
+// state. The hardware detail is deliberately absent — it goes on its own line
+// via Hardware, where there is room to read it.
+func (n NIC) Label() string {
+	speed := n.Speed
+	if speed == "" {
+		speed = "—"
+	}
+	state := n.State
+	if state == "" {
+		state = "unknown"
+	}
+	return fmt.Sprintf("%s  %s  %s", n.Name, speed, state)
+}
+
+// Hardware renders the card's identity for the detail line, degrading from the
+// database name to the driver and device id rather than to "unknown". An
+// operator needs something to match against the card in their hand.
+func (n NIC) Hardware() string {
+	if hw := strings.TrimSpace(n.Vendor + " " + n.Model); hw != "" {
+		return hw
+	}
+	switch {
+	case n.Driver != "" && n.DeviceID != "":
+		return n.Driver + " [" + n.DeviceID + "]"
+	case n.Driver != "":
+		return n.Driver
+	case n.DeviceID != "":
+		return n.DeviceID
+	}
+	return "no hardware detail"
+}
+
+// linkUpTimeout bounds how long Probe waits after bringing links up for speed
+// and carrier to settle. A 25gbe port negotiating against a cold switch can
+// take a second or two to report a speed.
+var linkUpTimeout = 2 * time.Second
+
+// Probe enumerates non-loopback physical interfaces and enriches each from
+// sysfs and the udev hardware database, ordered so the ports worth assigning
+// come first. Links are brought up first: a down port reports no speed and no
+// carrier, which would leave every fast port unidentified at exactly the moment
+// the operator needs to pick one.
+//
+// Nothing the probe depends on needs a running systemd. The live installer
+// boots straight into spinifex-init with only udevd started, so networkctl —
+// which has to reach systemd-networkd — enriches gaps and is never the source.
+func Probe() ([]NIC, error) {
+	ifaces, err := listInterfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	var names, physical []string
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagLoopback != 0 {
+			continue
+		}
+		if len(iface.HardwareAddr) == 0 {
+			continue
+		}
+		names = append(names, iface.Name)
+		if hasDevice(iface.Name) {
+			physical = append(physical, iface.Name)
+		}
+	}
+
+	// Bridges, bonds and veth pairs carry a MAC like any port but have no
+	// device behind them, and offering them would let a plane be bound to
+	// something that does not exist until after the install. Keep them only if
+	// the filter found nothing, so an unusual platform still gets a list.
+	if len(physical) > 0 {
+		names = physical
+	}
+
+	brought := false
+	for _, name := range names {
+		if _, err := commandOutput("ip", "link", "set", "dev", name, "up"); err != nil {
+			slog.Debug("netprobe: could not bring link up", "iface", name, "err", err)
+			continue
+		}
+		brought = true
+	}
+	if brought {
+		time.Sleep(linkUpTimeout)
+	}
+
+	nics := make([]NIC, 0, len(names))
+	for _, name := range names {
+		nics = append(nics, describe(name))
+	}
+	slices.SortFunc(nics, byUsefulness)
+	return nics, nil
+}
+
+// describe reads everything about one interface that needs no daemon, then lets
+// networkctl fill the gaps if it happens to be answering.
+func describe(name string) NIC {
+	nic := NIC{
+		Name:     name,
+		Driver:   sysLinkBase(name, "device/driver"),
+		DeviceID: deviceID(name),
+		MTU:      sysInt(name, "mtu"),
+		IsWiFi:   isWiFi(name),
+	}
+
+	// The kernel reports -1 for a port that cannot state its speed; treat that
+	// as unknown rather than letting it sort below every real figure.
+	nic.SpeedMbps = max(sysInt(name, "speed"), 0)
+	nic.Speed = formatSpeed(nic.SpeedMbps)
+	nic.Carrier = sysAttr(name, "carrier") == "1"
+	nic.State = linkState(name, nic.Carrier)
+
+	props := udevProperties(name)
+	nic.Vendor = props["ID_VENDOR_FROM_DATABASE"]
+	nic.Model = props["ID_MODEL_FROM_DATABASE"]
+
+	// udev's predictable names are the ones that appear in cabling notes; the
+	// path-based name is the one networkctl lists first.
+	for _, key := range []string{"ID_NET_NAME_PATH", "ID_NET_NAME_ONBOARD", "ID_NET_NAME_SLOT"} {
+		if alt := props[key]; alt != "" && alt != name {
+			nic.AltName = alt
+			break
+		}
+	}
+
+	if out, err := commandOutput("networkctl", "status", "--no-pager", name); err == nil {
+		fill(&nic, ParseStatus(string(out)))
+	} else {
+		slog.Debug("netprobe: networkctl unavailable, using sysfs only", "iface", name, "err", err)
+	}
+	return nic
+}
+
+// fill copies parsed hardware detail onto a NIC without overwriting anything
+// sysfs already established. State is never taken: the carrier bit is a truer
+// answer to "is this cable live" than networkd's configuration status.
+func fill(dst *NIC, src NIC) {
+	if dst.Vendor == "" {
+		dst.Vendor = src.Vendor
+	}
+	if dst.Model == "" {
+		dst.Model = src.Model
+	}
+	if dst.AltName == "" {
+		dst.AltName = src.AltName
+	}
+	if dst.Speed == "" {
+		dst.Speed = src.Speed
+	}
+	if dst.MTU == 0 {
+		dst.MTU = src.MTU
+	}
+}
+
+// byUsefulness orders the picker so the interfaces an operator would actually
+// assign come first: a live cable, then wired over wireless, then the fastest
+// port. The rest still appear, just below the plausible choices.
+func byUsefulness(a, b NIC) int {
+	if a.Carrier != b.Carrier {
+		return boolOrder(a.Carrier)
+	}
+	if a.IsWiFi != b.IsWiFi {
+		return boolOrder(b.IsWiFi)
+	}
+	if a.SpeedMbps != b.SpeedMbps {
+		return cmp.Compare(b.SpeedMbps, a.SpeedMbps)
+	}
+	return cmp.Compare(a.Name, b.Name)
+}
+
+// boolOrder sorts true ahead of false.
+func boolOrder(first bool) int {
+	if first {
+		return -1
+	}
+	return 1
+}
+
+// formatSpeed renders the kernel's Mbit/s figure the way a datasheet does.
+func formatSpeed(mbps int) string {
+	switch {
+	case mbps <= 0:
+		return ""
+	case mbps%1000 == 0:
+		return fmt.Sprintf("%d Gbps", mbps/1000)
+	default:
+		return fmt.Sprintf("%d Mbps", mbps)
+	}
+}
+
+// linkState collapses the kernel's flags into the three outcomes an operator
+// acts on: the port is live, the port is enabled but unplugged, or the port
+// never came up at all. operstate reports "down" for the latter two alike.
+func linkState(name string, carrier bool) string {
+	switch {
+	case carrier:
+		return "online"
+	case adminUp(name):
+		return "no cable"
+	default:
+		return "down"
+	}
+}
+
+// adminUp reads the IFF_UP bit out of the interface flags.
+func adminUp(name string) bool {
+	flags, err := strconv.ParseUint(strings.TrimPrefix(sysAttr(name, "flags"), "0x"), 16, 64)
+	return err == nil && flags&0x1 != 0
+}
+
+// deviceID reads the PCI or USB "vendor:device" pair. A NIC's device link
+// points at the PCI function directly; on USB it points at the interface, whose
+// parent carries the ids.
+func deviceID(name string) string {
+	dev := filepath.Join(sysClassNet, name, "device")
+	if id := idPair(dev, "vendor", "device"); id != "" {
+		return id
+	}
+	return idPair(filepath.Join(dev, ".."), "idVendor", "idProduct")
+}
+
+func idPair(dir, vendorAttr, productAttr string) string {
+	vendor := readTrimmed(filepath.Join(dir, vendorAttr))
+	product := readTrimmed(filepath.Join(dir, productAttr))
+	if vendor == "" || product == "" {
+		return ""
+	}
+	return strings.TrimPrefix(vendor, "0x") + ":" + strings.TrimPrefix(product, "0x")
+}
+
+// udevProperties reads the interface's udev database entry, which is where the
+// hardware-database vendor and model names live. udevd populates it during the
+// installer's device enumeration, so this works with no systemd running.
+func udevProperties(name string) map[string]string {
+	out, err := commandOutput("udevadm", "info", "--query=property", "--path="+filepath.Join(sysClassNet, name))
+	if err != nil {
+		slog.Debug("netprobe: udevadm info failed", "iface", name, "err", err)
+		return nil
+	}
+
+	props := make(map[string]string)
+	for line := range strings.SplitSeq(string(out), "\n") {
+		if key, value, ok := strings.Cut(strings.TrimSpace(line), "="); ok {
+			props[key] = value
+		}
+	}
+	return props
+}
+
+// ParseStatus extracts the hardware fields from `networkctl status <iface>`
+// output. The format is a set of right-aligned "Key: value" lines, with
+// Alternative Names continuing across subsequent indented lines.
+func ParseStatus(out string) NIC {
+	var nic NIC
+	var inAltNames bool
+
+	for line := range strings.SplitSeq(out, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" {
+			inAltNames = false
+			continue
+		}
+
+		key, value, found := strings.Cut(trimmed, ": ")
+		if !found {
+			// A continuation line under "Alternative Names:" — take the first
+			// only, which is the one operators see in cabling notes.
+			if inAltNames && nic.AltName == "" {
+				nic.AltName = trimmed
+			}
+			continue
+		}
+		inAltNames = false
+
+		key = strings.TrimSpace(key)
+		value = strings.TrimSpace(value)
+		switch key {
+		case "Vendor":
+			nic.Vendor = value
+		case "Model":
+			nic.Model = value
+		case "Speed":
+			nic.Speed = value
+		case "Online state":
+			// Rendered as "online" or "online (configured)".
+			if f := strings.Fields(value); len(f) > 0 {
+				nic.State = f[0]
+			}
+		case "MTU":
+			// Rendered as "9000 (min: 68, max: 9978)".
+			if f := strings.Fields(value); len(f) > 0 {
+				if mtu, err := strconv.Atoi(f[0]); err == nil {
+					nic.MTU = mtu
+				}
+			}
+		case "Alternative Names":
+			inAltNames = true
+			if f := strings.Fields(value); len(f) > 0 && nic.AltName == "" {
+				nic.AltName = f[0]
+			}
+		}
+	}
+	return nic
+}
+
+// hasDevice reports whether the interface is backed by real hardware, which
+// sysfs shows as a device link to the PCI, USB or platform node behind it.
+func hasDevice(name string) bool {
+	_, err := os.Stat(filepath.Join(sysClassNet, name, "device"))
+	return err == nil
+}
+
+// isWiFi reports whether the interface has a wireless subdirectory in sysfs.
+func isWiFi(name string) bool {
+	_, err := os.Stat(filepath.Join(sysClassNet, name, "wireless"))
+	return err == nil
+}
+
+// sysAttr reads one sysfs attribute. A missing or unreadable attribute is not
+// an error worth surfacing: speed and carrier both fail on a port with no link,
+// which the state column already conveys.
+func sysAttr(name, attr string) string {
+	return readTrimmed(filepath.Join(sysClassNet, name, attr))
+}
+
+func sysInt(name, attr string) int {
+	n, err := strconv.Atoi(sysAttr(name, attr))
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+// sysLinkBase resolves a sysfs symlink and returns its final element, which is
+// how the driver name is exposed.
+func sysLinkBase(name, attr string) string {
+	target, err := os.Readlink(filepath.Join(sysClassNet, name, attr))
+	if err != nil {
+		return ""
+	}
+	return filepath.Base(target)
+}
+
+func readTrimmed(path string) string {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(b))
+}

cmd/installer/netprobe/netprobe_test.go

@@ -0,0 +1,292 @@
+package netprobe
+
+import (
+	"errors"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Captured from hydrogen (HP DL325 Gen10, dual 25gbe Mellanox) in us-west-1-az1.
+const mellanox25g = `● 6: ens1f0np0
+                   Link File: /usr/lib/systemd/network/99-default.link
+                Network File: /etc/systemd/network/12-spinifex-lan-nic.network
+                       State: enslaved (configured)
+                Online state: online
+                        Type: ether
+                        Path: pci-0000:84:00.0
+                      Driver: mlx5_core
+                      Vendor: Mellanox Technologies
+                       Model: MT27710 Family [ConnectX-4 Lx]
+           Alternative Names: enp132s0f0np0
+                              enx040973e5ce60
+            Hardware Address: 04:09:73:e5:ce:60 (Hewlett Packard Enterprise)
+                         MTU: 9000 (min: 68, max: 9978)
+                       QDisc: mq
+                      Master: br-lan
+IPv6 Address Generation Mode: none
+    Number of Queues (Tx/Rx): 384/48
+            Auto negotiation: yes
+                       Speed: 25Gbps
+                      Duplex: full
+           Activation Policy: up
+         Required For Online: yes
+                Connected To: MikroTik (MikroTik RouterOS 7.11.3 (stable) Sep/27/2023 13:09:44 CRS504-4XQ) on port bridge/qsfp28-3-3
+`
+
+// A link with no carrier reports neither Speed nor an online state, which is
+// the case Probe brings links up to avoid.
+const linkDown = `● 3: eno6
+                   Link File: /usr/lib/systemd/network/99-default.link
+                       State: off
+                Online state: off
+                        Type: ether
+                      Driver: bnxt_en
+                      Vendor: Broadcom Inc. and subsidiaries
+                       Model: NetXtreme BCM5720 Gigabit Ethernet PCIe
+            Hardware Address: 04:09:73:e5:ce:5f
+                         MTU: 1500 (min: 60, max: 9000)
+`
+
+func TestParseStatusMellanox(t *testing.T) {
+	nic := ParseStatus(mellanox25g)
+
+	if nic.Vendor != "Mellanox Technologies" {
+		t.Errorf("Vendor = %q, want %q", nic.Vendor, "Mellanox Technologies")
+	}
+	if nic.Model != "MT27710 Family [ConnectX-4 Lx]" {
+		t.Errorf("Model = %q, want %q", nic.Model, "MT27710 Family [ConnectX-4 Lx]")
+	}
+	if nic.Speed != "25Gbps" {
+		t.Errorf("Speed = %q, want %q", nic.Speed, "25Gbps")
+	}
+	if nic.State != "online" {
+		t.Errorf("State = %q, want %q", nic.State, "online")
+	}
+	if nic.MTU != 9000 {
+		t.Errorf("MTU = %d, want 9000", nic.MTU)
+	}
+	// Only the first alternative name is taken; enx040973e5ce60 is ignored.
+	if nic.AltName != "enp132s0f0np0" {
+		t.Errorf("AltName = %q, want %q", nic.AltName, "enp132s0f0np0")
+	}
+}
+
+func TestParseStatusLinkDown(t *testing.T) {
+	nic := ParseStatus(linkDown)
+
+	if nic.Speed != "" {
+		t.Errorf("Speed = %q, want empty for a down link", nic.Speed)
+	}
+	if nic.State != "off" {
+		t.Errorf("State = %q, want %q", nic.State, "off")
+	}
+	if nic.Model != "NetXtreme BCM5720 Gigabit Ethernet PCIe" {
+		t.Errorf("Model = %q, want the Broadcom model", nic.Model)
+	}
+	if nic.MTU != 1500 {
+		t.Errorf("MTU = %d, want 1500", nic.MTU)
+	}
+}
+
+func TestParseStatusEmpty(t *testing.T) {
+	// A truncated or unparseable status must not panic or invent values.
+	nic := ParseStatus("● 2: eth0\n     Online state:\n     Alternative Names:\n")
+	if nic.State != "" || nic.AltName != "" {
+		t.Errorf("expected empty fields from valueless keys, got State=%q AltName=%q", nic.State, nic.AltName)
+	}
+}
+
+func TestLabelFallsBackWhenSpeedUnknown(t *testing.T) {
+	got := NIC{Name: "eth0", State: "no cable"}.Label()
+	want := "eth0  —  no cable"
+	if got != want {
+		t.Errorf("Label() = %q, want %q", got, want)
+	}
+}
+
+// Hardware must never render "unknown": an operator standing at the console
+// needs something to match against the card in their hand, and sysfs always has
+// at least a driver or a device id.
+func TestHardwareDegradesRatherThanUnknown(t *testing.T) {
+	tests := []struct {
+		name string
+		nic  NIC
+		want string
+	}{
+		{
+			name: "database name wins",
+			nic:  NIC{Vendor: "Mellanox Technologies", Model: "MT27710", Driver: "mlx5_core", DeviceID: "15b3:1015"},
+			want: "Mellanox Technologies MT27710",
+		},
+		{"vendor only", NIC{Vendor: "Intel Corporation"}, "Intel Corporation"},
+		{"driver and id", NIC{Driver: "igb", DeviceID: "8086:1533"}, "igb [8086:1533]"},
+		{"driver only", NIC{Driver: "virtio_net"}, "virtio_net"},
+		{"id only", NIC{DeviceID: "0bda:8153"}, "0bda:8153"},
+		{"nothing at all", NIC{Name: "eth0"}, "no hardware detail"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.nic.Hardware(); got != tt.want {
+				t.Errorf("Hardware() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestFormatSpeed(t *testing.T) {
+	tests := []struct {
+		mbps int
+		want string
+	}{
+		{25000, "25 Gbps"},
+		{1000, "1 Gbps"},
+		{100, "100 Mbps"},
+		{2500, "2500 Mbps"},
+		{0, ""},
+		{-1, ""},
+	}
+
+	for _, tt := range tests {
+		if got := formatSpeed(tt.mbps); got != tt.want {
+			t.Errorf("formatSpeed(%d) = %q, want %q", tt.mbps, got, tt.want)
+		}
+	}
+}
+
+// A live cable outranks everything, then wired over wireless, then speed. This
+// is what puts the ports an operator would actually assign at indices 0..2,
+// where newModel pre-fills wan, lan and vpc from.
+func TestByUsefulnessOrdersPluggedPortsFirst(t *testing.T) {
+	nics := []NIC{
+		{Name: "eno1", SpeedMbps: 1000},
+		{Name: "wlp2s0", Carrier: true, IsWiFi: true, SpeedMbps: 866},
+		{Name: "ens1f1np1", SpeedMbps: 25000},
+		{Name: "ens1f0np0", Carrier: true, SpeedMbps: 25000},
+		{Name: "eno2", Carrier: true, SpeedMbps: 1000},
+	}
+	slices.SortFunc(nics, byUsefulness)
+
+	want := []string{"ens1f0np0", "eno2", "wlp2s0", "ens1f1np1", "eno1"}
+	for i, name := range want {
+		if nics[i].Name != name {
+			t.Fatalf("order = %v, want %v", names(nics), want)
+		}
+	}
+}
+
+func names(nics []NIC) []string {
+	out := make([]string, len(nics))
+	for i, n := range nics {
+		out[i] = n.Name
+	}
+	return out
+}
+
+func TestProbeReadsSysfsWithoutSystemd(t *testing.T) {
+	root := t.TempDir()
+	sysClassNet = root
+	linkUpTimeout = 0
+	t.Cleanup(func() {
+		sysClassNet = "/sys/class/net"
+		linkUpTimeout = 2 * time.Second
+	})
+
+	// ens1f0np0: a 25gbe Mellanox with a live cable, identified by the hardware
+	// database. eno1: a 1gbe port that is up but unplugged, and absent from the
+	// database, so only its driver and PCI id identify it.
+	writeIface(t, root, "ens1f0np0", map[string]string{
+		"speed": "25000", "carrier": "1", "mtu": "9000", "flags": "0x1003",
+		"device/vendor": "0x15b3", "device/device": "0x1015",
+	}, "mlx5_core")
+	writeIface(t, root, "eno1", map[string]string{
+		"speed": "1000", "carrier": "0", "mtu": "1500", "flags": "0x1003",
+		"device/vendor": "0x8086", "device/device": "0x1533",
+	}, "igb")
+
+	listInterfaces = func() ([]net.Interface, error) {
+		return []net.Interface{
+			{Name: "lo", Flags: net.FlagLoopback},
+			{Name: "eno1", HardwareAddr: net.HardwareAddr{4, 9, 0x73, 0xe5, 0xce, 0x5f}},
+			{Name: "ens1f0np0", HardwareAddr: net.HardwareAddr{4, 9, 0x73, 0xe5, 0xce, 0x60}},
+			{Name: "veth0"}, // no hardware address, so not a physical port
+			{Name: "br-lan", HardwareAddr: net.HardwareAddr{2, 0, 0, 0, 0, 1}}, // a bridge, so no device behind it
+		}, nil
+	}
+	t.Cleanup(func() { listInterfaces = net.Interfaces })
+
+	// Only udevadm answers. networkctl fails the way it does in the live
+	// installer, where there is no systemd-networkd for it to reach.
+	commandOutput = func(name string, args ...string) ([]byte, error) {
+		switch {
+		case name == "ip":
+			return nil, nil
+		case name == "udevadm" && strings.HasSuffix(args[len(args)-1], "ens1f0np0"):
+			return []byte("ID_VENDOR_FROM_DATABASE=Mellanox Technologies\n" +
+				"ID_MODEL_FROM_DATABASE=MT27710 Family [ConnectX-4 Lx]\n" +
+				"ID_NET_NAME_PATH=enp132s0f0np0\n"), nil
+		case name == "udevadm":
+			return []byte("ID_NET_NAME_PATH=enp1s0f0\n"), nil
+		default:
+			return nil, errors.New("Failed to connect to bus: No such file or directory")
+		}
+	}
+	t.Cleanup(func() {
+		commandOutput = func(name string, args ...string) ([]byte, error) {
+			return exec.Command(name, args...).Output()
+		}
+	})
+
+	nics, err := Probe()
+	if err != nil {
+		t.Fatalf("Probe: %v", err)
+	}
+	if got := names(nics); !slices.Equal(got, []string{"ens1f0np0", "eno1"}) {
+		t.Fatalf("probed %v, want the two physical ports with the plugged one first", got)
+	}
+
+	mlx := nics[0]
+	if mlx.Label() != "ens1f0np0  25 Gbps  online" {
+		t.Errorf("Label() = %q", mlx.Label())
+	}
+	if want := "Mellanox Technologies MT27710 Family [ConnectX-4 Lx]"; mlx.Hardware() != want {
+		t.Errorf("Hardware() = %q, want %q", mlx.Hardware(), want)
+	}
+	if mlx.AltName != "enp132s0f0np0" || mlx.MTU != 9000 || !mlx.Carrier {
+		t.Errorf("unexpected detail: %+v", mlx)
+	}
+
+	// Up but unplugged, and unknown to the database: the driver and PCI id are
+	// all the operator gets, which is still enough to identify the card.
+	igb := nics[1]
+	if igb.Label() != "eno1  1 Gbps  no cable" {
+		t.Errorf("Label() = %q", igb.Label())
+	}
+	if igb.Hardware() != "igb [8086:1533]" {
+		t.Errorf("Hardware() = %q", igb.Hardware())
+	}
+}
+
+// writeIface lays out the sysfs attributes Probe reads for one interface,
+// including the driver symlink it resolves the module name from.
+func writeIface(t *testing.T, root, name string, attrs map[string]string, driver string) {
+	t.Helper()
+	for attr, value := range attrs {
+		path := filepath.Join(root, name, attr)
+		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(path, []byte(value+"\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := os.Symlink("../../../bus/pci/drivers/"+driver, filepath.Join(root, name, "device/driver")); err != nil {
+		t.Fatal(err)
+	}
+}

cmd/installer/systemd/systemd.go

@@ -6,44 +6,38 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
-// WriteLANBridgeUnit installs a non-critical oneshot service that activates
-// br-lan via networkctl *after* network-online.target. br-lan has
-// ActivationPolicy=manual in its networkd .network file, so networkd creates
-// the bridge at boot but does not bring it up automatically. This service
-// triggers the activation once the WAN path is confirmed online, ensuring a
-// missing LAN cable or DHCP timeout on the secondary bridge never stalls
+// WriteBridgeUnit installs a non-critical oneshot service that activates an
+// east-west bridge (br-lan, br-vpc) via networkctl *after* network-online.target.
+// Those bridges carry ActivationPolicy=manual in their networkd .network file,
+// so networkd creates them at boot but does not bring them up automatically.
+// This service triggers the activation once the WAN path is confirmed online,
+// ensuring a missing cable or DHCP timeout on a secondary bridge never stalls
 // the management interface or firstboot.
-func WriteLANBridgeUnit(root string) error {
-	unit := `[Unit]
-Description=Spinifex LAN bridge (non-critical)
+func WriteBridgeUnit(root, bridge string) error {
+	name := "spinifex-" + strings.TrimPrefix(bridge, "br-") + "-bridge.service"
+	unit := fmt.Sprintf(`[Unit]
+Description=Spinifex %s bridge (non-critical)
 After=network-online.target
 Wants=network-online.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/networkctl up br-lan
+ExecStart=/usr/bin/networkctl up %s
 RemainAfterExit=yes
 # Failure is non-critical — cable unplugged or switch not ready at boot.
 SuccessExitStatus=0 1
 
 [Install]
 WantedBy=multi-user.target
-`
-	unitPath := filepath.Join(root, "etc/systemd/system/spinifex-lan-bridge.service")
+`, bridge, bridge)
+	unitPath := filepath.Join(root, "etc/systemd/system", name)
 	if err := os.WriteFile(unitPath, []byte(unit), 0o644); err != nil {
 		return err
 	}
-	wantsDir := filepath.Join(root, "etc/systemd/system/multi-user.target.wants")
-	if err := os.MkdirAll(wantsDir, 0o755); err != nil {
-		return err
-	}
-	link := filepath.Join(wantsDir, "spinifex-lan-bridge.service")
-	if err := os.Remove(link); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("remove stale symlink %s: %w", link, err)
-	}
-	return os.Symlink("/etc/systemd/system/spinifex-lan-bridge.service", link)
+	return EnableUnit(root, name)
 }
 
 // WriteFirstbootUnit writes the spinifex-firstboot.service oneshot unit that

cmd/installer/ui/network.go

@@ -0,0 +1,696 @@
+package ui
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	"github.com/charmbracelet/bubbles/textinput"
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/charmbracelet/lipgloss"
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+	"github.com/mulgadc/spinifex/cmd/installer/netprobe"
+)
+
+// foldedNIC marks a role with no interface of its own, which collapses onto
+// the plane above it (vpc <- lan <- wan).
+const foldedNIC = -1
+
+// roleField identifies one editable element on the per-role screen. Which of
+// these are actually shown depends on the plane, the addressing mode, whether
+// the NIC is wireless, and whether Advanced is on.
+type roleField int
+
+const (
+	roleFieldNIC roleField = iota
+	roleFieldMethod
+	roleFieldIP
+	roleFieldMask
+	roleFieldGateway
+	roleFieldDNS
+	roleFieldVLAN
+	roleFieldMTU
+	roleFieldSSID
+	roleFieldWiFiPass
+)
+
+// roleForm is the editable state for one plane. The three forms are the whole
+// of the installer's network configuration.
+type roleForm struct {
+	plane install.Plane
+
+	// nic indexes into model.nics, or foldedNIC when the role is folded.
+	nic   int
+	dhcp  bool
+	focus roleField
+
+	ip       textinput.Model
+	mask     textinput.Model
+	gateway  textinput.Model
+	dns      textinput.Model
+	vlan     textinput.Model
+	mtu      textinput.Model
+	ssid     textinput.Model
+	wifiPass textinput.Model
+}
+
+func newRoleForm(plane install.Plane, nic int) roleForm {
+	text := func(placeholder string, limit int) textinput.Model {
+		in := textinput.New()
+		in.Prompt = ""
+		in.Placeholder = placeholder
+		if limit > 0 {
+			in.CharLimit = limit
+		}
+		return in
+	}
+
+	f := roleForm{
+		plane:    plane,
+		nic:      nic,
+		dhcp:     plane == install.PlaneWAN,
+		ip:       text(defaultIPPlaceholder(plane), 0),
+		mask:     text("255.255.255.0", 0),
+		gateway:  text("192.168.1.1", 0),
+		dns:      text("1.1.1.1, 8.8.8.8", 0),
+		vlan:     text("untagged", 4),
+		mtu:      text("1500", 5),
+		ssid:     text("Network SSID", 64),
+		wifiPass: text("WiFi password", 128),
+	}
+	f.wifiPass.EchoMode = textinput.EchoPassword
+	return f
+}
+
+// defaultIPPlaceholder hints the addressing convention for each plane so an
+// operator filling in a rack does not have to remember it.
+func defaultIPPlaceholder(plane install.Plane) string {
+	switch plane {
+	case install.PlaneLAN:
+		return "10.0.0.3"
+	case install.PlaneVPC:
+		return "10.1.0.3"
+	default:
+		return "192.168.1.10"
+	}
+}
+
+func (f *roleForm) folded() bool { return f.nic == foldedNIC }
+
+// canFold reports whether this role may be left unbound. wan always needs an
+// uplink, so only lan and vpc can fold.
+func (f *roleForm) canFold() bool { return f.plane != install.PlaneWAN }
+
+// visibleFields returns the fields shown for this form, in tab order. Hiding
+// rather than disabling keeps the default screen narrow: a DHCP role shows two
+// lines, and VLAN/MTU only appear once Advanced is on.
+func (f *roleForm) visibleFields(isWiFi, advanced bool) []roleField {
+	if f.folded() {
+		return []roleField{roleFieldNIC}
+	}
+	fields := []roleField{roleFieldNIC, roleFieldMethod}
+	if !f.dhcp {
+		fields = append(fields, roleFieldIP, roleFieldMask)
+		// Default route and resolvers are properties of the node, not of each
+		// plane, and wan is the only one that carries either: it holds the
+		// default route, so it is the only plane that can reach an upstream
+		// resolver. On DHCP the lease supplies both.
+		if f.plane == install.PlaneWAN {
+			fields = append(fields, roleFieldGateway, roleFieldDNS)
+		}
+	}
+	if advanced {
+		fields = append(fields, roleFieldVLAN, roleFieldMTU)
+	}
+	if isWiFi {
+		fields = append(fields, roleFieldSSID, roleFieldWiFiPass)
+	}
+	return fields
+}
+
+// input returns the textinput backing a field, or nil for the non-text fields.
+func (f *roleForm) input(field roleField) *textinput.Model {
+	switch field {
+	case roleFieldIP:
+		return &f.ip
+	case roleFieldMask:
+		return &f.mask
+	case roleFieldGateway:
+		return &f.gateway
+	case roleFieldDNS:
+		return &f.dns
+	case roleFieldVLAN:
+		return &f.vlan
+	case roleFieldMTU:
+		return &f.mtu
+	case roleFieldSSID:
+		return &f.ssid
+	case roleFieldWiFiPass:
+		return &f.wifiPass
+	default:
+		return nil
+	}
+}
+
+// blurAll drops focus from every input so only the focused field has a cursor.
+func (f *roleForm) blurAll() {
+	for _, field := range []roleField{
+		roleFieldIP, roleFieldMask, roleFieldGateway, roleFieldDNS,
+		roleFieldVLAN, roleFieldMTU, roleFieldSSID, roleFieldWiFiPass,
+	} {
+		f.input(field).Blur()
+	}
+}
+
+// focusCurrent gives the cursor to the focused field, if it is a text field.
+func (f *roleForm) focusCurrent() {
+	f.blurAll()
+	if in := f.input(f.focus); in != nil {
+		in.Focus()
+	}
+}
+
+// toRole converts the form into the install package's role model.
+func (f *roleForm) toRole(nics []netprobe.NIC) install.NetworkRole {
+	if f.folded() || f.nic >= len(nics) {
+		return install.NetworkRole{}
+	}
+	role := install.NetworkRole{
+		Interface: nics[f.nic].Name,
+		DHCPMode:  f.dhcp,
+		VLAN:      atoiOr(f.vlan.Value(), 0),
+		MTU:       atoiOr(f.mtu.Value(), 0),
+	}
+	if !f.dhcp {
+		role.Address = strings.TrimSpace(f.ip.Value())
+		role.Mask = strings.TrimSpace(f.mask.Value())
+		role.DNS = parseDNS(f.dns.Value())
+		if f.plane == install.PlaneWAN {
+			role.Gateway = strings.TrimSpace(f.gateway.Value())
+		}
+	}
+	if nics[f.nic].IsWiFi {
+		role.WiFiSSID = strings.TrimSpace(f.ssid.Value())
+		role.WiFiPass = f.wifiPass.Value()
+	}
+	return role
+}
+
+func atoiOr(s string, fallback int) int {
+	n, err := strconv.Atoi(strings.TrimSpace(s))
+	if err != nil {
+		return fallback
+	}
+	return n
+}
+
+// ── Roles overview screen ─────────────────────────────────────────────────────
+
+// continueRow is the cursor position below the three role rows.
+const continueRow = 3
+
+func (m model) handleRolesKey(key string) (tea.Model, tea.Cmd) {
+	switch key {
+	case "up", "k":
+		if m.roleCursor > 0 {
+			m.roleCursor--
+		}
+	case "down", "j":
+		if m.roleCursor < continueRow {
+			m.roleCursor++
+		}
+	case "a":
+		// Advanced reveals VLAN and MTU on every role at once, rather than
+		// per-role, so the operator flips it on for the whole node.
+		m.advanced = !m.advanced
+		m.validationErr = ""
+	case "enter":
+		if m.roleCursor == continueRow {
+			if err := m.buildConfig().Validate(); err != nil {
+				m.validationErr = capitalise(err.Error())
+				return m, nil
+			}
+			m.validationErr = ""
+			m.screen = screenIdentity
+			m.hostnameInput.Focus()
+			return m, nil
+		}
+		m.validationErr = ""
+		m.screen = screenNetworkRole
+		m.roles[m.roleCursor].focus = roleFieldNIC
+		m.roles[m.roleCursor].blurAll()
+	case "esc":
+		m.screen = screenDiskConfirm
+		m.eraseInput.Focus()
+	}
+	return m, nil
+}
+
+func (m model) viewNetworkRoles(w int) string {
+	title := styleTitle.Render("Network Planes")
+	subtitle := styleMuted.Render("A role left unbound folds onto the plane above it.")
+
+	adv := styleMuted.Render("Advanced: off")
+	if m.advanced {
+		adv = styleSuccess.Render("Advanced: on")
+	}
+
+	header := fmt.Sprintf("  %-*s%-*s", colRole, "ROLE", colIface, "INTERFACE")
+	if m.advanced {
+		header += fmt.Sprintf("%-*s%-*s", colVLAN, "VLAN", colMTU, "MTU")
+	}
+	header += "ADDRESS"
+
+	lines := []string{title, subtitle, "", adv, "", styleMuted.Render(header)}
+
+	for i := range m.roles {
+		row := m.renderRoleRow(i)
+		if i == m.roleCursor {
+			lines = append(lines, styleSelected.Render("> "+row))
+		} else {
+			lines = append(lines, "  "+row)
+		}
+		if detail := m.roleHardware(i, bodyWidth(w)); detail != "" {
+			lines = append(lines, detail)
+		}
+	}
+
+	lines = append(lines, "")
+	cont := "  Continue"
+	if m.roleCursor == continueRow {
+		cont = styleSelected.Render("> Continue")
+	}
+	lines = append(lines, cont)
+
+	if m.validationErr != "" {
+		lines = append(lines, "", styleError.Render(m.validationErr))
+	}
+	lines = append(lines, styleHelp.Render("↑/↓ move • Enter edit • a advanced • Esc back"))
+
+	body := lipgloss.JoinVertical(lipgloss.Left, lines...)
+	return lipgloss.Place(w, m.height, lipgloss.Center, lipgloss.Center,
+		styleBox.Width(min(w-4, 78)).Render(body),
+	)
+}
+
+// renderRoleRow renders one line of the overview table. A folded role names
+// the plane it collapsed onto rather than showing a blank, so the operator can
+// always see where the traffic will actually go.
+func (m model) renderRoleRow(i int) string {
+	f := &m.roles[i]
+	name := fmt.Sprintf("%-5s", f.plane)
+
+	if f.folded() {
+		_, landed := m.buildConfig().Resolve(f.plane)
+		return fmt.Sprintf("%-*s%s", colRole, name, styleMuted.Render("folds onto "+string(landed)))
+	}
+
+	iface := styleMuted.Render("—")
+	if f.nic < len(m.nics) {
+		iface = nicSummary(m.nics[f.nic])
+	}
+
+	addr := "DHCP"
+	if !f.dhcp {
+		addr = formatCIDR(f.ip.Value(), f.mask.Value())
+		if addr == "" {
+			addr = styleWarning.Render("not set")
+		}
+	}
+
+	row := padCell(name, colRole) + padCell(iface, colIface)
+	if m.advanced {
+		row += padCell(valueOr(f.vlan.Value(), "—"), colVLAN) + padCell(valueOr(f.mtu.Value(), "—"), colMTU)
+	}
+	return row + addr
+}
+
+// roleHardware is the vendor/model line beneath a role row, indented under the
+// interface column. It is what tells an operator that wan really is the 25gbe
+// Mellanox and not the onboard 1gbe.
+func (m model) roleHardware(i, width int) string {
+	f := &m.roles[i]
+	if f.folded() || f.nic >= len(m.nics) {
+		return ""
+	}
+	indent := 2 + colRole
+	return strings.Repeat(" ", indent) + styleMuted.Render(nicHardware(m.nics[f.nic], width-indent))
+}
+
+// nicSummary renders name, speed and link state, colouring the state so a port
+// with no cable stands out rather than having to be read for.
+func nicSummary(n netprobe.NIC) string {
+	state := styleWarning.Render(n.State)
+	if n.Carrier {
+		state = styleSuccess.Render(n.State)
+	}
+	speed := n.Speed
+	if speed == "" {
+		speed = "—"
+	}
+	return fmt.Sprintf("%s  %s  %s", n.Name, speed, state)
+}
+
+// nicHardware is the detail line: what the card is, plus the predictable name
+// it also answers to, which is the one written on cabling notes. The alt name
+// is dropped rather than truncated when the two will not fit together.
+func nicHardware(n netprobe.NIC, width int) string {
+	hw := n.Hardware()
+	const separator = "  ·  "
+	if n.AltName != "" && len([]rune(hw))+len(separator)+len([]rune(n.AltName)) <= width {
+		return hw + separator + n.AltName
+	}
+	return truncate(hw, width)
+}
+
+// bodyWidth is the text width inside the framed body. Detail lines are trimmed
+// to it rather than left to wrap, which would push the table out of alignment.
+func bodyWidth(w int) int {
+	return min(w-4, 78) - 4
+}
+
+// padCell pads a cell to width by display width, which the %-*s verb cannot do
+// once a cell carries the ANSI escapes of a style.
+func padCell(s string, width int) string {
+	return lipgloss.PlaceHorizontal(width, lipgloss.Left, s)
+}
+
+// Column widths for the overview table, shared by the header and the rows.
+const (
+	colRole  = 6
+	colIface = 32
+	colVLAN  = 7
+	colMTU   = 7
+)
+
+// focusMark gives the focused text field a visible caret without every field
+// rendering its own prompt.
+func focusMark(focused bool) string {
+	if focused {
+		return styleLabel.Render("> ")
+	}
+	return "  "
+}
+
+// formatCIDR renders an address and mask the way an operator writes them,
+// falling back to the raw mask when it is not dotted-decimal.
+func formatCIDR(addr, mask string) string {
+	addr = strings.TrimSpace(addr)
+	mask = strings.TrimSpace(mask)
+	if addr == "" {
+		return ""
+	}
+	if mask == "" {
+		return addr
+	}
+	if ip := net.ParseIP(mask); ip != nil && ip.To4() != nil {
+		if ones, bits := net.IPMask(ip.To4()).Size(); bits != 0 {
+			return fmt.Sprintf("%s/%d", addr, ones)
+		}
+	}
+	return addr + "/" + strings.TrimPrefix(mask, "/")
+}
+
+func valueOr(v, fallback string) string {
+	if v = strings.TrimSpace(v); v != "" {
+		return v
+	}
+	return fallback
+}
+
+// ── Per-role editor screen ────────────────────────────────────────────────────
+
+func (m model) handleRoleKey(key string, msg tea.KeyMsg) (tea.Model, tea.Cmd) {
+	f := &m.roles[m.roleCursor]
+	fields := f.visibleFields(m.nicIsWiFi(f.nic), m.advanced)
+	pos := fieldIndex(fields, f.focus)
+
+	switch key {
+	case "esc":
+		f.blurAll()
+		m.screen = screenNetworkRoles
+		m.validationErr = ""
+		return m, nil
+
+	case "enter":
+		if err := m.validateRole(m.roleCursor); err != "" {
+			m.validationErr = err
+			return m, nil
+		}
+		f.blurAll()
+		m.screen = screenNetworkRoles
+		m.validationErr = ""
+		return m, nil
+
+	case "tab", "down":
+		pos = (pos + 1) % len(fields)
+		f.focus = fields[pos]
+		f.focusCurrent()
+		return m, nil
+
+	case "shift+tab", "up":
+		pos = (pos - 1 + len(fields)) % len(fields)
+		f.focus = fields[pos]
+		f.focusCurrent()
+		return m, nil
+
+	case "left", "right":
+		// Only the two selector fields consume arrows; text fields need them
+		// for cursor movement.
+		switch f.focus {
+		case roleFieldNIC:
+			return m.cycleNIC(m.roleCursor, key == "right"), nil
+		case roleFieldMethod:
+			f.dhcp = key == "left"
+			f.focusCurrent()
+			return m, nil
+		}
+	}
+
+	if in := f.input(f.focus); in != nil {
+		var cmd tea.Cmd
+		*in, cmd = in.Update(msg)
+		return m, cmd
+	}
+	return m, nil
+}
+
+// cycleNIC steps the interface selection, including the fold option for lan
+// and vpc. Wrapping keeps a single key usable on a node with one NIC.
+func (m model) cycleNIC(idx int, forward bool) model {
+	f := &m.roles[idx]
+	low := 0
+	if f.canFold() {
+		low = foldedNIC
+	}
+	high := len(m.nics) - 1
+
+	if forward {
+		if f.nic >= high {
+			f.nic = low
+		} else {
+			f.nic++
+		}
+		return m
+	}
+	if f.nic <= low {
+		f.nic = high
+	} else {
+		f.nic--
+	}
+	return m
+}
+
+func (m model) nicIsWiFi(idx int) bool {
+	return idx >= 0 && idx < len(m.nics) && m.nics[idx].IsWiFi
+}
+
+func (m model) viewNetworkRole(w int) string {
+	f := &m.roles[m.roleCursor]
+	title := styleTitle.Render(fmt.Sprintf("Configure %s plane", strings.ToUpper(string(f.plane))))
+	subtitle := styleMuted.Render(planeDescription(f.plane))
+
+	lines := []string{title, subtitle, ""}
+	for _, field := range f.visibleFields(m.nicIsWiFi(f.nic), m.advanced) {
+		lines = append(lines, m.renderRoleField(f, field, bodyWidth(w)), "")
+	}
+
+	if m.validationErr != "" {
+		lines = append(lines, styleError.Render(m.validationErr), "")
+	}
+	lines = append(lines, styleHelp.Render("Tab/↑↓ move • ←/→ change • Enter done • Esc cancel"))
+
+	body := lipgloss.JoinVertical(lipgloss.Left, lines...)
+	return lipgloss.Place(w, m.height, lipgloss.Center, lipgloss.Center,
+		styleBox.Width(min(w-4, 78)).Render(body),
+	)
+}
+
+// planeDescription explains what rides each plane, so the choice is not just a
+// three-letter name.
+func planeDescription(p install.Plane) string {
+	switch p {
+	case install.PlaneWAN:
+		return "Public ingress and egress, EIPs and NAT (br-wan)."
+	case install.PlaneLAN:
+		return "Storage replication, NATS mesh and OVN control (br-lan)."
+	default:
+		return "Geneve tunnels carrying EC2 traffic between nodes (br-vpc)."
+	}
+}
+
+// roleLabelWidth is the field-label column on the editor screen, shared so a
+// wrapped value can indent to line up underneath it.
+const roleLabelWidth = 14
+
+func (m model) renderRoleField(f *roleForm, field roleField, width int) string {
+	focused := f.focus == field
+	label := styleLabel.Render(fmt.Sprintf("%-*s", roleLabelWidth, roleFieldLabel(field)))
+
+	switch field {
+	case roleFieldNIC:
+		return label + m.renderNICChoice(f, focused, width)
+
+	case roleFieldMethod:
+		return label + renderToggle([]string{"DHCP (automatic)", "Static"}, boolToIdx(!f.dhcp), focused)
+
+	default:
+		if in := f.input(field); in != nil {
+			return label + focusMark(focused) + in.View()
+		}
+		return label
+	}
+}
+
+func roleFieldLabel(field roleField) string {
+	switch field {
+	case roleFieldNIC:
+		return "Interface"
+	case roleFieldMethod:
+		return "IP method"
+	case roleFieldIP:
+		return "IP address"
+	case roleFieldMask:
+		return "Subnet mask"
+	case roleFieldGateway:
+		return "Gateway"
+	case roleFieldDNS:
+		return "DNS"
+	case roleFieldVLAN:
+		return "VLAN id"
+	case roleFieldMTU:
+		return "MTU"
+	case roleFieldSSID:
+		return "WiFi SSID"
+	default:
+		return "WiFi password"
+	}
+}
+
+// renderNICChoice shows the selected interface on the field line and its
+// hardware identity on a line of its own beneath. Vendor and model are what
+// make a port identifiable, and they do not fit alongside the selector.
+func (m model) renderNICChoice(f *roleForm, focused bool, width int) string {
+	var text, detail string
+	switch {
+	case f.folded():
+		_, landed := m.buildConfig().Resolve(f.plane)
+		text = fmt.Sprintf("(fold onto %s)", landed)
+	case f.nic < len(m.nics):
+		text = nicSummary(m.nics[f.nic])
+		detail = nicHardware(m.nics[f.nic], width-roleLabelWidth-2)
+	default:
+		text = "(none detected)"
+	}
+
+	if focused {
+		text = styleSelected.Render(" ← " + text + " → ")
+	} else {
+		text = styleLabel.Render("[ " + text + " ]")
+	}
+	if detail == "" {
+		return text
+	}
+	return text + "\n" + strings.Repeat(" ", roleLabelWidth+2) + styleMuted.Render(detail)
+}
+
+func renderToggle(options []string, selected int, focused bool) string {
+	var parts []string
+	for i, opt := range options {
+		switch {
+		case i == selected && focused:
+			parts = append(parts, styleSelected.Render(" "+opt+" "))
+		case i == selected:
+			parts = append(parts, styleLabel.Render("["+opt+"]"))
+		default:
+			parts = append(parts, styleMuted.Render(opt))
+		}
+	}
+	return strings.Join(parts, "  ")
+}
+
+func boolToIdx(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+func fieldIndex(fields []roleField, want roleField) int {
+	for i, f := range fields {
+		if f == want {
+			return i
+		}
+	}
+	return 0
+}
+
+// validateRole checks one plane's entries. Cross-role rules (a shared NIC
+// without distinct VLAN ids, an unbound wan) belong to install.Config.Validate
+// and run when the operator continues.
+func (m model) validateRole(idx int) string {
+	f := &m.roles[idx]
+	if f.folded() {
+		return ""
+	}
+	if len(m.nics) == 0 {
+		return "No network interfaces detected"
+	}
+
+	if !f.dhcp {
+		if net.ParseIP(strings.TrimSpace(f.ip.Value())) == nil {
+			return "Enter a valid IP address"
+		}
+		if !validSubnetMask(f.mask.Value()) {
+			return "Enter a dotted-decimal subnet mask, e.g. 255.255.255.0 or 255.255.0.0"
+		}
+		if f.plane == install.PlaneWAN && net.ParseIP(strings.TrimSpace(f.gateway.Value())) == nil {
+			return "Enter a valid gateway address"
+		}
+	}
+
+	if v := strings.TrimSpace(f.vlan.Value()); v != "" {
+		id, err := strconv.Atoi(v)
+		if err != nil || id < 1 || id > 4094 {
+			return "VLAN id must be between 1 and 4094"
+		}
+	}
+	if v := strings.TrimSpace(f.mtu.Value()); v != "" {
+		mtu, err := strconv.Atoi(v)
+		if err != nil || mtu < 68 || mtu > 9216 {
+			return "MTU must be between 68 and 9216"
+		}
+	}
+	if m.nicIsWiFi(f.nic) && strings.TrimSpace(f.ssid.Value()) == "" {
+		return "Enter the WiFi network SSID"
+	}
+	return ""
+}
+
+func capitalise(s string) string {
+	if s == "" {
+		return s
+	}
+	return strings.ToUpper(s[:1]) + s[1:]
+}

cmd/installer/ui/network_test.go

@@ -0,0 +1,357 @@
+package ui
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+	"github.com/mulgadc/spinifex/cmd/installer/netprobe"
+)
+
+func nics(n int) []netprobe.NIC {
+	out := make([]netprobe.NIC, 0, n)
+	names := []string{"eno1", "ens1f0np0", "ens1f1np1", "ens2f0np0"}
+	for i := range n {
+		out = append(out, netprobe.NIC{Name: names[i], Speed: "25Gbps", Vendor: "Mellanox", State: "online"})
+	}
+	return out
+}
+
+// The pre-fill is the whole of the simple path: an operator with a
+// conventional node should be able to press Continue without editing anything.
+func TestNewModelPrefillsRolesByNICCount(t *testing.T) {
+	tests := []struct {
+		name             string
+		count            int
+		wantLAN, wantVPC int
+	}{
+		{"one NIC folds everything onto wan", 1, foldedNIC, foldedNIC},
+		{"two NICs dedicate the second to lan, vpc folds", 2, 1, foldedNIC},
+		{"three NICs give each plane its own", 3, 1, 2},
+		{"four NICs still only bind three", 4, 1, 2},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(tt.count))
+
+			if got := m.roles[0].nic; got != 0 {
+				t.Errorf("wan nic = %d, want 0", got)
+			}
+			if got := m.roles[1].nic; got != tt.wantLAN {
+				t.Errorf("lan nic = %d, want %d", got, tt.wantLAN)
+			}
+			if got := m.roles[2].nic; got != tt.wantVPC {
+				t.Errorf("vpc nic = %d, want %d", got, tt.wantVPC)
+			}
+		})
+	}
+}
+
+// Hiding rather than disabling is what keeps the default screen narrow.
+func TestVisibleFields(t *testing.T) {
+	tests := []struct {
+		name     string
+		mutate   func(*roleForm)
+		isWiFi   bool
+		advanced bool
+		want     []roleField
+	}{
+		{
+			name:   "wan on DHCP asks two questions",
+			mutate: func(f *roleForm) { f.dhcp = true },
+			want:   []roleField{roleFieldNIC, roleFieldMethod},
+		},
+		{
+			name:   "wan static adds a gateway",
+			mutate: func(f *roleForm) { f.dhcp = false },
+			want:   []roleField{roleFieldNIC, roleFieldMethod, roleFieldIP, roleFieldMask, roleFieldGateway, roleFieldDNS},
+		},
+		{
+			name:     "advanced reveals VLAN and MTU",
+			mutate:   func(f *roleForm) { f.dhcp = true },
+			advanced: true,
+			want:     []roleField{roleFieldNIC, roleFieldMethod, roleFieldVLAN, roleFieldMTU},
+		},
+		{
+			name:   "wireless adds credentials",
+			mutate: func(f *roleForm) { f.dhcp = true },
+			isWiFi: true,
+			want:   []roleField{roleFieldNIC, roleFieldMethod, roleFieldSSID, roleFieldWiFiPass},
+		},
+		{
+			name:   "a folded role has nothing to configure",
+			mutate: func(f *roleForm) { f.nic = foldedNIC },
+			want:   []roleField{roleFieldNIC},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newRoleForm(install.PlaneWAN, 0)
+			tt.mutate(&f)
+			got := f.visibleFields(tt.isWiFi, tt.advanced)
+
+			if len(got) != len(tt.want) {
+				t.Fatalf("visibleFields() = %v, want %v", got, tt.want)
+			}
+			for i := range got {
+				if got[i] != tt.want[i] {
+					t.Fatalf("visibleFields() = %v, want %v", got, tt.want)
+				}
+			}
+		})
+	}
+}
+
+// A non-wan role must be able to reach the folded state, and wan must not.
+func TestCycleNICFoldReachableOnlyOffWAN(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(2))
+
+	// lan: 1 -> 0 -> folded -> wraps to the last NIC.
+	m.roles[1].nic = 1
+	for _, want := range []int{0, foldedNIC, 1} {
+		m = m.cycleNIC(1, false)
+		if m.roles[1].nic != want {
+			t.Fatalf("lan nic = %d, want %d", m.roles[1].nic, want)
+		}
+	}
+
+	// wan cycles only through real interfaces.
+	m.roles[0].nic = 0
+	for range 4 {
+		m = m.cycleNIC(0, false)
+		if m.roles[0].nic == foldedNIC {
+			t.Fatal("wan reached the folded state, but it always needs an uplink")
+		}
+	}
+}
+
+func TestToRoleReadsVLANAndMTU(t *testing.T) {
+	f := newRoleForm(install.PlaneLAN, 0)
+	f.dhcp = false
+	f.ip.SetValue("10.0.0.2")
+	f.mask.SetValue("255.255.0.0")
+	f.vlan.SetValue("10")
+	f.mtu.SetValue("9000")
+
+	role := f.toRole(nics(1))
+
+	if role.VLAN != 10 || role.MTU != 9000 {
+		t.Errorf("VLAN/MTU = %d/%d, want 10/9000", role.VLAN, role.MTU)
+	}
+	if role.Link() != "eno1.10" {
+		t.Errorf("Link() = %q, want eno1.10", role.Link())
+	}
+	// Only wan carries a default route, even if a gateway was typed earlier.
+	f.gateway.SetValue("10.0.0.1")
+	if got := f.toRole(nics(1)).Gateway; got != "" {
+		t.Errorf("lan Gateway = %q, want empty", got)
+	}
+}
+
+func TestToRoleFoldedIsEmpty(t *testing.T) {
+	f := newRoleForm(install.PlaneVPC, foldedNIC)
+	if !f.toRole(nics(2)).Folded() {
+		t.Error("a folded form must produce a folded role")
+	}
+}
+
+func TestValidateRole(t *testing.T) {
+	tests := []struct {
+		name    string
+		plane   install.Plane
+		mutate  func(*roleForm)
+		wantErr string
+	}{
+		{"valid static lan", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("10.0.0.2")
+			f.mask.SetValue("255.255.0.0")
+		}, ""},
+		{"bad ip", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("not-an-ip")
+			f.mask.SetValue("255.255.0.0")
+		}, "valid IP"},
+		{"wan needs a gateway", install.PlaneWAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("216.218.163.99")
+			f.mask.SetValue("255.255.255.224")
+		}, "gateway"},
+		{"VLAN out of range", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = true
+			f.vlan.SetValue("9000")
+		}, "VLAN id must be"},
+		{"MTU out of range", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = true
+			f.mtu.SetValue("70000")
+		}, "MTU must be"},
+		{"folded roles skip validation", install.PlaneVPC, func(f *roleForm) {
+			f.nic = foldedNIC
+			f.ip.SetValue("nonsense")
+		}, ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(3))
+			m.roles[0] = newRoleForm(tt.plane, 0)
+			tt.mutate(&m.roles[0])
+
+			got := m.validateRole(0)
+			if tt.wantErr == "" {
+				if got != "" {
+					t.Fatalf("validateRole() = %q, want no error", got)
+				}
+				return
+			}
+			if !strings.Contains(got, tt.wantErr) {
+				t.Fatalf("validateRole() = %q, want it to mention %q", got, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Continue must refuse a layout install.Config.Validate would reject, rather
+// than failing later during the install itself.
+func TestContinueBlocksOnSharedNICWithoutVLANs(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(2))
+	m.screen = screenNetworkRoles
+	m.roleCursor = continueRow
+	m.roles[1].nic = 0 // lan onto the wan NIC, both untagged
+
+	next, _ := m.handleRolesKey("enter")
+	got := next.(model)
+
+	if got.screen != screenNetworkRoles {
+		t.Error("expected to stay on the roles screen when the layout is invalid")
+	}
+	if !strings.Contains(got.validationErr, "both bind eno1") {
+		t.Errorf("validationErr = %q, want it to name the shared interface", got.validationErr)
+	}
+
+	// Distinct VLAN ids make the same pairing legal.
+	got.roles[0].vlan.SetValue("30")
+	got.roles[1].vlan.SetValue("10")
+	next, _ = got.handleRolesKey("enter")
+	if next.(model).screen != screenIdentity {
+		t.Error("distinct VLAN ids should allow two roles to share one NIC")
+	}
+}
+
+func TestAdvancedToggle(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(3))
+	m.screen = screenNetworkRoles
+
+	next, _ := m.handleRolesKey("a")
+	if !next.(model).advanced {
+		t.Fatal("'a' should turn Advanced on")
+	}
+	next, _ = next.(model).handleRolesKey("a")
+	if next.(model).advanced {
+		t.Fatal("'a' should turn Advanced back off")
+	}
+}
+
+// The default route and the resolvers are node-level, and wan is the only plane
+// that carries either, so the east-west editors must not ask for them. Their
+// bridges are link-local and activate manually — a resolver pinned there is
+// unreachable, and a gateway would fight the real default route.
+func TestVisibleFieldsOmitsRoutingOffWAN(t *testing.T) {
+	for _, plane := range []install.Plane{install.PlaneLAN, install.PlaneVPC} {
+		t.Run(string(plane), func(t *testing.T) {
+			f := newRoleForm(plane, 1)
+			f.dhcp = false
+			got := f.visibleFields(false, false)
+
+			want := []roleField{roleFieldNIC, roleFieldMethod, roleFieldIP, roleFieldMask}
+			if len(got) != len(want) {
+				t.Fatalf("visibleFields() = %v, want %v", got, want)
+			}
+			for i := range got {
+				if got[i] != want[i] {
+					t.Fatalf("visibleFields() = %v, want %v", got, want)
+				}
+			}
+		})
+	}
+}
+
+// The mask field must accept only what netgen can consume. It previously
+// advertised and accepted a prefix length, which passed the TUI and then failed
+// the install itself with "invalid mask: 24".
+func TestValidSubnetMaskRejectsPrefixLength(t *testing.T) {
+	for _, bad := range []string{"24", "/24", "8", "", "255.255.0", "255.0.255.0", "::1"} {
+		if validSubnetMask(bad) {
+			t.Errorf("validSubnetMask(%q) = true, want false", bad)
+		}
+	}
+	for _, good := range []string{"255.255.255.0", "255.255.0.0", "255.0.0.0", "255.255.255.252"} {
+		if !validSubnetMask(good) {
+			t.Errorf("validSubnetMask(%q) = false, want true", good)
+		}
+	}
+}
+
+// The hardware detail is the point of the NIC table: an operator has to be able
+// to tell which physical port each plane landed on. It gets a line of its own,
+// under the interface column, rather than being squeezed into the row.
+func TestRoleHardwareRendersOnItsOwnLine(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, []netprobe.NIC{{
+		Name: "ens1f0np0", Vendor: "Mellanox Technologies", Model: "MT27710",
+		AltName: "enp132s0f0np0", Speed: "25 Gbps", Carrier: true, State: "online",
+	}})
+
+	view := m.viewNetworkRoles(100)
+	if !strings.Contains(view, "ens1f0np0  25 Gbps  online") {
+		t.Errorf("expected the identity line in:\n%s", view)
+	}
+	detail := m.roleHardware(0, bodyWidth(100))
+	if !strings.Contains(detail, "Mellanox Technologies MT27710") {
+		t.Errorf("roleHardware = %q, want the vendor and model", detail)
+	}
+	if !strings.Contains(detail, "enp132s0f0np0") {
+		t.Errorf("roleHardware = %q, want the alternative name", detail)
+	}
+	// Every line has to fit the box, or lipgloss wraps it and the columns shear.
+	for line := range strings.SplitSeq(view, "\n") {
+		if len([]rune(line)) > 100 {
+			t.Errorf("line exceeds the terminal width: %q", line)
+		}
+	}
+}
+
+// A narrow box drops the alternative name before it truncates the model, since
+// half a model name identifies nothing.
+func TestNICHardwareDropsAltNameBeforeTruncating(t *testing.T) {
+	n := netprobe.NIC{Vendor: "Broadcom Inc. and subsidiaries", Model: "NetXtreme BCM5720", AltName: "enp1s0f1"}
+
+	if got := nicHardware(n, 60); got != "Broadcom Inc. and subsidiaries NetXtreme BCM5720" {
+		t.Errorf("nicHardware(60) = %q, want the name without the alt name", got)
+	}
+	if got := nicHardware(n, 20); got != "Broadcom Inc. and s…" {
+		t.Errorf("nicHardware(20) = %q, want a marked truncation", got)
+	}
+}
+
+// A role bound to a port with no hardware database entry still gets something
+// to identify the card by, never the bare word "unknown".
+func TestRoleHardwareFallsBackToDriver(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, []netprobe.NIC{{
+		Name: "eno1", Driver: "bnxt_en", DeviceID: "14e4:165f", State: "no cable",
+	}})
+
+	if got := m.roleHardware(0, bodyWidth(100)); !strings.Contains(got, "bnxt_en [14e4:165f]") {
+		t.Errorf("roleHardware = %q, want the driver and device id", got)
+	}
+}
+
+// A folded role has no interface, so it must not render an orphan detail line
+// under the plane it collapsed onto.
+func TestRoleHardwareEmptyWhenFolded(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(1))
+	if got := m.roleHardware(1, bodyWidth(100)); got != "" {
+		t.Errorf("roleHardware for a folded role = %q, want empty", got)
+	}
+}

cmd/lb-agent/main.go

@@ -19,11 +19,17 @@ import (
 	"syscall"
 
 	"github.com/mulgadc/spinifex/spinifex/lbagent"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 
 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 func main() {
+	// Without this the agent falls back to slog's default text handler, which folds
+	// attributes into the message so the console scraper recovers a level and nothing
+	// else. No OTLP export: the agent's only egress is the gateway it is watching.
+	otelsetup.SetDefaultJSONLogger(slog.LevelInfo)
+
 	var (
 		lbID       string
 		gatewayURL string

cmd/rds-agent/agent.go

@@ -0,0 +1,254 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/gwsign"
+	"github.com/mulgadc/spinifex/internal/rdsgw"
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Overridable via -ldflags "-X main.version=...".
+var version = "dev"
+
+// The gateway resolves the authoritative DB instance from the request's
+// credentials, so DBInstanceIdentifier is an assertion to be checked, not
+// trusted.
+type identity struct {
+	DBInstanceIdentifier string
+	AgentVersion         string
+	EngineVersion        string
+}
+
+// Every method is a SigV4-signed Query request, so the NATS bus stays
+// host-internal.
+type controlPlane interface {
+	Register(ctx context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error)
+	SubmitState(ctx context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error)
+	// The first call of an instance's life returns Mode=initialize with the
+	// master password; every later call returns Mode=attach without it.
+	GetBootstrapConfig(ctx context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error)
+	PollCommands(ctx context.Context, id identity, replies []handlers_rds.CommandReply, wait time.Duration) ([]handlers_rds.Command, error)
+}
+
+// Bounds a register, heartbeat or bootstrap request. The long poll sets its own
+// deadline from the wait it asked the gateway to hold.
+const callTimeout = 15 * time.Second
+
+// Keeps the poll's deadline past the requested wait, so a gateway answering at
+// the end of its window is not cut off by the client.
+const pollSlack = 10 * time.Second
+
+type gatewayControlPlane struct {
+	client *rdsgw.Client
+}
+
+var _ controlPlane = (*gatewayControlPlane)(nil)
+
+// Signs with the instance-role credentials the SDK chain resolves from IMDS,
+// against the pinned gateway CA.
+func newGatewayControlPlane(cfg config) (*gatewayControlPlane, error) {
+	signer, err := gwsign.NewIMDS(context.Background(), cfg.Region)
+	if err != nil {
+		return nil, fmt.Errorf("build IMDS signer: %w", err)
+	}
+	// The client timeout is the ceiling for the longest call, the long poll;
+	// shorter calls carry their own deadline.
+	client, err := rdsgw.New(cfg.GatewayURL, cfg.GatewayCA, signer, cfg.Region, cfg.PollWait+pollSlack)
+	if err != nil {
+		return nil, err
+	}
+	return &gatewayControlPlane{client: client}, nil
+}
+
+func (g *gatewayControlPlane) Register(ctx context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error) {
+	params := url.Values{}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	setIfPresent(params, "AgentVersion", id.AgentVersion)
+	setIfPresent(params, "EngineVersion", id.EngineVersion)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.RegisterDBInstanceOutput
+	if err := g.client.Call(ctx, "RegisterDBInstance", params, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func (g *gatewayControlPlane) SubmitState(ctx context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error) {
+	params := url.Values{"EngineHealth": {string(health)}}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	setIfPresent(params, "EngineVersion", id.EngineVersion)
+	setIfPresent(params, "Message", message)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.SubmitDBStateChangeOutput
+	if err := g.client.Call(ctx, "SubmitDBStateChange", params, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func (g *gatewayControlPlane) GetBootstrapConfig(ctx context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error) {
+	params := url.Values{}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.GetDBBootstrapConfigOutput
+	if err := g.client.Call(ctx, "GetDBBootstrapConfig", params, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+// Declared here so the in-guest binary does not link the server side of the API.
+type pollDBCommandsOutput struct {
+	Commands []handlers_rds.Command `xml:"Commands>member"`
+}
+
+func (g *gatewayControlPlane) PollCommands(ctx context.Context, id identity, replies []handlers_rds.CommandReply, wait time.Duration) ([]handlers_rds.Command, error) {
+	params := url.Values{"WaitTimeSeconds": {strconv.FormatInt(int64(wait.Seconds()), 10)}}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	// Replies ride the poll rather than a separate action, so replying and
+	// asking for the next command are one round trip.
+	for i, reply := range replies {
+		member := fmt.Sprintf("Replies.member.%d.", i+1)
+		params.Set(member+"CommandId", reply.CommandID)
+		params.Set(member+"Status", reply.Status)
+		setIfPresent(params, member+"Message", reply.Message)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, wait+pollSlack)
+	defer cancel()
+
+	var out pollDBCommandsOutput
+	if err := g.client.Call(ctx, "PollDBCommands", params, &out); err != nil {
+		return nil, err
+	}
+	return out.Commands, nil
+}
+
+// Only sets a parameter that has a value, so the gateway can tell "no
+// assertion" from "asserted as blank".
+func setIfPresent(params url.Values, key, value string) {
+	if value != "" {
+		params.Set(key, value)
+	}
+}
+
+type Agent struct {
+	cfg           config
+	id            identity
+	cp            controlPlane
+	probe         *engineProbe
+	engine        *postgresEngine
+	handoffWriter func(string, *handlers_rds.GetDBBootstrapConfigOutput) error
+
+	hb    *heartbeater
+	cmd   *commander
+	guard *paramGuard
+}
+
+// Assembles from already-built parts, so tests can pass fakes; New builds the
+// production control plane and delegates here.
+func newAgent(cfg config, cp controlPlane, probe *engineProbe) *Agent {
+	id := identity{
+		DBInstanceIdentifier: cfg.DBInstanceIdentifier,
+		AgentVersion:         version,
+		EngineVersion:        cfg.EngineVersion,
+	}
+	a := &Agent{cfg: cfg, id: id, cp: cp, probe: probe, handoffWriter: writeHandoff}
+	a.engine = newPostgresEngine(cfg, execCommandRunner, execSessionRunner)
+	a.hb = newHeartbeater(cp, probe, handlers_rds.HeartbeatInterval)
+	a.cmd = newCommander(cp, newCommandRegistry(a.engine, newGuestStorage(cfg, execCommandRunner)), cfg.PollWait)
+	a.guard = newParamGuard(a.engine, probe)
+	return a
+}
+
+// Does not wait for IMDS: the register loop rides out a datapath still coming up.
+func New(cfg config) (*Agent, error) {
+	if cfg.GatewayURL == "" {
+		return nil, fmt.Errorf("no gateway URL configured (RDS_GATEWAY_URL)")
+	}
+	cp, err := newGatewayControlPlane(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return newAgent(cfg, cp, newEngineProbe(cfg, execProbeRunner)), nil
+}
+
+func (a *Agent) Run(ctx context.Context) error {
+	// Named before the first dial. When register cannot reach the gateway the
+	// retry loop is all that survives, and the address it is dialing is what
+	// separates a mgmt NIC that came up without an address from a control plane
+	// that is genuinely down.
+	slog.Info("rds-agent: starting",
+		"gatewayURL", a.cfg.GatewayURL, "dbInstanceIdentifier", a.id.DBInstanceIdentifier,
+		"agentVersion", a.id.AgentVersion)
+
+	// Registration comes first: its response is the agent's identity.
+	if err := a.register(ctx); err != nil {
+		return err
+	}
+	a.hb.id, a.cmd.id = a.id, a.id
+
+	// Beating before the bootstrap keeps a stuck boot visible as a live VM with
+	// a down engine rather than as silence.
+	go a.hb.Run(ctx)
+
+	if err := a.bootstrap(ctx); err != nil {
+		return err
+	}
+
+	// Directives are only meaningful against a bootstrapped engine.
+	go a.cmd.Run(ctx)
+
+	// Started after the handoff, so the window it measures covers the engine's
+	// own start rather than the time rds-init spent waiting for this agent.
+	go a.guard.Run(ctx)
+
+	<-ctx.Done()
+	return nil
+}
+
+// Boot-retry bounds. Tight to start, since the usual cause is a control plane
+// still creating this instance's record, and capped so an outage does not turn
+// a booting fleet into a retry storm.
+const (
+	retryMin = 1 * time.Second
+	retryMax = 30 * time.Second
+)
+
+// Never gives up on its own: an agent that stopped trying would leave the
+// control plane with no signal at all.
+func retry(ctx context.Context, what string, fn func(context.Context) error) error {
+	delay := retryMin
+	for attempt := 1; ; attempt++ {
+		err := fn(ctx)
+		if err == nil {
+			return nil
+		}
+		if ctx.Err() != nil {
+			return fmt.Errorf("%s: %w", what, ctx.Err())
+		}
+		slog.Warn("rds-agent: "+what+" failed, retrying", "attempt", attempt, "retryIn", delay, "err", err)
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("%s: %w", what, ctx.Err())
+		case <-time.After(delay):
+		}
+		delay = min(delay*2, retryMax)
+	}
+}

cmd/rds-agent/agent_test.go

@@ -0,0 +1,472 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/gwsign"
+	"github.com/mulgadc/spinifex/internal/rdsgw"
+	gateway_rds "github.com/mulgadc/spinifex/spinifex/gateway/rds"
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+)
+
+// fakeControlPlane is a scriptable controlPlane. Every call records what the
+// agent sent, so a test asserts on the request as well as the reaction.
+type fakeControlPlane struct {
+	mu sync.Mutex
+
+	registerOut  *handlers_rds.RegisterDBInstanceOutput
+	registerErrs []error
+	registerReqs []identity
+
+	bootstrapOut  *handlers_rds.GetDBBootstrapConfigOutput
+	bootstrapErrs []error
+	bootstrapReqs []identity
+
+	states []submittedState
+
+	pollReplies [][]handlers_rds.CommandReply
+	pollQueue   [][]handlers_rds.Command
+	pollErr     error
+	polled      chan struct{}
+}
+
+type submittedState struct {
+	id      identity
+	health  handlers_rds.EngineHealth
+	message string
+}
+
+var _ controlPlane = (*fakeControlPlane)(nil)
+
+func newFakeControlPlane() *fakeControlPlane {
+	return &fakeControlPlane{
+		registerOut: &handlers_rds.RegisterDBInstanceOutput{
+			DBInstanceIdentifier: "db-resolved", HeartbeatIntervalSeconds: 30,
+		},
+		polled: make(chan struct{}, 64),
+	}
+}
+
+// nextErr pops the next scripted error, so a test can fail a call n times and
+// then let it succeed.
+func nextErr(errs *[]error) error {
+	if len(*errs) == 0 {
+		return nil
+	}
+	err := (*errs)[0]
+	*errs = (*errs)[1:]
+	return err
+}
+
+func (f *fakeControlPlane) Register(_ context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.registerReqs = append(f.registerReqs, id)
+	if err := nextErr(&f.registerErrs); err != nil {
+		return nil, err
+	}
+	return f.registerOut, nil
+}
+
+func (f *fakeControlPlane) SubmitState(_ context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.states = append(f.states, submittedState{id: id, health: health, message: message})
+	// Zero means "keep the cadence you have", so a test's fast interval is not
+	// reset by every beat.
+	return &handlers_rds.SubmitDBStateChangeOutput{Acknowledged: true}, nil
+}
+
+func (f *fakeControlPlane) GetBootstrapConfig(_ context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.bootstrapReqs = append(f.bootstrapReqs, id)
+	if err := nextErr(&f.bootstrapErrs); err != nil {
+		return nil, err
+	}
+	return f.bootstrapOut, nil
+}
+
+func (f *fakeControlPlane) PollCommands(ctx context.Context, _ identity, replies []handlers_rds.CommandReply, _ time.Duration) ([]handlers_rds.Command, error) {
+	f.mu.Lock()
+	f.pollReplies = append(f.pollReplies, slices.Clone(replies))
+	err := f.pollErr
+	var out []handlers_rds.Command
+	if len(f.pollQueue) > 0 {
+		out, f.pollQueue = f.pollQueue[0], f.pollQueue[1:]
+	}
+	f.mu.Unlock()
+
+	select {
+	case f.polled <- struct{}{}:
+	default:
+	}
+	if err != nil {
+		return nil, err
+	}
+	if len(out) > 0 {
+		return out, nil
+	}
+	// An idle window: block until the agent hangs up, as the gateway's long poll
+	// does, rather than spinning the caller's loop.
+	<-ctx.Done()
+	return nil, ctx.Err()
+}
+
+func (f *fakeControlPlane) snapshotStates() []submittedState {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]submittedState(nil), f.states...)
+}
+
+func (f *fakeControlPlane) snapshotReplies() [][]handlers_rds.CommandReply {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([][]handlers_rds.CommandReply, len(f.pollReplies))
+	for i := range f.pollReplies {
+		out[i] = slices.Clone(f.pollReplies[i])
+	}
+	return out
+}
+
+// testConfig points the agent's file output at a temp dir and its probe at a
+// name no real binary answers to.
+func testConfig(t *testing.T) config {
+	t.Helper()
+	return config{
+		GatewayURL: "https://gw.test",
+		HandoffDir: filepath.Join(t.TempDir(), "spinifex-rds"),
+		EngineHost: defaultEngineHost,
+		EnginePort: defaultEnginePort,
+		PGIsReady:  "pg_isready",
+		PollWait:   time.Second,
+	}
+}
+
+// staticProbe is a probeRunner that always reports the same exit code.
+func staticProbe(code int) probeRunner {
+	return func(context.Context, string, ...string) (int, error) { return code, nil }
+}
+
+func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput {
+	mode := handlers_rds.BootstrapModeAttach
+	if password != nil {
+		mode = handlers_rds.BootstrapModeInitialize
+	}
+	return &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:                 mode,
+		DBInstanceIdentifier: "db-resolved",
+		Engine:               "postgres",
+		MasterUsername:       "master",
+		MasterUserPassword:   password,
+		Port:                 5432,
+	}
+}
+
+// runAgent runs the agent until its bootstrap has landed, then cancels. It
+// returns once Run has exited so no goroutine outlives the test.
+func runAgent(t *testing.T, a *Agent) error {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool {
+		_, err := os.Stat(filepath.Join(a.cfg.HandoffDir, handoffEnvFile))
+		return err == nil
+	}, "handoff file to be written")
+
+	cancel()
+	select {
+	case err := <-done:
+		return err
+	case <-time.After(5 * time.Second):
+		t.Fatal("Run did not exit after cancellation")
+		return nil
+	}
+}
+
+// waitFor polls cond until it holds or the test times out.
+func waitFor(t *testing.T, cond func() bool, what string) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatalf("timed out waiting for %s", what)
+}
+
+func TestRun_RegistersThenDeliversBootstrap(t *testing.T) {
+	password := "s3cret"
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(&password)
+
+	cfg := testConfig(t)
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	if err := runAgent(t, a); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	if len(cp.registerReqs) != 1 {
+		t.Fatalf("registered %d times, want 1", len(cp.registerReqs))
+	}
+	// The agent had no configured identifier, so the gateway's resolution is
+	// what every later call must carry.
+	if got := cp.bootstrapReqs[0].DBInstanceIdentifier; got != "db-resolved" {
+		t.Errorf("bootstrap sent identifier %q, want the registered db-resolved", got)
+	}
+
+	env := readFile(t, filepath.Join(cfg.HandoffDir, handoffEnvFile))
+	if !strings.Contains(env, "RDS_MODE='initialize'") || !strings.Contains(env, "RDS_MASTER_PASSWORD='s3cret'") {
+		t.Errorf("handoff env = %q, want initialize mode carrying the password", env)
+	}
+	// The probe follows the port the control plane assigned, not the default.
+	if got := a.probe.port.Load(); got != 5432 {
+		t.Errorf("probe port = %d, want 5432", got)
+	}
+}
+
+// A reboot re-fetches in attach mode. The agent must write that payload as it
+// stands rather than refusing it or inventing a password.
+func TestRun_AttachModeNeedsNoPassword(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	env := readFile(t, filepath.Join(cfg.HandoffDir, handoffEnvFile))
+	if !strings.Contains(env, "RDS_MODE='attach'") {
+		t.Errorf("handoff env = %q, want attach mode", env)
+	}
+	if strings.Contains(env, "RDS_MASTER_PASSWORD") {
+		t.Errorf("handoff env = %q, want no password key at all in attach mode", env)
+	}
+}
+
+// The record may not exist yet when a freshly launched VM first calls, so both
+// boot-critical calls have to ride out failures instead of exiting.
+func TestRun_RetriesRegisterAndBootstrap(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cp.registerErrs = []error{errors.New("DBInstanceNotFound")}
+	cp.bootstrapErrs = []error{errors.New("ServerInternal")}
+
+	cfg := testConfig(t)
+	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	if len(cp.registerReqs) != 2 {
+		t.Errorf("register attempts = %d, want 2 (one failure, one success)", len(cp.registerReqs))
+	}
+	if len(cp.bootstrapReqs) != 2 {
+		t.Errorf("bootstrap attempts = %d, want 2 (one failure, one success)", len(cp.bootstrapReqs))
+	}
+}
+
+func TestBootstrap_RetriesHandoffWithoutRefetchingConsumedPassword(t *testing.T) {
+	cp := newFakeControlPlane()
+	password := "one-shot-secret"
+	cp.bootstrapOut = bootstrapOutput(&password)
+	cfg := testConfig(t)
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+
+	writes := 0
+	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
+		writes++
+		if got.MasterUserPassword == nil || *got.MasterUserPassword != password {
+			t.Fatalf("handoff attempt %d lost the initialize password", writes)
+		}
+		if writes == 1 {
+			return errors.New("temporary tmpfs write error")
+		}
+		return nil
+	}
+
+	if err := a.bootstrap(t.Context()); err != nil {
+		t.Fatalf("bootstrap: %v", err)
+	}
+	if len(cp.bootstrapReqs) != 1 {
+		t.Errorf("bootstrap fetches = %d, want 1", len(cp.bootstrapReqs))
+	}
+	if writes != 2 {
+		t.Errorf("handoff writes = %d, want 2", writes)
+	}
+}
+
+// A configured identifier is asserted on the wire so a mis-provisioned VM is
+// rejected by the gateway instead of adopting whatever it resolves to.
+func TestRun_SendsConfiguredIdentifier(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.DBInstanceIdentifier = "db-configured"
+	cfg.EngineVersion = "18.1"
+	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	sent := cp.registerReqs[0]
+	if sent.DBInstanceIdentifier != "db-configured" || sent.EngineVersion != "18.1" {
+		t.Errorf("register sent %+v, want the configured identifier and engine version", sent)
+	}
+}
+
+func TestRun_HeartbeatsEngineHealth(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	// Declining to set a cadence leaves the agent's own, which the test shortens
+	// rather than waiting out the 30s production interval.
+	cp.registerOut.HeartbeatIntervalSeconds = 0
+
+	cfg := testConfig(t)
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a.hb.interval = 10 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool { return len(cp.snapshotStates()) > 0 }, "a heartbeat")
+	cancel()
+	<-done
+
+	beat := cp.snapshotStates()[0]
+	if beat.health != handlers_rds.EngineHealthHealthy {
+		t.Errorf("heartbeat health = %q, want healthy", beat.health)
+	}
+	if beat.id.DBInstanceIdentifier != "db-resolved" {
+		t.Errorf("heartbeat identifier = %q, want db-resolved", beat.id.DBInstanceIdentifier)
+	}
+}
+
+// The cadence is the control plane's to set: it feeds the staleness window the
+// recovery reconciler judges an instance against, so the guest adopts it rather
+// than keeping its own.
+func TestRegister_AdoptsControlPlaneCadence(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cp.registerOut.HeartbeatIntervalSeconds = 45
+
+	cfg := testConfig(t)
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	if err := runAgent(t, a); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if a.hb.interval != 45*time.Second {
+		t.Errorf("heartbeat interval = %v, want the 45s the control plane returned", a.hb.interval)
+	}
+}
+
+// The agent being up says nothing about the engine. A down engine has to reach
+// the control plane as such, or auto-recovery has nothing to act on.
+func TestRun_HeartbeatReportsDownEngine(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cp.registerOut.HeartbeatIntervalSeconds = 0
+
+	cfg := testConfig(t)
+	probe := newEngineProbe(cfg, staticProbe(2))
+	// The engine has served before, so a silent one now is a failure rather
+	// than a boot still in progress.
+	probe.seenHealthy = true
+
+	a := newAgent(cfg, cp, probe)
+	a.hb.interval = 10 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool { return len(cp.snapshotStates()) > 0 }, "a heartbeat")
+	cancel()
+	<-done
+
+	beat := cp.snapshotStates()[0]
+	if beat.health != handlers_rds.EngineHealthUnhealthy {
+		t.Errorf("heartbeat health = %q, want unhealthy while the agent is up", beat.health)
+	}
+	if beat.message == "" {
+		t.Error("unhealthy heartbeat carried no message explaining the probe result")
+	}
+}
+
+// The mirror the agent decodes poll responses into has to stay aligned with the
+// gateway type that produces them; nothing else in the build ties the two.
+func TestPollDBCommandsOutput_MatchesGatewayShape(t *testing.T) {
+	issued := time.Date(2026, 7, 28, 1, 2, 3, 0, time.UTC)
+	// Rendered exactly as the gateway's typed adapter renders it, so a change to
+	// either side's tags fails here.
+	payload := utils.GenerateIAMXMLPayload("PollDBCommands", &gateway_rds.PollDBCommandsOutput{
+		Commands: []handlers_rds.Command{{
+			CommandID:  "cmd-1",
+			Type:       "reload-parameters",
+			Parameters: []handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}},
+			IssuedAt:   &issued,
+		}},
+	})
+	body, err := utils.MarshalToXML(payload)
+	if err != nil {
+		t.Fatalf("marshal poll result: %v", err)
+	}
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/xml")
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	client, err := rdsgw.New(srv.URL, "", gwsign.NewStatic("AKIATEST", "secret"), "us-east-1", time.Second)
+	if err != nil {
+		t.Fatalf("rdsgw.New: %v", err)
+	}
+
+	var got pollDBCommandsOutput
+	if err := client.Call(context.Background(), "PollDBCommands", nil, &got); err != nil {
+		t.Fatalf("Call: %v", err)
+	}
+	if len(got.Commands) != 1 {
+		t.Fatalf("decoded %d commands, want 1", len(got.Commands))
+	}
+	cmd := got.Commands[0]
+	if cmd.CommandID != "cmd-1" || cmd.Type != "reload-parameters" {
+		t.Errorf("command = %+v, want cmd-1/reload-parameters", cmd)
+	}
+	if len(cmd.Parameters) != 1 || cmd.Parameters[0].Value != "8MB" {
+		t.Errorf("Parameters = %+v, want the seeded member", cmd.Parameters)
+	}
+	if cmd.IssuedAt == nil || !cmd.IssuedAt.Equal(issued) {
+		t.Errorf("IssuedAt = %v, want %v", cmd.IssuedAt, issued)
+	}
+}
+
+func readFile(t *testing.T, path string) string {
+	t.Helper()
+	body, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	return string(body)
+}

cmd/rds-agent/bootstrap.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Fetches the boot material and writes the handoff rds-init is blocked waiting
+// for, then points the health probe at the assigned port.
+func (a *Agent) bootstrap(ctx context.Context) error {
+	var cfg *handlers_rds.GetDBBootstrapConfigOutput
+	if err := retry(ctx, "bootstrap fetch", func(ctx context.Context) error {
+		fetched, err := a.cp.GetBootstrapConfig(ctx, a.id)
+		if err != nil {
+			return err
+		}
+		if fetched == nil {
+			return fmt.Errorf("bootstrap fetch returned no config")
+		}
+		cfg = fetched
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	// Once an initialize response reaches the guest, keep retrying that same
+	// payload. Re-fetching after a local write failure returns attach because the
+	// control plane has already consumed the one-shot password.
+	if err := retry(ctx, "bootstrap handoff", func(context.Context) error {
+		return a.handoffWriter(a.cfg.HandoffDir, cfg)
+	}); err != nil {
+		return err
+	}
+	if cfg.Port > 0 {
+		a.probe.setPort(int(cfg.Port))
+		a.engine.setPort(int(cfg.Port))
+	}
+	// Mode is logged but never branched on: rds-init decides whether to initdb
+	// from the state of the datadir.
+	slog.Info("rds-agent: bootstrap config delivered",
+		"mode", cfg.Mode, "port", cfg.Port, "parameters", len(cfg.Parameters))
+	return nil
+}
+
+// The handoff files rds-init reads. They live on tmpfs, so nothing survives a
+// reboot and the next boot re-fetches rather than reusing a stale password or a
+// re-minted cert.
+const (
+	handoffEnvFile    = "bootstrap.env"
+	handoffParamsFile = "parameters.conf"
+	handoffCertFile   = "server.crt"
+	handoffKeyFile    = "server.key"
+
+	// Root-only: only rds-init reads these, and one holds the master password.
+	handoffMode    = 0o600
+	handoffDirMode = 0o700
+)
+
+// bootstrap.env is written last and renamed into place, so its appearance —
+// which rds-init waits on — means the whole handoff is complete.
+func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) error {
+	if cfg.MasterUsername == "" {
+		return fmt.Errorf("bootstrap config carries no master username")
+	}
+	if err := os.MkdirAll(dir, handoffDirMode); err != nil {
+		return fmt.Errorf("create handoff dir %s: %w", dir, err)
+	}
+	// MkdirAll leaves an existing directory's mode alone, and this one may have
+	// been created more permissively earlier in the boot.
+	if err := os.Chmod(dir, handoffDirMode); err != nil {
+		return fmt.Errorf("secure handoff dir %s: %w", dir, err)
+	}
+
+	if err := writeHandoffFile(dir, handoffParamsFile, renderParameters(cfg.Parameters)); err != nil {
+		return err
+	}
+	// Cert and key are written as a pair; half of one would start the engine
+	// with TLS configured against a key it does not have.
+	if cfg.ServingCertificate != "" && cfg.ServingPrivateKey != "" {
+		if err := writeHandoffFile(dir, handoffCertFile, cfg.ServingCertificate); err != nil {
+			return err
+		}
+		if err := writeHandoffFile(dir, handoffKeyFile, cfg.ServingPrivateKey); err != nil {
+			return err
+		}
+	}
+
+	return writeHandoffFile(dir, handoffEnvFile, renderBootstrapEnv(cfg))
+}
+
+// Every value is single-quoted because rds-init sources this with `.`: an
+// unquoted password holding a space, `$` or `;` would be word-split or executed.
+func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
+	var b strings.Builder
+	b.WriteString("# Written by rds-agent. Regenerated on every boot; edits are lost.\n")
+	writeEnvLine(&b, "RDS_MODE", cfg.Mode)
+	writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
+	// Present only in initialize mode: an attach fetch has no password to write
+	// and rds-init must not find a stale one.
+	if cfg.MasterUserPassword != nil {
+		writeEnvLine(&b, "RDS_MASTER_PASSWORD", *cfg.MasterUserPassword)
+	}
+	if cfg.DBName != "" {
+		writeEnvLine(&b, "RDS_DB_NAME", cfg.DBName)
+	}
+	if cfg.Port > 0 {
+		writeEnvLine(&b, "RDS_PORT", strconv.FormatInt(cfg.Port, 10))
+	}
+	return b.String()
+}
+
+func writeEnvLine(b *strings.Builder, key, value string) {
+	b.WriteString(key)
+	b.WriteString("=")
+	b.WriteString(shellQuote(value))
+	b.WriteString("\n")
+}
+
+// Single quotes suppress every shell expansion; an embedded quote is closed,
+// escaped and reopened.
+func shellQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+// postgresql.conf syntax. Values are quoted so a setting with a space or unit
+// suffix survives; the engine accepts quoted numerics and booleans too.
+func renderParameters(params []handlers_rds.Parameter) string {
+	var b strings.Builder
+	b.WriteString("# Resolved parameter group, written by rds-agent.\n")
+	for _, p := range params {
+		if p.Name == "" {
+			continue
+		}
+		fmt.Fprintf(&b, "%s = '%s'\n", p.Name, strings.ReplaceAll(p.Value, "'", "''"))
+	}
+	return b.String()
+}
+
+// Atomic: a temp file in the same directory, renamed over the target. The mode
+// is set at creation, so content is never briefly readable at the process umask.
+func writeHandoffFile(dir, name, content string) error {
+	path := filepath.Join(dir, name)
+	tmp := path + ".new"
+
+	f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, handoffMode)
+	if err != nil {
+		return fmt.Errorf("create %s: %w", tmp, err)
+	}
+	// A failed write must not leave the temp file behind holding a password.
+	if _, err := f.WriteString(content); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return fmt.Errorf("write %s: %w", tmp, err)
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("close %s: %w", tmp, err)
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("install %s: %w", path, err)
+	}
+	return nil
+}

cmd/rds-agent/bootstrap_test.go

@@ -0,0 +1,202 @@
+package main
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	password := "s3cret"
+
+	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:               handlers_rds.BootstrapModeInitialize,
+		MasterUsername:     "master",
+		MasterUserPassword: &password,
+		DBName:             "appdb",
+		Port:               5433,
+		Parameters: []handlers_rds.Parameter{
+			{Name: "max_connections", Value: "200"},
+			{Name: "shared_buffers", Value: "256MB"},
+		},
+		ServingCertificate: "CERT",
+		ServingPrivateKey:  "KEY",
+	})
+	if err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	env := readFile(t, filepath.Join(dir, handoffEnvFile))
+	for _, want := range []string{
+		"RDS_MODE='initialize'",
+		"RDS_MASTER_USERNAME='master'",
+		"RDS_MASTER_PASSWORD='s3cret'",
+		"RDS_DB_NAME='appdb'",
+		"RDS_PORT='5433'",
+	} {
+		if !strings.Contains(env, want) {
+			t.Errorf("handoff env missing %s:\n%s", want, env)
+		}
+	}
+
+	params := readFile(t, filepath.Join(dir, handoffParamsFile))
+	if !strings.Contains(params, "max_connections = '200'") || !strings.Contains(params, "shared_buffers = '256MB'") {
+		t.Errorf("parameters.conf = %q, want both resolved settings", params)
+	}
+	if got := readFile(t, filepath.Join(dir, handoffCertFile)); got != "CERT" {
+		t.Errorf("server.crt = %q, want CERT", got)
+	}
+	if got := readFile(t, filepath.Join(dir, handoffKeyFile)); got != "KEY" {
+		t.Errorf("server.key = %q, want KEY", got)
+	}
+
+	// The handoff holds a master password and a private key; anything readable
+	// beyond root is a disclosure to every process in the guest.
+	if mode := statMode(t, dir); mode != handoffDirMode {
+		t.Errorf("handoff dir mode = %#o, want %#o", mode, handoffDirMode)
+	}
+	for _, name := range []string{handoffEnvFile, handoffParamsFile, handoffCertFile, handoffKeyFile} {
+		if mode := statMode(t, filepath.Join(dir, name)); mode != handoffMode {
+			t.Errorf("%s mode = %#o, want %#o", name, mode, handoffMode)
+		}
+	}
+}
+
+// Attach mode carries no password, and the key must be absent rather than
+// present and empty — rds-init reads an empty value as "no password" only by
+// accident, and a later boot would inherit it.
+func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:           handlers_rds.BootstrapModeAttach,
+		MasterUsername: "master",
+		Port:           5432,
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	if env := readFile(t, filepath.Join(dir, handoffEnvFile)); strings.Contains(env, "RDS_MASTER_PASSWORD") {
+		t.Errorf("handoff env = %q, want no RDS_MASTER_PASSWORD key", env)
+	}
+}
+
+// The env file is sourced by a shell, so a password holding shell metacharacters
+// must survive as data rather than being split, expanded or executed.
+func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	password := `a b$(id);'"\` + "`whoami`"
+
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:               handlers_rds.BootstrapModeInitialize,
+		MasterUsername:     "master",
+		MasterUserPassword: &password,
+		Port:               5432,
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	env := readFile(t, filepath.Join(dir, handoffEnvFile))
+	line := findLine(t, env, "RDS_MASTER_PASSWORD=")
+	if got := shellSource(t, line, "RDS_MASTER_PASSWORD"); got != password {
+		t.Errorf("sourced password = %q, want %q (line was %s)", got, password, line)
+	}
+}
+
+// A serving cert with no key would have rds-init configure TLS against a key
+// that is not there; the pair is delivered together or not at all.
+func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:               handlers_rds.BootstrapModeAttach,
+		MasterUsername:     "master",
+		Port:               5432,
+		ServingCertificate: "CERT",
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	if _, err := os.Stat(filepath.Join(dir, handoffCertFile)); !os.IsNotExist(err) {
+		t.Errorf("server.crt exists without a key; stat err = %v", err)
+	}
+}
+
+// A config with no master username cannot bootstrap anything, and writing it
+// would only move the failure into rds-init after the boot has committed to it.
+func TestWriteHandoff_RejectsConfigWithNoMasterUser(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode: handlers_rds.BootstrapModeAttach,
+		Port: 5432,
+	})
+	if err == nil {
+		t.Fatal("writeHandoff accepted a config with no master username, want an error")
+	}
+	if _, statErr := os.Stat(filepath.Join(dir, handoffEnvFile)); !os.IsNotExist(statErr) {
+		t.Errorf("handoff env was written despite the rejection; stat err = %v", statErr)
+	}
+}
+
+// A directory left over from an earlier boot at a looser mode must be tightened,
+// not accepted as it is found.
+func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatalf("seed dir: %v", err)
+	}
+
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:           handlers_rds.BootstrapModeAttach,
+		MasterUsername: "master",
+		Port:           5432,
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+	if mode := statMode(t, dir); mode != handoffDirMode {
+		t.Errorf("handoff dir mode = %#o, want it tightened to %#o", mode, handoffDirMode)
+	}
+}
+
+func TestShellQuote_EscapesEmbeddedQuotes(t *testing.T) {
+	for _, in := range []string{"", "plain", "it's", `''`, "new\nline", `$HOME`} {
+		if got := shellSource(t, "V="+shellQuote(in), "V"); got != in {
+			t.Errorf("shellQuote(%q) sourced back as %q", in, got)
+		}
+	}
+}
+
+// shellSource evaluates one KEY=value assignment in a real shell and returns
+// what the variable ends up holding. The quoting is only correct if a shell
+// agrees, so the test asks one instead of re-implementing its rules.
+func shellSource(t *testing.T, line, name string) string {
+	t.Helper()
+	out, err := exec.Command("/bin/sh", "-c", line+"\nprintf %s \"$"+name+"\"\n").Output()
+	if err != nil {
+		t.Fatalf("sourcing %q: %v", line, err)
+	}
+	return string(out)
+}
+
+func statMode(t *testing.T, path string) os.FileMode {
+	t.Helper()
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatalf("stat %s: %v", path, err)
+	}
+	return info.Mode().Perm()
+}
+
+func findLine(t *testing.T, body, prefix string) string {
+	t.Helper()
+	for line := range strings.SplitSeq(body, "\n") {
+		if strings.HasPrefix(line, prefix) {
+			return line
+		}
+	}
+	t.Fatalf("no line starting %q in:\n%s", prefix, body)
+	return ""
+}

cmd/rds-agent/command.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strconv"
+	"strings"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Executes one control-plane directive inside the guest and returns the message
+// reported back to the issuer.
+type commandHandler func(ctx context.Context, cmd handlers_rds.Command) (string, error)
+
+// A registry rather than a dispatch switch, so guest operations — password
+// apply, parameter reload, storage grow, snapshot quiesce — add an entry each.
+type commandRegistry map[string]commandHandler
+
+// An unregistered type is replied to as unsupported, so a control plane ahead
+// of the guest gets a clear answer rather than a timeout.
+func newCommandRegistry(engine engineOps, storage storageOps) commandRegistry {
+	return commandRegistry{
+		handlers_rds.CommandSetPassword: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			params := commandParams(cmd)
+			return "", engine.SetPassword(ctx,
+				params[handlers_rds.CommandParamMasterUsername],
+				params[handlers_rds.CommandParamMasterUserPassword])
+		},
+		handlers_rds.CommandApplyParams: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			pending, err := engine.ApplyParameters(ctx, cmd.Parameters)
+			if err != nil {
+				return "", err
+			}
+			// The reply carries no structured payload, so the settings still
+			// awaiting a restart come back as the message itself.
+			return strings.Join(pending, ","), nil
+		},
+		handlers_rds.CommandStopEngine: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			return "", engine.Stop(ctx)
+		},
+		handlers_rds.CommandGrowFilesystem: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			return storage.GrowFilesystem(ctx)
+		},
+		handlers_rds.CommandQuiesce: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			params := commandParams(cmd)
+			hold, err := quiesceHoldFrom(params[handlers_rds.CommandParamQuiesceDeadlineSeconds])
+			if err != nil {
+				return "", err
+			}
+			return "", engine.Quiesce(ctx, params[handlers_rds.CommandParamQuiesceLabel], hold)
+		},
+		handlers_rds.CommandUnquiesce: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			return "", engine.Unquiesce(ctx)
+		},
+	}
+}
+
+// The control plane's hold deadline. A missing or unreadable one is refused
+// rather than defaulted: a hold the guest chose the length of would not be the
+// bound the control plane is relying on.
+func quiesceHoldFrom(value string) (time.Duration, error) {
+	seconds, err := strconv.Atoi(strings.TrimSpace(value))
+	if err != nil || seconds <= 0 {
+		return 0, fmt.Errorf("%s must be a positive number of seconds, got %q",
+			handlers_rds.CommandParamQuiesceDeadlineSeconds, value)
+	}
+	return time.Duration(seconds) * time.Second, nil
+}
+
+// A later value wins, so a malformed command carrying a parameter twice cannot
+// leave the handler acting on the first of two conflicting values.
+func commandParams(cmd handlers_rds.Command) map[string]string {
+	params := make(map[string]string, len(cmd.Parameters))
+	for _, p := range cmd.Parameters {
+		params[p.Name] = p.Value
+	}
+	return params
+}
+
+// Spaces retries after a failed poll, so a broken channel is not re-polled at
+// line rate for as long as it stays broken.
+const pollErrorBackoff = 5 * time.Second
+
+// The agent's command channel: a long poll that carries back the replies for
+// whatever the previous poll delivered.
+type commander struct {
+	cp       controlPlane
+	id       identity
+	registry commandRegistry
+	wait     time.Duration
+	// Cleared only once a poll has carried them, so a failed poll re-delivers
+	// rather than dropping the result of work the guest actually did.
+	pending []handlers_rds.CommandReply
+}
+
+func newCommander(cp controlPlane, registry commandRegistry, wait time.Duration) *commander {
+	if wait <= 0 {
+		wait = defaultPollWait
+	}
+	return &commander{cp: cp, registry: registry, wait: wait}
+}
+
+func (c *commander) Run(ctx context.Context) {
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+		commands, err := c.cp.PollCommands(ctx, c.id, c.pending, c.wait)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			slog.Warn("rds-agent: command poll failed", "err", err)
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(pollErrorBackoff):
+			}
+			continue
+		}
+		c.pending = nil
+
+		for _, cmd := range commands {
+			c.pending = append(c.pending, c.execute(ctx, cmd))
+		}
+	}
+}
+
+// Every command gets a reply: the issuer is blocked on this command ID, so a
+// silent drop costs it a full timeout.
+func (c *commander) execute(ctx context.Context, cmd handlers_rds.Command) handlers_rds.CommandReply {
+	handler, ok := c.registry[cmd.Type]
+	if !ok {
+		slog.Warn("rds-agent: unsupported command", "commandID", cmd.CommandID, "type", cmd.Type)
+		return failedReply(cmd, fmt.Sprintf("unsupported command type %q", cmd.Type))
+	}
+
+	slog.Info("rds-agent: executing command", "commandID", cmd.CommandID, "type", cmd.Type)
+	message, err := handler(ctx, cmd)
+	if err != nil {
+		slog.Error("rds-agent: command failed", "commandID", cmd.CommandID, "type", cmd.Type, "err", err)
+		return failedReply(cmd, err.Error())
+	}
+	return handlers_rds.CommandReply{
+		CommandID: cmd.CommandID,
+		Status:    handlers_rds.CommandStatusSucceeded,
+		Message:   message,
+	}
+}
+
+func failedReply(cmd handlers_rds.Command, message string) handlers_rds.CommandReply {
+	return handlers_rds.CommandReply{
+		CommandID: cmd.CommandID,
+		Status:    handlers_rds.CommandStatusFailed,
+		Message:   message,
+	}
+}

cmd/rds-agent/command_test.go

@@ -0,0 +1,173 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCommander_DispatchesByNameAndReplies(t *testing.T) {
+	var ran handlers_rds.Command
+	registry := commandRegistry{
+		"reload-parameters": func(_ context.Context, cmd handlers_rds.Command) (string, error) {
+			ran = cmd
+			return "reloaded", nil
+		},
+	}
+
+	reply := newCommander(nil, registry, time.Second).execute(context.Background(), handlers_rds.Command{
+		CommandID:  "cmd-1",
+		Type:       "reload-parameters",
+		Parameters: []handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}},
+	})
+
+	if ran.CommandID != "cmd-1" || len(ran.Parameters) != 1 {
+		t.Errorf("handler received %+v, want the command with its parameters", ran)
+	}
+	if reply.Status != handlers_rds.CommandStatusSucceeded || reply.Message != "reloaded" {
+		t.Errorf("reply = %+v, want succeeded/reloaded", reply)
+	}
+	if reply.CommandID != "cmd-1" {
+		t.Errorf("reply command ID = %q, want cmd-1", reply.CommandID)
+	}
+}
+
+func TestCommander_FailedHandlerRepliesWithItsError(t *testing.T) {
+	registry := commandRegistry{
+		"grow-storage": func(context.Context, handlers_rds.Command) (string, error) {
+			return "", errors.New("no space left on device")
+		},
+	}
+
+	reply := newCommander(nil, registry, time.Second).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-2", Type: "grow-storage"})
+
+	if reply.Status != handlers_rds.CommandStatusFailed {
+		t.Errorf("reply status = %q, want failed", reply.Status)
+	}
+	if reply.Message != "no space left on device" {
+		t.Errorf("reply message = %q, want the handler's error", reply.Message)
+	}
+}
+
+// A control plane ahead of the guest issues types this build has no handler
+// for. The issuer is blocked on the command ID, so it gets an answer rather
+// than a timeout.
+func TestCommander_UnknownTypeRepliesFailedNotDropped(t *testing.T) {
+	reply := newCommander(nil, newCommandRegistry(&fakeEngine{}, &fakeStorage{}), time.Second).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-3", Type: "quiesce"})
+
+	if reply.CommandID != "cmd-3" || reply.Status != handlers_rds.CommandStatusFailed {
+		t.Errorf("reply = %+v, want a failed reply for cmd-3", reply)
+	}
+	if reply.Message == "" {
+		t.Error("unsupported-command reply carried no message")
+	}
+}
+
+// The reply for work the guest actually did rides the next poll. A poll that
+// fails must not consume it, or the issuer never learns the outcome.
+func TestCommander_RunCarriesRepliesOnTheNextPoll(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollQueue = [][]handlers_rds.Command{
+		{{CommandID: "cmd-1", Type: "reload-parameters"}},
+	}
+	registry := commandRegistry{
+		"reload-parameters": func(context.Context, handlers_rds.Command) (string, error) { return "done", nil },
+	}
+
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	waitFor(t, func() bool {
+		replies := cp.snapshotReplies()
+		return len(replies) >= 2 && len(replies[1]) == 1
+	}, "the reply to ride the second poll")
+	cancel()
+	<-done
+
+	replies := cp.snapshotReplies()
+	if len(replies[0]) != 0 {
+		t.Errorf("first poll carried replies %+v, want none", replies[0])
+	}
+	if got := replies[1][0]; got.CommandID != "cmd-1" || got.Status != handlers_rds.CommandStatusSucceeded {
+		t.Errorf("second poll carried %+v, want the succeeded reply for cmd-1", got)
+	}
+}
+
+func TestCommander_RetainsAndRetriesTheCompletePendingBatch(t *testing.T) {
+	cp := newFakeControlPlane()
+	pending := []handlers_rds.CommandReply{
+		{CommandID: "cmd-1", Status: handlers_rds.CommandStatusSucceeded},
+		{CommandID: "cmd-2", Status: handlers_rds.CommandStatusFailed, Message: "disk full"},
+	}
+	c := newCommander(cp, commandRegistry{
+		"reload-parameters": func(context.Context, handlers_rds.Command) (string, error) {
+			return "reloaded", nil
+		},
+	}, 10*time.Millisecond)
+	c.pending = append([]handlers_rds.CommandReply(nil), pending...)
+	cp.pollErr = errors.New("reply publication failed")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+	<-cp.polled
+	cancel()
+	<-done
+
+	assert.Equal(t, pending, c.pending)
+	replies := cp.snapshotReplies()
+	require.Len(t, replies, 1)
+	assert.Equal(t, pending, replies[0])
+
+	cp.mu.Lock()
+	cp.pollErr = nil
+	cp.pollQueue = [][]handlers_rds.Command{{{
+		CommandID: "cmd-3",
+		Type:      "reload-parameters",
+	}}}
+	cp.mu.Unlock()
+
+	ctx, cancel = context.WithCancel(context.Background())
+	done = make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+	waitFor(t, func() bool {
+		return len(cp.snapshotReplies()) >= 3
+	}, "the retained batch to succeed and the new reply to reach the following poll")
+	cancel()
+	<-done
+
+	replies = cp.snapshotReplies()
+	assert.Equal(t, pending, replies[1], "the complete failed batch must be retried in order")
+	require.Len(t, replies[2], 1)
+	assert.Equal(t, "cmd-3", replies[2][0].CommandID)
+	assert.Equal(t, handlers_rds.CommandStatusSucceeded, replies[2][0].Status)
+}
+
+// A poll error must not spin the loop: without a backoff a gateway that is down
+// would be re-polled at line rate for as long as it stayed down.
+func TestCommander_BacksOffAfterAFailedPoll(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollErr = errors.New("gateway unreachable")
+
+	c := newCommander(cp, newCommandRegistry(&fakeEngine{}, &fakeStorage{}), 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+	<-done
+
+	// pollErrorBackoff is 5s, so the window admits the first poll and no more.
+	if polls := len(cp.snapshotReplies()); polls > 1 {
+		t.Errorf("polled %d times in 200ms against a broken channel, want 1", polls)
+	}
+}

cmd/rds-agent/config.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/guestenv"
+)
+
+const (
+	defaultEnvFile    = "/etc/spinifex-rds/agent.env"
+	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
+	defaultHandoffDir = "/run/spinifex-rds"
+	defaultEngineHost = "127.0.0.1"
+	defaultEnginePort = 5432
+	defaultPGIsReady  = "pg_isready"
+
+	// The guest layout rds-init lays down. Overridable for the same reason it is
+	// there: a sibling engine preset points at its own paths.
+	defaultPGBin         = "/usr/libexec/postgresql18"
+	defaultPGData        = "/var/lib/postgresql/18/data"
+	defaultSocketDir     = "/run/postgresql"
+	defaultPGUser        = "postgres"
+	defaultRCService     = "rc-service"
+	defaultEngineSvcName = "postgresql"
+	// The long-poll window the agent asks the gateway to hold a request open
+	// for. The gateway caps it at 20s.
+	defaultPollWait = 20 * time.Second
+)
+
+// Static settings delivered per-instance by cloud-init. It carries no secrets:
+// IMDS is readable by anything in the guest, so the master password only
+// arrives via GetDBBootstrapConfig.
+type config struct {
+	GatewayURL string
+	GatewayCA  string
+	Region     string
+	// Optional — the gateway resolves the instance from the caller's
+	// credentials. When set it is sent, so a mis-provisioned VM is rejected.
+	DBInstanceIdentifier string
+	// What this image actually ships. Empty leaves the control plane's recorded
+	// version alone rather than clearing it.
+	EngineVersion string
+	HandoffDir    string
+	EngineHost    string
+	EnginePort    int
+	// Overridable so a test or a sibling engine preset can point at its own.
+	PGIsReady string
+	PollWait  time.Duration
+
+	// Where the command handlers reach the engine: the client binaries, the
+	// datadir the parameter file is installed into, the socket they connect
+	// over, the OS user they drop to, and the service the stop goes through.
+	PGBin         string
+	PGData        string
+	SocketDir     string
+	PGUser        string
+	RCService     string
+	EngineService string
+
+	// Where the data volume is mounted, and the two kernel surfaces a storage
+	// grow resolves its device from. Overridable so a test can point them at
+	// fixtures rather than at the running host's.
+	DataMount  string
+	MountsFile string
+	SysBlock   string
+}
+
+// Reads the cloud-init env file, then lets real env vars override.
+func loadConfig(envFile string) config {
+	get := guestenv.Load(envFile).Get
+
+	cfg := config{
+		GatewayURL:           get("RDS_GATEWAY_URL"),
+		GatewayCA:            get("RDS_GATEWAY_CA"),
+		Region:               get("RDS_REGION"),
+		DBInstanceIdentifier: get("RDS_DB_INSTANCE_IDENTIFIER"),
+		EngineVersion:        get("RDS_ENGINE_VERSION"),
+		HandoffDir:           get("RDS_HANDOFF_DIR"),
+		EngineHost:           get("RDS_ENGINE_HOST"),
+		PGIsReady:            get("RDS_PG_ISREADY"),
+		PGBin:                get("RDS_PG_BIN"),
+		PGData:               get("RDS_PGDATA"),
+		SocketDir:            get("RDS_SOCKET_DIR"),
+		PGUser:               get("RDS_PG_USER"),
+		RCService:            get("RDS_RC_SERVICE"),
+		EngineService:        get("RDS_ENGINE_SERVICE"),
+		DataMount:            get("RDS_DATA_MOUNT"),
+		EnginePort:           defaultEnginePort,
+		PollWait:             defaultPollWait,
+	}
+	if cfg.GatewayCA == "" {
+		cfg.GatewayCA = defaultGatewayCA
+	}
+	if cfg.HandoffDir == "" {
+		cfg.HandoffDir = defaultHandoffDir
+	}
+	if cfg.EngineHost == "" {
+		cfg.EngineHost = defaultEngineHost
+	}
+	if cfg.PGIsReady == "" {
+		cfg.PGIsReady = defaultPGIsReady
+	}
+	if cfg.PGBin == "" {
+		cfg.PGBin = defaultPGBin
+	}
+	if cfg.PGData == "" {
+		cfg.PGData = defaultPGData
+	}
+	if cfg.SocketDir == "" {
+		cfg.SocketDir = defaultSocketDir
+	}
+	if cfg.PGUser == "" {
+		cfg.PGUser = defaultPGUser
+	}
+	if cfg.RCService == "" {
+		cfg.RCService = defaultRCService
+	}
+	if cfg.EngineService == "" {
+		cfg.EngineService = defaultEngineSvcName
+	}
+	// rds-datadir reads the same RDS_DATA_MOUNT, so the two agree on where the
+	// volume landed without either asserting it to the other.
+	if cfg.DataMount == "" {
+		cfg.DataMount = defaultDataMount
+	}
+	cfg.MountsFile = defaultMountsFile
+	cfg.SysBlock = defaultSysBlock
+	// The authoritative port comes from the bootstrap config; this is only what
+	// the probe uses until that first fetch lands.
+	if v := get("RDS_ENGINE_PORT"); v != "" {
+		if p, err := strconv.Atoi(v); err == nil && p > 0 {
+			cfg.EnginePort = p
+		}
+	}
+	if v := get("RDS_POLL_WAIT"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.PollWait = d
+		}
+	}
+	return cfg
+}

cmd/rds-agent/config_test.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestLoadConfig_Defaults(t *testing.T) {
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+
+	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
+		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
+	}
+	if cfg.EngineHost != defaultEngineHost || cfg.EnginePort != defaultEnginePort {
+		t.Errorf("engine endpoint = %s:%d, want the local default", cfg.EngineHost, cfg.EnginePort)
+	}
+	if cfg.PGIsReady != defaultPGIsReady || cfg.PollWait != defaultPollWait {
+		t.Errorf("probe/pollWait = %q/%v, want the built-in defaults", cfg.PGIsReady, cfg.PollWait)
+	}
+	// Identity is optional: the gateway resolves it from the caller's
+	// credentials, so an agent with no configured identifier is normal.
+	if cfg.DBInstanceIdentifier != "" {
+		t.Errorf("DBInstanceIdentifier = %q, want empty when unconfigured", cfg.DBInstanceIdentifier)
+	}
+}
+
+func TestLoadConfig_ReadsEnvFile(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "agent.env")
+	body := "RDS_GATEWAY_URL=https://gw.internal:9999\n" +
+		"RDS_REGION=ap-southeast-2\n" +
+		"RDS_DB_INSTANCE_IDENTIFIER=db-1\n" +
+		"RDS_ENGINE_VERSION=18.1\n" +
+		"RDS_ENGINE_PORT=6543\n" +
+		"RDS_POLL_WAIT=5s\n"
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatalf("write env file: %v", err)
+	}
+
+	cfg := loadConfig(path)
+	if cfg.GatewayURL != "https://gw.internal:9999" || cfg.Region != "ap-southeast-2" {
+		t.Errorf("gateway/region = %q/%q, want the delivered values", cfg.GatewayURL, cfg.Region)
+	}
+	if cfg.DBInstanceIdentifier != "db-1" || cfg.EngineVersion != "18.1" {
+		t.Errorf("identity = %q/%q, want db-1/18.1", cfg.DBInstanceIdentifier, cfg.EngineVersion)
+	}
+	if cfg.EnginePort != 6543 || cfg.PollWait != 5*time.Second {
+		t.Errorf("port/pollWait = %d/%v, want 6543/5s", cfg.EnginePort, cfg.PollWait)
+	}
+}
+
+// An unparseable override falls back to the default rather than leaving the
+// agent with a zero port or a zero-length long poll.
+func TestLoadConfig_IgnoresUnusableOverrides(t *testing.T) {
+	t.Setenv("RDS_ENGINE_PORT", "not-a-port")
+	t.Setenv("RDS_POLL_WAIT", "forever")
+
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	if cfg.EnginePort != defaultEnginePort || cfg.PollWait != defaultPollWait {
+		t.Errorf("port/pollWait = %d/%v, want the defaults", cfg.EnginePort, cfg.PollWait)
+	}
+}

cmd/rds-agent/engine.go

@@ -0,0 +1,411 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The in-guest half of the engine seam. Everything PostgreSQL-shaped that a
+// control-plane command needs lives behind this, so the command registry above
+// stays engine-agnostic.
+type engineOps interface {
+	// Rotates the master role's password live. Never persisted anywhere in the
+	// guest: it exists only for the length of this call.
+	SetPassword(ctx context.Context, username, password string) error
+	// Installs the resolved parameter set and reloads. Returns the settings the
+	// engine accepted but will not honour until it restarts.
+	ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error)
+	// Shuts the engine down cleanly, so the data volume is checkpointed before
+	// the VM is stopped or snapshotted.
+	Stop(ctx context.Context) error
+	// Puts the engine into backup mode so the datadir can be snapshotted at a
+	// checkpoint. Releases itself after hold, whatever happens to the caller.
+	Quiesce(ctx context.Context, label string, hold time.Duration) error
+	// Takes the engine back out of backup mode. Fails when no hold is active,
+	// which is how a hold that expired mid-snapshot becomes visible.
+	Unquiesce(ctx context.Context) error
+}
+
+// The rollback half of the parameter path, kept off engineOps because it is not
+// a control-plane directive: nothing issues it, the agent runs it when the
+// engine fails to come up after a parameter change.
+type parameterRecovery interface {
+	// Reports whether the installed set differed and was replaced.
+	RestoreLastKnownGoodParameters() (bool, error)
+	Restart(ctx context.Context) error
+}
+
+// One child process. Env replaces the agent's own environment rather than
+// extending it, so a secret placed here reaches only the process that needs it.
+type command struct {
+	Name  string
+	Args  []string
+	Env   []string
+	Stdin string
+	// The OS user to drop to before exec. Empty runs as the agent itself.
+	User string
+}
+
+// Returns stdout. stderr is folded into the error, because psql reports the
+// actual SQL failure there while exiting with a bare status.
+type commandRunner func(ctx context.Context, c command) (string, error)
+
+func execCommandRunner(ctx context.Context, c command) (string, error) {
+	cmd := exec.CommandContext(ctx, c.Name, c.Args...)
+	cmd.Env = c.Env
+	if c.Stdin != "" {
+		cmd.Stdin = strings.NewReader(c.Stdin)
+	}
+	if c.User != "" {
+		credential, err := lookupCredential(c.User)
+		if err != nil {
+			return "", err
+		}
+		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: credential}
+	}
+
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout, cmd.Stderr = &stdout, &stderr
+	if err := cmd.Run(); err != nil {
+		message := strings.TrimSpace(stderr.String())
+		if message == "" {
+			message = err.Error()
+		}
+		return stdout.String(), fmt.Errorf("%s: %s", filepath.Base(c.Name), message)
+	}
+	return stdout.String(), nil
+}
+
+func lookupCredential(username string) (*syscall.Credential, error) {
+	u, err := user.Lookup(username)
+	if err != nil {
+		return nil, fmt.Errorf("resolve user %s: %w", username, err)
+	}
+	uid, err := strconv.ParseUint(u.Uid, 10, 32)
+	if err != nil {
+		return nil, fmt.Errorf("parse uid of %s: %w", username, err)
+	}
+	gid, err := strconv.ParseUint(u.Gid, 10, 32)
+	if err != nil {
+		return nil, fmt.Errorf("parse gid of %s: %w", username, err)
+	}
+	return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil
+}
+
+// The engine is reached over its unix socket under peer authentication, so the
+// agent drops to the postgres OS user rather than holding a password of its own.
+type postgresEngine struct {
+	run       commandRunner
+	startSess sessionRunner
+	psql      string
+	rcService string
+	service   string
+	pgData    string
+	socketDir string
+	osUser    string
+	// Set from the bootstrap config, on a different goroutine than the commands
+	// that read it.
+	port atomic.Int64
+
+	// The backup mode currently held, if any. Guarded because the expiry timer
+	// releases it on its own goroutine.
+	mu   sync.Mutex
+	held *quiesceHold
+}
+
+var _ engineOps = (*postgresEngine)(nil)
+
+func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner) *postgresEngine {
+	e := &postgresEngine{
+		run:       run,
+		startSess: startSess,
+		psql:      filepath.Join(cfg.PGBin, "psql"),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		pgData:    cfg.PGData,
+		socketDir: cfg.SocketDir,
+		osUser:    cfg.PGUser,
+	}
+	e.port.Store(int64(cfg.EnginePort))
+	return e
+}
+
+func (e *postgresEngine) setPort(port int) {
+	e.port.Store(int64(port))
+}
+
+// The role name and the password ride the environment and are re-quoted by
+// psql, so neither reaches a shell word or an argv another process can read.
+func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// The statement below runs as the cluster superuser over the socket under
+	// peer auth, so a reserved name in the command payload would hand the
+	// customer the bootstrap superuser rather than rotate their own role.
+	if err := handlers_rds.EnginePostgres().ValidateUsernameNotReserved(username); err != nil {
+		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
+	}
+	const sql = `\getenv master RDS_MASTER_USERNAME
+\getenv password RDS_MASTER_PASSWORD
+ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
+`
+	_, err := e.psqlRun(ctx, sql,
+		"RDS_MASTER_USERNAME="+username,
+		"RDS_MASTER_PASSWORD="+password,
+	)
+	if err != nil {
+		// psql echoes the failing statement, which here would carry the new
+		// password back to the control plane and into the event ring.
+		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
+	}
+	return nil
+}
+
+// The include the resolved parameter set is rendered to, and the copy of the
+// last one the engine accepted. Both live beside the data rather than in /etc,
+// matching rds-init: a class change boots a fresh root volume, which would
+// otherwise revert them when a fresh root volume is used.
+const (
+	parametersFileName = "10-rds-parameters.conf"
+	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
+	// copy must not be read as a second set of settings.
+	lastGoodFileName = "10-rds-parameters.last-good"
+)
+
+// Installs the resolved set, validates it with the engine's own config parser
+// before the engine ever adopts it, and reloads. A value the engine refuses is
+// rolled back here rather than left on the data volume, where it would survive
+// every VM replace and turn the next restart into a boot loop.
+func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	// The check has to run against the file in place, because the engine parses
+	// the datadir's own include_dir. The window that leaves is closed from both
+	// ends: the rollback below, and the last-known-good restore the agent runs at
+	// boot when the engine does not come up.
+	previous, restore, err := e.installParameters(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := e.checkConfig(ctx); err != nil {
+		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restore())
+	}
+	if _, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n"); err != nil {
+		return nil, errors.Join(fmt.Errorf("reload the engine configuration: %w", err), restore())
+	}
+
+	// Recorded only once the engine has both parsed and adopted it, so the
+	// rollback target is a set that is known to work rather than the last one
+	// written.
+	if err := e.recordLastKnownGood(previous); err != nil {
+		return nil, err
+	}
+
+	// Read after the reload: a setting only becomes pending_restart once the
+	// engine has seen the new value and declined to adopt it live.
+	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
+	if err != nil {
+		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
+	}
+	var pending []string
+	for line := range strings.SplitSeq(out, "\n") {
+		if name := strings.TrimSpace(line); name != "" {
+			pending = append(pending, name)
+		}
+	}
+	return pending, nil
+}
+
+func (e *postgresEngine) parametersPath() string {
+	return filepath.Join(e.pgData, "conf.d", parametersFileName)
+}
+
+func (e *postgresEngine) lastGoodPath() string {
+	return filepath.Join(e.pgData, "conf.d", lastGoodFileName)
+}
+
+// Written as root and handed to the engine's user, at the same path and mode
+// rds-init installs it at, so a later boot overwrites rather than shadows it.
+// Returns the content that was there and a func putting it back, so the caller
+// can undo the install without re-rendering anything.
+func (e *postgresEngine) installParameters(params []handlers_rds.Parameter) (previous []byte, restore func() error, err error) {
+	path := e.parametersPath()
+	// A missing file is the first-ever apply, and its rollback is a removal.
+	previous, readErr := os.ReadFile(path)
+	if readErr != nil && !os.IsNotExist(readErr) {
+		return nil, nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
+	}
+	existed := readErr == nil
+
+	if err := e.writeEngineFile(path, []byte(renderParameters(params))); err != nil {
+		return nil, nil, err
+	}
+	return previous, func() error {
+		if !existed {
+			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
+			}
+			return nil
+		}
+		return e.writeEngineFile(path, previous)
+	}, nil
+}
+
+// Keeps the rollback target in step with what the engine is actually running.
+// The first apply of an instance's life has no predecessor, so the freshly
+// accepted set becomes the target instead.
+func (e *postgresEngine) recordLastKnownGood(previous []byte) error {
+	content := previous
+	if content == nil {
+		var err error
+		if content, err = os.ReadFile(e.parametersPath()); err != nil {
+			return fmt.Errorf("read the accepted parameters: %w", err)
+		}
+	}
+	return e.writeEngineFile(e.lastGoodPath(), content)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change: the instance comes back on the old parameters rather
+// than boot-looping into recovery. Reports whether anything was restored.
+func (e *postgresEngine) RestoreLastKnownGoodParameters() (bool, error) {
+	lastGood, err := os.ReadFile(e.lastGoodPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	current, err := os.ReadFile(e.parametersPath())
+	if err != nil && !os.IsNotExist(err) {
+		return false, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	if bytes.Equal(current, lastGood) {
+		return false, nil
+	}
+	if err := e.writeEngineFile(e.parametersPath(), lastGood); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// Atomic: a temp file in the same directory, renamed over the target. The temp
+// name deliberately does not end in .conf, so a crash between write and rename
+// cannot leave the engine reading a half-written include.
+func (e *postgresEngine) writeEngineFile(path string, content []byte) error {
+	tmp := path + ".new"
+	if err := os.WriteFile(tmp, content, 0o600); err != nil {
+		return fmt.Errorf("write %s: %w", tmp, err)
+	}
+	if err := e.chownToEngine(tmp); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("install %s: %w", path, err)
+	}
+	return nil
+}
+
+// The engine's own parser, run offline against the datadir. Reading one setting
+// back is enough: postgres parses postgresql.conf and every include first, and
+// exits non-zero naming the file and line of an unknown parameter or a value
+// outside its range.
+func (e *postgresEngine) checkConfig(ctx context.Context) error {
+	if _, err := e.run(ctx, command{
+		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
+		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
+		Env:  []string{"PATH=" + defaultGuestPath},
+		User: e.osUser,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+// The engine reads its own config, so a root-owned file would be unreadable to
+// it. A guest with no such user is a broken image, not something to work around.
+func (e *postgresEngine) chownToEngine(path string) error {
+	credential, err := lookupCredential(e.osUser)
+	if err != nil {
+		return err
+	}
+	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
+		return fmt.Errorf("hand %s to %s: %w", path, e.osUser, err)
+	}
+	return nil
+}
+
+// Through the service manager rather than pg_ctl, so the supervisor records the
+// engine as stopped and does not restart it underneath a VM that is going down.
+func (e *postgresEngine) Stop(ctx context.Context) error {
+	return e.serviceAction(ctx, "stop")
+}
+
+// Only the parameter rollback calls this: a restart the control plane wants goes
+// through RebootDBInstance, which cycles the VM.
+func (e *postgresEngine) Restart(ctx context.Context) error {
+	return e.serviceAction(ctx, "restart")
+}
+
+func (e *postgresEngine) serviceAction(ctx context.Context, action string) error {
+	if _, err := e.run(ctx, command{
+		Name: e.rcService,
+		Args: []string{e.service, action},
+		Env:  []string{"PATH=" + defaultGuestPath},
+	}); err != nil {
+		return fmt.Errorf("%s the %s service: %w", action, e.service, err)
+	}
+	return nil
+}
+
+// A minimal environment for a child that is not carrying secrets.
+const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+// Feeds sql on stdin rather than as an argument, so a statement is never
+// visible in the process table.
+func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.psql,
+		Args:  e.psqlArgs(),
+		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
+		Stdin: sql,
+		User:  e.osUser,
+	})
+}
+
+// Reading the script from stdin is what lets one invocation serve both a
+// one-shot run and a session held open across several statements.
+func (e *postgresEngine) psqlArgs() []string {
+	return []string{
+		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
+		"-v", "ON_ERROR_STOP=1",
+		"-h", e.socketDir,
+		"-p", strconv.FormatInt(e.port.Load(), 10),
+		"-U", e.osUser,
+		"-d", "postgres",
+		"-f", "-",
+	}
+}
+
+func redact(s, secret string) string {
+	if secret == "" {
+		return s
+	}
+	return strings.ReplaceAll(s, secret, "[REDACTED]")
+}

cmd/rds-agent/engine_test.go

@@ -0,0 +1,307 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"os/user"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Stands in for the engine in the registry tests, where what matters is that a
+// command reaches the right method with the right arguments.
+type fakeEngine struct {
+	username string
+	password string
+	params   []handlers_rds.Parameter
+	pending  []string
+	stopped  bool
+	label    string
+	hold     time.Duration
+	released bool
+	err      error
+}
+
+var _ engineOps = (*fakeEngine)(nil)
+
+func (f *fakeEngine) SetPassword(_ context.Context, username, password string) error {
+	f.username, f.password = username, password
+	return f.err
+}
+
+func (f *fakeEngine) ApplyParameters(_ context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	f.params = params
+	return f.pending, f.err
+}
+
+func (f *fakeEngine) Stop(context.Context) error {
+	f.stopped = true
+	return f.err
+}
+
+func (f *fakeEngine) Quiesce(_ context.Context, label string, hold time.Duration) error {
+	f.label, f.hold = label, hold
+	return f.err
+}
+
+func (f *fakeEngine) Unquiesce(context.Context) error {
+	f.released = true
+	return f.err
+}
+
+func TestCommandRegistry_SetPasswordCarriesBothParameters(t *testing.T) {
+	engine := &fakeEngine{}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(), handlers_rds.Command{
+		CommandID: "cmd-1",
+		Type:      handlers_rds.CommandSetPassword,
+		Parameters: []handlers_rds.Parameter{
+			{Name: handlers_rds.CommandParamMasterUsername, Value: "mulgamaster"},
+			{Name: handlers_rds.CommandParamMasterUserPassword, Value: "n3w-pw"},
+		},
+	})
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if engine.username != "mulgamaster" || engine.password != "n3w-pw" {
+		t.Errorf("engine got %q/%q, want mulgamaster/n3w-pw", engine.username, engine.password)
+	}
+	// The reply is recorded by the control plane; the new password must not be
+	// in it.
+	if strings.Contains(reply.Message, "n3w-pw") {
+		t.Errorf("reply message %q carries the new password", reply.Message)
+	}
+}
+
+// CommandReply has no structured payload, so the settings still awaiting a
+// restart come back as the message the issuer parses.
+func TestCommandRegistry_ApplyParamsReportsPendingRestart(t *testing.T) {
+	engine := &fakeEngine{pending: []string{"shared_buffers", "max_connections"}}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(), handlers_rds.Command{
+		CommandID:  "cmd-2",
+		Type:       handlers_rds.CommandApplyParams,
+		Parameters: []handlers_rds.Parameter{{Name: "shared_buffers", Value: "256MB"}},
+	})
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if reply.Message != "shared_buffers,max_connections" {
+		t.Errorf("reply message = %q, want the pending-restart list", reply.Message)
+	}
+	if len(engine.params) != 1 || engine.params[0].Name != "shared_buffers" {
+		t.Errorf("engine got %+v, want the command's parameters", engine.params)
+	}
+}
+
+func TestCommandRegistry_StopEngineReachesTheEngine(t *testing.T) {
+	engine := &fakeEngine{}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-3", Type: handlers_rds.CommandStopEngine})
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if !engine.stopped {
+		t.Error("stop-engine did not reach the engine")
+	}
+}
+
+func TestCommandRegistry_FailureRepliesFailed(t *testing.T) {
+	engine := &fakeEngine{err: errors.New("the engine did not stop")}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-4", Type: handlers_rds.CommandStopEngine})
+
+	if reply.Status != handlers_rds.CommandStatusFailed {
+		t.Fatalf("reply = %+v, want failed", reply)
+	}
+	if !strings.Contains(reply.Message, "the engine did not stop") {
+		t.Errorf("reply message = %q, want the engine's error", reply.Message)
+	}
+}
+
+// Records what a postgresEngine would have run, so the SQL and the argv are
+// asserted without a live cluster.
+type recordingRunner struct {
+	calls []command
+	out   string
+	err   error
+}
+
+func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
+	r.calls = append(r.calls, c)
+	return r.out, r.err
+}
+
+func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
+	t.Helper()
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	cfg.PGData = t.TempDir()
+	if err := os.MkdirAll(filepath.Join(cfg.PGData, "conf.d"), 0o700); err != nil {
+		t.Fatalf("create conf.d: %v", err)
+	}
+	// The guest's postgres user does not exist here, and the chown of the
+	// installed parameter file has to resolve to something real.
+	current, err := user.Current()
+	if err != nil {
+		t.Fatalf("resolve the current user: %v", err)
+	}
+	cfg.PGUser = current.Username
+	return newPostgresEngine(cfg, run, nil)
+}
+
+// The password reaches psql through the environment and is re-quoted there,
+// so it is never a shell word or an argv another process can read.
+func TestPostgresEngine_SetPasswordKeepsTheSecretOutOfArgv(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	call := runner.calls[0]
+	for _, arg := range call.Args {
+		if strings.Contains(arg, "n3w-pw") {
+			t.Fatalf("argv %v carries the password", call.Args)
+		}
+	}
+	if strings.Contains(call.Stdin, "n3w-pw") {
+		t.Errorf("SQL %q carries the password literally, want it read from the environment", call.Stdin)
+	}
+	if !slices.Contains(call.Env, "RDS_MASTER_PASSWORD=n3w-pw") {
+		t.Errorf("env %v does not carry the password", call.Env)
+	}
+	if !strings.Contains(call.Stdin, "ALTER ROLE") {
+		t.Errorf("SQL %q does not alter the role", call.Stdin)
+	}
+}
+
+// psql echoes the failing statement, which here would carry the new password
+// back into a reply the control plane records.
+func TestPostgresEngine_SetPasswordRedactsTheSecretFromAFailure(t *testing.T) {
+	runner := &recordingRunner{err: errors.New(`ERROR: syntax error at or near "n3w-pw"`)}
+	engine := newTestEngine(t, runner.run)
+
+	err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw")
+	if err == nil {
+		t.Fatal("SetPassword succeeded against a failing psql")
+	}
+	if strings.Contains(err.Error(), "n3w-pw") {
+		t.Errorf("error %q leaks the password", err)
+	}
+	if !strings.Contains(err.Error(), "[REDACTED]") {
+		t.Errorf("error %q does not mark the redaction", err)
+	}
+}
+
+func TestPostgresEngine_SetPasswordRejectsAnIncompleteCommand(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", ""); err == nil {
+		t.Error("SetPassword accepted an empty password")
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands for an incomplete request, want 0", len(runner.calls))
+	}
+}
+
+// The apply runs as the cluster superuser under peer auth, so a reserved
+// name arriving in a command payload would rotate the bootstrap superuser's
+// password rather than the customer's own role.
+func TestPostgresEngine_SetPasswordRefusesReservedRoles(t *testing.T) {
+	for _, username := range []string{"postgres", "rds_superuser", "rdsadmin", "pg_toast_owner", "PostGres"} {
+		t.Run(username, func(t *testing.T) {
+			runner := &recordingRunner{}
+			engine := newTestEngine(t, runner.run)
+
+			err := engine.SetPassword(context.Background(), username, "n3w-pw")
+			if err == nil {
+				t.Fatalf("SetPassword accepted the reserved role %q", username)
+			}
+			if len(runner.calls) != 0 {
+				t.Errorf("ran %d commands for a reserved role, want 0", len(runner.calls))
+			}
+		})
+	}
+}
+
+// The parameters land where rds-init installs them, so the next boot overwrites
+// the file rather than shadowing it with a second copy.
+func TestPostgresEngine_ApplyParametersInstallsAndReloads(t *testing.T) {
+	runner := &recordingRunner{out: "shared_buffers\nmax_connections\n"}
+	engine := newTestEngine(t, runner.run)
+
+	pending, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "shared_buffers", Value: "256MB"}})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+
+	installed, err := os.ReadFile(filepath.Join(engine.pgData, "conf.d", "10-rds-parameters.conf"))
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "shared_buffers = '256MB'") {
+		t.Errorf("installed parameters = %q, want the resolved value", installed)
+	}
+	if len(runner.calls) != 3 {
+		t.Fatalf("ran %d commands, want a config check, a reload and a pending-restart read", len(runner.calls))
+	}
+	// The check runs before the reload, so a value the engine would refuse never
+	// reaches a running cluster.
+	if !slices.Contains(runner.calls[0].Args, "-C") {
+		t.Errorf("first command = %v, want the offline config check", runner.calls[0].Args)
+	}
+	if !strings.Contains(runner.calls[1].Stdin, "pg_reload_conf") {
+		t.Errorf("second statement = %q, want the reload", runner.calls[1].Stdin)
+	}
+	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
+		t.Errorf("third statement = %q, want the pending-restart read", runner.calls[2].Stdin)
+	}
+	if len(pending) != 2 || pending[0] != "shared_buffers" || pending[1] != "max_connections" {
+		t.Errorf("pending = %v, want both settings the engine reported", pending)
+	}
+}
+
+// A reload that failed leaves the engine on its old values, so reporting the
+// apply as successful would tell the customer a change took effect that did not.
+func TestPostgresEngine_ApplyParametersSurfacesAFailedReload(t *testing.T) {
+	runner := &recordingRunner{err: errors.New("could not connect to server")}
+	engine := newTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}}); err == nil {
+		t.Fatal("ApplyParameters succeeded against a failing reload")
+	}
+}
+
+// Through the service manager, so the supervisor records the engine as stopped
+// and does not restart it underneath a VM that is going down.
+func TestPostgresEngine_StopGoesThroughTheServiceManager(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.Stop(context.Background()); err != nil {
+		t.Fatalf("Stop: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	call := runner.calls[0]
+	if filepath.Base(call.Name) != "rc-service" {
+		t.Errorf("ran %q, want rc-service", call.Name)
+	}
+	if len(call.Args) != 2 || call.Args[0] != "postgresql" || call.Args[1] != "stop" {
+		t.Errorf("args = %v, want [postgresql stop]", call.Args)
+	}
+}

cmd/rds-agent/health.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"sync/atomic"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// A non-zero exit is a result, not a fault, so it comes back as a code; err is
+// reserved for the probe failing to run at all.
+type probeRunner func(ctx context.Context, name string, args ...string) (int, error)
+
+func execProbeRunner(ctx context.Context, name string, args ...string) (int, error) {
+	err := exec.CommandContext(ctx, name, args...).Run()
+	if err == nil {
+		return 0, nil
+	}
+	var exitErr *exec.ExitError
+	if errors.As(err, &exitErr) {
+		return exitErr.ExitCode(), nil
+	}
+	return -1, err
+}
+
+// Shells out to pg_isready rather than dialling the port, since only a startup
+// exchange separates an engine that is serving from one still in recovery.
+type engineProbe struct {
+	run    probeRunner
+	binary string
+	host   string
+	// Set from the bootstrap config once it lands, from a different goroutine
+	// than the heartbeat that reads it.
+	port atomic.Int64
+	// Latches once the engine has answered. The agent is up well before initdb
+	// finishes, so a down engine is "starting" until it has served once.
+	seenHealthy bool
+}
+
+func newEngineProbe(cfg config, run probeRunner) *engineProbe {
+	p := &engineProbe{run: run, binary: cfg.PGIsReady, host: cfg.EngineHost}
+	p.port.Store(int64(cfg.EnginePort))
+	return p
+}
+
+func (p *engineProbe) setPort(port int) {
+	p.port.Store(int64(port))
+}
+
+// What the engine is doing, before the seenHealthy latch collapses "never up"
+// and "was up and went away" into one health. The parameter rollback needs them
+// apart: a postmaster that is up and replaying WAL will come back on its own, one
+// that is not running at all after a parameter change will not.
+type engineState int
+
+const (
+	// Not answering at all, or the probe could not run.
+	engineAbsent engineState = iota
+	// The postmaster is up and rejecting connections: startup or crash recovery.
+	engineRecovering
+	engineServing
+)
+
+// Only the heartbeat calls this, so the seenHealthy latch is single-goroutine
+// state.
+func (p *engineProbe) Check(ctx context.Context) (handlers_rds.EngineHealth, string) {
+	state, message := p.state(ctx)
+	switch state {
+	case engineServing:
+		p.seenHealthy = true
+		return handlers_rds.EngineHealthHealthy, ""
+	case engineRecovering:
+		return handlers_rds.EngineHealthStarting, message
+	default:
+		return p.degraded(), message
+	}
+}
+
+// The raw probe result. Safe to call from a goroutine other than the
+// heartbeat's: it touches no latched state.
+func (p *engineProbe) state(ctx context.Context) (engineState, string) {
+	port := strconv.FormatInt(p.port.Load(), 10)
+	code, err := p.run(ctx, p.binary, "-h", p.host, "-p", port, "-q")
+	switch {
+	case err != nil:
+		// A missing binary or broken image. Reporting healthy on the strength of
+		// nothing would hide it, so this reads as absent like an engine that did
+		// not answer.
+		return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
+	case code == 0:
+		return engineServing, ""
+	case code == 1:
+		return engineRecovering, "engine is rejecting connections (startup or recovery)"
+	default:
+		return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", p.host, port)
+	}
+}
+
+// A non-answering engine: starting until it has answered once, unhealthy after.
+func (p *engineProbe) degraded() handlers_rds.EngineHealth {
+	if p.seenHealthy {
+		return handlers_rds.EngineHealthUnhealthy
+	}
+	return handlers_rds.EngineHealthStarting
+}

cmd/rds-agent/health_test.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
+	tests := []struct {
+		name        string
+		code        int
+		runErr      error
+		seenHealthy bool
+		want        handlers_rds.EngineHealth
+	}{
+		{name: "accepting connections", code: 0, want: handlers_rds.EngineHealthHealthy},
+		{name: "rejecting during startup", code: 1, want: handlers_rds.EngineHealthStarting},
+		{name: "silent before first success", code: 2, want: handlers_rds.EngineHealthStarting},
+		{name: "silent after first success", code: 2, seenHealthy: true, want: handlers_rds.EngineHealthUnhealthy},
+		{name: "no attempt made", code: 3, want: handlers_rds.EngineHealthStarting},
+		{name: "probe missing before first success", runErr: errors.New("executable not found"), want: handlers_rds.EngineHealthStarting},
+		{name: "probe missing after first success", runErr: errors.New("executable not found"), seenHealthy: true, want: handlers_rds.EngineHealthUnhealthy},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
+				return tt.code, tt.runErr
+			})
+			probe.seenHealthy = tt.seenHealthy
+
+			got, message := probe.Check(context.Background())
+			if got != tt.want {
+				t.Errorf("health = %q, want %q", got, tt.want)
+			}
+			if got != handlers_rds.EngineHealthHealthy && message == "" {
+				t.Error("a non-healthy result carried no message explaining it")
+			}
+			if got == handlers_rds.EngineHealthHealthy && message != "" {
+				t.Errorf("a healthy result carried a message: %q", message)
+			}
+		})
+	}
+}
+
+// The latch is what separates a boot still in progress from a failure, and it
+// has to flip the first time the engine answers.
+func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
+	code := 0
+	probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
+		return code, nil
+	})
+
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthHealthy {
+		t.Fatalf("first check = %q, want healthy", got)
+	}
+	code = 2
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthUnhealthy {
+		t.Errorf("check after the engine went away = %q, want unhealthy", got)
+	}
+}
+
+// The port the engine listens on comes from the bootstrap config, so the probe
+// must follow it rather than keep asking the default.
+func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
+	var gotArgs []string
+	probe := newEngineProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, error) {
+		gotArgs = args
+		return 0, nil
+	})
+
+	probe.setPort(6543)
+	probe.Check(context.Background())
+
+	if !strings.Contains(strings.Join(gotArgs, " "), "-p 6543") {
+		t.Errorf("probe args = %v, want the assigned port 6543", gotArgs)
+	}
+}
+
+func testProbeConfig() config {
+	return config{EngineHost: defaultEngineHost, EnginePort: defaultEnginePort, PGIsReady: defaultPGIsReady}
+}

cmd/rds-agent/heartbeat.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The beat carries the probe's result, not the agent's own liveness: an agent
+// up while the engine is down is what the recovery reconciler must see.
+type heartbeater struct {
+	cp       controlPlane
+	probe    *engineProbe
+	id       identity
+	interval time.Duration
+}
+
+func newHeartbeater(cp controlPlane, probe *engineProbe, interval time.Duration) *heartbeater {
+	if interval <= 0 {
+		interval = handlers_rds.HeartbeatInterval
+	}
+	return &heartbeater{cp: cp, probe: probe, interval: interval}
+}
+
+// Called before Run starts and from Run's own goroutine after, so the interval
+// is never written concurrently with the loop reading it.
+func (h *heartbeater) setInterval(d time.Duration) {
+	if d > 0 {
+		h.interval = d
+	}
+}
+
+func (h *heartbeater) beat(ctx context.Context) {
+	health, message := h.probe.Check(ctx)
+
+	out, err := h.cp.SubmitState(ctx, h.id, health, message)
+	if err != nil {
+		// Not escalated: the control plane already treats a missing heartbeat as
+		// staleness, and the next tick retries.
+		slog.Warn("rds-agent: heartbeat failed", "health", health, "err", err)
+		return
+	}
+	slog.Debug("rds-agent: heartbeat", "health", health, "persisted", out.Persisted)
+	h.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
+}
+
+// A timer rather than a ticker, so a cadence change takes effect on the next
+// beat, not at a restart.
+func (h *heartbeater) Run(ctx context.Context) {
+	timer := time.NewTimer(h.interval)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			h.beat(ctx)
+			timer.Reset(h.interval)
+		}
+	}
+}

cmd/rds-agent/main.go

@@ -0,0 +1,40 @@
+// rds-agent runs inside a Spinifex RDS instance. It registers the VM with the
+// control plane over TLS+SigV4 (never NATS), writes the bootstrap handoff the
+// engine's first-boot script consumes, then heartbeats health and polls for
+// directives.
+//
+// It is the only path by which a secret reaches the VM: the master password is
+// served once, to an authenticated caller. Static config is read from the
+// cloud-init env file /etc/spinifex-rds/agent.env; real env vars override it.
+package main
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)
+
+func main() {
+	cfg := loadConfig(defaultEnvFile)
+
+	agent, err := New(cfg)
+	if err != nil {
+		slog.Error("rds-agent: startup failed", "err", err)
+		os.Exit(1)
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	// A shutdown signal cancels whatever boot retry loop was running; that is a
+	// clean stop, not a failure.
+	if err := agent.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
+		slog.Error("rds-agent: run failed", "err", err)
+		os.Exit(1)
+	}
+}

cmd/rds-agent/paramrecovery.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"time"
+)
+
+// How long the engine is given to come up after a boot before the installed
+// parameter set is treated as the reason it has not. Generous, because an
+// ordinary cold start also runs crash recovery, and a rollback that fired
+// against a cluster still replaying WAL would restart it for nothing.
+const parameterRollbackAfter = 5 * time.Minute
+
+// How often the wait re-checks. The engine reports through the same probe the
+// heartbeat uses, so this costs a pg_isready per tick and nothing else.
+const parameterRollbackPoll = 10 * time.Second
+
+// The boot-time half of the parameter safety net. A static parameter only takes
+// effect at a restart, so a value the engine's config check accepted can still
+// stop it from starting — a shared_buffers the host cannot allocate is the usual
+// shape. When that happens the bad config is on the persistent data volume, so
+// every replace and every recovery boot fails the same way.
+//
+// This breaks that loop: once the engine has failed to come up for long enough,
+// the last set it accepted is put back and the engine is restarted. The failure
+// is reported rather than swallowed, so the instance comes back on the old
+// parameters and the control plane still sees that the change did not land.
+type paramGuard struct {
+	engine parameterRecovery
+	probe  *engineProbe
+	after  time.Duration
+	poll   time.Duration
+}
+
+func newParamGuard(engine parameterRecovery, probe *engineProbe) *paramGuard {
+	return &paramGuard{engine: engine, probe: probe, after: parameterRollbackAfter, poll: parameterRollbackPoll}
+}
+
+// Runs once per agent lifetime: a rollback that did not bring the engine back
+// means the parameters were not the cause, and repeating it would only churn a
+// cluster that is failing for another reason.
+func (g *paramGuard) Run(ctx context.Context) {
+	if !g.waitForEngineDown(ctx) {
+		return
+	}
+
+	restored, err := g.engine.RestoreLastKnownGoodParameters()
+	if err != nil {
+		slog.Error("rds-agent: could not restore the last known good parameters", "err", err)
+		return
+	}
+	if !restored {
+		// The engine is already running the set it last accepted, so whatever is
+		// keeping it down is not the parameters.
+		slog.Warn("rds-agent: engine is down but its parameters are the last accepted set; not rolling back")
+		return
+	}
+
+	slog.Error("rds-agent: engine did not start after a parameter change; rolled back to the last accepted set",
+		"after", g.after)
+	if err := g.engine.Restart(ctx); err != nil {
+		slog.Error("rds-agent: restart after the parameter rollback failed", "err", err)
+	}
+}
+
+// Reports true once the engine has been continuously absent for the whole
+// window, false if it came up or the agent is shutting down. A postmaster that
+// is up and replaying WAL resets the clock: it is making progress on its own, and
+// restarting it would only throw the recovery away and start it again.
+func (g *paramGuard) waitForEngineDown(ctx context.Context) bool {
+	deadline := time.Now().Add(g.after)
+	timer := time.NewTimer(g.poll)
+	defer timer.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return false
+		case <-timer.C:
+		}
+		switch state, _ := g.probe.state(ctx); state {
+		case engineServing:
+			return false
+		case engineRecovering:
+			deadline = time.Now().Add(g.after)
+		}
+		if time.Now().After(deadline) {
+			return true
+		}
+		timer.Reset(g.poll)
+	}
+}

cmd/rds-agent/paramrecovery_test.go

@@ -0,0 +1,230 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// A runner that fails only the offline config check, so the check's own
+// rejection is separable from a reload or a psql failure.
+type checkFailingRunner struct {
+	recordingRunner
+
+	checkErr error
+}
+
+func (r *checkFailingRunner) run(ctx context.Context, c command) (string, error) {
+	if slices.Contains(c.Args, "-C") {
+		r.calls = append(r.calls, c)
+		return "", r.checkErr
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+// The failure this closes: a value the engine refuses would otherwise sit on the
+// data volume, where it survives every VM replace and turns the next restart
+// into a boot loop.
+func TestPostgresEngine_ApplyParametersRollsBackAValueTheEngineRejects(t *testing.T) {
+	runner := &checkFailingRunner{checkErr: errors.New(`unrecognized configuration parameter "work_men"`)}
+	engine := newTestEngine(t, runner.run)
+
+	// A first apply the engine accepted, which is what the rejected one has to
+	// roll back to.
+	runner.checkErr = nil
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "4096"}}); err != nil {
+		t.Fatalf("first ApplyParameters: %v", err)
+	}
+	runner.checkErr = errors.New(`unrecognized configuration parameter "work_men"`)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "999999999"}}); err == nil {
+		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
+	}
+
+	installed, err := os.ReadFile(engine.parametersPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "work_mem = '4096'") {
+		t.Errorf("installed parameters = %q, want the previous accepted set restored", installed)
+	}
+}
+
+// The first apply of an instance's life has nothing to roll back to, so a
+// rejection has to leave no include at all rather than an empty one.
+func TestPostgresEngine_ApplyParametersWithdrawsTheFirstRejectedSet(t *testing.T) {
+	runner := &checkFailingRunner{checkErr: errors.New("configuration file contains errors")}
+	engine := newTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "0"}}); err == nil {
+		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
+	}
+	if _, err := os.Stat(engine.parametersPath()); !os.IsNotExist(err) {
+		t.Errorf("the rejected include is still installed (stat err = %v)", err)
+	}
+}
+
+// The rollback target has to be a set the engine actually adopted, so a second
+// accepted apply leaves the first one as what a failed restart falls back to.
+func TestPostgresEngine_LastKnownGoodTracksTheAcceptedSet(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	for _, value := range []string{"4096", "8192"} {
+		if _, err := engine.ApplyParameters(context.Background(),
+			[]handlers_rds.Parameter{{Name: "work_mem", Value: value}}); err != nil {
+			t.Fatalf("ApplyParameters(%s): %v", value, err)
+		}
+	}
+
+	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read the last known good parameters: %v", err)
+	}
+	if !strings.Contains(string(lastGood), "work_mem = '4096'") {
+		t.Errorf("last known good = %q, want the set the engine ran before the latest apply", lastGood)
+	}
+
+	// include_dir globs *.conf, so the rollback copy must not read as a second
+	// set of settings alongside the live one.
+	if strings.HasSuffix(engine.lastGoodPath(), ".conf") {
+		t.Errorf("the rollback copy at %s would be included by the engine", engine.lastGoodPath())
+	}
+}
+
+func TestPostgresEngine_RestoreLastKnownGoodParameters(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	// Nothing to restore before anything has ever been applied.
+	restored, err := engine.RestoreLastKnownGoodParameters()
+	if err != nil {
+		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+	}
+	if restored {
+		t.Error("restored a set with no last known good on disk")
+	}
+
+	for _, value := range []string{"4096", "8192"} {
+		if _, err := engine.ApplyParameters(context.Background(),
+			[]handlers_rds.Parameter{{Name: "work_mem", Value: value}}); err != nil {
+			t.Fatalf("ApplyParameters(%s): %v", value, err)
+		}
+	}
+
+	restored, err = engine.RestoreLastKnownGoodParameters()
+	if err != nil {
+		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+	}
+	if !restored {
+		t.Fatal("did not restore a set that differs from the last known good")
+	}
+	installed, err := os.ReadFile(engine.parametersPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "work_mem = '4096'") {
+		t.Errorf("installed = %q, want the last accepted set", installed)
+	}
+
+	// Idempotent: the second call finds them already equal and does nothing, so a
+	// repeated rollback cannot churn a cluster that is failing for another reason.
+	restored, err = engine.RestoreLastKnownGoodParameters()
+	if err != nil {
+		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+	}
+	if restored {
+		t.Error("restored again when the installed set already matched")
+	}
+}
+
+// Stands in for the engine in the guard's tests: what matters is whether the
+// rollback and the restart were reached at all.
+type fakeRecovery struct {
+	restored   bool
+	restoreErr error
+	restarts   int
+}
+
+func (f *fakeRecovery) RestoreLastKnownGoodParameters() (bool, error) {
+	return f.restored, f.restoreErr
+}
+
+func (f *fakeRecovery) Restart(context.Context) error {
+	f.restarts++
+	return nil
+}
+
+// A probe whose result the test drives directly, standing in for pg_isready.
+func stubProbe(t *testing.T, code int, err error) *engineProbe {
+	t.Helper()
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	return newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
+		return code, err
+	})
+}
+
+func newTestGuard(engine parameterRecovery, probe *engineProbe) *paramGuard {
+	g := newParamGuard(engine, probe)
+	// Real budgets are minutes, which is the point of them; the behaviour under
+	// test is the decision, not the wait.
+	g.after, g.poll = 20*time.Millisecond, time.Millisecond
+	return g
+}
+
+// pg_isready exit 2 is an engine that is not there at all, which after a
+// parameter change is the boot-loop shape this breaks.
+func TestParamGuard_RollsBackAndRestartsWhenTheEngineNeverComesUp(t *testing.T) {
+	engine := &fakeRecovery{restored: true}
+	newTestGuard(engine, stubProbe(t, 2, nil)).Run(t.Context())
+
+	if engine.restarts != 1 {
+		t.Errorf("restarts = %d, want the engine restarted on the rolled-back set", engine.restarts)
+	}
+}
+
+func TestParamGuard_DoesNothingWhenTheEngineIsServing(t *testing.T) {
+	engine := &fakeRecovery{restored: true}
+	newTestGuard(engine, stubProbe(t, 0, nil)).Run(t.Context())
+
+	if engine.restarts != 0 {
+		t.Errorf("restarts = %d, want none against a healthy engine", engine.restarts)
+	}
+}
+
+// pg_isready exit 1 is a postmaster that is up and replaying WAL. It is making
+// progress on its own, and restarting it would throw the recovery away.
+func TestParamGuard_LeavesARecoveringEngineAlone(t *testing.T) {
+	engine := &fakeRecovery{restored: true}
+	guard := newTestGuard(engine, stubProbe(t, 1, nil))
+	// Bounded so the deadline-resetting loop cannot run forever if it stops
+	// resetting; the assertion is that the rollback was never reached.
+	ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+	defer cancel()
+	guard.Run(ctx)
+
+	if engine.restarts != 0 {
+		t.Errorf("restarts = %d, want none while the engine is recovering", engine.restarts)
+	}
+}
+
+// The parameters are only the cause if they differ from the set the engine last
+// accepted; restarting on an identical set would churn for nothing.
+func TestParamGuard_DoesNotRestartWhenTheParametersAreAlreadyTheAcceptedSet(t *testing.T) {
+	engine := &fakeRecovery{restored: false}
+	newTestGuard(engine, stubProbe(t, 2, nil)).Run(t.Context())
+
+	if engine.restarts != 0 {
+		t.Errorf("restarts = %d, want none when there was nothing to roll back", engine.restarts)
+	}
+}

cmd/rds-agent/quiesce.go

@@ -0,0 +1,259 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"os/exec"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+)
+
+// PostgreSQL's non-exclusive backup API ties the backup to the session that
+// started it: when that session ends, for any reason, the backup is aborted.
+// That is what makes the hold self-expiring — a control plane that dies mid
+// snapshot cannot leave the engine in backup mode forever.
+type engineSession interface {
+	// Runs sql and waits for the engine to finish it.
+	Exec(ctx context.Context, sql string) error
+	// Ends the session, releasing whatever the engine tied to it.
+	Close() error
+}
+
+// Starts a child process whose lifetime the caller controls, rather than one
+// that is run to completion. ctx governs the process, not one statement.
+type sessionRunner func(ctx context.Context, c command) (engineSession, error)
+
+// Written after every statement so the reader knows the engine is done with it.
+// The statements this carries return LSNs, so it cannot appear in a result.
+const sessionSentinel = "--rds-agent-ready--"
+
+// The backup mode currently held, and the timer that ends it regardless of what
+// happens to the control plane.
+type quiesceHold struct {
+	label   string
+	session engineSession
+	expiry  *time.Timer
+}
+
+// Puts the engine into backup mode: the datadir is checkpointed and the engine
+// stops writing over the pages a snapshot is about to read. The hold is released
+// by Unquiesce, or by its own deadline, whichever comes first.
+func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if label == "" {
+		return errors.New("quiesce requires a backup label")
+	}
+	if hold <= 0 {
+		return errors.New("quiesce requires a positive hold deadline")
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:  e.psql,
+		Args:  e.psqlArgs(),
+		Env:   []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
+		Stdin: "",
+		User:  e.osUser,
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// fast forces an immediate checkpoint rather than spreading it over the
+	// checkpoint interval, which would hold the snapshot open for minutes.
+	const sql = `\getenv label RDS_BACKUP_LABEL
+SELECT pg_backup_start(:'label', fast => true);
+`
+	if err := session.Exec(ctx, sql); err != nil {
+		if closeErr := session.Close(); closeErr != nil {
+			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+		}
+		return fmt.Errorf("put the engine into backup mode: %w", err)
+	}
+
+	e.held = &quiesceHold{
+		label:   label,
+		session: session,
+		expiry:  time.AfterFunc(hold, func() { e.expireQuiesce(label) }),
+	}
+	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold", hold)
+	return nil
+}
+
+// Ends the backup cleanly. A missing hold is an error rather than a silent
+// success: it means the deadline fired first, so the snapshot the control plane
+// just took was not taken against a held checkpoint.
+func (e *postgresEngine) Unquiesce(ctx context.Context) error {
+	e.mu.Lock()
+	held := e.held
+	e.held = nil
+	e.mu.Unlock()
+
+	if held == nil {
+		return errors.New("the engine is not quiesced; the backup hold had already expired")
+	}
+	held.expiry.Stop()
+
+	// The stop has to run on the session that started the backup, and the session
+	// is closed either way — the engine aborts an unstopped backup with it.
+	execErr := held.session.Exec(ctx, "SELECT pg_backup_stop();\n")
+	closeErr := held.session.Close()
+	if execErr != nil {
+		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
+	}
+	if closeErr != nil {
+		return fmt.Errorf("close the backup session: %w", closeErr)
+	}
+	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
+	return nil
+}
+
+// The deadline. Ending the session is enough — the engine aborts the backup it
+// was holding — so this never has to reach the engine itself.
+func (e *postgresEngine) expireQuiesce(label string) {
+	e.mu.Lock()
+	held := e.held
+	if held == nil || held.label != label {
+		e.mu.Unlock()
+		return
+	}
+	e.held = nil
+	e.mu.Unlock()
+
+	slog.Warn("rds-agent: backup hold expired; releasing the engine", "label", label)
+	if err := held.session.Close(); err != nil {
+		slog.Error("rds-agent: closing an expired backup session failed", "label", label, "err", err)
+	}
+}
+
+// A psql child with its stdin held open, so statements can be fed to it one at a
+// time and the engine keeps seeing a single session.
+type psqlSession struct {
+	cmd    *exec.Cmd
+	stdin  io.WriteCloser
+	stdout *bufio.Reader
+	stderr *bytes.Buffer
+
+	mu     sync.Mutex
+	closed bool
+}
+
+var _ engineSession = (*psqlSession)(nil)
+
+func execSessionRunner(ctx context.Context, c command) (engineSession, error) {
+	cmd := exec.CommandContext(ctx, c.Name, c.Args...)
+	cmd.Env = c.Env
+	if c.User != "" {
+		credential, err := lookupCredential(c.User)
+		if err != nil {
+			return nil, err
+		}
+		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: credential}
+	}
+
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return nil, fmt.Errorf("open stdin: %w", err)
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, fmt.Errorf("open stdout: %w", err)
+	}
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("start %s: %w", c.Name, err)
+	}
+	return &psqlSession{cmd: cmd, stdin: stdin, stdout: bufio.NewReader(stdout), stderr: &stderr}, nil
+}
+
+// Feeds sql and waits for the sentinel that follows it. ON_ERROR_STOP makes psql
+// exit on a failed statement, so a sentinel that never arrives is the failure
+// itself and the engine's own message is on stderr.
+func (s *psqlSession) Exec(ctx context.Context, sql string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.closed {
+		return errors.New("the backup session is closed")
+	}
+	if _, err := io.WriteString(s.stdin, sql+"\\echo "+sessionSentinel+"\n"); err != nil {
+		return fmt.Errorf("%w: %s", err, strings.TrimSpace(s.stderr.String()))
+	}
+
+	// Read on its own goroutine so a psql that never answers costs the caller its
+	// deadline rather than blocking it forever.
+	done := make(chan error, 1)
+	go func() { done <- s.awaitSentinel() }()
+	select {
+	case err := <-done:
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (s *psqlSession) awaitSentinel() error {
+	for {
+		line, err := s.stdout.ReadString('\n')
+		if strings.TrimSpace(line) == sessionSentinel {
+			return nil
+		}
+		if err != nil {
+			if message := strings.TrimSpace(s.stderr.String()); message != "" {
+				return errors.New(message)
+			}
+			return fmt.Errorf("the backup session ended: %w", err)
+		}
+	}
+}
+
+// Closing stdin ends the script, and the wait reaps the child. A session that
+// will not end is killed, because leaving it is what would keep the engine in
+// backup mode.
+func (s *psqlSession) Close() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.closed {
+		return nil
+	}
+	s.closed = true
+
+	if err := s.stdin.Close(); err != nil {
+		slog.Debug("rds-agent: closing the backup session stdin", "err", err)
+	}
+	done := make(chan error, 1)
+	go func() { done <- s.cmd.Wait() }()
+	select {
+	case err := <-done:
+		if err != nil {
+			return fmt.Errorf("%w: %s", err, strings.TrimSpace(s.stderr.String()))
+		}
+		return nil
+	case <-time.After(sessionCloseGrace):
+		if err := s.cmd.Process.Kill(); err != nil {
+			return fmt.Errorf("kill the backup session: %w", err)
+		}
+		<-done
+		return nil
+	}
+}
+
+// How long a backup session gets to end on its own before it is killed. The
+// engine releases the backup on either, so this only bounds the wait.
+const sessionCloseGrace = 10 * time.Second

cmd/rds-agent/quiesce_test.go

@@ -0,0 +1,246 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// fakeSession stands in for the held psql child. Closing it is what the engine
+// treats as the end of the backup, so a test asserts on that rather than on any
+// statement the release would have sent.
+type fakeSession struct {
+	mu       sync.Mutex
+	executed []string
+	closed   int
+	execErr  error
+	closeErr error
+}
+
+var _ engineSession = (*fakeSession)(nil)
+
+func (f *fakeSession) Exec(_ context.Context, sql string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.executed = append(f.executed, sql)
+	return f.execErr
+}
+
+func (f *fakeSession) Close() error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.closed++
+	return f.closeErr
+}
+
+func (f *fakeSession) statements() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.executed...)
+}
+
+func (f *fakeSession) closes() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.closed
+}
+
+// newQuiesceEngine wires an engine whose backup sessions are the fake, and
+// returns the commands each session was started with.
+func newQuiesceEngine(t *testing.T, session *fakeSession) (*postgresEngine, *[]command) {
+	t.Helper()
+	started := &[]command{}
+	engine := newTestEngine(t, (&recordingRunner{}).run)
+	engine.startSess = func(_ context.Context, c command) (engineSession, error) {
+		*started = append(*started, c)
+		return session, nil
+	}
+	return engine, started
+}
+
+// The label rides the environment and is re-quoted by psql, so it never reaches
+// a shell word or an argv another process can read.
+func TestPostgresEngine_QuiesceStartsTheBackupOnAHeldSession(t *testing.T) {
+	session := &fakeSession{}
+	engine, started := newQuiesceEngine(t, session)
+
+	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	if len(*started) != 1 {
+		t.Fatalf("started %d sessions, want 1", len(*started))
+	}
+	if !slices.Contains((*started)[0].Env, "RDS_BACKUP_LABEL=orders-db-pre-upgrade") {
+		t.Errorf("env %v does not carry the backup label", (*started)[0].Env)
+	}
+	statements := session.statements()
+	if len(statements) != 1 || !strings.Contains(statements[0], "pg_backup_start") {
+		t.Fatalf("statements = %q, want a pg_backup_start", statements)
+	}
+	// A spread checkpoint would hold the snapshot open for minutes.
+	if !strings.Contains(statements[0], "fast => true") {
+		t.Errorf("statement %q does not force an immediate checkpoint", statements[0])
+	}
+	if session.closes() != 0 {
+		t.Error("the session was closed while the backup was meant to be held")
+	}
+}
+
+func TestPostgresEngine_UnquiesceStopsTheBackupAndEndsTheSession(t *testing.T) {
+	session := &fakeSession{}
+	engine, _ := newQuiesceEngine(t, session)
+	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	if err := engine.Unquiesce(context.Background()); err != nil {
+		t.Fatalf("Unquiesce: %v", err)
+	}
+
+	statements := session.statements()
+	if len(statements) != 2 || !strings.Contains(statements[1], "pg_backup_stop") {
+		t.Fatalf("statements = %q, want a pg_backup_stop", statements)
+	}
+	if session.closes() != 1 {
+		t.Errorf("closed %d times, want the session ended exactly once", session.closes())
+	}
+}
+
+// A hold that expired mid-snapshot means the snapshot was not taken against a
+// held checkpoint, so the release reports it rather than succeeding silently.
+func TestPostgresEngine_UnquiesceWithoutAHoldIsAnError(t *testing.T) {
+	engine, _ := newQuiesceEngine(t, &fakeSession{})
+
+	err := engine.Unquiesce(context.Background())
+	if err == nil {
+		t.Fatal("Unquiesce succeeded with no hold to release")
+	}
+	if !strings.Contains(err.Error(), "expired") {
+		t.Errorf("error %q does not say the hold had expired", err)
+	}
+}
+
+// The engine aborts the backup when the session ends, which is what makes the
+// hold self-expiring: a control plane that dies cannot leave it held forever.
+func TestPostgresEngine_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
+	session := &fakeSession{}
+	engine, _ := newQuiesceEngine(t, session)
+
+	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+	deadline := time.Now().Add(2 * time.Second)
+	for session.closes() == 0 && time.Now().Before(deadline) {
+		time.Sleep(time.Millisecond)
+	}
+
+	if session.closes() != 1 {
+		t.Fatal("the backup session outlived its deadline")
+	}
+	// The hold is gone with it, so a later release is honest about that.
+	if err := engine.Unquiesce(context.Background()); err == nil {
+		t.Error("Unquiesce succeeded after the hold expired")
+	}
+}
+
+// One backup at a time: a second start would open a concurrent backup alongside
+// the first rather than extending it.
+func TestPostgresEngine_QuiesceRefusesASecondHold(t *testing.T) {
+	session := &fakeSession{}
+	engine, started := newQuiesceEngine(t, session)
+	if err := engine.Quiesce(context.Background(), "first", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	err := engine.Quiesce(context.Background(), "second", time.Minute)
+	if err == nil {
+		t.Fatal("a second quiesce was accepted while the first was held")
+	}
+	if !strings.Contains(err.Error(), "first") {
+		t.Errorf("error %q does not name the backup already holding the engine", err)
+	}
+	if len(*started) != 1 {
+		t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
+	}
+}
+
+// A failed start leaves nothing held, so a retry is not refused by a hold that
+// never took effect.
+func TestPostgresEngine_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
+	session := &fakeSession{execErr: errors.New(`ERROR:  a backup is already in progress`)}
+	engine, _ := newQuiesceEngine(t, session)
+
+	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err == nil {
+		t.Fatal("Quiesce succeeded against an engine that refused the backup")
+	}
+	if session.closes() != 1 {
+		t.Errorf("closed %d times, want the failed session ended", session.closes())
+	}
+	if engine.held != nil {
+		t.Error("a failed quiesce left a hold behind")
+	}
+}
+
+func TestCommandRegistry_QuiesceCarriesTheLabelAndDeadline(t *testing.T) {
+	engine := &fakeEngine{}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),
+		handlers_rds.Command{
+			CommandID: "cmd-5",
+			Type:      handlers_rds.CommandQuiesce,
+			Parameters: []handlers_rds.Parameter{
+				{Name: handlers_rds.CommandParamQuiesceLabel, Value: "orders-db-pre-upgrade"},
+				{Name: handlers_rds.CommandParamQuiesceDeadlineSeconds, Value: "360"},
+			},
+		})
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if engine.label != "orders-db-pre-upgrade" || engine.hold != 6*time.Minute {
+		t.Errorf("engine got %q/%s, want the command's label and deadline", engine.label, engine.hold)
+	}
+}
+
+// A hold the guest chose the length of would not be the bound the control plane
+// is relying on, so a missing or unreadable deadline is refused.
+func TestCommandRegistry_QuiesceRejectsAnUnusableDeadline(t *testing.T) {
+	for _, value := range []string{"", "soon", "0", "-30"} {
+		engine := &fakeEngine{}
+		reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),
+			handlers_rds.Command{
+				CommandID: "cmd-6",
+				Type:      handlers_rds.CommandQuiesce,
+				Parameters: []handlers_rds.Parameter{
+					{Name: handlers_rds.CommandParamQuiesceLabel, Value: "orders-db-pre-upgrade"},
+					{Name: handlers_rds.CommandParamQuiesceDeadlineSeconds, Value: value},
+				},
+			})
+
+		if reply.Status != handlers_rds.CommandStatusFailed {
+			t.Errorf("deadline %q: reply = %+v, want failed", value, reply)
+		}
+		if engine.label != "" {
+			t.Errorf("deadline %q: the engine was quiesced anyway", value)
+		}
+	}
+}
+
+func TestCommandRegistry_UnquiesceReachesTheEngine(t *testing.T) {
+	engine := &fakeEngine{}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-7", Type: handlers_rds.CommandUnquiesce})
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if !engine.released {
+		t.Error("unquiesce did not reach the engine")
+	}
+}

cmd/rds-agent/register.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"time"
+)
+
+// Adopts the authoritative DB instance identifier and heartbeat cadence from
+// the response. Idempotent, so a restart re-registers unconditionally.
+func (a *Agent) register(ctx context.Context) error {
+	return retry(ctx, "register", func(ctx context.Context) error {
+		out, err := a.cp.Register(ctx, a.id)
+		if err != nil {
+			return err
+		}
+		if out.DBInstanceIdentifier != "" {
+			a.id.DBInstanceIdentifier = out.DBInstanceIdentifier
+		}
+		a.hb.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
+
+		slog.Info("rds-agent: registered",
+			"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat", a.hb.interval)
+		return nil
+	})
+}

cmd/rds-agent/storage.go

@@ -0,0 +1,175 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// The in-guest half of a storage grow. The control plane has already
+// grown the block device by the time this runs; all that is left is to put the
+// filesystem on the capacity that is now there. Both ext4 and XFS grow while
+// mounted, so this needs no ordering against the engine start.
+type storageOps interface {
+	// Extends the filesystem at the data mount onto its whole device, and
+	// reports what it did for the command reply.
+	GrowFilesystem(ctx context.Context) (string, error)
+}
+
+// Tool names rather than paths: the image installs them from
+// cloud-utils-growpart, e2fsprogs-extra and xfsprogs, all on PATH.
+const (
+	defaultGrowpart   = "growpart"
+	defaultResize2fs  = "resize2fs"
+	defaultXFSGrowfs  = "xfs_growfs"
+	defaultMountsFile = "/proc/mounts"
+	defaultSysBlock   = "/sys/class/block"
+	// Where rds-datadir mounts the data volume; the datadir sits one level in.
+	defaultDataMount = "/var/lib/postgresql"
+	// growpart says this when the partition already fills the disk, which a
+	// resumed grow is the normal way to reach.
+	growpartNoChange = "NOCHANGE"
+)
+
+type guestStorage struct {
+	run        commandRunner
+	dataMount  string
+	mountsFile string
+	sysBlock   string
+}
+
+var _ storageOps = (*guestStorage)(nil)
+
+func newGuestStorage(cfg config, run commandRunner) *guestStorage {
+	return &guestStorage{
+		run:        run,
+		dataMount:  cfg.DataMount,
+		mountsFile: cfg.MountsFile,
+		sysBlock:   cfg.SysBlock,
+	}
+}
+
+// The device and filesystem behind the data mount, resolved from the kernel's
+// own mount table rather than from anything the control plane asserted.
+type dataMount struct {
+	device string
+	fstype string
+}
+
+func (g *guestStorage) GrowFilesystem(ctx context.Context) (string, error) {
+	mount, err := g.resolveDataMount()
+	if err != nil {
+		return "", err
+	}
+	// rds-datadir formats the whole device and refuses a disk that already
+	// carries a partition table, so the data volume is normally unpartitioned
+	// and there is no partition to push out first.
+	if err := g.growPartition(ctx, mount.device); err != nil {
+		return "", err
+	}
+
+	switch {
+	case strings.HasPrefix(mount.fstype, "ext"):
+		// resize2fs grows a mounted ext filesystem online, and takes the device.
+		if _, err := g.run(ctx, command{
+			Name: defaultResize2fs,
+			Args: []string{mount.device},
+			Env:  []string{"PATH=" + defaultGuestPath},
+		}); err != nil {
+			return "", fmt.Errorf("grow the %s filesystem on %s: %w", mount.fstype, mount.device, err)
+		}
+	case mount.fstype == "xfs":
+		// xfs_growfs is addressed by mount point, and XFS can only grow mounted.
+		if _, err := g.run(ctx, command{
+			Name: defaultXFSGrowfs,
+			Args: []string{g.dataMount},
+			Env:  []string{"PATH=" + defaultGuestPath},
+		}); err != nil {
+			return "", fmt.Errorf("grow the xfs filesystem at %s: %w", g.dataMount, err)
+		}
+	default:
+		// Guessing a tool here would either do nothing or corrupt a filesystem
+		// this image was never meant to be holding.
+		return "", fmt.Errorf("%s holds an unsupported filesystem %q; refusing to grow it", mount.device, mount.fstype)
+	}
+
+	slog.Info("rds-agent: data filesystem grown", "device", mount.device, "fstype", mount.fstype, "mount", g.dataMount)
+	return fmt.Sprintf("grew %s (%s) at %s", mount.device, mount.fstype, g.dataMount), nil
+}
+
+// Pushes the partition out to the end of its disk when the data mount happens
+// to sit on one. A device that is not a partition needs nothing, and a
+// partition that already fills its disk is the normal state of a resumed grow.
+func (g *guestStorage) growPartition(ctx context.Context, device string) error {
+	disk, number, ok := g.partitionOf(device)
+	if !ok {
+		return nil
+	}
+	out, err := g.run(ctx, command{
+		Name: defaultGrowpart,
+		Args: []string{disk, number},
+		Env:  []string{"PATH=" + defaultGuestPath},
+	})
+	if err == nil {
+		return nil
+	}
+	if strings.Contains(out, growpartNoChange) || strings.Contains(err.Error(), growpartNoChange) {
+		return nil
+	}
+	return fmt.Errorf("grow partition %s of %s: %w", number, disk, err)
+}
+
+// Whether device is a partition, and if so its disk and number. Read from
+// sysfs, which states it directly rather than inferring it from a trailing
+// digit — nvme0n1 is a disk and vdb1 is a partition on the same rule.
+func (g *guestStorage) partitionOf(device string) (disk, number string, ok bool) {
+	name := filepath.Base(device)
+	raw, err := os.ReadFile(filepath.Join(g.sysBlock, name, "partition"))
+	if err != nil {
+		return "", "", false
+	}
+	number = strings.TrimSpace(string(raw))
+	if number == "" {
+		return "", "", false
+	}
+	// A partition's sysfs entry hangs off its disk's, so the parent directory
+	// names the disk without any parsing of the device name.
+	target, err := os.Readlink(filepath.Join(g.sysBlock, name))
+	if err != nil {
+		return "", "", false
+	}
+	return filepath.Join(filepath.Dir(device), filepath.Base(filepath.Dir(target))), number, true
+}
+
+// The mount table entry for the data mount. An absent one means the data volume
+// is not mounted, which is a broken guest rather than something to grow.
+func (g *guestStorage) resolveDataMount() (dataMount, error) {
+	f, err := os.Open(g.mountsFile)
+	if err != nil {
+		return dataMount{}, fmt.Errorf("read %s: %w", g.mountsFile, err)
+	}
+	defer f.Close()
+
+	// Later entries shadow earlier ones at the same point, so the last match is
+	// the filesystem actually visible there.
+	found := dataMount{}
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		if len(fields) < 3 || fields[1] != g.dataMount {
+			continue
+		}
+		found = dataMount{device: fields[0], fstype: fields[2]}
+	}
+	if err := scanner.Err(); err != nil {
+		return dataMount{}, fmt.Errorf("read %s: %w", g.mountsFile, err)
+	}
+	if found.device == "" {
+		return dataMount{}, fmt.Errorf("no filesystem is mounted at %s; the data volume is not attached", g.dataMount)
+	}
+	return found, nil
+}

cmd/rds-agent/storage_test.go

@@ -0,0 +1,226 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Stands in for the guest's storage in the registry tests, where what matters
+// is that the command reaches it rather than what it does to a filesystem.
+type fakeStorage struct {
+	grown   bool
+	message string
+	err     error
+}
+
+var _ storageOps = (*fakeStorage)(nil)
+
+func (f *fakeStorage) GrowFilesystem(context.Context) (string, error) {
+	f.grown = true
+	return f.message, f.err
+}
+
+// runLog records what a grow shelled out to, in order, so the test asserts on
+// the tools invoked rather than on a filesystem it would have to create.
+type runLog struct {
+	calls []command
+	// errOn fails the first command whose name matches, which is how growpart's
+	// tolerated failure and resize2fs's fatal one are told apart.
+	errOn  string
+	errOut string
+	err    error
+}
+
+func (r *runLog) run(_ context.Context, c command) (string, error) {
+	r.calls = append(r.calls, c)
+	if r.errOn != "" && c.Name == r.errOn {
+		return r.errOut, r.err
+	}
+	return "", nil
+}
+
+func (r *runLog) names() []string {
+	names := make([]string, 0, len(r.calls))
+	for _, c := range r.calls {
+		names = append(names, c.Name)
+	}
+	return names
+}
+
+// A guest whose data volume is mounted as fstype from device, with a sysfs tree
+// that reports the device as a whole disk unless the test says otherwise.
+func newTestStorage(t *testing.T, device, fstype string, run commandRunner) *guestStorage {
+	t.Helper()
+	dir := t.TempDir()
+	mounts := filepath.Join(dir, "mounts")
+	content := "proc /proc proc rw 0 0\n"
+	if device != "" {
+		content += device + " " + defaultDataMount + " " + fstype + " rw,relatime 0 0\n"
+	}
+	if err := os.WriteFile(mounts, []byte(content), 0o600); err != nil {
+		t.Fatalf("write the mount table: %v", err)
+	}
+	cfg := config{DataMount: defaultDataMount, MountsFile: mounts, SysBlock: filepath.Join(dir, "block")}
+	if err := os.MkdirAll(cfg.SysBlock, 0o755); err != nil {
+		t.Fatalf("create the sysfs tree: %v", err)
+	}
+	return newGuestStorage(cfg, run)
+}
+
+// Makes name look like partition number of disk, the way sysfs states it: a
+// "partition" file under the device, hanging off its disk's directory.
+func seedPartition(t *testing.T, g *guestStorage, disk, name, number string) {
+	t.Helper()
+	// sysfs holds the real entry under the device tree and links to it from
+	// /sys/class/block, so the partition's disk is its link target's parent.
+	target := filepath.Join("..", "devices", disk, name)
+	if err := os.MkdirAll(filepath.Join(g.sysBlock, target), 0o755); err != nil {
+		t.Fatalf("create the partition tree: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(g.sysBlock, target, "partition"), []byte(number+"\n"), 0o600); err != nil {
+		t.Fatalf("write the partition number: %v", err)
+	}
+	if err := os.Symlink(target, filepath.Join(g.sysBlock, name)); err != nil {
+		t.Fatalf("link the partition: %v", err)
+	}
+}
+
+// The data volume is formatted whole, so the usual grow is resize2fs against
+// the device with no partition to push out first.
+func TestGrowFilesystem_ResizesAnExt4DeviceOnline(t *testing.T) {
+	log := &runLog{}
+	g := newTestStorage(t, "/dev/vdb", "ext4", log.run)
+
+	message, err := g.GrowFilesystem(context.Background())
+	if err != nil {
+		t.Fatalf("GrowFilesystem: %v", err)
+	}
+
+	if got := log.names(); len(got) != 1 || got[0] != defaultResize2fs {
+		t.Fatalf("ran %v, want just %s", got, defaultResize2fs)
+	}
+	if args := log.calls[0].Args; len(args) != 1 || args[0] != "/dev/vdb" {
+		t.Errorf("resize2fs args = %v, want the device", args)
+	}
+	if !strings.Contains(message, "/dev/vdb") || !strings.Contains(message, defaultDataMount) {
+		t.Errorf("reply message = %q, want the device and mount it grew", message)
+	}
+}
+
+// XFS is addressed by mount point and can only grow mounted, unlike ext.
+func TestGrowFilesystem_GrowsXFSByMountPoint(t *testing.T) {
+	log := &runLog{}
+	g := newTestStorage(t, "/dev/vdb", "xfs", log.run)
+
+	if _, err := g.GrowFilesystem(context.Background()); err != nil {
+		t.Fatalf("GrowFilesystem: %v", err)
+	}
+
+	if got := log.names(); len(got) != 1 || got[0] != defaultXFSGrowfs {
+		t.Fatalf("ran %v, want just %s", got, defaultXFSGrowfs)
+	}
+	if args := log.calls[0].Args; len(args) != 1 || args[0] != defaultDataMount {
+		t.Errorf("xfs_growfs args = %v, want the mount point", args)
+	}
+}
+
+// A partitioned data device has to have the partition pushed out to the end of
+// the disk before the filesystem can be extended onto it.
+func TestGrowFilesystem_PushesThePartitionOutFirst(t *testing.T) {
+	log := &runLog{}
+	g := newTestStorage(t, "/dev/vdb1", "ext4", log.run)
+	seedPartition(t, g, "vdb", "vdb1", "1")
+
+	if _, err := g.GrowFilesystem(context.Background()); err != nil {
+		t.Fatalf("GrowFilesystem: %v", err)
+	}
+
+	if got := log.names(); len(got) != 2 || got[0] != defaultGrowpart || got[1] != defaultResize2fs {
+		t.Fatalf("ran %v, want growpart then resize2fs", got)
+	}
+	if args := log.calls[0].Args; len(args) != 2 || args[0] != "/dev/vdb" || args[1] != "1" {
+		t.Errorf("growpart args = %v, want the disk and the partition number", args)
+	}
+}
+
+// A resumed grow finds the partition already filling its disk, which growpart
+// reports as a failure. Treating it as one would fail a grow that has nothing
+// left to do at that step.
+func TestGrowFilesystem_ToleratesAnAlreadyFullPartition(t *testing.T) {
+	log := &runLog{errOn: defaultGrowpart, errOut: "NOCHANGE: partition 1 is size 41940992. it cannot be grown", err: errors.New("exit status 1")}
+	g := newTestStorage(t, "/dev/vdb1", "ext4", log.run)
+	seedPartition(t, g, "vdb", "vdb1", "1")
+
+	if _, err := g.GrowFilesystem(context.Background()); err != nil {
+		t.Fatalf("GrowFilesystem: %v", err)
+	}
+	if got := log.names(); len(got) != 2 || got[1] != defaultResize2fs {
+		t.Errorf("ran %v, want the resize to follow the tolerated growpart", got)
+	}
+}
+
+// Any other growpart failure is a real one, and extending a filesystem onto
+// capacity the partition does not have would be the corrupting move.
+func TestGrowFilesystem_FailsOnARealGrowpartError(t *testing.T) {
+	log := &runLog{errOn: defaultGrowpart, errOut: "unable to read partition table", err: errors.New("exit status 2")}
+	g := newTestStorage(t, "/dev/vdb1", "ext4", log.run)
+	seedPartition(t, g, "vdb", "vdb1", "1")
+
+	if _, err := g.GrowFilesystem(context.Background()); err == nil {
+		t.Fatal("GrowFilesystem succeeded on a failed growpart, want an error")
+	}
+	if got := log.names(); len(got) != 1 {
+		t.Errorf("ran %v, want nothing after the failed growpart", got)
+	}
+}
+
+// Guessing a tool for an unrecognised filesystem would either do nothing or
+// corrupt one this image was never meant to be holding.
+func TestGrowFilesystem_RefusesAnUnsupportedFilesystem(t *testing.T) {
+	log := &runLog{}
+	g := newTestStorage(t, "/dev/vdb", "btrfs", log.run)
+
+	_, err := g.GrowFilesystem(context.Background())
+	if err == nil || !strings.Contains(err.Error(), "btrfs") {
+		t.Fatalf("err = %v, want a refusal naming the filesystem", err)
+	}
+	if len(log.calls) != 0 {
+		t.Errorf("ran %v against an unsupported filesystem, want nothing", log.names())
+	}
+}
+
+// Nothing mounted at the data mount is a broken guest rather than a grow to
+// attempt against whatever else happens to be there.
+func TestGrowFilesystem_FailsWhenTheDataVolumeIsNotMounted(t *testing.T) {
+	log := &runLog{}
+	g := newTestStorage(t, "", "", log.run)
+
+	if _, err := g.GrowFilesystem(context.Background()); err == nil {
+		t.Fatal("GrowFilesystem succeeded with no data volume mounted, want an error")
+	}
+	if len(log.calls) != 0 {
+		t.Errorf("ran %v with nothing mounted, want nothing", log.names())
+	}
+}
+
+// The control plane's grow-filesystem directive has to reach the guest's
+// storage, or a grown volume never becomes usable capacity.
+func TestCommandRegistry_GrowFilesystemReachesTheGuestStorage(t *testing.T) {
+	storage := &fakeStorage{message: "grew /dev/vdb (ext4) at /var/lib/postgresql"}
+
+	reply := newCommander(nil, newCommandRegistry(&fakeEngine{}, storage), 0).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-9", Type: handlers_rds.CommandGrowFilesystem})
+
+	if !storage.grown {
+		t.Error("the grow-filesystem command did not reach the guest storage")
+	}
+	if reply.Status != handlers_rds.CommandStatusSucceeded || reply.Message != storage.message {
+		t.Errorf("reply = %+v, want succeeded carrying what the grow reported", reply)
+	}
+}

cmd/spinifex/cmd/admin.go

@@ -117,8 +117,10 @@ var nodeDrainCmd = &cobra.Command{
 	Long: `Run the GATE and DRAIN shutdown phases against the local node only: power down
 its guests via QMP and unmount their volumes (flushing the viperblock WAL) while
 every service is still running. STORAGE/PERSIST/INFRA are left to systemd's
-ordered unit teardown. This is the ExecStop of spinifex-shutdown.service, so a
-systemctl stop or host reboot drains guests before any storage service stops.`,
+ordered unit teardown. This is what spinifex-shutdown.service's ExecStop runs,
+with --only-if-host-stopping so it only drains on a genuine host
+shutdown/reboot -- not on a plain "systemctl restart/stop spinifex.target".
+Run by hand without that flag, it always drains unconditionally.`,
 	Run: runNodeDrainLocal,
 }
 
@@ -281,6 +283,9 @@ func init() {
 	nodeCmd.AddCommand(nodeDrainCmd)
 	nodeDrainCmd.Flags().Bool("local", false, "Drain the local node only (required)")
 	nodeDrainCmd.Flags().Duration("timeout", 120*time.Second, "Maximum time to wait per phase")
+	nodeDrainCmd.Flags().Bool("only-if-host-stopping", false, "Skip the drain unless systemd is unwinding into shutdown.target (real reboot/poweroff); unset, always drains")
+	nodeCmd.AddCommand(nodeJSProbeCmd)
+	nodeJSProbeCmd.Flags().Duration("timeout", 10*time.Second, "Maximum time to wait for the canary round-trip")
 
 	adminCmd.AddCommand(imagesCmd)
 	imagesCmd.AddCommand(imagesImportCmd)
@@ -354,6 +359,8 @@ func init() {
 	adminJoinCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)")
 	adminJoinCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)")
 	adminJoinCmd.Flags().String("token", "", "Join token from the init node (required)")
+	adminJoinCmd.Flags().Bool("force", false, "Join even though this node is already initialized, discarding its own CA and master key")
+	adminJoinCmd.Flags().Duration("join-timeout", 20*time.Minute, "How long to keep retrying while the formation server is unreachable")
 	adminJoinCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all)")
 	adminJoinCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during join (default: enabled)")
 	adminJoinCmd.Flags().String("email", "", "Operator email address (used for update and security notifications)")
@@ -391,6 +398,8 @@ func init() {
 	}
 }
 
+const bytesPerGiB = 1024 * 1024 * 1024
+
 // amiVolumeSizeGiB returns the smallest whole GiB that still holds sizeBytes.
 //
 // Rounding up is load-bearing. The image is copied into a root volume of
@@ -405,7 +414,6 @@ func init() {
 // raises a caller's requested size to this value, so it inherits the mistake
 // rather than catching it.
 func amiVolumeSizeGiB(sizeBytes int64) uint64 {
-	const bytesPerGiB = 1024 * 1024 * 1024
 	if sizeBytes <= 0 {
 		return 0
 	}
@@ -903,8 +911,10 @@ func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	fmt.Printf("✅ Removed AMI %s (freed %s across %d objects).\n",
-		imageID, utils.HumanBytes(utils.SafeInt64ToUint64(res.BytesFreed)), res.ObjectsDeleted)
+	// BytesDeleted is logical: predastore reclaims the underlying disk space
+	// asynchronously via background compaction, not at delete time.
+	fmt.Printf("✅ Removed AMI %s (%d objects, %s marked for deletion; disk space is reclaimed by background compaction).\n",
+		imageID, res.ObjectsDeleted, utils.HumanBytes(utils.SafeInt64ToUint64(res.BytesDeleted)))
 }
 
 func printDependents(w io.Writer, d admin.Dependents) {
@@ -1310,7 +1320,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 
 	// Create config directory
-	if err := os.MkdirAll(configDir, 0700); err != nil {
+	if err := EnsureConfigDir(configDir); err != nil {
 		fmt.Fprintf(os.Stderr, "Error creating config directory: %v\n", err)
 		os.Exit(1)
 	}
@@ -1935,6 +1945,54 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	fmt.Println()
 }
 
+// joinRetryInterval paces retries against an unreachable formation server. The
+// primary may still be booting, so this is measured in "how long until the other
+// machine finishes starting", not in fractions of a second.
+const joinRetryInterval = 5 * time.Second
+
+// joinRetryable separates a formation server that is not up yet from one that
+// has answered and said no. Bare-metal nodes cannot be sequenced reliably, so a
+// joiner racing ahead of the primary must wait rather than fail. A rejected
+// token or a duplicate node name answers the same way on every attempt, so
+// those are reported at once instead of after the whole timeout.
+func joinRetryable(err error, statusCode int) bool {
+	if err != nil {
+		// Connection refused, DNS failure, TLS handshake, timeout: the primary
+		// is not listening yet.
+		return true
+	}
+	return statusCode >= 500
+}
+
+// checkJoinPreconditions rejects a join that would silently destroy this node's
+// existing cluster identity. Joining adopts the primary's CA and master key,
+// overwriting whatever is here: correct on a freshly installed node, and on one
+// that has been in service it orphans every fragment and volume sealed under the
+// old key. runAdminInit guards the same way before re-initializing.
+func checkJoinPreconditions(configDir string, force bool) error {
+	if force {
+		return nil
+	}
+	tomlPath := filepath.Join(configDir, "spinifex.toml")
+	if !admin.FileExists(tomlPath) {
+		return nil
+	}
+	return fmt.Errorf("this node is already initialized: %s", tomlPath)
+}
+
+// joinDiscardsIdentityMsg spells out what a forced join throws away. Kept out of
+// the error string so that stays short enough to wrap in another.
+const joinDiscardsIdentityMsg = `Joining will discard this node's own cluster identity:
+  - CA certificate and key
+  - master key, and any data sealed under it
+  - viperblock key, and any volumes encrypted under it
+
+That is safe on a freshly installed node — an ISO install initializes a
+single-node cluster at first boot, and nothing has been sealed under these keys
+yet. On a node that has been in service it is unrecoverable data loss.
+
+To proceed: spx admin join --force ...`
+
 func runAdminJoin(cmd *cobra.Command, args []string) {
 	if os.Getuid() != 0 {
 		fmt.Fprintln(os.Stderr, "⚠️  Warning: 'spx admin join' is not running as root.")
@@ -1955,6 +2013,8 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	clusterBind, _ := cmd.Flags().GetString("cluster-bind")
 	services, _ := cmd.Flags().GetStringSlice("services")
 	compactionInterval, _ := cmd.Flags().GetInt("predastore-compaction-interval")
+	force, _ := cmd.Flags().GetBool("force")
+	joinTimeout, _ := cmd.Flags().GetDuration("join-timeout")
 
 	email, _ := cmd.Flags().GetString("email")
 	email = strings.TrimSpace(email)
@@ -2003,6 +2063,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	// Checked before any network call so a node that will not join says so
+	// immediately. Unlike init this exits non-zero: a node that did not join
+	// must not look like success to a provisioning script.
+	if err := checkJoinPreconditions(configDir, force); err != nil {
+		fmt.Fprintf(os.Stderr, "⚠️  %v\n\n%s\n", err, joinDiscardsIdentityMsg)
+		os.Exit(1)
+	}
+
 	// Default cluster-bind to bind IP if not specified
 	if clusterBind == "" {
 		clusterBind = bindIP
@@ -2049,19 +2117,47 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}
 
 	joinURL := fmt.Sprintf("https://%s/formation/join", leaderHost)
-	req, err := http.NewRequest(http.MethodPost, joinURL, bytes.NewBuffer(reqBody))
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error creating join request: %v\n", err)
-		os.Exit(1)
-	}
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Authorization", "Bearer "+joinToken)
+	deadline := time.Now().Add(joinTimeout)
 
-	resp, err := client.Do(req)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error connecting to formation server: %v\n", err)
-		fmt.Fprintf(os.Stderr, "Make sure the leader node has run 'spx admin init' and is accessible at %s\n", leaderHost)
-		os.Exit(1)
+	// A fresh body per attempt: bytes.Buffer is consumed by the first send.
+	var resp *http.Response
+	for attempt := 1; ; attempt++ {
+		req, reqErr := http.NewRequest(http.MethodPost, joinURL, bytes.NewBuffer(reqBody))
+		if reqErr != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error creating join request: %v\n", reqErr)
+			os.Exit(1)
+		}
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Authorization", "Bearer "+joinToken)
+
+		var doErr error
+		resp, doErr = client.Do(req)
+		statusCode := 0
+		if doErr == nil {
+			statusCode = resp.StatusCode
+		}
+		if !joinRetryable(doErr, statusCode) {
+			break
+		}
+		if doErr == nil {
+			resp.Body.Close()
+		}
+
+		if time.Now().After(deadline) {
+			if doErr != nil {
+				fmt.Fprintf(os.Stderr, "❌ Error connecting to formation server: %v\n", doErr)
+			} else {
+				fmt.Fprintf(os.Stderr, "❌ Formation server returned status %d\n", statusCode)
+			}
+			fmt.Fprintf(os.Stderr, "Gave up after %s (%d attempts).\n", joinTimeout, attempt)
+			fmt.Fprintf(os.Stderr, "Make sure the leader node has run 'spx admin init' and is accessible at %s\n", leaderHost)
+			os.Exit(1)
+		}
+		if attempt == 1 {
+			fmt.Printf("⏳ Formation server at %s not ready, retrying every %s (up to %s)...\n",
+				leaderHost, joinRetryInterval, joinTimeout)
+		}
+		time.Sleep(joinRetryInterval)
 	}
 
 	body, err := io.ReadAll(resp.Body)
@@ -2170,7 +2266,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		configDir = DefaultConfigDir()
 	}
 
-	if err := os.MkdirAll(configDir, 0700); err != nil {
+	if err := EnsureConfigDir(configDir); err != nil {
 		fmt.Fprintf(os.Stderr, "Error creating config directory: %v\n", err)
 		os.Exit(1)
 	}
@@ -2431,7 +2527,7 @@ func buildRemoteNodes(allNodes map[string]formation.NodeInfo, localNode, northst
 			continue
 		}
 		// Prefer the peer's advertise IP (off-host dial target); fall back
-		// to BindIP for pre-siv-8 joiners that didn't send AdvertiseIP.
+		// to BindIP for joiners that did not send AdvertiseIP.
 		host := n.AdvertiseIP
 		if host == "" {
 			host = n.BindIP
@@ -2790,7 +2886,11 @@ func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region
 	admin.CreateServiceDirectories(dataDir)
 
 	if os.Getuid() == 0 {
-		admin.SetServiceOwnership()
+		if err := admin.SetServiceOwnership(); err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: service ownership not fully applied: %v\n", err)
+		} else {
+			fmt.Println("✅ Service ownership set")
+		}
 	}
 }
 

cmd/spinifex/cmd/admin_acm.go

@@ -0,0 +1,170 @@
+package cmd
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	handlers_acm "github.com/mulgadc/spinifex/spinifex/handlers/acm"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/spf13/cobra"
+)
+
+var certCreateTenantCACmd = &cobra.Command{
+	Use:   "create-tenant-ca",
+	Short: "Create the ACM tenant private CA for PRIVATE_CA certificate issuance",
+	Long: `Creates the tenant root CA that ACM's PRIVATE_CA validation mode signs leaf
+certificates from. This is a separate, independent root from the platform CA
+(ca.pem/ca.key generated by 'spx admin init') — it is never used for node-to-node
+trust, ipsec, or system VM guests, and installing it into device trust stores
+is a one-time, per-deployment action.
+
+--domain is required and repeatable. The domains given here are baked into the
+CA as x509 name constraints and are frozen at creation: PRIVATE_CA certificate
+requests are only honoured for these domains (and their subdomains), and
+widening the list later requires regenerating the root.
+
+Running this command against an already-existing tenant CA reports what is
+already there rather than silently regenerating — rotating the root invalidates
+every device's trust store at once, so that can only ever be an explicit,
+operator-driven action via --regenerate.`,
+	Run: runCertCreateTenantCA,
+}
+
+// certForceRenewCmd reissues a PRIVATE_CA certificate immediately, over NATS
+// against the running daemon's renewal worker — distinct from certRenewCmd
+// ("spx admin cert renew"), which regenerates the node's own server TLS
+// certificate offline and is unrelated to ACM.
+var certForceRenewCmd = &cobra.Command{
+	Use:   "force-renew",
+	Short: "Reissue an ACM PRIVATE_CA certificate immediately, bypassing its renewal window",
+	Long: `Reissues the given ACM certificate under its existing ARN right now, instead of
+waiting for the renewal worker's proportional window to elapse. Only PRIVATE_CA
+certificates (tenant-CA-issued) can be force-renewed; imported and public
+certificates are refused.
+
+This talks to the running daemon over NATS, taking the same per-certificate
+lease the scheduled renewal worker uses, so it is safe to run against a live
+deployment. It exists primarily to verify the ELBv2 fan-out — that every load
+balancer referencing the certificate re-renders with the new leaf — without
+waiting out the real renewal window.`,
+	Run: runCertForceRenew,
+}
+
+func init() {
+	certCmd.AddCommand(certCreateTenantCACmd)
+	certCreateTenantCACmd.Flags().StringSlice("domain", nil, "Domain the tenant CA is permitted to sign for (repeatable)")
+	certCreateTenantCACmd.Flags().Bool("regenerate", false, "Replace an existing tenant CA with a fresh root (invalidates every device's trust store — requires confirmation)")
+	certCreateTenantCACmd.Flags().Bool("yes", false, "Skip the --regenerate confirmation prompt")
+	if err := certCreateTenantCACmd.MarkFlagRequired("domain"); err != nil {
+		panic(err)
+	}
+
+	certCmd.AddCommand(certForceRenewCmd)
+	certForceRenewCmd.Flags().String("arn", "", "Certificate ARN to force-renew (required)")
+	certForceRenewCmd.Flags().String("account", "", "Account ID that owns the certificate; defaults to the bootstrap account")
+	if err := certForceRenewCmd.MarkFlagRequired("arn"); err != nil {
+		panic(err)
+	}
+}
+
+func runCertForceRenew(cmd *cobra.Command, _ []string) {
+	certArn, _ := cmd.Flags().GetString("arn")
+	accountID, _ := cmd.Flags().GetString("account")
+	if accountID == "" {
+		accountID = admin.DefaultAccountID()
+	}
+
+	_, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	fmt.Printf("Force-renewing %s (account %s)...\n", certArn, accountID)
+	out, err := utils.NATSRequest[handlers_acm.ForceRenewCertificateOutput](
+		ctx, nc, "acm.ForceRenewCertificate",
+		&handlers_acm.ForceRenewCertificateInput{CertificateArn: certArn},
+		30*time.Second, accountID,
+	)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("Reissued %s\n", out.CertificateArn)
+	fmt.Printf("  Serial:    %s\n", out.Serial)
+	fmt.Printf("  NotAfter:  %s\n", out.NotAfter.Format(time.RFC3339))
+}
+
+func runCertCreateTenantCA(cmd *cobra.Command, _ []string) {
+	domains, _ := cmd.Flags().GetStringSlice("domain")
+	regenerate, _ := cmd.Flags().GetBool("regenerate")
+	yes, _ := cmd.Flags().GetBool("yes")
+
+	configDir, _ := cmd.Root().Flags().GetString("config-dir")
+	certPath := admin.TenantCACertPath(configDir)
+	keyPath := admin.TenantCAKeyPath(configDir)
+
+	existed := admin.FileExists(certPath) && admin.FileExists(keyPath)
+
+	if existed && regenerate {
+		fmt.Println("⚠️  About to regenerate the tenant CA. This invalidates trust for every")
+		fmt.Println("   device that has installed the current root — each one will need the new")
+		fmt.Println("   certificate reinstalled before it trusts PRIVATE_CA leaves again.")
+		fmt.Printf("   Current CA:      %s\n", certPath)
+		fmt.Printf("   New domains:     %s\n", strings.Join(domains, ", "))
+		fmt.Println()
+		if !yes {
+			fmt.Print("Type 'yes' to proceed: ")
+			reader := bufio.NewReader(os.Stdin)
+			answer, _ := reader.ReadString('\n')
+			if strings.TrimSpace(answer) != "yes" {
+				fmt.Println("Aborted.")
+				return
+			}
+		}
+		// Regeneration is a deliberate, explicit action only reachable via
+		// --regenerate (plus confirmation) — LoadOrCreateTenantCA itself never
+		// regenerates over an existing root, so the old files are removed first
+		// to put it back on the "create" path.
+		if err := os.Remove(certPath); err != nil {
+			fmt.Fprintf(os.Stderr, "Error removing existing tenant CA cert: %v\n", err)
+			os.Exit(1)
+		}
+		if err := os.Remove(keyPath); err != nil {
+			fmt.Fprintf(os.Stderr, "Error removing existing tenant CA key: %v\n", err)
+			os.Exit(1)
+		}
+	} else if existed {
+		fmt.Println("Tenant CA already exists — reporting current state (use --regenerate to replace it):")
+	}
+
+	ca, err := handlers_acm.LoadOrCreateTenantCA(certPath, keyPath, domains)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	if existed && !regenerate {
+		fmt.Println("✅ No changes made.")
+	} else {
+		fmt.Println("✅ Tenant CA ready.")
+	}
+	fmt.Printf("   Certificate:      %s\n", certPath)
+	fmt.Printf("   Key:              %s (0600)\n", keyPath)
+	fmt.Printf("   Permitted domains: %s\n", strings.Join(ca.PermittedDomains(), ", "))
+	fmt.Println()
+	fmt.Println("Install the certificate into every client's trust store so PRIVATE_CA")
+	fmt.Println("certificates are trusted. The tenant CA is only loaded at daemon startup, so")
+	fmt.Println("restart spinifex-daemon on any node that started before this CA existed:")
+	fmt.Println("  sudo systemctl restart spinifex-daemon")
+}

cmd/spinifex/cmd/admin_jsprobe.go

@@ -0,0 +1,94 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/spf13/cobra"
+
+	"github.com/mulgadc/spinifex/spinifex/daemon"
+)
+
+// canaryKey is written and read back by the JetStream write probe. It lives in
+// the same bucket the control plane depends on, under a reserved prefix no
+// other consumer reads.
+const canaryKey = "_watchdog.canary"
+
+var nodeJSProbeCmd = &cobra.Command{
+	Use:   "js-probe",
+	Short: "Verify JetStream still accepts writes on this node",
+	Long: `Round-trip a canary key through the cluster-state KV bucket and exit non-zero
+if the write or the read-back fails.
+
+This exists because JetStream can latch a write error per message block and keep
+serving reads: nats-server's filestore records the failure on the block and
+returns it on every later write, and nothing clears it short of a restart. The
+HTTP health endpoint reports the server-wide JetStream state, which that latch
+never touches, so only an actual write observes it.`,
+	Run: runNodeJSProbe,
+}
+
+func runNodeJSProbe(cmd *cobra.Command, _ []string) {
+	timeout, _ := cmd.Flags().GetDuration("timeout")
+	if timeout <= 0 {
+		timeout = 10 * time.Second
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), timeout)
+	defer cancel()
+
+	if err := probeJetStreamWrite(ctx); err != nil {
+		fmt.Fprintf(os.Stderr, "js-probe: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Println("js-probe: ok")
+}
+
+// probeJetStreamWrite writes a canary value and reads it back, returning the
+// first error. It deliberately probes the existing cluster-state bucket rather
+// than a bucket of its own: a dedicated bucket is a separate stream that can be
+// placed on a different node, so it could stay writable while the bucket the
+// control plane actually depends on is wedged.
+func probeJetStreamWrite(ctx context.Context) error {
+	_, nc, err := loadConfigAndConnect()
+	if err != nil {
+		return fmt.Errorf("connect: %w", err)
+	}
+	defer nc.Close()
+
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return fmt.Errorf("jetstream: %w", err)
+	}
+
+	kv, err := js.KeyValue(ctx, daemon.ClusterStateBucket)
+	if err != nil {
+		return fmt.Errorf("open bucket %s: %w", daemon.ClusterStateBucket, err)
+	}
+	return canaryRoundTrip(ctx, kv)
+}
+
+// canaryRoundTrip writes a unique value and reads it back, returning the first
+// error. The write is what matters: a latched block returns its stored error on
+// every subsequent write, so this is the only operation that observes it.
+func canaryRoundTrip(ctx context.Context, kv jetstream.KeyValue) error {
+	want := []byte(time.Now().UTC().Format(time.RFC3339Nano))
+	if _, err := kv.Put(ctx, canaryKey, want); err != nil {
+		return fmt.Errorf("write canary: %w", err)
+	}
+
+	entry, err := kv.Get(ctx, canaryKey)
+	if err != nil {
+		return fmt.Errorf("read back canary: %w", err)
+	}
+
+	// A stale value means the write was accepted but not durable, which is the
+	// same operational problem as an outright write failure.
+	if string(entry.Value()) != string(want) {
+		return fmt.Errorf("canary read back stale: wrote %q, read %q", want, entry.Value())
+	}
+	return nil
+}