release: update go.mod for v1.16.0

* feat: render predastore templates with the cluster and S3 planes split

The templates set one bind_addr per host, which put raft and blob wherever
S3 was published. A host now binds the cluster plane on the address peers
dial and the gate carries its own bind_addr for the S3 API, so the installer
can hand S3 the WAN without exposing replication with it.

The single-node template also drops to one node of each role. Extra blob
nodes only hash-partition keys across more badger indexes on the same drive,
and extra meta replicas add fsyncs for a quorum that dies with the machine.

* docs: regenerate the AWS model operation coverage table

* fix: repin predastore to the merged dev commit

.github/actions/e2e-analyze/README.md

@@ -70,7 +70,7 @@ When the report format changes intentionally, regenerate the snapshots:
 go test ./.github/actions/e2e-analyze/ -update
 ```
 
-## Per-failure bundles (Stage 2)
+## Per-failure bundles
 
 After parsing, the action materialises a bundle directory per failure
 under `<log-dir>/analysis/NN-<TestName>/`. The report links to it from

.github/actions/e2e-analyze/analyze.go

@@ -1,6 +1,6 @@
 // Package main implements the e2e-analyze GitHub Action.
 //
-// Stage 1: cluster failures from go-junit-report's junit-*.xml by error signature,
+// Cluster failures from go-junit-report's junit-*.xml by error signature,
 // pick the earliest non-cascade failure per suite as the likely root cause,
 // and render a human-readable report.
 //
@@ -73,8 +73,14 @@ type Failure struct {
 	// `test-<suite>.start` file written by the workflow before the suite
 	// runs) can recompute StartAt without re-parsing the XML.
 	OffsetFromSuiteStart time.Duration
+	// SuiteSpan is the suite's own wall-clock span, set only when the suite
+	// ran its tests in parallel. Overlapping tests make the cumulative
+	// offset above pure fiction — it lands minutes past the real failure —
+	// so consumers widen the journal window to the suite instead of
+	// slicing a plausible-looking window around the wrong minute.
+	SuiteSpan time.Duration
 	// BundlePath is the relative path (from log-dir) to the per-failure
-	// bundle written by Stage 2's WriteBundles. Empty when bundles were
+	// bundle written by WriteBundles. Empty when bundles were
 	// not generated (e.g. unit tests that call Render directly).
 	BundlePath string
 }
@@ -88,6 +94,12 @@ type SuiteReport struct {
 	Root      *Failure    // nil if no failures
 	Cascades  []Failure   // everything else that failed in this suite
 	Buckets   [][]Failure // failures grouped by signature (root's bucket first)
+	// EndedAt is go-junit-report's <testsuite timestamp>, which records when
+	// the XML was produced — i.e. when the suite finished, not when it began.
+	EndedAt time.Time
+	// TopLevelTotal is the summed duration of the parentless tests. Longer than
+	// the suite's wall clock means they overlapped.
+	TopLevelTotal time.Duration
 }
 
 // Report is the full analysis across every junit file.
@@ -119,6 +131,28 @@ var cascadeMarkers = []string{
 	"Expected value not to be nil",
 }
 
+// teardownPatterns match the content a test logs from t.Cleanup or from the
+// harness's on-failure diagnostics — i.e. after it has already failed. They are
+// the LAST file:line lines in the failure body, so extractErrorLine's
+// keep-the-last rule picks them over the assertion unless they are skipped.
+// Anchored deliberately: a fragment loose enough to match an assertion would
+// blank out the one line the report exists to show.
+var teardownPatterns = []*regexp.Regexp{
+	regexp.MustCompile(`^DB instance \S+ is gone$`),
+	regexp.MustCompile(`^db diagnostics \S+:`),
+}
+
+// isTeardownLine reports whether a candidate error line was emitted after the
+// failure it would otherwise be mistaken for.
+func isTeardownLine(content string) bool {
+	for _, re := range teardownPatterns {
+		if re.MatchString(content) {
+			return true
+		}
+	}
+	return false
+}
+
 // noiseReplacers normalise dynamic identifiers so that "ten failures, same
 // bug" bucket together. Pattern ordering matters: longer/more-specific
 // patterns must precede the catch-alls.
@@ -206,11 +240,14 @@ func extractErrorLine(body string) string {
 		if m := goFileLineRe.FindStringSubmatch(l); m != nil {
 			// Keep the LAST occurrence: t.Log / harness traces emit
 			// progress lines from the same file:line pattern before the
-			// failing assertion finally fires.
-			goFileLine = strings.TrimSpace(m[1])
+			// failing assertion finally fires. Cleanup runs after it, though,
+			// so its lines are last of all and have to be skipped.
+			if content := strings.TrimSpace(m[1]); !isTeardownLine(content) {
+				goFileLine = content
+			}
 			continue
 		}
-		if fileHintRe.MatchString(l) {
+		if fileHintRe.MatchString(l) && !isTeardownLine(l) {
 			lastTestLine = l
 		}
 	}
@@ -254,11 +291,19 @@ func extractFileHint(body string) string {
 	if m := errorTraceRe.FindStringSubmatch(body); m != nil {
 		return m[1]
 	}
-	all := fileHintRe.FindAllString(body, -1)
-	if len(all) == 0 {
-		return ""
+	// Scanned line by line rather than across the body so a cleanup line's own
+	// file:line does not become the site the report points a reader at.
+	hint := ""
+	for _, raw := range strings.Split(body, "\n") {
+		l := strings.TrimSpace(raw)
+		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isTeardownLine(strings.TrimSpace(m[1])) {
+			continue
+		}
+		if all := fileHintRe.FindAllString(l, -1); len(all) > 0 {
+			hint = all[len(all)-1]
+		}
 	}
-	return all[len(all)-1]
+	return hint
 }
 
 // signature normalises an error line so different runs of the same root
@@ -313,6 +358,47 @@ func computeLeafSet(cases []junitTC) map[string]bool {
 	return leaf
 }
 
+// computeRolledUpSet returns the testcase names whose failure is already
+// reported by a failing descendant. go-junit-report gives such a parent a
+// testcase of its own, and its body is the parent's t.Log output rather than an
+// assertion — a cleanup line like "DB instance … is gone", which passing tests
+// emit too, reads to extractErrorLine exactly like a failure site.
+func computeRolledUpSet(cases []junitTC) map[string]bool {
+	rolledUp := make(map[string]bool, len(cases))
+	for _, tc := range cases {
+		if tc.Failure == nil && tc.Error == nil {
+			continue
+		}
+		// Mark every ancestor: the failure belongs to the deepest case that
+		// carries it, not to the tests it is nested under.
+		name := tc.Name
+		for {
+			i := strings.LastIndex(name, "/")
+			if i <= 0 {
+				break
+			}
+			name = name[:i]
+			rolledUp[name] = true
+		}
+	}
+	return rolledUp
+}
+
+// topLevelTotal sums the durations of the tests with no parent, which are the
+// units t.Parallel overlaps. Compared against the suite's wall clock it says
+// whether the suite ran its tests concurrently — see ApplySuiteStartFiles.
+// suite.Time cannot answer that: go-junit-report sets it to the sum of every
+// testcase, parents included, so it is never less than any subset of itself.
+func topLevelTotal(cases []junitTC) time.Duration {
+	total := 0.0
+	for _, tc := range cases {
+		if !strings.Contains(tc.Name, "/") {
+			total += tc.Time
+		}
+	}
+	return time.Duration(total * float64(time.Second))
+}
+
 // parseStartTime turns a junit timestamp attr (RFC3339) into a time.Time;
 // zero value if unparseable.
 func parseStartTime(s string) time.Time {
@@ -359,6 +445,9 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 		// summing every testcase double-counts. Pre-compute the leaf set so
 		// only leaves contribute to the suite's running offset.
 		isLeaf := computeLeafSet(suite.Cases)
+		rolledUp := computeRolledUpSet(suite.Cases)
+		rep.EndedAt = start
+		rep.TopLevelTotal += topLevelTotal(suite.Cases)
 		cumul := 0.0
 		for _, tc := range suite.Cases {
 			rep.Total++
@@ -376,11 +465,12 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 				continue
 			}
 
-			// Skip the synthetic parent-test "Failed" stub that go-junit-report
-			// emits for any Go test whose subtests failed. It has an empty body
-			// and would otherwise outrank real failures by XML order.
+			// Skip the parent-test entry go-junit-report emits for any Go test
+			// whose subtests failed: it is a rollup of failures already counted
+			// below it, and it would outrank them by XML order. An empty body is
+			// the same stub in the case where the parent logged nothing.
 			trimmed := strings.TrimSpace(stripANSI(body))
-			if trimmed == "" {
+			if trimmed == "" || rolledUp[tc.Name] {
 				if leaf {
 					cumul += tc.Time
 				}
@@ -462,14 +552,24 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 	return rep, nil
 }
 
-// Render writes the markdown report (Stage 1).
+// Render writes the markdown report.
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
 
-	totalFail := 0
+	totalFail, totalTests := 0, 0
 	for _, s := range r.Suites {
 		totalFail += s.FailCount
+		totalTests += s.Total
+	}
+
+	// A cell that dies before the suites run emits junit with no testcases at
+	// all. Reporting that as clean turns "nothing ran" into "nothing wrong", so
+	// name it and point above the suites.
+	if totalTests == 0 {
+		b.WriteString("⚠️ No tests ran — no testcases in any junit file. " +
+			"The failure is upstream of the suites (provisioning, bootstrap, or reimage); check the job log.\n")
+		return b.String()
 	}
 	if totalFail == 0 {
 		b.WriteString("✅ No failures across any suite.\n")
@@ -484,12 +584,22 @@ func Render(r Report) string {
 		fmt.Fprintf(&b, "### Suite `%s`: %d failed, 1 root cause likely\n\n", s.Label, s.FailCount)
 
 		if s.Root != nil {
-			b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			// XML order is completion order, so "earliest" only means earliest
+			// in time for a suite whose tests ran one after another.
+			if s.Root.SuiteSpan > 0 {
+				b.WriteString("**First non-cascade failure** (the suite ran in parallel, so this is report order, not time order)\n\n")
+			} else {
+				b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			}
 			fmt.Fprintf(&b, "- Test: `%s`\n", s.Root.Name)
-			if !s.Root.StartAt.IsZero() {
+			switch {
+			case s.Root.SuiteSpan > 0:
+				fmt.Fprintf(&b, "- Start: unknown, the suite ran in parallel (duration %.1fs)\n",
+					s.Root.Duration)
+			case !s.Root.StartAt.IsZero():
 				fmt.Fprintf(&b, "- Start: %s (duration %.1fs)\n",
 					s.Root.StartAt.Format("15:04:05"), s.Root.Duration)
-			} else {
+			default:
 				fmt.Fprintf(&b, "- Duration: %.1fs\n", s.Root.Duration)
 			}
 			if s.Root.Error != "" {

.github/actions/e2e-analyze/analyze_test.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 )
 
 // updateGolden lets `go test ./.github/actions/e2e-analyze/ -update` rewrite
@@ -124,6 +125,22 @@ func TestExtractErrorLine_FallsBackToLastTestLine(t *testing.T) {
 	}
 }
 
+// A test's cleanup logs after it has failed, so its lines are the last in the
+// body — and the cleanup line here is one passing tests emit too.
+func TestExtractErrorLine_SkipsCleanupLoggedAfterTheFailure(t *testing.T) {
+	body := `    failure_test.go:42: DB instance rds-e2e-failure-1 reached status available
+    failure_test.go:60: EventuallyErr: condition not met within 4m0s: rds-e2e-failure-1 status=available want=failed
+    rds.go:364: db diagnostics rds-e2e-failure-1: no VM to capture a console from (<nil>)
+    main_test.go:325: DB instance rds-e2e-failure-1 is gone`
+	got := extractErrorLine(body)
+	if !strings.HasPrefix(got, "EventuallyErr:") {
+		t.Errorf("extractErrorLine = %q, want the assertion at failure_test.go:60", got)
+	}
+	if hint := extractFileHint(body); hint != "failure_test.go:60" {
+		t.Errorf("extractFileHint = %q, want failure_test.go:60", hint)
+	}
+}
+
 func TestExtractFileHint_PicksLastMatch(t *testing.T) {
 	body := "ec2helpers.go:50: setup\n    vpc_test.go:227: Eventually: condition not met"
 	got := extractFileHint(body)
@@ -157,6 +174,116 @@ func TestParseFile_SkipsEmptyParentFailure(t *testing.T) {
 	}
 }
 
+func TestParseFile_SkipsParentRolledUpBySubtests(t *testing.T) {
+	// A parent that logs during cleanup gets a non-empty body, and a t.Log like
+	// "DB instance … is gone" — which passing tests emit too — reads exactly
+	// like an assertion site. Its subtest holds the real failure.
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="2">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    main_test.go:325: DB instance rds-e2e-backup-1 is gone
+]]></failure></testcase>
+  <testcase name="TestX/RealFailure" time="0.5"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 {
+		t.Fatalf("FailCount = %d, want 1 (the parent rollup should be skipped)", sr.FailCount)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX/RealFailure" {
+		t.Fatalf("expected the subtest as root, got %+v", sr.Root)
+	}
+}
+
+// A parent that fails on its own after its subtests passed is a real failure,
+// and dropping it would lose the only record of it.
+func TestParseFile_KeepsParentWithNoFailingSubtest(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    foo_test.go:42:
+        Messages: the parent asserted after its subtests
+]]></failure></testcase>
+  <testcase name="TestX/Passing" time="0.5"></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX" {
+		t.Fatalf("expected the parent as root, got %+v", sr.Root)
+	}
+}
+
+// suiteFixture writes a junit file plus its .start sidecar into a temp log dir
+// and returns the parsed, start-file-corrected report for one suite.
+func suiteFixture(t *testing.T, start string, xml []byte) SuiteReport {
+	t.Helper()
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "test-x.start"), []byte(start), 0o644); err != nil {
+		t.Fatalf("write start file: %v", err)
+	}
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	rep := Report{Suites: []SuiteReport{sr}}
+	ApplySuiteStartFiles(&rep, dir)
+	return rep.Suites[0]
+}
+
+// Parallel tests overlap, so summing the ones before a failure puts its start
+// minutes past where it really was — and the journal slice cut from that window
+// misses the failure entirely. The tell is the top-level tests taking longer in
+// total than the suite's own wall clock.
+func TestApplySuiteStartFiles_DropsTheOffsetWhenTheSuiteRanInParallel(t *testing.T) {
+	// timestamp is when go-junit-report wrote the XML, i.e. the suite's end.
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestSlow" time="100.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 2*time.Minute {
+		t.Errorf("SuiteSpan = %s, want 2m0s to mark the suite parallel", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 0, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start %s rather than a summed offset", s.Root.StartAt, want)
+	}
+}
+
+// The sequential case must keep its offsets: they are the only thing that makes
+// a per-failure journal slice narrow enough to read.
+func TestApplySuiteStartFiles_KeepsTheOffsetWhenTheSuiteRanSequentially(t *testing.T) {
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestFirst" time="30.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 0 {
+		t.Errorf("SuiteSpan = %s, want 0 for a sequential suite", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 30, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start plus 30s of preceding tests %s", s.Root.StartAt, want)
+	}
+}
+
 func TestParseFile_CascadeFallbackWhenAllCascades(t *testing.T) {
 	// If every failure is a cascade marker, the earliest one becomes root
 	// so the report isn't blank.
@@ -189,3 +316,25 @@ func TestRender_NoFailuresIsClean(t *testing.T) {
 		t.Errorf("expected zero-failure banner, got:\n%s", out)
 	}
 }
+
+// A cell that fails in provisioning writes junit with no testcases. Rendering
+// the clean banner there reports "nothing ran" as "nothing wrong".
+func TestRender_NoTestsRanIsNotClean(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		suites []SuiteReport
+	}{
+		{"empty junit file", []SuiteReport{{File: "junit-baremetal.xml", Label: "baremetal", Total: 0, FailCount: 0}}},
+		{"no junit files at all", nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			out := Render(Report{Title: "E2E failure analysis", Suites: tc.suites})
+			if strings.Contains(out, "No failures across any suite") {
+				t.Errorf("clean banner rendered when no test ran, got:\n%s", out)
+			}
+			if !strings.Contains(out, "No tests ran") {
+				t.Errorf("expected a no-tests-ran warning, got:\n%s", out)
+			}
+		})
+	}
+}

.github/actions/e2e-analyze/bundle.go

@@ -17,9 +17,9 @@ import (
 )
 
 // bundlePad is the per-failure window padding either side of the
-// [StartAt, StartAt+Duration] testcase span. Five seconds matches the
-// plan doc: long enough to catch the daemon's pre-test setup and
-// post-test cleanup chatter, short enough to keep the slice focused.
+// [StartAt, StartAt+Duration] testcase span. Five seconds catches the
+// daemon's pre-test setup and post-test cleanup chatter while keeping
+// the slice focused.
 const bundlePad = 5 * time.Second
 
 // nameSanitiseRe replaces characters unsafe in directory names with `_`.
@@ -58,21 +58,57 @@ func ApplySuiteStartFiles(rep *Report, logDir string) {
 			fmt.Fprintf(os.Stderr, "::warning::e2e-analyze: parse %s: %v\n", startPath, err)
 			continue
 		}
-		if s.Root != nil {
-			s.Root.StartAt = ts.Add(s.Root.OffsetFromSuiteStart)
-		}
-		for ci := range s.Cascades {
-			s.Cascades[ci].StartAt = ts.Add(s.Cascades[ci].OffsetFromSuiteStart)
-		}
-		for bi := range s.Buckets {
-			for fi := range s.Buckets[bi] {
-				f := &s.Buckets[bi][fi]
-				f.StartAt = ts.Add(f.OffsetFromSuiteStart)
+		// The start file is the first wall clock the analyzer has, so it is also
+		// the first chance to tell a parallel suite from a sequential one — and
+		// a parallel one's per-test offsets are fiction, since overlapping tests
+		// never started at the sum of the ones before them.
+		span := suiteSpanIfParallel(s, ts)
+		for _, f := range collectFailures(s) {
+			f.SuiteSpan = span
+			if span > 0 {
+				f.StartAt = ts
+				continue
 			}
+			f.StartAt = ts.Add(f.OffsetFromSuiteStart)
 		}
 	}
 }
 
+// suiteSpanIfParallel returns the suite's wall-clock span when its top-level
+// tests overlapped, and zero when they ran one after another. Consumers read a
+// non-zero span as "no per-test offset is derivable from this suite".
+func suiteSpanIfParallel(s *SuiteReport, start time.Time) time.Duration {
+	if s.EndedAt.IsZero() || s.TopLevelTotal == 0 {
+		return 0
+	}
+	wall := s.EndedAt.Sub(start)
+	// The 5% margin keeps per-testcase rounding from calling a sequential
+	// suite parallel; real overlap runs to several times the wall clock.
+	if wall <= 0 || s.TopLevelTotal <= time.Duration(float64(wall)*1.05) {
+		return 0
+	}
+	return wall
+}
+
+// collectFailures returns every failure in the report by pointer. Root, cascades
+// and buckets hold independent copies of the same failures, so a field that has
+// to reach the bundle writer must be set on all three.
+func collectFailures(s *SuiteReport) []*Failure {
+	var all []*Failure
+	if s.Root != nil {
+		all = append(all, s.Root)
+	}
+	for ci := range s.Cascades {
+		all = append(all, &s.Cascades[ci])
+	}
+	for bi := range s.Buckets {
+		for fi := range s.Buckets[bi] {
+			all = append(all, &s.Buckets[bi][fi])
+		}
+	}
+	return all
+}
+
 // WriteBundles materialises per-failure bundles under <logDir>/analysis/.
 // Mutates rep in-place so each Failure.BundlePath holds the relative
 // path the renderer will link from the summary.
@@ -143,7 +179,11 @@ func writeOneBundle(logDir, base, suiteLabel string, f *Failure) {
 
 	writeFailureSummary(dir, suiteLabel, f)
 
-	if !f.StartAt.IsZero() {
+	// A parallel suite offers no derivable per-test window, and the narrowing is
+	// the whole point of the slice: cutting one from the wrong minute leaves out
+	// the lines that explain the failure, and widening it to the suite just
+	// copies the artifact's own journal into every bundle.
+	if !f.StartAt.IsZero() && f.SuiteSpan == 0 {
 		start := f.StartAt.Add(-bundlePad)
 		end := f.StartAt.Add(time.Duration(f.Duration * float64(time.Second))).Add(bundlePad)
 		journalSrc := filepath.Join(logDir, "spinifex-journal.log")
@@ -188,7 +228,11 @@ func writeFailureSummary(dir, suiteLabel string, f *Failure) {
 	var b strings.Builder
 	fmt.Fprintf(&b, "Suite:    %s\n", suiteLabel)
 	fmt.Fprintf(&b, "Test:     %s\n", f.Name)
-	if !f.StartAt.IsZero() {
+	switch {
+	case f.SuiteSpan > 0:
+		b.WriteString("Start:    unknown, the suite ran in parallel; " +
+			"no journal window is derivable — read ../../spinifex-journal.log\n")
+	case !f.StartAt.IsZero():
 		fmt.Fprintf(&b, "Start:    %s\n", f.StartAt.Format(time.RFC3339))
 	}
 	fmt.Fprintf(&b, "Duration: %.3fs\n", f.Duration)

.github/actions/e2e-analyze/main.go

@@ -45,7 +45,7 @@ func main() {
 		report.Suites = append(report.Suites, sr)
 	}
 
-	// Stage 2: prefer authoritative per-suite start times written by the
+	// Prefer authoritative per-suite start times written by the
 	// workflow over junit-report's conversion-time `timestamp` attribute,
 	// then materialise per-failure bundles so Render can link them.
 	if logDir != "" {

.github/actions/e2e-analyze/slice.go

@@ -1,6 +1,6 @@
 package main
 
-// Stage 2: per-failure log slicers used by bundle.go to materialise time-window /
+// Per-failure log slicers used by bundle.go to materialise time-window /
 // testcase-window log excerpts next to each named failure.
 
 import (

.github/actions/otel-job-id/action.yml

@@ -0,0 +1,52 @@
+name: Resolve OTEL job id
+description: >-
+  Append ci.job_id to SPINIFEX_OTEL_ATTRS so telemetry emitted by this job can
+  be pivoted straight back to its GitHub job log. Must run before the step that
+  bakes SPINIFEX_OTEL_ATTRS into a VM's telemetry.env, and requires
+  `actions: read` on the calling job.
+
+inputs:
+  job-name:
+    description: >-
+      Rendered name of the calling job, exactly as it appears in the Actions UI.
+      github.job is only the YAML key and no context exposes the numeric id, so
+      the id has to be looked up by name.
+    required: true
+  token:
+    description: Token used for the Actions jobs API lookup.
+    required: false
+    default: ${{ github.token }}
+
+runs:
+  using: composite
+  steps:
+    - name: Resolve CI job id (best-effort)
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.token }}
+        JOB_NAME: ${{ inputs.job-name }}
+        REPO: ${{ github.repository }}
+        RUN_ID: ${{ github.run_id }}
+      run: |
+        # Never fail the caller: on any lookup error or name mismatch ci.job_id
+        # is simply omitted and the run proceeds with the labels it already has.
+        set +e
+
+        if [ -z "${SPINIFEX_OTEL_ATTRS:-}" ]; then
+          echo "SPINIFEX_OTEL_ATTRS is unset or empty — telemetry is disabled, nothing to label"
+          exit 0
+        fi
+
+        # env.JOB_NAME rather than shell interpolation into the jq filter: job
+        # names carry quotes, slashes and non-ASCII separators that would
+        # otherwise break the expression.
+        JOB_ID=$(gh api "repos/${REPO}/actions/runs/${RUN_ID}/jobs" --paginate \
+          --jq '.jobs[] | select(.name == env.JOB_NAME) | .id' 2>/dev/null | head -n1)
+
+        if [ -n "$JOB_ID" ]; then
+          echo "SPINIFEX_OTEL_ATTRS=${SPINIFEX_OTEL_ATTRS},ci.job_id=${JOB_ID}" >> "$GITHUB_ENV"
+          echo "resolved ci.job_id=${JOB_ID} for job '${JOB_NAME}'"
+        else
+          echo "::warning::could not resolve numeric job id for '${JOB_NAME}' — continuing without ci.job_id"
+        fi
+        exit 0

.github/dependabot.yml

@@ -5,7 +5,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -19,7 +19,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -38,7 +38,7 @@ updates:
     directory: /spinifex/services/spinifexui/frontend
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3

.github/scripts/suite-sets.sh

@@ -0,0 +1,62 @@
+# shellcheck shell=bash
+# shellcheck disable=SC2034  # every variable here is consumed by the sourcing workflow
+# Source from a workflow `run:` block:
+#   . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+#
+# Which e2e suites each topology can run. Sourced by every workflow that drives
+# suites (e2e.yml, e2e-suites.yml, e2e-nightly.yml) so the lists cannot drift
+# apart — they previously disagreed three ways.
+#
+# Eligibility is a property of the suite, not of the workflow: a suite belongs
+# to a topology when all of its assertions are reachable there. lb is
+# multi-only for exactly that reason — on a single node every internet-facing
+# subtest skips for want of a peer, so a single-node lb run reports green
+# having exercised a fraction of the suite.
+
+# Suites runnable against a one-node environment.
+#
+# partialblock gates merges because it earned it, not because it is new: on one
+# node, same binary and workload, it fails against the pre-fix engine (1758 of
+# 8192 halves reverted a generation) and passes against the fixed one (0 of
+# 8192). A suite that has never been shown to go red for the reason it claims
+# does not belong here.
+# rds is single-only for the opposite reason to lb: v1 has no multi-AZ
+# behaviour, so a second node adds no assertion. Its DB VMs are capped at four
+# concurrent by the suite's own semaphore, which is what keeps it inside the
+# job's 50-minute budget alongside the suites above.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
+
+# Suites runnable against a multi-node environment.
+E2E_SUITES_MULTI="multinode lb cert"
+
+# grep -xE alternation form, for narrowing a requested set to the eligible one.
+E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
+E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
+
+# e2e-nightly runs a narrower set than the two sets above, deliberately: its
+# ~19 cells each have a 35-minute budget and exist to prove that every
+# install / network / host-OS permutation boots and serves. Suite breadth is
+# the PR gate's job, and it already runs on every push. These are subsets, and
+# e2e_suites_assert_subset below enforces that.
+E2E_SUITES_NIGHTLY_SINGLE="single cert"
+E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
+
+# Fail loudly when a narrowed list names a suite its topology cannot run. This
+# is the guard that stops the lists drifting again: a suite added to a nightly
+# subset without also being declared eligible above stops the job here rather
+# than silently skipping half its assertions in CI.
+#   e2e_suites_assert_subset <label> <subset> <eligible-set>
+e2e_suites_assert_subset() {
+  local label="$1" subset="$2" eligible="$3" suite found eligible_suite
+  for suite in ${subset}; do
+    found=0
+    for eligible_suite in ${eligible}; do
+      if [ "${suite}" = "${eligible_suite}" ]; then found=1; break; fi
+    done
+    if [ "${found}" -ne 1 ]; then
+      echo "::error::${label} names suite '${suite}', which is not eligible for that topology (eligible: ${eligible}). Fix the list in .github/scripts/suite-sets.sh." >&2
+      return 1
+    fi
+  done
+  return 0
+}

.github/workflows/apt-packages.yml

@@ -0,0 +1,48 @@
+name: Apt Packages
+
+# Path-filtered like image-scripts.yml: this resolves setup.sh's runtime package
+# list against each supported suite's real indices, so a renamed, dropped, or
+# ambiguously virtual package fails here rather than on a user's machine.
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  apt_packages:
+    name: Resolve apt list (${{ matrix.image }})
+    runs-on: ubuntu-26.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # The suites the nightly E2E matrix actually installs on. Ubuntu 22.04 is
+        # accepted by detect_os but has never been exercised and does not resolve.
+        image:
+          - ubuntu:24.04
+          - ubuntu:26.04
+          - debian:13
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Resolve runtime package list
+        run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}

.github/workflows/e2e-baremetal.yml

@@ -9,6 +9,11 @@ on:
         required: false
         type: string
         default: ""
+      otel-attrs:
+        description: "OTEL_RESOURCE_ATTRIBUTES value for this run, e.g. ci.run_id=123,ci.cell=20 (caller-specific, so it's an input rather than a workflow-level default)"
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -16,6 +21,10 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  # Same central ES/APM sink as the other four e2e workflows. Bare-metal had
+  # no default here before — this cell's telemetry never reached the sink at
+  # all (not just missing ci.cell), see bm-bootstrap.sh's telemetry.env step.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
 
 jobs:
   e2e_baremetal:
@@ -28,6 +37,9 @@ jobs:
     env:
       BM_ISO_CACHE: /var/cache/spinifex-iso-builder
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-20-${{ github.run_id }}
+      # Caller-supplied (workflow_call input); empty for a manual
+      # workflow_dispatch run, which bm-bootstrap.sh treats as "no attrs".
+      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs }}
     steps:
       - name: Clean Workspace
         run: |

.github/workflows/e2e-ddil.yml

@@ -1,8 +1,7 @@
 name: E2E DDIL
 
-# Phase 1 trigger: workflow_dispatch only. Nightly cron + on-main-merge
-# land in mulga-siv-5.8 once ≥4 scenarios are asserting. See
-# docs/development/improvements/ddil-e2e-test-harness.md §4 / §Dependencies.
+# This workflow is manually triggered while the DDIL scenarios are being
+# validated.
 on:
   workflow_dispatch:
     inputs:
@@ -13,6 +12,9 @@ on:
 
 permissions:
   contents: read
+  # actions: read lets the otel-job-id action look up this job's numeric id via
+  # the Actions API — github.job is only the YAML key, no context exposes it.
+  actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
@@ -28,6 +30,10 @@ jobs:
     timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
+      # Job-level env wins over the workflow-level default. ci.cell=ddil marks
+      # this workflow's telemetry so a DDIL run's partition-induced errors are
+      # never mistaken for a healthy multi-node run's in the sink.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=ddil
     steps:
       - name: Clean Workspace
         run: sudo rm -rf "$GITHUB_WORKSPACE"/*
@@ -72,6 +78,13 @@ jobs:
           echo "${BIN_DIR}" >> "${GITHUB_PATH}"
           "${BIN_DIR}/gotestsum" --version
 
+      # Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
+      # the VMs, so a telemetry doc points back at this job's log directly.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: DDIL Real Multi-Node
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |

.github/workflows/e2e-nightly.yml

@@ -11,12 +11,16 @@ on:
 
 permissions:
   contents: read
+  # actions: read lets the "Resolve CI job id" step below look up this job's
+  # numeric id via the Actions API — github.job is only the YAML key, no
+  # context exposes the id itself.
+  actions: read
 
 # Scheduled runs are loaded from the default branch (main), so the schedule
 # trigger must stay in sync on main for cron to fire.
 env:
   GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
-  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
+  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
   # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
   # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
@@ -32,20 +36,25 @@ jobs:
       matrix:
         include:
           - { cell: 1,  install: binary, topo: single,     net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
+          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
+          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
           - { cell: 8,  install: binary, topo: real-multi, net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
+          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
+          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
           - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
           - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
           - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }
 
     runs-on: [ self-hosted, "${{ matrix.runner }}" ]
-    timeout-minutes: 35
+    timeout-minutes: ${{ matrix.extra == 'tofu-examples' && 45 || 35 }}
     env:
       TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
+      # Job-level env wins over the workflow-level default above, so telemetry
+      # from this cell carries ci.cell/ci.matrix. Nine cells run under one
+      # run id and every cell names its hosts node1/node2/node3, so without
+      # this every cell's telemetry collapses into one unattributable bucket.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}
 
     steps:
       # blank or scheduled run = full matrix.
@@ -92,6 +101,16 @@ jobs:
         with:
           path: spinifex
 
+      # Nine cells share one run id and every cell names its hosts
+      # node1/node2/node3, so ci.cell above separates them and ci.job_id here
+      # links each cell's telemetry to the job log that produced it. Runs
+      # before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
+      - name: Resolve CI job id
+        if: steps.gate.outputs.selected == 'true'
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
+
       - name: Build Install Artifacts
         if: steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
@@ -109,10 +128,13 @@ jobs:
         run: |
           export TF_VAR_git_branch="${GIT_BRANCH}"
 
+          # Match the family, not a pinned version: an exact match silently stops
+          # overriding the image when the matrix bumps release, dropping the cell
+          # back onto the debian gold.
           TOFU_VAR_ARGS=""
-          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
-            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
-          fi
+          case "$MATRIX_HOST" in
+            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
+          esac
 
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
@@ -147,13 +169,20 @@ jobs:
         if: steps.gate.outputs.selected == 'true' && matrix.topo == 'single' && matrix.extra == ''
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          set -euo pipefail
+          # Nightly runs a documented subset of the single-node set; the assert
+          # fails the cell if that subset ever names a suite this topology
+          # cannot fully exercise.
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          e2e_suites_assert_subset "e2e-nightly single" "${E2E_SUITES_NIGHTLY_SINGLE}" "${E2E_SUITES_SINGLE}"
+
           INVENTORY=$(tofu output -json inventory)
           WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
           SSH_KEY="${SSH_KEY/#\~/$HOME}"
           ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
             "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
-             for SUITE in single cert lb; do \
+             for SUITE in ${E2E_SUITES_NIGHTLY_SINGLE}; do \
                echo \"::group::suite \$SUITE\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
@@ -190,6 +219,11 @@ jobs:
         if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          set -euo pipefail
+          # See the single-node step: same subset guard, multi-node set.
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          e2e_suites_assert_subset "e2e-nightly multi" "${E2E_SUITES_NIGHTLY_MULTI}" "${E2E_SUITES_MULTI}"
+
           INVENTORY=$(tofu output -json inventory)
           NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
@@ -204,7 +238,7 @@ jobs:
 
           ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
             "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
-             for SUITE in multinode cert lb; do \
+             for SUITE in ${E2E_SUITES_NIGHTLY_MULTI}; do \
                echo \"::group::suite \$SUITE\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
@@ -217,6 +251,18 @@ jobs:
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
              done; exit \$FAIL"
 
+      - name: Import RDS PostgreSQL AMI
+        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "spx admin images import --name spinifex-rds-postgres"
+
       - name: Run Tofu Workbooks
         if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
         working-directory: mulga/scripts/tofu-cluster
@@ -339,10 +385,12 @@ jobs:
         env:
           MATRIX_HOST: ${{ matrix.host }}
         run: |
+          # Must mirror the Provision step's override, or destroy runs against a
+          # different image path than apply did.
           TOFU_VAR_ARGS=""
-          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
-            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
-          fi
+          case "$MATRIX_HOST" in
+            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
+          esac
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
@@ -373,4 +421,8 @@ jobs:
     uses: ./.github/workflows/e2e-baremetal.yml
     with:
       ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+      # A reusable-workflow call gets none of this workflow's context (env:
+      # above, matrix.cell) for free — cell-20 needs its own attrs string.
+      # There's no matrix leg here, just the one fixed baremetal cell.
+      otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
     secrets: inherit

.github/workflows/e2e-suites.yml

@@ -1,13 +1,14 @@
-# E2E (per-suite blocks) — EXPERIMENTAL, workflow_dispatch only.
+# E2E per-suite blocks. Each suite appears as its own node in the Actions
+# graph:
 #
-# Restructures the looped e2e.yml into env-grouped job blocks so each suite
-# surfaces as its own node in the Actions graph (mulga-siv-283):
-#
-#   single_provision ─┬─ single_suite (matrix: single, cert, lb, eks, iam)
+#   single_provision ─┬─ single_suite (matrix: the single-node-eligible suites)
 #                     └─ single_teardown (if: always)
-#   multi_provision  ─┬─ multi_suite  (matrix: multinode)
+#   multi_provision  ─┬─ multi_suite  (matrix: the multi-node-eligible suites)
 #                     └─ multi_teardown (if: always)
 #
+# Both matrices come from .github/scripts/suite-sets.sh, the one definition of
+# which suites a topology can run, shared with e2e.yml and e2e-nightly.yml.
+#
 # Constraint: tofu state is runner-filesystem-local, so *_provision and
 # *_teardown must execute on the same self-hosted runner (one runner per label
 # today). Teardown reuses the persisted mulga/ tofu-cluster workspace and does
@@ -25,9 +26,13 @@ on:
   workflow_dispatch:
     inputs:
       single_suites:
-        description: "Single-node suites to run as matrix legs"
+        description: "Single-node suites to run as matrix legs (empty = every single-node-eligible suite, per .github/scripts/suite-sets.sh)"
         required: false
-        default: "single cert lb eks iam"
+        default: ""
+      multi_suites:
+        description: "Multi-node suites to run as matrix legs (empty = every multi-node-eligible suite, per .github/scripts/suite-sets.sh)"
+        required: false
+        default: ""
       run_multinode:
         description: "Also run the multinode env block"
         type: boolean
@@ -40,6 +45,10 @@ on:
 
 permissions:
   contents: read
+  # actions: read lets the otel-job-id action look up a provision job's numeric
+  # id via the Actions API — github.job is only the YAML key, no context
+  # exposes it.
+  actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
@@ -64,6 +73,13 @@ jobs:
       matrix: ${{ steps.suites.outputs.matrix }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
+      # This job (not single_suite) is where "Bootstrap via Binary Install"
+      # bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env, before the
+      # suite matrix even exists — single_suite's legs share this one
+      # already-booted VM, so daemon telemetry can't be scoped any finer
+      # than "single env" per suite leg. ci.cell=single distinguishes it
+      # from multi_provision's VMs within the same run id.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -115,12 +131,22 @@ jobs:
       - name: Re-enable Go toolchain auto-download
         run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
 
+      # Default and allow-list both come from the shared definition, so this
+      # workflow cannot drift from e2e.yml about what a single node can run.
       - name: Compute suite matrix
         id: suites
+        env:
+          REQUESTED: ${{ inputs.single_suites }}
         run: |
-          LIST="${{ inputs.single_suites || 'single cert lb eks iam' }}"
+          set -euo pipefail
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          LIST="${REQUESTED:-${E2E_SUITES_SINGLE}}"
+          e2e_suites_assert_subset "single_suites" "${LIST}" "${E2E_SUITES_SINGLE}"
           echo "list=${LIST}" >> "$GITHUB_OUTPUT"
-          JSON=$(printf '%s\n' $LIST | jq -R . | jq -cs .)
+          # Unquoted on purpose: the space-separated list must word-split into
+          # one JSON element per suite.
+          # shellcheck disable=SC2086
+          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
           echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
 
       - name: Reclaim docker build cache
@@ -130,6 +156,14 @@ jobs:
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 
+      # ci.job_id names the provision job, not the suite job that later drives
+      # the env: the suite legs share this env and its baked telemetry.env, so
+      # the provisioner is the only job the daemon telemetry can belong to.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: Single env · provision
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
@@ -497,8 +531,13 @@ jobs:
       node1_ip: ${{ steps.cluster.outputs.node1_ip }}
       ssh_key: ${{ steps.cluster.outputs.ssh_key }}
       nodes_csv: ${{ steps.cluster.outputs.nodes_csv }}
+      suites: ${{ steps.suites.outputs.list }}
+      matrix: ${{ steps.suites.outputs.matrix }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
+      # See single_provision above — daemon telemetry is baked in here, at
+      # provision time, before the multi_suite matrix exists.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=multi
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -554,6 +593,12 @@ jobs:
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 
+      # See the matching note on single_provision.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: Multi env · provision
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
@@ -601,14 +646,36 @@ jobs:
           echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
           echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
 
-      - name: Build multinode binary on runner
+      # Mirrors single_provision's step: the multi leg previously ran multinode
+      # alone, silently dropping the lb and cert coverage that only a
+      # multi-node env can complete.
+      - name: Compute suite matrix
+        id: suites
+        env:
+          REQUESTED: ${{ inputs.multi_suites }}
+        run: |
+          set -euo pipefail
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          LIST="${REQUESTED:-${E2E_SUITES_MULTI}}"
+          e2e_suites_assert_subset "multi_suites" "${LIST}" "${E2E_SUITES_MULTI}"
+          echo "list=${LIST}" >> "$GITHUB_OUTPUT"
+          # Unquoted on purpose: the space-separated list must word-split into
+          # one JSON element per suite.
+          # shellcheck disable=SC2086
+          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
+          echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
+
+      - name: Build multinode binaries on runner
         working-directory: spinifex/tests/e2e
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          banner "build multinode, teardown"
+          # teardown always builds: the run-scoped sweep binary runs in the
+          # teardown job, it is never a selectable suite leg.
+          SUITES_BUILD="${{ steps.suites.outputs.list }} teardown"
+          banner "build ${SUITES_BUILD// /, }"
           group_open "go test -c"
-          make e2e-build SUITES_BUILD="multinode teardown"
+          make e2e-build SUITES_BUILD="${SUITES_BUILD}"
           group_close
           ok "$(basename _bin/multinode.test) $(stat -c '%s' _bin/multinode.test | numfmt --to=iec)"
 
@@ -652,7 +719,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: [multinode]
+        suite: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
     env:
       NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
       SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}

.github/workflows/e2e.yml

@@ -15,7 +15,7 @@ on:
   workflow_dispatch:
     inputs:
       suites:
-        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam')"
+        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam e2e-rds')"
         required: false
         default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
       test_run:
@@ -26,10 +26,13 @@ on:
 permissions:
   contents: read
   statuses: write
+  # actions: read lets the otel-job-id action look up a job's numeric id via
+  # the Actions API — github.job is only the YAML key, no context exposes it.
+  actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
-  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam' }}
+  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam e2e-partialblock' }}
   TEST_RUN: ${{ inputs.test_run || '' }}
   # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
   # empty string disables telemetry entirely.
@@ -37,57 +40,28 @@ env:
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
-  # In-process integration front gate. Provisions nothing -- no tofu, no
-  # docker, no Spinifex daemons, no fixed ports -- so it runs directly on the
-  # runner host rather than inside a provisioned VM, and completes in seconds.
-  #
-  # It runs on ci-single rather than a GitHub-hosted runner so it shares the
-  # toolchain and sibling-repo build of the live suites, and so the run stays
-  # entirely on the LAN that hosts the Elastic sink. Holding a ci-single slot
-  # for seconds is negligible against the 50-minute e2e_single that follows.
-  #
-  # e2e_single and e2e_multi both require this job to succeed, so a red
-  # integration run is caught before either provisions a tofu env or pays the
-  # Spinifex install -- which is the expensive half, not the tofu apply.
-  integration:
-    name: Integration (front gate)
-    runs-on: [self-hosted, ci-single]
-    timeout-minutes: 10
-    steps:
-      # This runner is shared with e2e_single, which leaves root-owned tofu and
-      # mount artifacts behind; a plain checkout would fail on them. Unmount
-      # anything still bound under the workspace before wiping it.
-      - name: Clean Workspace
-        run: |
-          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
-            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
-          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
-
-      - uses: actions/checkout@v7
-      - uses: actions/setup-go@v7
-        with:
-          go-version-file: go.mod
-          cache: false
-      - name: Run in-process integration tests
-        run: make test-integration
-
   e2e_single:
     name: Single-Node E2E
-    needs: [integration]
-    # The integration front gate must be green: a red integration run stops
-    # here, before this job provisions a tofu env or pays the Spinifex install.
-    #
     # The dispatch check gates on exactly the suites this job's Run step
-    # allow-lists (single|iam|cert|eks|ecs|storagegrowth). Gating on a suite the
-    # job won't run — lb, which only executes in the multinode job — would
+    # allow-lists, i.e. E2E_SUITES_SINGLE in .github/scripts/suite-sets.sh —
+    # an `if:` expression cannot source that file, so this one list is
+    # hand-mirrored and must be updated with it. Gating on a suite the job
+    # won't run — lb, which only executes in the multinode job — would
     # provision a tofu VM and pay the Spinifex install only to run nothing and
-    # tear down. Push triggers keep running via the null-check on inputs.suites.
-    if: needs.integration.result == 'success' && (github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth'))
+    # tear down. Push triggers keep running via the null-check on
+    # inputs.suites.
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
     runs-on: [self-hosted, ci-single]
     timeout-minutes: 50
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
+      # Job-level env wins over the workflow-level default. This job and
+      # e2e_multi share one run id and both name their hosts node1/node2/node3,
+      # so without ci.cell their telemetry collapses into one bucket. This job
+      # is where "Bootstrap via Binary Install" bakes SPINIFEX_OTEL_ATTRS into
+      # the VM's telemetry.env, so the label has to be set here.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -152,13 +126,21 @@ jobs:
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 
+      # Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
+      # the VM, so a telemetry doc points back at this job's log directly.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: Single-Node E2E
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
           ./build-install-artifacts.sh "${GIT_BRANCH}" \
             "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
             "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar"
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
 
       - name: Provision VM
         working-directory: mulga/scripts/tofu-cluster
@@ -268,6 +250,54 @@ jobs:
           EOF
           ok "kubectl installed on VM"
 
+      # CreateDBInstance boots the DB VM from the rds-postgres AMI, so that image
+      # must be in the VM's image store before the suite runs — same split as the
+      # eks-node pair above: build on the runner (needs source + network), import
+      # on the VM (needs predastore/NATS). The suite's psql client needs no step
+      # here: it runs inside a client VM that installs postgresql-client from its
+      # own user-data, because a private-only endpoint is unreachable from the
+      # runner and from the cluster host alike.
+      - name: Build rds-postgres AMI (libguestfs, runner)
+        if: contains(env.SUITES, 'e2e-rds')
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          banner "build rds-postgres AMI (libguestfs — no host devices)"
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
+          export LIBGUESTFS_BACKEND=direct
+          sudo chmod 0644 /boot/vmlinuz-* || true
+          group_open "make build-rds-postgres-image"
+          make build-rds-postgres-image
+          group_close
+          ok "raw: $(du -h /tmp/rds-postgres-image-build/rds-postgres-alpine.raw | cut -f1)"
+
+      - name: Import rds-postgres AMI on VM
+        if: contains(env.SUITES, 'e2e-rds')
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+            spinifex/scripts/images/rds-postgres/manifest.conf)"
+          # SYSTEM_TAG carries three tags, not one: managed-by marks it a system
+          # image and engine/engine-version are what an EngineVersion request
+          # resolves against. --tag is repeatable, so expand rather than pass the
+          # whole string as a single value.
+          TAG_ARGS=""
+          for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          banner "import rds-postgres AMI to VM image store"
+          scp "${SSH_OPTS[@]}" \
+            /tmp/rds-postgres-image-build/rds-postgres-alpine.raw \
+            "$VM":/tmp/rds-postgres-alpine.raw
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "spx admin images import --file /tmp/rds-postgres-alpine.raw \
+               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+               --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
+          ok "rds-postgres AMI imported"
+
       - name: Build E2E test binaries on runner
         working-directory: spinifex/tests/e2e
         run: |
@@ -317,8 +347,11 @@ jobs:
           set +e
           # Allow-list, not a deny-list: a suite runs here only if named. Deny-lists
           # silently fan a newly added suite out to every job, which is how lb ended up
-          # running twice per e2e run.
-          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE 'single|iam|cert|eks|ecs|storagegrowth' | tr '\n' ' ')
+          # running twice per e2e run. The allow-list itself is shared with the other
+          # e2e workflows so the three cannot disagree about what a topology can run;
+          # suite-sets.sh also carries the per-suite rationale.
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_SINGLE_RE}" | tr '\n' ' ')
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no in-VM suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -495,19 +528,19 @@ jobs:
 
   e2e_multi:
     name: Multi-Node E2E
-    needs: [integration]
-    # Gated on the integration front gate for the same reason as e2e_single:
-    # the 3-node env is never stood up if the control plane is already broken.
     # The dispatch check gates on exactly the suites this job's Run step
     # allow-lists (multinode|lb|cert) so lb — which only runs here — still
     # provisions the 3-node env when requested on its own, and the env stays
     # unprovisioned when none of them are asked for.
-    if: needs.integration.result == 'success' && (github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-cert'))
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-cert')
     runs-on: [self-hosted, ci-multi]
     timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
+      # See the matching note on e2e_single: ci.cell is what separates this
+      # job's 3-node telemetry from the single-node job's under one run id.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=multi
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -572,13 +605,20 @@ jobs:
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 
+      # See the matching note on e2e_single.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: Multi-Node E2E
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
           ./build-install-artifacts.sh "${GIT_BRANCH}" \
             "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
             "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar"
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
 
       - name: Provision VMs
         working-directory: mulga/scripts/tofu-cluster
@@ -714,7 +754,8 @@ jobs:
         run: |
           set +e
           # Allow-list, not a deny-list — see the single-node job for why.
-          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE 'multinode|lb|cert' | tr '\n' ' ')
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_MULTI_RE}" | tr '\n' ' ')
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -930,7 +971,7 @@ jobs:
   e2e_status:
     name: Report E2E Status
     if: always()
-    needs: [integration, e2e_single, e2e_multi]
+    needs: [e2e_single, e2e_multi]
     runs-on: ubuntu-26.04
     timeout-minutes: 5
     steps:
@@ -939,22 +980,15 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           REPORT_SHA: ${{ github.sha }}
         run: |
-          # integration must have actually succeeded -- unlike e2e_single/multi,
-          # "skipped" is never an acceptable result for it (it has no `if:` of
-          # its own), so a red or skipped integration run fails the gate here
-          # even though e2e_single/e2e_multi themselves report "skipped" (their
-          # `if:` never let them start).
-          integration_ok=true
-          case "${{ needs.integration.result }}" in success) ;; *) integration_ok=false ;; esac
           single_ok=true; multi_ok=true
           case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
           case "${{ needs.e2e_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
-          if $integration_ok && $single_ok && $multi_ok; then
+          if $single_ok && $multi_ok; then
             STATE="success"
             DESC="E2E suites passed"
           else
             STATE="failure"
-            DESC="E2E failed: integration=${{ needs.integration.result }} single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
+            DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
           fi
           gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \
             -f state="$STATE" \
@@ -971,7 +1005,7 @@ jobs:
   # status-report check.
   e2e_es_summary:
     name: ES CI Run Summary
-    needs: [integration, e2e_single, e2e_multi]
+    needs: [e2e_single, e2e_multi]
     if: always()
     runs-on: [self-hosted, ci-single]
     timeout-minutes: 5
@@ -1011,7 +1045,6 @@ jobs:
           BRANCH: ${{ github.head_ref || github.ref_name }}
           SHA: ${{ github.sha }}
           SERVER_URL: ${{ github.server_url }}
-          INTEGRATION_RESULT: ${{ needs.integration.result }}
           SINGLE_RESULT: ${{ needs.e2e_single.result }}
           MULTI_RESULT: ${{ needs.e2e_multi.result }}
           SINGLE_DIR: ${{ github.workspace }}/es-summary-single
@@ -1040,20 +1073,15 @@ jobs:
           FAILED_JSON=$(printf '%s' "$FAILED_SUITES" | jq -R -s 'split("\n") | map(select(length > 0))')
           FAILED_COUNT=$(printf '%s' "$FAILED_JSON" | jq 'length')
 
-          # Mirror e2e_status's integration_ok/single_ok/multi_ok logic exactly
-          # so the PR status and this ES doc can never disagree on overall
-          # pass/fail. Like e2e_status, "skipped" is only acceptable for
-          # single/multi (their `if:` may legitimately not select them); a
-          # skipped or failed integration always fails the gate.
-          integration_ok=true
-          case "$INTEGRATION_RESULT" in success) ;; *) integration_ok=false ;; esac
+          # Mirror e2e_status's single_ok/multi_ok logic exactly so the PR
+          # status and this ES doc can never disagree on overall pass/fail.
           single_ok=true
           case "$SINGLE_RESULT" in success | skipped) ;; *) single_ok=false ;; esac
           multi_ok=true
           case "$MULTI_RESULT" in success | skipped) ;; *) multi_ok=false ;; esac
 
           STATE=success
-          if [ "$integration_ok" != true ] || [ "$single_ok" != true ] || [ "$multi_ok" != true ] || [ "$FAILED_COUNT" -gt 0 ]; then
+          if [ "$single_ok" != true ] || [ "$multi_ok" != true ] || [ "$FAILED_COUNT" -gt 0 ]; then
             STATE=failure
           fi
 
@@ -1070,7 +1098,6 @@ jobs:
             --arg sha "${SHA}" \
             --arg run_attempt "${RUN_ATTEMPT}" \
             --arg state "${STATE}" \
-            --arg integration_result "${INTEGRATION_RESULT}" \
             --arg single_result "${SINGLE_RESULT}" \
             --arg multi_result "${MULTI_RESULT}" \
             --argjson duration "${DURATION}" \
@@ -1081,7 +1108,6 @@ jobs:
               labels: { ci_run_id: $run_id },
               ci: { workflow: $workflow, branch: $branch, sha: $sha, run_attempt: $run_attempt },
               state: $state,
-              integration_result: $integration_result,
               single_result: $single_result,
               multi_result: $multi_result,
               failed_suites: $failed_suites,

.github/workflows/gpu-e2e.yml

@@ -1,6 +1,6 @@
 # GPU E2E — drives wattle's persistent, shared-host Spinifex stack over SSH to
 # exercise GPU passthrough (raw EC2, ECS task GPU exposure, EKS pod GPU
-# exposure). See docs/development/feature/wattle-deploy-and-ci.md (Part B).
+# exposure).
 #
 # ---------------------------------------------------------------------------
 # ONE-TIME HOST-SIDE SETUP — NOT part of this repo, a human must do this:
@@ -32,7 +32,6 @@
 #      to skip cleanly (job succeeds, all work steps skipped) instead of
 #      queuing behind manual work indefinitely.
 #
-#   Runbook: docs/development/feature/wattle-deploy-and-ci.md (Part B).
 # ---------------------------------------------------------------------------
 
 name: GPU E2E
@@ -44,32 +43,6 @@ on:
         description: "Optional -test.run regex scoped to tests/e2e/gpu (empty = run all GPU tests)"
         required: false
         default: ""
-  push:
-    branches:
-      - main
-      - dev
-    paths: &gpu_paths
-      - "spinifex/gpu/**"
-      - "spinifex/instancetypes/**"
-      - "spinifex/daemon/gpu_adapter.go"
-      - "spinifex/daemon/gpu_pool_reconciler.go"
-      - "spinifex/handlers/ecs/gpu_report.go"
-      - "spinifex/handlers/eks/nodegroup.go"
-      - "spinifex/handlers/eks/nodegroup_userdata.go"
-      - "spinifex/vm/vm.go"
-      - "spinifex/vm/lifecycle.go"
-      - "cmd/ecs-agent/gpu_discovery.go"
-      - "cmd/ecs-agent/gpu_ledger.go"
-      - "cmd/spinifex/cmd/admin_gpu.go"
-      - "tests/e2e/gpu/**"
-      - "scripts/images/eks-node-gpu/**"
-      - "scripts/images/ecs-agent-gpu/**"
-      - ".github/workflows/gpu-e2e.yml"
-  pull_request:
-    branches:
-      - main
-      - dev
-    paths: *gpu_paths
 
 permissions:
   contents: read
@@ -88,7 +61,11 @@ env:
   # the GPU suite's own daemon-side spans are emitted by wattle's already-
   # configured obs-agent, not by this job.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
-  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
+  # ci.cell=gpu keeps this workflow's telemetry separable from the tofu-env
+  # workflows'. Nothing in this job bakes these attrs into a daemon (see the
+  # note above), so the label only takes effect once wattle's obs-agent is
+  # given them; there is deliberately no ci.job_id lookup for the same reason.
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=gpu
   WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
   WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
   TEST_RUN: ${{ inputs.test_run || '' }}
@@ -258,9 +235,8 @@ jobs:
       # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
       # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
       # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
-      # the advertised gateway (mulga-7suds), so the override is required
-      # here (proper fix tracked separately: tests/e2e/harness/env.go should
-      # prefer spinifex.toml's advertised bridge).
+      # the advertised gateway, so the override is required here. The harness
+      # should prefer spinifex.toml's advertised bridge.
       - name: Run GPU e2e suite on wattle
         id: e2e
         if: steps.gate.outputs.paused != 'true'

.github/workflows/image-scripts.yml

@@ -0,0 +1,39 @@
+name: Image Scripts
+
+# Separate from spinifex.yml on purpose: scripts/images/ and images/ are shell
+# and mkosi assets baked into system images, not Go, so they get their own
+# path-filtered gate instead of running on every Go contributor's push (see
+# ui.yml for the same pattern applied to the frontend).
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  image_scripts:
+    name: Image Scripts
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Run image script tests + shellcheck
+        run: make test-images

.github/workflows/publish-system-ami.yml

@@ -1,17 +1,19 @@
-name: Publish EKS + ECS node AMIs
+name: Publish System AMIs
 
 # Builds the system AMIs (libguestfs, on the runner) and publishes each
 # compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Four images ship:
+# iso.mulgadc.com/system-ami/. Five images ship:
 #   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
 #   spinifex-ecs-node       Alpine ECS container instance
 #   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
 #   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
+#   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
 # Operators then install them on any cluster with:
 #   spx admin images import --name spinifex-eks-node
 #   spx admin images import --name spinifex-ecs-node
 #   spx admin images import --name spinifex-eks-node-gpu
 #   spx admin images import --name spinifex-ecs-node-gpu
+#   spx admin images import --name spinifex-rds-postgres
 #
 # Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
 # action, not something every push should trigger. The `variant` input scopes
@@ -29,6 +31,7 @@ on:
           - all
           - cpu
           - gpu
+          - rds
       dry_run:
         description: "Build + convert only; skip the R2 upload"
         type: boolean
@@ -56,10 +59,12 @@ jobs:
           set -euo pipefail
           cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
           gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
+          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"}'
           case "${VARIANT}" in
             cpu) items="${cpu}" ;;
             gpu) items="${gpu}" ;;
-            *)   items="${cpu},${gpu}" ;;
+            rds) items="${rds}" ;;
+            *)   items="${cpu},${gpu},${rds}" ;;
           esac
           echo "images={\"include\":[${items}]}" >> "$GITHUB_OUTPUT"
 

.github/workflows/segscan-oracle.yml

@@ -0,0 +1,81 @@
+# Segscan storage oracle — push-triggered (main/dev) + manual dispatch only,
+# deliberately NOT wired into spinifex.yml's pull_request path.
+#
+# tests/integration/segscan_oracle_test.go needs the mulga umbrella repo's
+# scripts/segscan checked out alongside spinifex (it is a separate Go module,
+# `package main`, and cannot be imported — see spinifex/testutil/segscanoracle).
+# spinifex.yml's jobs check out only spinifex/viperblock/predastore and run on
+# every PR; adding a second-repo checkout there would make every spinifex PR
+# depend on mulga main being healthy and pay that checkout cost on every run,
+# for one storage-accounting oracle test. Mirrors e2e.yml's push-only trigger
+# This never gates a PR before merge, matching how e2e.yml's own
+# "front gate" job works.
+name: Segscan Storage Oracle
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  segscan_oracle:
+    name: Segscan Oracle
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - name: Checkout Mulga (scripts/segscan only)
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/segscan
+          path: mulga
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore
+
+      - name: Run Segscan Oracle Test
+        env:
+          SEGSCAN_DIR: ${{ github.workspace }}/mulga/scripts/segscan
+        run: make -C spinifex test-segscan-oracle

.github/workflows/spinifex.yml

@@ -68,15 +68,6 @@ jobs:
       - name: Manifest check + drift lint
         run: make -C spinifex manifest-check manifest-lint
 
-      # In-process integration tier (gateway.SetupRoutes over embedded NATS
-      # JetStream). Runs here, on every PR, rather than only relying on
-      # e2e.yml's "Integration (front gate)" job: that job is push-triggered
-      # against main/dev on a self-hosted runner, so it never blocks a PR
-      # before merge. This step provisions nothing and completes in seconds,
-      # so it belongs as an ordinary required PR check, not a separate job.
-      - name: Run Integration Tests
-        run: make -C spinifex test-integration
-
       - name: Run Tests with Coverage
         run: make -C spinifex test-cover
 
@@ -86,6 +77,55 @@ jobs:
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
 
+  # In-process integration tier (gateway.SetupRoutes over embedded NATS
+  # JetStream). Its own job so it runs in parallel with the unit tiers rather
+  # than adding to their wall time, and it still blocks a PR — unlike e2e.yml's
+  # "Integration (front gate)" job, which is push-triggered against main/dev on
+  # a self-hosted runner and so never gates a PR before merge.
+  integration_tests:
+    name: Integration Tests
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore
+
+      - name: Run Integration Tests
+        run: make -C spinifex test-integration
+
   race_detection:
     name: Race Detection
     runs-on: ubuntu-26.04

.github/workflows/ui.yml

@@ -57,3 +57,6 @@ jobs:
 
       - name: Build Frontend
         run: pnpm build
+
+      - name: Check for generated route tree drift
+        run: git diff --exit-code -- src/routeTree.gen.ts

.gitignore

@@ -37,3 +37,6 @@ Thumbs.db
 
 # Test reports
 *-report.txt
+
+# Per-worktree golangci-lint cache
+/.cache/

.golangci.yml

@@ -1,5 +1,8 @@
 version: "2"
 
+issues:
+  max-same-issues: 10
+
 formatters:
   enable:
     - gofmt
@@ -14,7 +17,6 @@ linters:
     - bidichk # checks for dangerous unicode bidi character sequences
     - bodyclose # checks whether HTTP response body is closed successfully
     - canonicalheader # checks whether net/http.Header uses canonical header keys
-    - decorder # checks declaration order of types, consts, vars, and funcs
     - depguard # blocks imports of packages with a preferred replacement
     - durationcheck # checks for two durations multiplied together
     - embeddedstructfieldcheck # checks embedded sync.Mutex/RWMutex in structs
@@ -24,17 +26,20 @@ linters:
     - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
+    - forbidigo # blocks the legacy JetStream/KV symbols
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
     - godot # checks that declaration comments end in a period
     - goprintffuncname # checks that printf-like functions are named with f at the end
     - gosec # inspects source code for security problems
     - govet # reports suspicious constructs (Printf alignment, shadow, etc.)
+    - iface # detects interface pollution — duplicate interface declarations
     - ineffassign # detects assignments to existing variables that are not used
+    - iotamixing # flags const blocks that mix iota with non-iota declarations
     - makezero # finds slice declarations with non-zero initial length followed by append
     - mirror # catches wrong mirror patterns in bytes/strings usage
     - misspell # finds commonly misspelled English words
-    - musttag # checks structs passed to marshal/unmarshal have proper tags
     - modernize # modernizes code to use newer Go idioms (go fix analyzers)
+    - musttag # checks structs passed to marshal/unmarshal have proper tags
     - nakedret # finds naked returns in functions
     - nilerr # catches code that returns nil even after checking err != nil
     - nilnesserr # catches returning a different nil error variable after err != nil check
@@ -55,9 +60,6 @@ linters:
     - whitespace # detects leading and trailing whitespace in functions
 
   settings:
-    embeddedstructfieldcheck:
-      forbid-mutex: true
-
     depguard:
       rules:
         main:
@@ -67,11 +69,32 @@ linters:
             - pkg: math/rand$
               desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
 
+    embeddedstructfieldcheck:
+      forbid-mutex: true
+
     errcheck:
       check-type-assertions: true # replaces forcetypeassert
       exclude-functions:
         - (*github.com/nats-io/nats.go.Subscription).Unsubscribe # cleanup-only, errors not actionable
 
+    forbidigo:
+      analyze-types: true
+      forbid:
+        - pattern: '\.JetStream$'
+          pkg: '^github\.com/nats-io/nats\.go$'
+          msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
+        - pattern: '^nats\.(JetStreamContext|KeyValue.*|KeyWatcher|Err(KeyNotFound|KeyExists|BucketNotFound|NoKeysFound|StreamNotFound))$'
+          pkg: '^github\.com/nats-io/nats\.go$'
+          msg: legacy nats KV types/sentinels are superseded; use github.com/nats-io/nats.go/jetstream.
+
+    gosec:
+      excludes:
+        - G104 # unchecked errors — errcheck is stricter
+        - G116 # bidi chars — bidichk is more thorough
+        - G117 # use of net/http serve functions (we handle TLS separately)
+        - G703 # path traversal taint analysis — false positives on config file loading
+        - G705 # XSS taint analysis — false positives on internal error responses
+
     govet:
       enable-all: true
       disable:
@@ -79,6 +102,28 @@ linters:
         - shadow # too noisy to retrofit — all 50 findings are err re-declarations
         - unusedwrite # false positives on struct field assignments before marshal/send
 
+    modernize:
+      disable:
+        - newexpr # don't replace aws.String() with new() — AWS SDK idiom
+        - stringsbuilder # don't replace small string concatenations with strings.Builder
+
+    nakedret:
+      max-func-lines: 0
+
+    nolintlint:
+      require-explanation: true
+      require-specific: true
+
+    perfsprint:
+      concat-loop: false
+      err-error: false # don't flag err.Error()
+      errorf: false # don't convert fmt.Errorf to errors.New
+      strconcat: false
+
+    reassign:
+      patterns:
+        - ".*"
+
     staticcheck:
       checks:
         - all
@@ -90,40 +135,8 @@ linters:
         - -ST1020 # godoc on exported types — not enforcing yet
         - -ST1021 # godoc on exported type groups — not enforcing yet
         - -ST1022 # godoc on exported vars/consts — not enforcing yet
-        - -SA1019 # deprecated usage (handle separately)
         - -SA4006 # value overwritten — ineffassign + wastedassign cover this better
-        - -SA9005 # ineffective break (staticcheck false positives)
-
-    gosec:
-      excludes:
-        - G104 # unchecked errors — errcheck is stricter
-        - G116 # bidi chars — bidichk is more thorough
-        - G117 # use of net/http serve functions (we handle TLS separately)
-        - G703 # path traversal taint analysis — false positives on config file loading
-        - G705 # XSS taint analysis — false positives on internal error responses
-        - G706 # log injection taint analysis — false positives on structured slog usage
-
-    modernize:
-      disable:
-        - newexpr # don't replace aws.String() with new() — AWS SDK idiom
-        - stringsbuilder # don't replace small string concatenations with strings.Builder
-
-    perfsprint:
-      concat-loop: false
-      err-error: false # don't flag err.Error()
-      errorf: false # don't convert fmt.Errorf to errors.New
-      strconcat: false
-
-    nakedret:
-      max-func-lines: 0
-
-    nolintlint:
-      require-explanation: true
-      require-specific: true
-
-    reassign:
-      patterns:
-        - ".*"
+        - -SA9005 # marshaling structs with no exported fields — intended {} for empty API-response types
 
     testifylint:
       disable:
@@ -153,3 +166,8 @@ linters:
         linters: [gosec] # subprocess execution — legitimate in VM orchestration
       - text: "G304"
         linters: [gosec] # file path from variable — necessary for config loading
+
+      # aws-sdk-go v1 is deprecated wholesale (EOL 2025-07-31); the v1->v2
+      # migration is tracked separately, so suppress its SA1019 noise for now.
+      - text: "aws-sdk-go is deprecated"
+        linters: [staticcheck]

Makefile

@@ -1,6 +1,11 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
+# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
+# path, not worktree path, so parallel worktrees of this module collide and
+# surface each other's issues. Scope it per worktree.
+export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
+
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
 ifeq ($(ARCH),x86_64)
@@ -44,7 +49,7 @@ else
   _SECQ  = 2>&1 | tee
 endif
 
-build: go_build build-installer build-lb-agent
+build: go_build build-installer build-lb-agent generate-aws-model-coverage
 
 # Build spinifex-ui frontend (requires pnpm)
 build-ui:
@@ -100,6 +105,12 @@ build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd +
 import-ecs-node-image: ## Build + register the ecs-node AMI (requires a running cluster)
 	$(MAKE) build-system-image IMAGE=ecs-agent IMPORT=1
 
+build-rds-postgres-image: ## Build the spinifex-rds-postgres AMI (Alpine + PostgreSQL 18 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-postgres
+
+import-rds-postgres-image: ## Build + register the rds-postgres AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-postgres IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -120,10 +131,10 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
 
-# Preflight — runs the same checks as GitHub Actions (lint + vuln + tests).
-# Use this before committing to catch CI failures locally.
+# the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
+# integration and race tests skipped to keep quick, they run in CI.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-race test-harness test-integration
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-harness
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
@@ -134,25 +145,40 @@ test-harness:
 	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)
 
 # In-process integration tier: the real gateway router against embedded NATS
-# JetStream, with only the daemon-side NATS subjects stubbed. Build-tagged
-# `integration` so it's skipped by the default `go test ./spinifex/...` and by
-# `test-cover`/`test-race`. Nothing provisioned — no tofu, no docker, no
-# Spinifex daemons, so the whole package runs in well under a minute — part
-# of `preflight` (and its own PR-blocking CI step) rather than the
-# self-hosted, push-triggered live e2e tiers, so a regression here is caught
-# before it can be merged, not just after.
+# JetStream, with only the daemon-side NATS subjects stubbed.
+AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
+AWS_MODEL_CONFORMANCE_MODE ?= fail
+AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
-	$(_Q)LOG_IGNORE=1 go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
+	@$(MAKE) --no-print-directory generate-aws-model-coverage
+	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
+
+generate-aws-model-coverage:
+	@mkdir -p $(dir $(AWS_MODEL_OPERATION_COVERAGE_REPORT))
+	@go run ./cmd/aws-model-coverage > $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+
+aws-model-coverage: generate-aws-model-coverage
+	@cat $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+
+# Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
+# checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
+# is not the default local or CI layout, so this is a separate target from
+# test-integration rather than folded into it. Skips itself when segscan's
+# source isn't found.
+test-segscan-oracle:
+	@echo -e "\n....Running segscan storage oracle test...."
+	$(_Q)LOG_IGNORE=1 go test -tags=integration,segscanoracle -timeout 120s ./tests/integration/... -run TestSegscanOracle $(_RACEQ)
 
 # Validate docs/service-interfaces.yaml. Schema check + cross-reference
-# of services/suites/fixtures + on-disk path existence. Subject content
-# vs source is enforced separately in Bead 5 drift lint.
+# of services/suites/fixtures + on-disk path existence.
 manifest-check:
 	@echo -e "\n....Checking service-interfaces.yaml...."
 	@go run ./tests/e2e/manifest-check/cmd/manifest-check -repo-root . -manifest docs/service-interfaces.yaml
 
-# Drift guards (Bead 5): direct-create fixture lint + NATS subject lint,
+# Drift guards: direct-create fixture lint + NATS subject lint,
 # ratcheted against tests/e2e/manifest-lint/baseline.txt. Fails only on NEW
 # drift beyond the baseline.
 manifest-lint:
@@ -166,19 +192,19 @@ manifest-lint-update:
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	LOG_IGNORE=1 go test -timeout 120s ./spinifex/...
+	GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -timeout 120s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... $(_COVQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... $(_RACEQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -186,6 +212,39 @@ test-actions:
 	@echo -e "\n....Running action tests...."
 	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
 
+# Shell suites + shellcheck for scripts/images/ and images/mkosi.profiles/
+# helpers baked into system images. Kept out of `preflight` (a dedicated CI
+# job gates it on scripts/images/**+images/mkosi.profiles/** changes instead)
+# so image-asset churn doesn't run on every Go contributor's commit.
+#
+# images/mkosi.profiles/ ships scripts with no .sh suffix (mkosi's own
+# mkosi.*.chroot lifecycle hooks, and wrapper binaries like vllm-serve under
+# mkosi.extra/) so a plain `-name '*.sh'` glob silently skips exactly the
+# files most worth linting. Sources there are found by shebang instead of
+# name; this also means a profile that ships no scripts at all yields an
+# empty match rather than an error.
+test-images:
+	@echo -e "\n....Running scripts/images/**/*_test.sh...."
+	@for t in $$(find scripts/images -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running images/mkosi.profiles/**/*_test.sh...."
+	@for t in $$(find images/mkosi.profiles -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over scripts/images/**/*.sh...."
+	shellcheck -S warning $$(find scripts/images -name '*.sh' | sort)
+	@echo -e "\n....Running shellcheck over images/mkosi.profiles/** shell sources...."
+	@srcs="$$(grep -rlIE '^#!/bin/(bash|sh)' images/mkosi.profiles 2>/dev/null | sort)"; \
+	if [ -n "$$srcs" ]; then \
+		shellcheck -S warning $$srcs; \
+	else \
+		echo "  (no shell sources found under images/mkosi.profiles)"; \
+	fi
+	@echo "  test-images ok"
+
 # Check that new/changed code meets coverage threshold (runs tests first)
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
@@ -201,6 +260,20 @@ bench:
 # `make install-microvm` explicitly the first time.
 deploy: build
 	sudo install -m 755 bin/spx /usr/local/bin/spx
+	@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
+		if ! sudo /usr/local/bin/spx admin preflight; then \
+			echo ""; \
+			echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
+			echo "  grants don't match the binary just installed. Remediate with:"; \
+			echo "    scripts/update-nodes.sh   (from the mulga root)"; \
+			echo "  or:"; \
+			echo "    make reinstall"; \
+			echo "  Set SKIP_ASSET_PREFLIGHT=1 to bypass for a deliberate binary-only push."; \
+			exit 1; \
+		fi; \
+	else \
+		echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
+	fi
 	@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
 		$(MAKE) install-microvm; \
 	else \
@@ -223,7 +296,7 @@ install-system:
 	sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
 		-o Dpkg::Options::="--force-confdef" \
 		-o Dpkg::Options::="--force-confold" \
-		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils qemu-kvm \
+		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
 		ovmf qemu-efi-aarch64 \
 		libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
 		iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
@@ -255,7 +328,7 @@ quickinstall: install-system install-go install-aws
 
 lint:
 	@echo "Running golangci-lint..."
-	$(_Q)golangci-lint run ./...
+	$(_Q)scripts/run-gate.sh golangci-lint golangci-lint run ./...
 	@echo "  golangci-lint ok"
 
 fix:
@@ -263,9 +336,15 @@ fix:
 
 govulncheck:
 	@echo "Running govulncheck..."
-	$(_Q)go tool govulncheck ./...
+	$(_Q)scripts/run-gate.sh govulncheck go tool govulncheck ./...
 	@echo "  govulncheck ok"
 
+# NilAway — advisory nil-panic analysis. Not in preflight due to false positives
+nilaway:
+	@echo "Running nilaway..."
+	$(_Q)scripts/run-gate.sh nilaway go tool nilaway -include-pkgs=github.com/mulgadc/spinifex -exclude-test-files ./...
+	@echo "  nilaway ok"
+
 # Build release tarballs — use distro-ARCH for single arch, distro for both
 distro: distro-amd64 distro-arm64
 	@echo ""
@@ -309,8 +388,8 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness test-integration manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
-	lint fix govulncheck \
+	lint fix govulncheck nilaway \
 	distro distro-amd64 distro-arm64 distro-clean

README.md

@@ -5,14 +5,14 @@
 <p align="center">
   <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
-  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-docs.mulgadc.com-DC2626?style=flat-square" alt="Docs"></a>
+  <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
 </p>
 
 <p align="center">
   <a href="#what-is-spinifex">What is Spinifex?</a> ·
   <a href="#the-platform">Platform</a> ·
   <a href="#three-ways-to-deploy">Deploy</a> ·
-  <a href="#aws-compatibility">AWS services</a> ·
+  <a href="#aws-compatibility">AWS Compatibility</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
   <a href="#installation">Installation</a> ·
@@ -23,22 +23,13 @@
 
 # Spinifex: the open, AWS-compatible cloud you run yourself
 
-Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your
-software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control.
-Move workloads off the hyperscalers, run AI at a fraction of the cost, or take
-the cloud to the edge, without rewriting a line. The exact build you ship to AWS,
-Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
+Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line. The exact build you ship to AWS, Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
 
 ## What is Spinifex?
 
-Spinifex is an open-source infrastructure platform that recreates the AWS service
-surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises
-environments. Run cloud-native software without a hyperscaler.
+Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
 
-Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering
-underneath: distributed object storage with erasure coding, block storage built
-to survive failure, and compute on bare metal. That depth is the reason your
-software runs unchanged, instead of rewritten.
+Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering underneath: distributed object storage with erasure coding, block storage built to survive failure, and compute on bare metal. That depth is the reason your software runs unchanged, instead of rewritten.
 
 It's built for teams that need:
 
@@ -57,53 +48,41 @@ You change *where* your software runs, not *what* it is.
 
 From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
 
-## Three ways to deploy
+## Three Ways To Deploy
 
-One AWS-compatible surface, three deployment shapes. Pick the one that matches
-your reality; the platform on top is the same.
+One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
 
 ### Neocloud
 
-Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and
-onto our Neocloud partner ecosystem without rewriting them. GPU capacity
-(H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the
-software.
+Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and onto our Neocloud partner ecosystem without rewriting them. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
 
 ### On-premise
 
-Bring your own hardware and host Spinifex in your own data centre. A real
-multi-node HA cluster integrated with your storage and networking, with full
-control of stack, data, and jurisdiction. A predictable bill, no egress
-surprises, an auditable open-source core.
+Bring your own hardware and host Spinifex in your own data centre. A real multi-node HA cluster integrated with your storage and networking, with full control of stack, data, and jurisdiction. A predictable bill, no egress surprises, an auditable open-source core.
 
 ### Edge
 
-Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels,
-factories, and clinics. Compute next to the data, running through disconnection,
-on hardware you own. The same AWS APIs your software already uses.
+Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
 
-## AWS compatibility
+## AWS Compatibility
 
-Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform:
-everything you deploy on AWS deploys on Spinifex unchanged. At the edge,
-on-premise, or on a partner Neocloud.
+Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
 | **EC2** | Compute | Available |
-| **EBS** | Block storage | Available |
-| **S3** | Object storage | Available |
+| **EBS** | Block Storage | Available |
+| **S3** | Object Storage | Available |
 | **VPC** | Networking | Available |
-| **IAM** | Identity & auth | Available |
-| **ALB / NLB** | Load balancers | Available |
+| **IAM** | Identity & Auth | Available |
+| **ALB / NLB** | Load Balancers | Available |
 | **EKS** | Kubernetes | Available |
-| **ECR** | Container registry | Available |
-| **ECS** | Container service | Available |
-| **RDS** | Databases | Q3 2026 |
+| **ECR** | Container Registry | Available |
+| **ECS** | Container Service | Available |
+| **RDS** | Databases | Available |
+| **Bedrock** | AI Deployment | Q3 2026 |
 
-Roadmap items (Q3 2026) ship under the same AWS API surface. Code written for AWS
-today keeps working the moment they land.
-[Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
+Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 
 ## Core Components
 
@@ -133,17 +112,21 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 - Multipart uploads, streaming reads/writes
 - Data redundancy with Reed-Solomon encoding
 
+### Northstar (Authoritative DNS)
+
+[Northstar](https://github.com/mulgadc/northstar) is a lightweight authoritative DNS server that handles both internal service discovery and public-facing DNS for a Spinifex cluster. Zones are human-readable TOML files, served from the local filesystem or synced from Predastore, keeping DNS configuration in the same object storage as the rest of your infrastructure.
+
+- Authoritative DNS over UDP, TCP, and DNS-over-TLS
+- Service discovery for cluster components via SRV records
+- Zone files stored locally or in S3-compatible storage
+
 ## Architecture at a Glance
 
 <p align="center">
   <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
 </p>
 
-Every AWS API call is authenticated at the gateway, published to a NATS subject,
-and answered by whichever daemon claims it. Daemons are stateless: scale
-horizontally by starting more. No etcd, no Kubernetes, no external control plane.
-Just systemd units, a NATS cluster, and your hardware. Deep dive:
-**[`docs/DESIGN.md`](docs/DESIGN.md)**.
+Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
 ## Key Features
 
@@ -156,7 +139,7 @@ Just systemd units, a NATS cluster, and your hardware. Deep dive:
 
 ## Installation
 
-Installation requires an Ubuntu / Debian system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex.
+Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
 
 ### Bare Metal ISO
 
@@ -166,16 +149,11 @@ The recommended installation is a [bootable x86 installer](https://iso.mulgadc.c
 curl -fLO https://iso.mulgadc.com/spinifex.iso
 ```
 
-Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware. The install guide walks through the full process.
-
+Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware.
 
 ### Single Node Install
 
-The installation is straightforward to set up and running on a single node for testing purposes. Debian 13 is currently supported, additional Linux distributions are on the immediate roadmap.
-
->*Prerequisite:* Linux bridge for networking.
-
-Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) prerequisites for setup details.
+>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
 
 ```bash
 curl -fsSL https://install.mulgadc.com | bash
@@ -193,7 +171,7 @@ aws ec2 describe-instance-types
 
 ### Development Setup
 
-For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation
+For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
 
 ### Component Repositories
 
@@ -201,6 +179,7 @@ Spinifex coordinates these independent components:
 
 - **[Predastore](https://github.com/mulgadc/predastore)** - S3-compatible object storage
 - **[Viperblock](https://github.com/mulgadc/viperblock)** - EBS-compatible block storage
+- **[Northstar](https://github.com/mulgadc/northstar)** - Authoritative DNS server
 
 Each component can be developed independently. See component-specific documentation for focused development guides.
 
@@ -214,10 +193,6 @@ Spinifex ships with a built-in web console — an optional alternative to the AW
 
 The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
 
-```bash
-open https://YOUR_NODE_IP:3000
-```
-
 - **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
 - **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
 - **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.

build/logrotate/spinifex

@@ -1,4 +1,8 @@
 /var/log/spinifex/*.log {
+    # The directory is 0775 root:spinifex so each service user can write its own
+    # log. logrotate refuses to rotate inside a group-writable directory unless
+    # it is told which user to rotate as.
+    su root spinifex
     daily
     rotate 14
     compress

build/scripts/nats-js-watchdog.sh

@@ -1,21 +1,32 @@
 #!/bin/sh
 # JetStream liveness watchdog for spinifex-nats.
 #
-# NATS can disable JetStream at runtime (e.g. a filesystem permission error while
-# flushing stream state) and keep the process alive serving KV as read-only. That
-# is not a process failure, so Restart=on-failure never fires and KV-backed
-# services (awsgw IAM bucket init, etc.) stall until NATS is restarted by hand.
+# NATS can stop accepting JetStream writes at runtime while the process stays
+# alive and keeps serving reads. That is not a process failure, so
+# Restart=on-failure never fires and KV-backed services (awsgw IAM bucket init,
+# etc.) stall until NATS is restarted by hand.
 #
-# This probes the NATS monitoring endpoint's JS health and restarts the service
-# only when JetStream is sustainedly unhealthy while the service is running. The
-# multi-sample window avoids restarting during transient startup/quorum 503s.
+# Two distinct latches produce that symptom, and they need different probes:
+#
+#   1. Server-wide. handleOutOfSpace calls ShutdownJetStream and sets an
+#      internal disabled flag that only a restart clears. healthz reports it.
+#   2. Per message block. filestore records a write error on the block and
+#      returns it on every later write; nothing clears it short of a restart.
+#      healthz does NOT report it, because the server-wide state is untouched —
+#      only an actual write observes it.
+#
+# So a health check alone misses case 2 entirely. Both probes run here, and a
+# sustained failure of either restarts the service. The multi-sample window
+# avoids restarting during transient startup/quorum 503s.
 
 set -eu
 
 HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
 SERVICE="spinifex-nats.service"
+SPX="${SPX_BIN:-/usr/local/bin/spx}"
 SAMPLES=3        # consecutive failing samples required before acting
 SAMPLE_GAP=2     # seconds between samples
+PROBE_TIMEOUT=5  # seconds for one canary round-trip
 
 # Only act on a running service: a stopped/activating unit is systemd's to manage,
 # and try-restart on it would either no-op or fight an in-flight (re)start.
@@ -30,15 +41,29 @@ if ! command -v curl >/dev/null 2>&1; then
     exit 0
 fi
 
-# A single 200 means JetStream is healthy — exit fast on the common case.
 js_healthy() {
     curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
 }
 
+# Round-trips a canary key through the cluster-state bucket. Absent spx this
+# degrades to the health check alone rather than restarting on a missing tool.
+js_writable() {
+    if [ ! -x "$SPX" ]; then
+        return 0
+    fi
+    "$SPX" admin node js-probe --timeout "${PROBE_TIMEOUT}s" >/dev/null 2>&1
+}
+
+failure=""
 i=1
 while [ "$i" -le "$SAMPLES" ]; do
     if js_healthy; then
-        exit 0
+        if js_writable; then
+            exit 0
+        fi
+        failure="JetStream accepts no writes (canary round-trip failed) while $HEALTH_URL reports healthy"
+    else
+        failure="JetStream unhealthy at $HEALTH_URL"
     fi
     if [ "$i" -lt "$SAMPLES" ]; then
         sleep "$SAMPLE_GAP"
@@ -46,5 +71,5 @@ while [ "$i" -le "$SAMPLES" ]; do
     i=$((i + 1))
 done
 
-echo "nats-js-watchdog: JetStream unhealthy on $SAMPLES consecutive probes ($HEALTH_URL); restarting $SERVICE" >&2
+echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2
 systemctl try-restart "$SERVICE"

build/scripts/predastore-start.sh

@@ -1,30 +0,0 @@
-#!/bin/sh
-# Helper script for spinifex-predastore.service
-# Detects host, port, backend, and node-id from spinifex.toml
-# then execs spx. Falls back to single-node defaults if no config found.
-
-CONF=/etc/spinifex/spinifex.toml
-BIND="0.0.0.0:8443"
-# -1 = co-located mode (all [[db]] peers in one process). Single-node spinifex
-# templates list 3 DB peers on 127.0.0.1; only -1 can serve that topology.
-# Multi-node spinifex.toml emits node_id = N (N >= 1) which overrides below.
-# predastore rejects NODE_ID=0 since d3a02f0 (strconv.Atoi-of-garbage guard).
-NODE_ID="-1"
-
-if [ -f "$CONF" ]; then
-    H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
-    if [ -n "$H" ]; then
-        BIND="$H"
-    fi
-
-    N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
-    if [ -n "$N" ]; then
-        NODE_ID="$N"
-    fi
-fi
-
-export SPINIFEX_PREDASTORE_HOST="${BIND%%:*}"
-export SPINIFEX_PREDASTORE_PORT="${BIND##*:}"
-export SPINIFEX_PREDASTORE_NODE_ID="$NODE_ID"
-
-exec /usr/local/bin/spx service predastore start

build/systemd/regenerate-ssh-host-keys.service

@@ -7,6 +7,9 @@ Before=ssh.service
 Type=oneshot
 ExecStart=/usr/bin/ssh-keygen -A
 RemainAfterExit=yes
+# No ExecStop: the keygen process has already exited by the time this unit
+# is stopped, so there is nothing to signal.
+TimeoutStopSec=5
 
 [Install]
 WantedBy=multi-user.target

build/systemd/spinifex-awsgw.service

@@ -14,8 +14,15 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service awsgw start
 Restart=on-failure
 RestartSec=5
+# No SIGTERM handler in this process: it exits immediately on signal, so this
+# only bounds an unexpectedly wedged process before SIGKILL.
+TimeoutStopSec=10
 
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/
 Environment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem

build/systemd/spinifex-daemon.service

@@ -7,6 +7,8 @@ Wants=openvswitch-ipsec.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
+# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see
+EnvironmentFile=-/etc/spinifex/host.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-daemon
@@ -19,13 +21,17 @@ KillMode=process
 TimeoutStopSec=15
 # RG-4: infra tier — reaped only after every guest
 OOMScoreAdjust=-500
+# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead
+MemoryHigh=75%
+MemoryMax=85%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
 # Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)
 Environment=XDG_RUNTIME_DIR=/run/spinifex
 
-# RG-9: daemon tier — privileged by necessity (GPU vfio). NoNewPrivileges omitted (RG-10: sudo ip/ovs-vsctl/dhcpcd)
+# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,
+# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.
 AmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE
 ProtectSystem=full
 PrivateDevices=no

build/systemd/spinifex-nats-watchdog.service

@@ -5,10 +5,16 @@ After=spinifex-nats.service
 
 [Service]
 Type=oneshot
-# Runs as root: the probe is a local curl to the NATS monitoring port, and the
-# remedy is `systemctl try-restart spinifex-nats` when JetStream is disabled while
-# the process stays alive (which Restart=on-failure cannot catch).
+# Runs as root: the probes are a local curl to the NATS monitoring port and a
+# canary KV round-trip via spx, and the remedy is `systemctl try-restart
+# spinifex-nats` when JetStream stops accepting writes while the process stays
+# alive (which Restart=on-failure cannot catch).
 ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
+# Sized for the probe loop's own worst case (3 samples x (5s curl + 5s canary)
+# + 2 gaps x 2s = 34s): a stop mid-cycle lets the script finish rather than
+# cutting it off. Only the failing path runs that long; a healthy node exits on
+# the first sample.
+TimeoutStopSec=45
 
 # Hardening (kept compatible with talking to PID1 over /run/systemd).
 NoNewPrivileges=yes

build/systemd/spinifex-nats.service

@@ -12,7 +12,15 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service nats start
 Restart=on-failure
 RestartSec=5
+# nats-server installs its own SIGTERM handler (JetStream store shutdown, raft
+# stepdown, client drain) with no internal deadline; every dependent service
+# stops before nats does, so size this for nats's own graceful-stop needs.
+TimeoutStopSec=20
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf
 
 # Filesystem access

build/systemd/spinifex-northstar.service

@@ -13,7 +13,14 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service northstar start
 Restart=on-failure
 RestartSec=5
+# server.Shutdown() is called with an internal 5s context; margin covers the
+# NATS unsubscribe/close and process exit that follow it.
+TimeoutStopSec=10
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/northstar/
 

build/systemd/spinifex-predastore.service

@@ -14,10 +14,20 @@ Slice=spinifex-system.slice
 User=spinifex-storage
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
-ExecStart=/var/lib/spinifex/predastore-start.sh
+ExecStart=/usr/local/bin/spx service predastore start
 Restart=on-failure
 RestartSec=5
+# Matches viperblock's drain budget: viperblock's WAL flush writes into
+# predastore, and After= already stops viperblock first, but this is the
+# backstop if that ordering is ever bypassed (direct stop, future edit).
+TimeoutStopSec=30
 OOMScoreAdjust=-500
+# Legitimately the most memory-hungry infra service (S3 buffering/caching);
+# High/Max stay well above idle so normal load isn't throttled, while still
+# bounding a leak short of the slice-wide cap.
+MemoryHigh=40%
+MemoryMax=55%
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml
 Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
 Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem

build/systemd/spinifex-qmp-collector.service

@@ -16,6 +16,13 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service qmp-collector start
 Restart=on-failure
 RestartSec=5
+# Poll loop selects on ctx.Done() directly and closes NATS locally; shutdown
+# is near-instant, so this only bounds an unexpectedly wedged process.
+TimeoutStopSec=10
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 # Same runtime dir as the daemon: telemetry sockets + metadata live there

build/systemd/spinifex-shutdown.service

@@ -9,9 +9,13 @@ After=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.serv
 Type=oneshot
 RemainAfterExit=yes
 ExecStart=/bin/true
-# ExecStop only runs when the target/host goes down, not on a single-service
-# restart — so a daemon restart still leaves guests running (DDIL).
-ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s
+# ExecStop fires on ANY target stop, including the stop half of `systemctl
+# restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless
+# of whether the host is actually going down. --only-if-host-stopping drains
+# only when systemd itself is unwinding into shutdown.target; a plain restart
+# or `systemctl stop spinifex.target` falls through so QEMU/nbdkit survive
+# (KillMode=process) and the daemon reconnects to them on the way back up.
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --only-if-host-stopping
 TimeoutStopSec=180
 
 [Install]

build/systemd/spinifex-ui.service

@@ -12,16 +12,24 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service spinifex-ui start
 Restart=on-failure
 RestartSec=5
+# server.Shutdown() uses an internal 30s context (WriteTimeout=120s covers
+# long streaming proxy responses, e.g. console passthrough); add margin above it.
+TimeoutStopSec=40
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
+Environment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/
 
 # Filesystem access
 NoNewPrivileges=yes
 ProtectSystem=strict
 PrivateDevices=yes
 ReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key
-ReadWritePaths=/var/log/spinifex
+ReadWritePaths=/var/lib/spinifex/spinifex-ui /var/log/spinifex
 
 # Hardening
 ProtectHome=yes

build/systemd/spinifex-viperblock.service

@@ -15,6 +15,16 @@ ExecStart=/usr/local/bin/spx service viperblock start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=15%
+MemoryMax=25%
+# KillMode=process: nbdkit still serving a guest survives a restart here, the
+# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts
+# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.
+KillMode=process
+# Bounds this process's own WAL drain, which the plugin caps at 20s.
+TimeoutStopSec=30
 EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/

build/systemd/spinifex-vpcd.service

@@ -14,13 +14,23 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service vpcd start
 Restart=on-failure
 RestartSec=5
+# Waits for an in-flight OVN drift-reconcile pass to return before exiting;
+# no sub-timeout bounds that call today, so this covers a normal pass.
+TimeoutStopSec=20
 OOMScoreAdjust=-500
+# Per-service ceiling: bounds a leak here instead of the slice-wide cap.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/
 
 # RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no
 # longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem
-# sandbox is restored. NoNewPrivileges stays off (RG-10: sudo ip/ovs-vsctl/dhcpcd).
+# sandbox is restored.
+#
+# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and
+# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR
+# update that retires the last escalation path.
 AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
 CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
 NoNewPrivileges=no

cmd/aws-model-coverage/main.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/mulgadc/spinifex/internal/awsmodel"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+)
+
+func main() {
+	dispatch := gateway.AWSOperationInventory()
+	coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
+	for _, service := range awsmodel.Services() {
+		inventory, ok := dispatch[string(service)]
+		var modelInventory awsmodel.DispatchInventory
+		switch {
+		case ok:
+			modelInventory = awsmodel.DispatchInventory{
+				Registered:  inventory.Registered,
+				Stubbed:     inventory.Stubbed,
+				Unsupported: inventory.Unsupported,
+			}
+		case service == awsmodel.S3:
+			modelInventory = awsmodel.DispatchInventory{
+				Opaque: true,
+				Note:   "Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.",
+			}
+		default:
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: no gateway inventory for %s\n", service)
+			os.Exit(1)
+		}
+
+		coverage, err := awsmodel.CompareOperations(service, modelInventory)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
+			os.Exit(1)
+		}
+		coverages = append(coverages, coverage)
+	}
+
+	fmt.Print(awsmodel.RenderCoverageMarkdown(coverages))
+}

cmd/dhcptest/main.go

@@ -6,7 +6,7 @@
 // Usage (must run as root — needs AF_PACKET):
 //
 //	sudo dhcptest --iface=br-wan
-//	sudo dhcptest --iface=br-wan --mac=02:00:00:7c:b7:b9 --promisc
+//	sudo dhcptest --iface=br-wan --mac=02:54:df:71:22:5c --promisc
 //	sudo dhcptest --iface=br-wan --spinifex-id=gateway-wan1
 package main
 
@@ -23,6 +23,7 @@ import (
 
 	"github.com/insomniacslk/dhcp/dhcpv4"
 	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
+	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
 )
 
 func main() {
@@ -36,6 +37,7 @@ func main() {
 	timeout := flag.Duration("timeout", 10*time.Second, "Per-attempt DHCP timeout")
 	retries := flag.Int("retries", 3, "Number of DORA attempts before giving up")
 	verbose := flag.Bool("verbose", false, "Print full DHCP packet contents")
+	broadcast := flag.Bool("broadcast", false, "Set the BOOTP broadcast flag, as vpcd's Acquire path does, so replies come back to ff:ff:ff:ff:ff:ff rather than unicast to the bridge MAC")
 	flag.Parse()
 
 	// Resolve the hardware address to use.
@@ -44,9 +46,11 @@ func main() {
 
 	switch {
 	case *spinifexID != "":
-		hwAddr, err = net.ParseMAC(generateMAC(*spinifexID))
+		// vpcd's own derivation, so the probe leases under the MAC the daemon
+		// would have used for this id.
+		hwAddr, err = dhcp.DeriveMAC(*spinifexID)
 		if err != nil {
-			slog.Error("Could not parse generated MAC", "spinifex-id", *spinifexID, "error", err)
+			slog.Error("Could not derive MAC", "spinifex-id", *spinifexID, "error", err)
 			os.Exit(1)
 		}
 		fmt.Printf("spinifex virtual MAC for %q: %s\n", *spinifexID, hwAddr)
@@ -109,16 +113,11 @@ func main() {
 	}
 	defer client.Close()
 
-	// Identity modifiers — client-id / hostname / vendor-class for upstream lease tagging.
-	var mods []dhcpv4.Modifier
-	if cid != "" {
-		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(cid))))
-	}
-	if *hostname != "" {
-		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptHostName(*hostname)))
-	}
-	if *vendorClass != "" {
-		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClassIdentifier(*vendorClass)))
+	// Identity modifiers — client-id / hostname / vendor-class for upstream lease
+	// tagging. Built by the vpcd code path so the probe cannot drift from it.
+	mods := dhcp.IdentityModifiers(cid, *hostname, *vendorClass, hwAddr)
+	if *broadcast {
+		mods = append(mods, dhcpv4.WithBroadcast(true))
 	}
 
 	fmt.Println("Sending DHCP Discover...")
@@ -153,16 +152,6 @@ func (slogPrintfer) Printf(format string, v ...any) {
 	slog.Info(fmt.Sprintf(format, v...), "component", "nclient4")
 }
 
-// generateMAC is the same deterministic hash used by vpcd's topology.go.
-// It produces a locally-administered MAC (02:00:00:xx:xx:xx) from a resource ID.
-func generateMAC(resourceID string) string {
-	h := uint32(0)
-	for _, c := range resourceID {
-		h = h*31 + uint32(c)
-	}
-	return fmt.Sprintf("02:00:00:%02x:%02x:%02x", (h>>16)&0xff, (h>>8)&0xff, h&0xff)
-}
-
 func ifaceMAC(name string) (net.HardwareAddr, error) {
 	ifc, err := net.InterfaceByName(name)
 	if err != nil {

cmd/ecr-credential-provider/main.go

@@ -12,17 +12,16 @@
 package main
 
 import (
-	"bufio"
 	"encoding/json"
 	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/guestenv"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 
 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
@@ -144,13 +143,7 @@ func cacheDurationFrom(expiresAt time.Time) string {
 // loadConfig reads the cloud-init env file then lets real env vars override. The
 // IMDS base is fixed (not in the env file) but overridable for tests.
 func loadConfig(envFile string) config {
-	env := parseEnvFile(envFile)
-	get := func(key string) string {
-		if v := os.Getenv(key); v != "" {
-			return v
-		}
-		return env[key]
-	}
+	get := guestenv.Load(envFile).Get
 
 	cfg := config{
 		GatewayURL: get("EKS_GATEWAY_URL"),
@@ -167,29 +160,6 @@ func loadConfig(envFile string) config {
 	return cfg
 }
 
-// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
-func parseEnvFile(path string) map[string]string {
-	out := map[string]string{}
-	f, err := os.Open(path)
-	if err != nil {
-		return out
-	}
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		key, val, ok := strings.Cut(line, "=")
-		if !ok {
-			continue
-		}
-		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
-	}
-	return out
-}
-
 // emitEmpty writes a valid CredentialProviderResponse with no auth entries.
 func emitEmpty(out io.Writer) {
 	resp := CredentialProviderResponse{

cmd/ecr-credential-provider/main_test.go

@@ -18,12 +18,14 @@ func fakeBackend(t *testing.T) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
 
-	// IMDSv2 token.
+	// IMDSv2 token. Real IMDS echoes the requested TTL back as a response
+	// header; the SDK client parses it from there, not the body.
 	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPut {
 			w.WriteHeader(http.StatusMethodNotAllowed)
 			return
 		}
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("imds-v2-token"))
 	})
 	// IMDS role name.

cmd/ecs-agent/config.go

@@ -1,11 +1,10 @@
 package main
 
 import (
-	"bufio"
-	"os"
 	"strconv"
-	"strings"
 	"time"
+
+	"github.com/mulgadc/spinifex/internal/guestenv"
 )
 
 const (
@@ -40,13 +39,7 @@ type config struct {
 
 // loadConfig reads the cloud-init env file then lets real env vars override.
 func loadConfig(envFile string) config {
-	env := parseEnvFile(envFile)
-	get := func(key string) string {
-		if v := os.Getenv(key); v != "" {
-			return v
-		}
-		return env[key]
-	}
+	get := guestenv.Load(envFile).Get
 
 	cfg := config{
 		GatewayURL:       get("ECS_GATEWAY_URL"),
@@ -94,26 +87,3 @@ func loadConfig(envFile string) config {
 	}
 	return cfg
 }
-
-// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
-func parseEnvFile(path string) map[string]string {
-	out := map[string]string{}
-	f, err := os.Open(path)
-	if err != nil {
-		return out
-	}
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		key, val, ok := strings.Cut(line, "=")
-		if !ok {
-			continue
-		}
-		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
-	}
-	return out
-}

cmd/ecs-agent/config_test.go

@@ -71,18 +71,3 @@ func TestLoadConfig_HeartbeatOverride(t *testing.T) {
 		t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
 	}
 }
-
-func TestParseEnvFile_SkipsBlankAndComments(t *testing.T) {
-	dir := t.TempDir()
-	p := filepath.Join(dir, "f.env")
-	if err := os.WriteFile(p, []byte("\n# c\nA=1\nnokeyval\nB = 2 \n"), 0o600); err != nil {
-		t.Fatal(err)
-	}
-	m := parseEnvFile(p)
-	if m["A"] != "1" || m["B"] != "2" {
-		t.Errorf("parse mismatch: %#v", m)
-	}
-	if _, ok := m["nokeyval"]; ok {
-		t.Errorf("line without = should be skipped")
-	}
-}

cmd/ecs-agent/credentials/imds.go

@@ -3,57 +3,39 @@ package credentials
 import (
 	"context"
 	"net/http"
-	"sync"
-	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 )
 
-// refreshMargin is how long before expiry the cache is considered stale.
-const refreshMargin = 5 * time.Minute
-
-// IMDSProvider fetches instance-role credentials from IMDS and caches them until
-// they near expiry. Safe for concurrent use.
+// IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
+// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache
+// (which refetches once the real expiry passes; ec2rolecreds also caps
+// Expires to at most an hour out). Safe for concurrent use.
 type IMDSProvider struct {
-	client *http.Client
-	base   string
-
-	mu     sync.Mutex
-	cached Credentials
+	cache *aws.CredentialsCache
 }
 
 var _ CredentialsProvider = (*IMDSProvider)(nil)
 
 // NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
-// "http://169.254.169.254/latest"). A nil client uses http.DefaultClient.
+// "http://169.254.169.254/latest"). A nil client uses the SDK IMDS client's
+// default HTTP client.
 func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
-	if client == nil {
-		client = http.DefaultClient
-	}
-	return &IMDSProvider{client: client, base: base}
+	provider := imdscreds.NewProvider(client, base)
+	return &IMDSProvider{cache: aws.NewCredentialsCache(provider)}
 }
 
 // Retrieve returns cached credentials when still valid, else fetches fresh ones.
 func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	if p.cached.Valid(refreshMargin) {
-		return p.cached, nil
-	}
-	if err := ctx.Err(); err != nil {
-		return Credentials{}, err
-	}
-
-	raw, err := imdscreds.Fetch(p.client, p.base)
+	creds, err := p.cache.Retrieve(ctx)
 	if err != nil {
 		return Credentials{}, err
 	}
-	p.cached = Credentials{
-		AccessKeyID:     raw.AccessKeyID,
-		SecretAccessKey: raw.SecretAccessKey,
-		SessionToken:    raw.SessionToken,
-		Expiration:      raw.Expiration,
-	}
-	return p.cached, nil
+	return Credentials{
+		AccessKeyID:     creds.AccessKeyID,
+		SecretAccessKey: creds.SecretAccessKey,
+		SessionToken:    creds.SessionToken,
+		Expiration:      creds.Expires,
+	}, nil
 }

cmd/ecs-agent/credentials/imds_test.go

@@ -10,14 +10,19 @@ import (
 	"time"
 )
 
+// imdsStub serves the IMDSv2 token + role + credentials endpoints under the
+// real /latest prefix the SDK IMDS client always requests.
 func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		// Real IMDS echoes the requested TTL back as a response header; the SDK
+		// client parses it from there, not the body.
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("v2-token"))
 	})
-	mux.HandleFunc("/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/meta-data/iam/security-credentials/" {
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
 			_, _ = w.Write([]byte("node-role"))
 			return
 		}
@@ -33,15 +38,17 @@ func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Serv
 
 func TestRetrieve_FetchesAndCaches(t *testing.T) {
 	var hits int32
-	exp := time.Now().Add(6 * time.Hour).UTC().Round(time.Second)
+	// Kept under an hour: ec2rolecreds caps Expires to now+1h.
+	exp := time.Now().Add(30 * time.Minute).UTC().Round(time.Second)
 	srv := imdsStub(t, &hits, map[string]string{
+		"Code":            "Success",
 		"AccessKeyId":     "AKIA",
 		"SecretAccessKey": "secret",
 		"Token":           "session",
 		"Expiration":      exp.Format(time.RFC3339),
 	})
 
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	got, err := p.Retrieve(context.Background())
 	if err != nil {
 		t.Fatalf("Retrieve: %v", err)
@@ -50,7 +57,7 @@ func TestRetrieve_FetchesAndCaches(t *testing.T) {
 		t.Errorf("creds mismatch: %+v", got)
 	}
 
-	// Second call within validity window must not re-hit IMDS.
+	// Second call well within validity must not re-hit IMDS.
 	if _, err := p.Retrieve(context.Background()); err != nil {
 		t.Fatalf("Retrieve 2: %v", err)
 	}
@@ -59,15 +66,17 @@ func TestRetrieve_FetchesAndCaches(t *testing.T) {
 	}
 }
 
-func TestRetrieve_RefetchesWhenStale(t *testing.T) {
+func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
 	var hits int32
-	// Expiry inside the refresh margin → never cached, always refetched.
+	// Already expired: aws.CredentialsCache refetches once the stored
+	// credentials' real expiry has passed, regardless of ExpiryWindow.
 	srv := imdsStub(t, &hits, map[string]string{
+		"Code":            "Success",
 		"AccessKeyId":     "AKIA",
 		"SecretAccessKey": "secret",
-		"Expiration":      time.Now().Add(time.Minute).UTC().Format(time.RFC3339),
+		"Expiration":      time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
 	})
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	if _, err := p.Retrieve(context.Background()); err != nil {
 		t.Fatal(err)
 	}
@@ -75,13 +84,13 @@ func TestRetrieve_RefetchesWhenStale(t *testing.T) {
 		t.Fatal(err)
 	}
 	if hits != 2 {
-		t.Errorf("hits = %d, want 2 (stale refetch)", hits)
+		t.Errorf("hits = %d, want 2 (expired refetch)", hits)
 	}
 }
 
 func TestRetrieve_CancelledContext(t *testing.T) {
-	srv := imdsStub(t, nil, map[string]string{"AccessKeyId": "A", "SecretAccessKey": "B"})
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 	if _, err := p.Retrieve(ctx); err == nil {
@@ -89,16 +98,10 @@ func TestRetrieve_CancelledContext(t *testing.T) {
 	}
 }
 
-func TestCredentials_Valid(t *testing.T) {
-	if (Credentials{}).Valid(0) {
-		t.Error("empty creds should be invalid")
-	}
-	noExp := Credentials{AccessKeyID: "a", SecretAccessKey: "b"}
-	if !noExp.Valid(time.Hour) {
-		t.Error("zero expiry should be treated valid")
-	}
-	soon := Credentials{AccessKeyID: "a", SecretAccessKey: "b", Expiration: time.Now().Add(time.Minute)}
-	if soon.Valid(5 * time.Minute) {
-		t.Error("expiry inside margin should be invalid")
+func TestRetrieve_CodeNotSuccessErrors(t *testing.T) {
+	srv := imdsStub(t, nil, map[string]string{"Code": "Failure", "AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
+	if _, err := p.Retrieve(context.Background()); err == nil {
+		t.Fatal("expected error when Code != Success")
 	}
 }

cmd/ecs-agent/credentials/provider.go

@@ -17,18 +17,6 @@ type Credentials struct {
 	Expiration      time.Time
 }
 
-// Valid reports whether the credentials are usable now, leaving a margin so a
-// caller does not start a request with a token about to expire mid-flight.
-func (c Credentials) Valid(margin time.Duration) bool {
-	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
-		return false
-	}
-	if c.Expiration.IsZero() {
-		return true
-	}
-	return time.Now().Add(margin).Before(c.Expiration)
-}
-
 // CredentialsProvider yields current credentials, refreshing as needed. Retrieve
 // must be safe for concurrent use.
 type CredentialsProvider interface {

cmd/ecs-agent/metadata.go

@@ -1,11 +1,14 @@
 package main
 
 import (
-	"encoding/json"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
 )
 
 // instanceMetadata is the IMDS-derived identity of the host the agent runs on.
@@ -15,83 +18,44 @@ type instanceMetadata struct {
 	AZ         string
 }
 
-// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2.
-// base is the IMDS root, e.g. "http://169.254.169.254/latest".
+// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2
+// via the SDK IMDS client. base is the IMDS root, e.g.
+// "http://169.254.169.254/latest".
 func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
-	token, err := imdsToken(client, base)
-	if err != nil {
-		return instanceMetadata{}, err
-	}
+	imdsClient := imdscreds.NewClient(client, base)
+	ctx := context.Background()
 
-	instanceID, err := imdsGet(client, base+"/meta-data/instance-id", token)
+	instanceID, err := getMetadata(ctx, imdsClient, "instance-id")
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
 	}
-	az, err := imdsGet(client, base+"/meta-data/placement/availability-zone", token)
+	az, err := getMetadata(ctx, imdsClient, "placement/availability-zone")
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
 	}
 
-	doc, err := imdsGet(client, base+"/dynamic/instance-identity/document", token)
+	doc, err := imdsClient.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
 	}
-	var ident struct {
-		AccountID string `json:"accountId"`
-	}
-	if err := json.Unmarshal([]byte(doc), &ident); err != nil {
-		return instanceMetadata{}, fmt.Errorf("parse identity document: %w", err)
-	}
 
 	return instanceMetadata{
-		AccountID:  ident.AccountID,
+		AccountID:  doc.AccountID,
 		InstanceID: strings.TrimSpace(instanceID),
 		AZ:         strings.TrimSpace(az),
 	}, nil
 }
 
-// imdsToken fetches an IMDSv2 session token.
-func imdsToken(client *http.Client, base string) (string, error) {
-	req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
+// getMetadata reads a single meta-data path and returns its body as a string.
+func getMetadata(ctx context.Context, imdsClient *imds.Client, path string) (string, error) {
+	out, err := imdsClient.GetMetadata(ctx, &imds.GetMetadataInput{Path: path})
 	if err != nil {
 		return "", err
 	}
-	req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("imds token: %w", err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
+	defer out.Content.Close()
+	body, err := io.ReadAll(out.Content)
 	if err != nil {
 		return "", err
 	}
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("imds token: status %d", resp.StatusCode)
-	}
-	return string(body), nil
-}
-
-// imdsGet performs a token-authenticated IMDS GET.
-func imdsGet(client *http.Client, url, token string) (string, error) {
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return "", err
-	}
-	if token != "" {
-		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
-	}
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("status %d", resp.StatusCode)
-	}
 	return string(body), nil
 }

cmd/ecs-agent/metadata_test.go

@@ -9,11 +9,14 @@ import (
 func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPut {
 			w.WriteHeader(http.StatusMethodNotAllowed)
 			return
 		}
+		// Real IMDS echoes the requested TTL back as a response header; the SDK
+		// client parses it from there, not the body.
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("v2-token"))
 	})
 	guard := func(w http.ResponseWriter, r *http.Request) bool {
@@ -23,17 +26,17 @@ func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 		}
 		return true
 	}
-	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte("i-0abc123"))
 		}
 	})
-	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte("us-east-1a"))
 		}
 	})
-	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte(`{"accountId":"123456789012","region":"us-east-1"}`))
 		}
@@ -45,7 +48,7 @@ func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 
 func TestFetchInstanceMetadata_OK(t *testing.T) {
 	srv := metadataStub(t, true)
-	got, err := fetchInstanceMetadata(srv.Client(), srv.URL)
+	got, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest")
 	if err != nil {
 		t.Fatalf("fetch: %v", err)
 	}
@@ -57,20 +60,23 @@ func TestFetchInstanceMetadata_OK(t *testing.T) {
 func TestFetchInstanceMetadata_TokenRequired(t *testing.T) {
 	// Server returns 405 for non-PUT token; a GET-only client would 401 downstream.
 	srv := metadataStub(t, true)
-	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err != nil {
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err != nil {
 		t.Fatalf("v2 path should succeed: %v", err)
 	}
 }
 
 func TestFetchInstanceMetadata_BadDocument(t *testing.T) {
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("t")) })
-	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
-	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
-	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("t"))
+	})
+	mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
+	mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
+	mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
 	srv := httptest.NewServer(mux)
 	t.Cleanup(srv.Close)
-	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err == nil {
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err == nil {
 		t.Fatal("expected parse error on bad identity document")
 	}
 }

cmd/ecs-agent/netns.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"hash/crc32"
+	"os"
 	"os/exec"
 	"strings"
 	"time"
@@ -21,6 +22,12 @@ const (
 	credEndpointCIDR = defaultCredEndpointIP + "/32"
 )
 
+// udhcpcScript is the config script busybox udhcpc execs to apply a lease. It
+// is the reason presence of the udhcpc binary is not enough to use it: Debian
+// and Ubuntu's busybox ship the applet without this script, and udhcpc then
+// takes the lease and exits nonzero having configured nothing.
+const udhcpcScript = "/usr/share/udhcpc/default.script"
+
 // netCmdRunner executes a host networking command. Abstracted so unit tests can
 // assert the ip/netns sequence without touching the kernel.
 type netCmdRunner interface {
@@ -43,10 +50,47 @@ type taskNetns struct {
 	run     netCmdRunner
 	nicWait time.Duration
 	poll    time.Duration
+	// Injected so the DHCP-client choice below is unit-testable on a host that
+	// has neither client installed.
+	lookPath   func(string) (string, error)
+	fileIsExec func(string) bool
 }
 
 func newTaskNetns(run netCmdRunner) *taskNetns {
-	return &taskNetns{run: run, nicWait: 30 * time.Second, poll: time.Second}
+	return &taskNetns{
+		run:        run,
+		nicWait:    30 * time.Second,
+		poll:       time.Second,
+		lookPath:   exec.LookPath,
+		fileIsExec: func(p string) bool { fi, err := os.Stat(p); return err == nil && fi.Mode()&0o111 != 0 },
+	}
+}
+
+// dhcpStep returns the in-netns DHCP command that leases the task ENI's
+// OVN-assigned address.
+//
+// This image ships on two distributions and they do not agree on a DHCP client.
+// Alpine has busybox udhcpc; Ubuntu has no udhcpc at all, and `ip netns exec`
+// exits 1 — not 127 — when it cannot exec the binary, so a hardcoded udhcpc
+// fails as an opaque "network setup failed" with the task stuck in STOPPED.
+// Resolve the client at setup time instead, preferring udhcpc so Alpine keeps
+// its existing behaviour exactly, then falling back to dhcpcd. Same order and
+// same reasoning as dhcp_oneshot() in mulga-mgmt-net.sh.
+//
+// Both invocations are one-shot: udhcpc -q exits once it has the lease and -n
+// gives up rather than backgrounding, and dhcpcd -1 exits after configuring the
+// interface. Neither leaves a daemon behind in the netns.
+func (n *taskNetns) dhcpStep(name, iface string) ([]string, error) {
+	if _, err := n.lookPath("udhcpc"); err == nil && n.fileIsExec(udhcpcScript) {
+		return []string{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"}, nil
+	}
+	// Lease state is keyed by interface name, which is unique per task here:
+	// each task ENI is a separate hot-plug, so it lands on its own PCI slot and
+	// gets its own predictable name before being moved into the netns.
+	if _, err := n.lookPath("dhcpcd"); err == nil {
+		return []string{"ip", "netns", "exec", name, "dhcpcd", "-q", "-1", "-t", "20", iface}, nil
+	}
+	return nil, fmt.Errorf("no DHCP client for %s: need udhcpc (with %s) or dhcpcd", iface, udhcpcScript)
 }
 
 func netnsName(taskID string) string { return "ecs-" + taskID }
@@ -68,11 +112,16 @@ func (n *taskNetns) Setup(taskID, mac string) (string, error) {
 	}
 	hostVeth := credVethName(taskID)
 	hostIP, taskIP := credSubnet(taskID)
+	dhcp, err := n.dhcpStep(name, iface)
+	if err != nil {
+		_ = n.Teardown(taskID)
+		return "", err
+	}
 	steps := [][]string{
 		{"ip", "link", "set", iface, "netns", name},
 		{"ip", "-n", name, "link", "set", "lo", "up"},
 		{"ip", "-n", name, "link", "set", iface, "up"},
-		{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"},
+		dhcp,
 		// Credentials path: a veth into the host netns plus a host-routed /32 to the
 		// credential endpoint, so the container reaches 169.254.170.2 without it
 		// leaking onto the task ENI's VPC route.

cmd/ecs-agent/netns_test.go

@@ -55,10 +55,27 @@ const linkWithENI = `1: lo: <LOOPBACK,UP> mtu 65536 ... link/loopback 00:00:00:0
 2: eth0: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:aa:bb:cc ...
 3: eth1: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:de:ad:01 ...`
 
+// withDHCPClients pins which DHCP clients the netns sees, so a test asserts the
+// branch it names rather than whatever happens to be installed on the machine
+// running it. Without this the DHCP step is host-dependent: an Alpine CI box and
+// an Ubuntu dev box would exercise different code from the same test.
+func withDHCPClients(n *taskNetns, present map[string]bool, scriptOK bool) {
+	n.lookPath = func(f string) (string, error) {
+		if present[f] {
+			return "/sbin/" + f, nil
+		}
+		return "", errors.New("not found")
+	}
+	n.fileIsExec = func(string) bool { return scriptOK }
+}
+
+// newTestNetns defaults to the Alpine arrangement — udhcpc plus its config
+// script — so the sequence assertions below keep describing that platform.
 func newTestNetns(f *fakeNetRunner) *taskNetns {
 	n := newTaskNetns(f)
 	n.nicWait = 50 * time.Millisecond
 	n.poll = 5 * time.Millisecond
+	withDHCPClients(n, map[string]bool{"udhcpc": true}, true)
 	return n
 }
 
@@ -142,6 +159,79 @@ func TestNetns_SetupTearsDownOnStepFailure(t *testing.T) {
 	}
 }
 
+// The image runs on two distributions with different DHCP clients, and picking
+// the wrong one fails as an opaque "network setup failed" with the task stuck
+// in STOPPED — `ip netns exec` exits 1, not 127, when it cannot exec the
+// binary. Each branch is asserted so that misresolution is a test failure.
+func TestNetns_DHCPStepPerPlatform(t *testing.T) {
+	tests := []struct {
+		name     string
+		present  map[string]bool
+		scriptOK bool
+		want     string
+		wantErr  bool
+	}{
+		{
+			name:     "alpine: udhcpc with its config script",
+			present:  map[string]bool{"udhcpc": true, "dhcpcd": true},
+			scriptOK: true,
+			want:     "ip netns exec ecs-t-001 udhcpc -i eth1 -q -n",
+		},
+		{
+			// Ubuntu ships no udhcpc at all — the case that broke awsvpc tasks.
+			name:    "ubuntu: no udhcpc, falls back to dhcpcd",
+			present: map[string]bool{"dhcpcd": true},
+			want:    "ip netns exec ecs-t-001 dhcpcd -q -1 -t 20 eth1",
+		},
+		{
+			// Debian/Ubuntu busybox ships the applet without the script, so
+			// presence of the binary alone must not select it.
+			name:     "udhcpc present but config script missing: still dhcpcd",
+			present:  map[string]bool{"udhcpc": true, "dhcpcd": true},
+			scriptOK: false,
+			want:     "ip netns exec ecs-t-001 dhcpcd -q -1 -t 20 eth1",
+		},
+		{
+			name:    "neither client: explicit error, not a silent bad command",
+			present: map[string]bool{},
+			wantErr: true,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			n := newTestNetns(&fakeNetRunner{linkOut: linkWithENI})
+			withDHCPClients(n, tc.present, tc.scriptOK)
+			got, err := n.dhcpStep("ecs-t-001", "eth1")
+			if tc.wantErr {
+				if err == nil {
+					t.Fatalf("want error with no DHCP client, got %v", got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("dhcpStep: %v", err)
+			}
+			if joined := strings.Join(got, " "); joined != tc.want {
+				t.Errorf("want %q, got %q", tc.want, joined)
+			}
+		})
+	}
+}
+
+// A missing DHCP client must not leave the half-built netns behind for the next
+// attempt to trip over, same as any other failed setup step.
+func TestNetns_SetupTearsDownWhenNoDHCPClient(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	withDHCPClients(n, map[string]bool{}, false)
+	if _, err := n.Setup("t-001", "52:54:00:de:ad:01"); err == nil {
+		t.Fatal("want error when no DHCP client is available")
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected teardown, got %v", f.joined())
+	}
+}
+
 func TestNetns_Teardown(t *testing.T) {
 	f := &fakeNetRunner{}
 	n := newTestNetns(f)

cmd/ecs-agent/reconcile.go

@@ -8,12 +8,18 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
-// reconcile re-adopts containers that survived an agent restart. It lists the
-// runtime's containers and, for each task still running in this cluster,
-// re-registers its IAM credentials, re-attaches the exit-wait, and refreshes its
-// RUNNING state on the gateway. The returned set seeds pollAssignments so the
+// reconcile re-syncs the gateway with what this instance is actually running
+// after an agent restart. It lists the runtime's containers and, for each task
+// in this cluster, either re-adopts it (still running: re-register IAM
+// credentials, re-attach the exit-wait, refresh RUNNING) or reports it STOPPED
+// and clears its leftovers. The returned set seeds pollAssignments so the
 // gateway's re-delivered assignments for these tasks are acked but not re-run.
 //
+// Reporting the dead ones is the load-bearing half. The exit-wait is cancelled
+// when the agent shuts down, so a container that dies while the agent is down
+// has nobody to report its exit; the gateway holds the task RUNNING forever,
+// its service never replaces it, and the target never recovers.
+//
 // It is best-effort: a nil runner or a List error returns an empty set and the
 // agent falls back to its pre-reconcile behaviour (poll + run).
 func (a *Agent) reconcile(ctx context.Context) map[string]bool {
@@ -27,15 +33,17 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 		return adopted
 	}
 
-	// Group this cluster's running containers back into their tasks. The Assign is
-	// reconstructed from labels — enough to re-register creds and tear down netns.
+	// Group this cluster's containers back into their tasks, running or not. The
+	// Assign is reconstructed from labels — enough to re-register creds and tear
+	// down netns.
 	type task struct {
-		as         *bus.Assign
-		containers []ctrruntime.Container
+		as      *bus.Assign
+		running []ctrruntime.Container
+		dead    []ctrruntime.Container
 	}
 	tasks := map[string]*task{}
 	for _, c := range containers {
-		if !c.Running || c.Labels[labelClusterName] != a.id.ClusterName {
+		if c.Labels[labelClusterName] != a.id.ClusterName {
 			continue
 		}
 		taskID := c.Labels[labelTaskID]
@@ -53,20 +61,50 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 			}}
 			tasks[taskID] = t
 		}
-		t.containers = append(t.containers, c)
+		if c.Running {
+			t.running = append(t.running, c)
+		} else {
+			t.dead = append(t.dead, c)
+		}
 	}
 
 	for taskID, t := range tasks {
+		// A task with nothing running is over, whatever the gateway still believes.
+		// Mixed running/dead is left to the running path: the exit-wait re-attached
+		// below reports the individual container when it is reaped.
+		if len(t.running) == 0 {
+			a.reportStoppedTask(ctx, t.as, t.dead)
+			adopted[taskID] = true
+			continue
+		}
+
 		a.registerTaskCreds(t.as)
-		statuses := make([]bus.ContainerStatus, 0, len(t.containers))
-		for _, c := range t.containers {
+		statuses := make([]bus.ContainerStatus, 0, len(t.running))
+		for _, c := range t.running {
 			name := c.Labels[labelContainerName]
 			go a.waitContainer(ctx, t.as, name, c.ID)
 			statuses = append(statuses, bus.ContainerStatus{Name: name, Status: bus.TaskStatusRunning, ContainerID: c.ID})
 		}
 		a.reportTaskState(t.as, bus.TaskStatusRunning, "", statuses)
 		adopted[taskID] = true
-		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.containers))
+		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.running))
 	}
 	return adopted
 }
+
+// reportStoppedTask clears a task's leftover containers and tells the gateway it
+// is STOPPED, so the scheduler stops counting a task this instance is not
+// running and its service can place a replacement.
+func (a *Agent) reportStoppedTask(ctx context.Context, as *bus.Assign, dead []ctrruntime.Container) {
+	statuses := make([]bus.ContainerStatus, 0, len(dead))
+	for _, c := range dead {
+		if err := a.runner.Remove(ctx, c.ID); err != nil {
+			slog.Warn("ecs-agent: reconcile remove failed", "task", as.TaskID, "container", c.ID, "err", err)
+		}
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
+		})
+	}
+	a.reportTaskState(as, bus.TaskStatusStopped, "Container exited while the agent was not running", statuses)
+	slog.Info("ecs-agent: reported stopped task", "task", as.TaskID, "containers", len(dead))
+}

cmd/ecs-agent/reconcile_test.go

@@ -72,6 +72,63 @@ func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
 	waitedFor(t, rt, containerID("t-001", "web"))
 }
 
+// A task whose container died while the agent was down must be reported STOPPED,
+// not silently skipped. The exit-wait is cancelled on agent shutdown, so nothing
+// else ever reports that exit: skipping it leaves the gateway holding the task
+// RUNNING forever, its service never places a replacement, and the load
+// balancer target stays unhealthy with no path back.
+func TestReconcile_ReportsTaskWhoseContainerDied(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-dead", "web", "default", "t-dead", testTaskRole, "52:54:00:de:ad:02", false),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusStopped); n != 1 {
+		t.Errorf("STOPPED reports for t-dead = %d, want 1", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusRunning); n != 0 {
+		t.Errorf("t-dead must not be reported RUNNING, got %d such reports", n)
+	}
+	// Seeded so a re-delivered assignment for the dead task is acked, not re-run.
+	if !adopted["t-dead"] {
+		t.Errorf("t-dead not seeded into the adopted set; got %v", adopted)
+	}
+	if want := containerID("t-dead", "web"); !slices.Contains(rt.Removed, want) {
+		t.Errorf("leftover container %s not removed; removed=%v", want, rt.Removed)
+	}
+}
+
+// A task with one live and one dead container is still running: the live half is
+// re-adopted and the re-attached exit-wait reports the dead one when it is
+// reaped, so reconcile must not declare the whole task STOPPED.
+func TestReconcile_PartiallyDeadTaskStaysRunning(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-mixed", "web", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", true),
+			adoptedContainer("t-mixed", "sidecar", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", false),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	a.reconcile(context.Background())
+
+	if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("RUNNING reports for t-mixed = %d, want 1", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusStopped); n != 0 {
+		t.Errorf("t-mixed must not be reported STOPPED while a container still runs, got %d", n)
+	}
+}
+
 // pollAssignments seeded with an adopted task acks its re-delivered assignment
 // but does not re-run it, while a genuinely new task still dispatches.
 func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
@@ -123,23 +180,35 @@ func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
 	}
 }
 
-// Containers for another cluster or with a stopped task are not adopted.
-func TestReconcile_FiltersOtherClusterAndStopped(t *testing.T) {
+// Containers belonging to another cluster are ignored entirely: this agent
+// speaks for its own cluster only, and must neither adopt nor report them.
+//
+// A stopped container of this cluster IS handled — see
+// TestReconcile_ReportsTaskWhoseContainerDied. It lands in the adopted set so
+// its re-delivered assignment is acked rather than re-run, after reconcile has
+// reported the task STOPPED.
+func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
+	cp := &fakeCP{}
 	rt := &ctrruntime.FakePuller{
 		WaitErr: errors.New("blocked"),
 		Containers: []ctrruntime.Container{
 			adoptedContainer("t-other", "web", "other", "", "", "", true),
-			adoptedContainer("t-stop", "web", "default", "", "", "", false),
+			adoptedContainer("t-other-dead", "web", "other", "", "", "", false),
 			adoptedContainer("t-ok", "web", "default", "", "", "", true),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
 
 	adopted := a.reconcile(context.Background())
 
 	if len(adopted) != 1 || !adopted["t-ok"] {
 		t.Fatalf("adopted = %v, want only t-ok", adopted)
 	}
+	for _, id := range []string{"t-other", "t-other-dead"} {
+		if n := countStatus(cp.taskStates(), id, bus.TaskStatusStopped); n != 0 {
+			t.Errorf("reported %d STOPPED states for another cluster's task %s, want 0", n, id)
+		}
+	}
 }
 
 // A nil runner or a List error degrades to an empty set without panicking.

cmd/ecs-agent/taskrun_test.go

@@ -70,8 +70,8 @@ func TestRunTask_PullRunReportsRunning(t *testing.T) {
 	}
 }
 
-// TestRunTask_GPUCarriedToRunSpec covers mulga-11opz (Epic C task C1): the
-// AssignContainer's GPU count reaches the runtime RunSpec unchanged; a
+// TestRunTask_GPUCarriedToRunSpec verifies that the AssignContainer's GPU count
+// reaches the runtime RunSpec unchanged; a
 // container with no GPU request keeps RunSpec.GPU at zero (regression).
 func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
 	cp := &fakeCP{}
@@ -534,7 +534,7 @@ func (sentinelResolver) Authorize(context.Context, string) (string, string, stri
 	return "", "", "", nil
 }
 
-// TestPullResolver covers siv-459: an assign with no execution role (or no
+// TestPullResolver verifies that an assign with no execution role (or no
 // credential endpoint) uses the instance resolver; with both, a distinct
 // exec-role resolver is built.
 func TestPullResolver(t *testing.T) {

cmd/installer/autoinstall/autoinstall.go

@@ -4,12 +4,14 @@
 package autoinstall
 
 import (
+	"cmp"
 	"fmt"
 	"log/slog"
 	"net"
 	"os"
 	"os/exec"
-	"sort"
+	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"syscall"
@@ -18,6 +20,10 @@ import (
 	"github.com/mulgadc/spinifex/cmd/installer/install"
 )
 
+// listDisks is the block-device scan, indirected so tests can drive selection
+// without real hardware.
+var listDisks = install.ListDisks
+
 // Load returns an install.Config built from SPINIFEX_* environment variables,
 // or nil if SPINIFEX_AUTO is not set to "1" (interactive mode).
 func Load() (*install.Config, error) {
@@ -65,9 +71,9 @@ func buildConfig() (*install.Config, error) {
 		hostname = "spinifex-node"
 	}
 
-	disk, err := resolveDisk(os.Getenv("SPINIFEX_DISK"))
+	storage, err := resolveStorage()
 	if err != nil {
-		return nil, fmt.Errorf("disk: %w", err)
+		return nil, fmt.Errorf("storage: %w", err)
 	}
 
 	wanIface, err := resolveNIC(os.Getenv("SPINIFEX_WAN_IFACE"), "")
@@ -76,73 +82,217 @@ func buildConfig() (*install.Config, error) {
 	}
 
 	cfg := &install.Config{
-		Disk:         disk,
+		Storage:      storage,
 		Hostname:     hostname,
 		RootPassword: password,
 		// Optional on the headless path — empty means no --email passed to
 		// spx admin init, which omits the operator identity from telemetry.
-		Email:        strings.TrimSpace(os.Getenv("SPINIFEX_EMAIL")),
-		WANInterface: wanIface,
-		WANDHCPMode:  strings.ToLower(os.Getenv("SPINIFEX_WAN_MODE")) != "static",
+		Email: strings.TrimSpace(os.Getenv("SPINIFEX_EMAIL")),
 	}
 
-	if !cfg.WANDHCPMode {
-		ip := os.Getenv("SPINIFEX_WAN_IP")
-		mask := os.Getenv("SPINIFEX_WAN_MASK")
-		gw := os.Getenv("SPINIFEX_WAN_GW")
-		if ip == "" || mask == "" || gw == "" {
-			return nil, fmt.Errorf("SPINIFEX_WAN_IP, SPINIFEX_WAN_MASK, SPINIFEX_WAN_GW required for static mode")
-		}
-		cfg.WANAddress = ip
-		cfg.WANMask = mask
-		cfg.WANGateway = gw
-		if dns := os.Getenv("SPINIFEX_WAN_DNS"); dns != "" {
-			cfg.WANDNS = strings.Split(dns, ",")
-		}
+	cfg.WAN, err = parseRole("WAN", wanIface)
+	if err != nil {
+		return nil, err
+	}
+	if cfg.WAN.DHCPMode {
+		cfg.WAN.Gateway = ""
+	} else if cfg.WAN.Gateway == "" {
+		return nil, fmt.Errorf("SPINIFEX_WAN_IP, SPINIFEX_WAN_MASK, SPINIFEX_WAN_GW required for static mode")
 	}
 
-	if lanIface := os.Getenv("SPINIFEX_LAN_IFACE"); lanIface != "" {
-		lan, err := resolveNIC(lanIface, wanIface)
+	// lan and vpc are optional: an unset _IFACE leaves the role folded, which
+	// is how a single- or two-NIC node collapses onto the planes above it.
+	for _, r := range []struct {
+		plane string
+		dst   *install.NetworkRole
+	}{{"LAN", &cfg.LAN}, {"VPC", &cfg.VPC}} {
+		name := os.Getenv("SPINIFEX_" + r.plane + "_IFACE")
+		if name == "" {
+			continue
+		}
+		iface, err := resolveNIC(name, wanIface)
 		if err != nil {
-			return nil, fmt.Errorf("LAN NIC: %w", err)
+			return nil, fmt.Errorf("%s NIC: %w", r.plane, err)
 		}
-		cfg.LANInterface = lan
-		cfg.LANDHCPMode = strings.ToLower(os.Getenv("SPINIFEX_LAN_MODE")) != "static"
-		if !cfg.LANDHCPMode {
-			cfg.LANAddress = os.Getenv("SPINIFEX_LAN_IP")
-			cfg.LANMask = os.Getenv("SPINIFEX_LAN_MASK")
-			if dns := os.Getenv("SPINIFEX_LAN_DNS"); dns != "" {
-				cfg.LANDNS = strings.Split(dns, ",")
-			}
+		if *r.dst, err = parseRole(r.plane, iface); err != nil {
+			return nil, err
 		}
+		// Only the wan plane installs a default route.
+		r.dst.Gateway = ""
+	}
+
+	if err := cfg.Validate(); err != nil {
+		return nil, fmt.Errorf("network roles: %w", err)
 	}
 
 	if os.Getenv("SPINIFEX_GPU_PASSTHROUGH") == "1" {
 		cfg.GPUPassthrough = true
 	}
 
-	role := strings.ToLower(os.Getenv("SPINIFEX_ROLE"))
-	if role == "" {
-		role = "init"
-	}
-	cfg.ClusterRole = role
-	skipFormation := os.Getenv("SPINIFEX_SKIP_FORMATION") == "1"
-	if skipFormation {
+	if os.Getenv("SPINIFEX_SKIP_FORMATION") == "1" {
 		cfg.SkipFormation = true
 	}
 
-	if role == "join" && !skipFormation {
-		joinAddr := os.Getenv("SPINIFEX_JOIN_ADDR")
-		if joinAddr == "" {
-			return nil, fmt.Errorf("SPINIFEX_JOIN_ADDR required when SPINIFEX_ROLE=join")
-		}
-		cfg.JoinAddr = joinAddr
-	}
-
 	return cfg, nil
 }
 
-// resolveDisk maps the spx_disk value from grub.cfg to a block device path.
+// resolveStorage builds the disk configuration from the kernel cmdline.
+//
+//	SPINIFEX_FS               ext4 (default) or one of the zfs-* topologies
+//	SPINIFEX_DISKS            explicit ordered member list, required for multi-disk pools
+//	SPINIFEX_DISK             the OS disk, as before
+//	SPINIFEX_DISK_SPINIFEX    ext4 only: dedicated disk for /var/lib/spinifex
+//	SPINIFEX_DISK_PREDASTORE  ext4 only: dedicated disk for /var/lib/spinifex/predastore
+func resolveStorage() (install.DiskConfig, error) {
+	cfg := install.DiskConfig{FS: install.FSExt4}
+	if v := strings.TrimSpace(os.Getenv("SPINIFEX_FS")); v != "" {
+		fs, err := install.ParseFSType(v)
+		if err != nil {
+			return cfg, err
+		}
+		cfg.FS = fs
+	}
+
+	var err error
+	switch list := strings.TrimSpace(os.Getenv("SPINIFEX_DISKS")); {
+	case list != "":
+		if cfg.Disks, err = disksByName(strings.Split(list, ",")); err != nil {
+			return cfg, err
+		}
+	case cfg.FS.MinDisks() > 1:
+		// No auto-selection for a multi-disk pool. Choosing which disks to
+		// erase is not a decision an unattended install gets to make, and the
+		// member order determines the RAID10 pairing.
+		return cfg, fmt.Errorf("%s needs at least %d disks — set SPINIFEX_DISKS to an explicit comma-separated list",
+			cfg.FS.Label(), cfg.FS.MinDisks())
+	default:
+		d, derr := resolveDisk(os.Getenv("SPINIFEX_DISK"))
+		if derr != nil {
+			return cfg, derr
+		}
+		cfg.Disks = []install.Disk{d}
+	}
+
+	if err := applyDiskRoles(&cfg); err != nil {
+		return cfg, err
+	}
+	if cfg.ZFS, err = parseZFSOpts(); err != nil {
+		return cfg, err
+	}
+	if err := cfg.Validate(); err != nil {
+		return cfg, err
+	}
+	slog.Info("autoinstall: storage selected", "fs", cfg.FS, "disks", cfg.Paths())
+	return cfg, nil
+}
+
+// roleVars maps each optional data role to the variable that assigns it. The OS
+// role is not here: it always comes from SPINIFEX_DISK.
+var roleVars = []struct {
+	role install.DiskRole
+	env  string
+}{
+	{install.RoleSpinifex, "SPINIFEX_DISK_SPINIFEX"},
+	{install.RolePredastore, "SPINIFEX_DISK_PREDASTORE"},
+}
+
+// applyDiskRoles attaches dedicated data drives to an ext4 install. Roles are
+// always set on ext4, even with no extra drives, so the OS assignment is
+// explicit rather than implied by position.
+func applyDiskRoles(cfg *install.DiskConfig) error {
+	if cfg.FS.IsZFS() {
+		for _, rv := range roleVars {
+			if strings.TrimSpace(os.Getenv(rv.env)) != "" {
+				return fmt.Errorf("%s applies to ext4 only — %s spans every member already", rv.env, cfg.FS.Label())
+			}
+		}
+		return nil
+	}
+	if len(cfg.Disks) != 1 {
+		return fmt.Errorf("ext4 takes one OS disk, %d given — set SPINIFEX_DISK, and SPINIFEX_DISK_SPINIFEX or SPINIFEX_DISK_PREDASTORE for dedicated drives",
+			len(cfg.Disks))
+	}
+
+	roles := []install.RoleMount{{Role: install.RoleOS, Disk: cfg.Disks[0]}}
+	for _, rv := range roleVars {
+		spec := strings.TrimSpace(os.Getenv(rv.env))
+		if spec == "" {
+			continue
+		}
+		// Resolved with the same fail-closed selector as the OS disk, so an
+		// ambiguous value lists the candidates rather than picking one.
+		d, err := resolveDisk(spec)
+		if err != nil {
+			return fmt.Errorf("%s: %w", rv.env, err)
+		}
+		roles = append(roles, install.RoleMount{Role: rv.role, Disk: d})
+	}
+	*cfg = cfg.WithRoles(roles)
+	return nil
+}
+
+// parseZFSOpts reads the advanced pool tunables. Every one is optional; unset
+// values are computed from the hardware at install time.
+func parseZFSOpts() (install.ZFSOpts, error) {
+	var o install.ZFSOpts
+	ints := []struct {
+		env string
+		dst *int
+	}{
+		{"SPINIFEX_ZFS_ASHIFT", &o.Ashift},
+		{"SPINIFEX_ZFS_COPIES", &o.Copies},
+		{"SPINIFEX_ZFS_ARC_MAX_MIB", &o.ARCMaxMiB},
+	}
+	for _, f := range ints {
+		v := strings.TrimSpace(os.Getenv(f.env))
+		if v == "" {
+			continue
+		}
+		n, err := strconv.Atoi(v)
+		if err != nil {
+			return o, fmt.Errorf("%s: %w", f.env, err)
+		}
+		*f.dst = n
+	}
+	o.Compress = strings.TrimSpace(os.Getenv("SPINIFEX_ZFS_COMPRESS"))
+	o.Checksum = strings.TrimSpace(os.Getenv("SPINIFEX_ZFS_CHECKSUM"))
+	return o, nil
+}
+
+// disksByName resolves an explicit member list. Names may be kernel names
+// ("sdb"), device paths or by-id paths; the order given is preserved because
+// RAID10 pairs members in it.
+func disksByName(names []string) ([]install.Disk, error) {
+	available, err := listDisks()
+	if err != nil {
+		return nil, err
+	}
+	out := make([]install.Disk, 0, len(names))
+	for _, raw := range names {
+		name := strings.TrimSpace(raw)
+		if name == "" {
+			continue
+		}
+		// A by-id path or a symlink resolves to the kernel device the scan
+		// reported, so both spellings name the same disk.
+		resolved := name
+		if !strings.HasPrefix(resolved, "/dev/") {
+			resolved = "/dev/" + resolved
+		}
+		if target, err := filepath.EvalSymlinks(resolved); err == nil {
+			resolved = target
+		}
+
+		idx := slices.IndexFunc(available, func(d install.Disk) bool { return d.Path == resolved })
+		if idx < 0 {
+			return nil, fmt.Errorf("%q is not an available disk — found:\n%s", name, diskList(available))
+		}
+		out = append(out, available[idx])
+	}
+	return out, nil
+}
+
+// resolveDisk maps the SPINIFEX_DISK value from grub.cfg to a disk.
 //
 // Supported values:
 //
@@ -151,141 +301,78 @@ func buildConfig() (*install.Config, error) {
 //	smallest        — smallest non-removable disk (typical OS-on-SSD pattern)
 //	nvme            — the only NVMe disk; fail if multiple found
 //	/dev/sda (etc.) — exact device path; fail if not present
-func resolveDisk(target string) (string, error) {
-	switch strings.ToLower(target) {
+func resolveDisk(target string) (install.Disk, error) {
+	disks, err := candidateDisks()
+	if err != nil {
+		return install.Disk{}, err
+	}
+	switch strings.ToLower(strings.TrimSpace(target)) {
 	case "", "auto":
-		return singleDisk()
-	case "largest":
-		return diskBySize(true)
-	case "smallest":
-		return diskBySize(false)
-	case "nvme":
-		return nvmeDisk()
-	default:
-		if _, err := os.Stat(target); err != nil {
-			return "", fmt.Errorf("%q not found", target)
+		if len(disks) != 1 {
+			return install.Disk{}, fmt.Errorf(
+				"expected exactly one disk, found %d — set SPINIFEX_DISK to one of:\n%s\n"+
+					"  largest   (largest disk)\n"+
+					"  smallest  (smallest disk)\n"+
+					"  nvme      (NVMe only)\n"+
+					"  /dev/sdX  (exact path)",
+				len(disks), diskList(disks))
 		}
-		return target, nil
+		return disks[0], nil
+	case "largest":
+		return pickBySize(disks, true)
+	case "smallest":
+		return pickBySize(disks, false)
+	case "nvme":
+		nvmes := slices.DeleteFunc(slices.Clone(disks), func(d install.Disk) bool {
+			return !strings.HasPrefix(filepath.Base(d.Path), "nvme")
+		})
+		if len(nvmes) != 1 {
+			return install.Disk{}, fmt.Errorf("expected exactly one NVMe disk, found %d:\n%s",
+				len(nvmes), diskList(nvmes))
+		}
+		return nvmes[0], nil
+	default:
+		found, err := disksByName([]string{target})
+		if err != nil {
+			return install.Disk{}, err
+		}
+		return found[0], nil
 	}
 }
 
-type diskCandidate struct {
-	dev   string
-	bytes int64
-}
-
-// nonRemovableDisks returns all non-removable, non-virtual block devices.
-func nonRemovableDisks() ([]diskCandidate, error) {
-	entries, err := os.ReadDir("/sys/block")
+// candidateDisks lists the disks an unattended install may erase: fixed, and
+// not the installer's own media.
+func candidateDisks() ([]install.Disk, error) {
+	all, err := listDisks()
 	if err != nil {
 		return nil, err
 	}
-	var out []diskCandidate
-	for _, e := range entries {
-		dev := e.Name()
-		switch {
-		case strings.HasPrefix(dev, "loop"),
-			strings.HasPrefix(dev, "ram"),
-			strings.HasPrefix(dev, "dm-"),
-			strings.HasPrefix(dev, "sr"):
-			continue
-		}
-		removable, _ := os.ReadFile("/sys/block/" + dev + "/removable")
-		if strings.TrimSpace(string(removable)) == "1" {
-			continue
-		}
-		sizeRaw, _ := os.ReadFile("/sys/block/" + dev + "/size")
-		sectors, parseErr := strconv.ParseInt(strings.TrimSpace(string(sizeRaw)), 10, 64)
-		if parseErr != nil || sectors == 0 {
-			continue
-		}
-		out = append(out, diskCandidate{dev: dev, bytes: sectors * 512})
-	}
-	return out, nil
+	return slices.DeleteFunc(all, func(d install.Disk) bool {
+		return d.Removable || d.LiveMedia
+	}), nil
 }
 
-// diskList returns a human-readable list of candidates for error messages.
-func diskList(disks []diskCandidate) string {
+func pickBySize(disks []install.Disk, largest bool) (install.Disk, error) {
+	if len(disks) == 0 {
+		return install.Disk{}, fmt.Errorf("no non-removable disks found")
+	}
+	sorted := slices.Clone(disks)
+	slices.SortFunc(sorted, func(a, b install.Disk) int { return cmp.Compare(a.Bytes, b.Bytes) })
+	if largest {
+		return sorted[len(sorted)-1], nil
+	}
+	return sorted[0], nil
+}
+
+// diskList renders candidates for an error message.
+func diskList(disks []install.Disk) string {
 	lines := make([]string, len(disks))
 	for i, d := range disks {
-		lines[i] = fmt.Sprintf("  /dev/%s (%dG)", d.dev, d.bytes>>30)
+		lines[i] = fmt.Sprintf("  %s (%s, %s)", d.Path, d.SizeHuman(), d.Content)
 	}
 	return strings.Join(lines, "\n")
 }
 
-// singleDisk selects the only non-removable disk, or fails listing all found.
-func singleDisk() (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	switch len(disks) {
-	case 0:
-		return "", fmt.Errorf("no non-removable disks found")
-	case 1:
-		slog.Info("autoinstall: single disk selected", "disk", disks[0].dev, "size_gb", disks[0].bytes>>30)
-		return "/dev/" + disks[0].dev, nil
-	default:
-		return "", fmt.Errorf(
-			"multiple disks found — set SPINIFEX_DISK to one of:\n%s\n"+
-				"  largest   (largest disk)\n"+
-				"  smallest  (smallest disk)\n"+
-				"  nvme      (NVMe only)\n"+
-				"  /dev/sdX  (exact path)",
-			diskList(disks),
-		)
-	}
-}
-
-// diskBySize selects the largest (largest=true) or smallest non-removable disk.
-func diskBySize(largest bool) (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	if len(disks) == 0 {
-		return "", fmt.Errorf("no non-removable disks found")
-	}
-	sort.Slice(disks, func(i, j int) bool {
-		if largest {
-			return disks[i].bytes > disks[j].bytes
-		}
-		return disks[i].bytes < disks[j].bytes
-	})
-	label := "smallest"
-	if largest {
-		label = "largest"
-	}
-	slog.Info("autoinstall: disk selected by size", "mode", label, "disk", disks[0].dev, "size_gb", disks[0].bytes>>30)
-	return "/dev/" + disks[0].dev, nil
-}
-
-// nvmeDisk selects the only NVMe disk, or fails listing all NVMe disks found.
-func nvmeDisk() (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	var nvmes []diskCandidate
-	for _, d := range disks {
-		if strings.HasPrefix(d.dev, "nvme") {
-			nvmes = append(nvmes, d)
-		}
-	}
-	switch len(nvmes) {
-	case 0:
-		return "", fmt.Errorf("no NVMe disks found")
-	case 1:
-		slog.Info("autoinstall: NVMe disk selected", "disk", nvmes[0].dev, "size_gb", nvmes[0].bytes>>30)
-		return "/dev/" + nvmes[0].dev, nil
-	default:
-		return "", fmt.Errorf(
-			"multiple NVMe disks found — set SPINIFEX_DISK to one of:\n%s",
-			diskList(nvmes),
-		)
-	}
-}
-
 // virtualNICPrefixes lists interface name prefixes that identify non-physical
 // interfaces (bridges, tunnels, container/OVS veth pairs, etc.). These are
 // skipped when auto-selecting a NIC so we don't configure docker0 or
@@ -297,6 +384,42 @@ var virtualNICPrefixes = []string{
 
 // resolveNIC returns the interface name to use. "auto" or empty picks the
 // first physical (non-loopback, non-virtual) interface that is not exclude.
+// parseRole reads the SPINIFEX_<PLANE>_* kernel-cmdline vars for one plane.
+// iface is the already-resolved interface name; _VLAN and _MTU are optional
+// and default to untagged at the link MTU.
+func parseRole(plane, iface string) (install.NetworkRole, error) {
+	role := install.NetworkRole{
+		Interface: iface,
+		DHCPMode:  strings.ToLower(os.Getenv("SPINIFEX_"+plane+"_MODE")) != "static",
+	}
+	if v := os.Getenv("SPINIFEX_" + plane + "_VLAN"); v != "" {
+		id, err := strconv.Atoi(v)
+		if err != nil {
+			return role, fmt.Errorf("SPINIFEX_%s_VLAN: %w", plane, err)
+		}
+		role.VLAN = id
+	}
+	if v := os.Getenv("SPINIFEX_" + plane + "_MTU"); v != "" {
+		mtu, err := strconv.Atoi(v)
+		if err != nil {
+			return role, fmt.Errorf("SPINIFEX_%s_MTU: %w", plane, err)
+		}
+		role.MTU = mtu
+	}
+	if !role.DHCPMode {
+		role.Address = os.Getenv("SPINIFEX_" + plane + "_IP")
+		role.Mask = os.Getenv("SPINIFEX_" + plane + "_MASK")
+		role.Gateway = os.Getenv("SPINIFEX_" + plane + "_GW")
+		if role.Address == "" || role.Mask == "" {
+			return role, fmt.Errorf("SPINIFEX_%s_IP and SPINIFEX_%s_MASK required for static mode", plane, plane)
+		}
+	}
+	if dns := os.Getenv("SPINIFEX_" + plane + "_DNS"); dns != "" {
+		role.DNS = strings.Split(dns, ",")
+	}
+	return role, nil
+}
+
 func resolveNIC(name, exclude string) (string, error) {
 	if name != "" && name != "auto" {
 		return name, nil

cmd/installer/autoinstall/autoinstall_test.go

@@ -1,12 +1,57 @@
 package autoinstall
 
 import (
+	"errors"
+	"fmt"
+	"maps"
+	"net"
 	"os"
+	"slices"
+	"strings"
 	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
 )
 
+const gib = 1 << 30
+
+// setEnv clears every SPINIFEX_* variable inherited from the test runner before
+// applying want, so one case cannot leak a kernel-cmdline value into the next.
+func setEnv(t *testing.T, want map[string]string) {
+	t.Helper()
+	for _, kv := range os.Environ() {
+		if k, _, ok := strings.Cut(kv, "="); ok && strings.HasPrefix(k, "SPINIFEX_") {
+			t.Setenv(k, "")
+		}
+	}
+	for k, v := range want {
+		t.Setenv(k, v)
+	}
+}
+
+// withDisks replaces the block-device scan. The clone matters: candidateDisks
+// filters with slices.DeleteFunc, which would otherwise mutate the fixture.
+func withDisks(t *testing.T, disks []install.Disk, err error) {
+	t.Helper()
+	prev := listDisks
+	listDisks = func() ([]install.Disk, error) { return slices.Clone(disks), err }
+	t.Cleanup(func() { listDisks = prev })
+}
+
+func mkDisk(path string, bytes int64) install.Disk {
+	return install.Disk{
+		Path:              path,
+		ByID:              "/dev/disk/by-id/test-" + strings.TrimPrefix(path, "/dev/"),
+		Bytes:             bytes,
+		Model:             "TESTDISK",
+		LogicalBlockSize:  512,
+		PhysicalBlockSize: 512,
+		Content:           "empty",
+	}
+}
+
 func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
-	os.Unsetenv("SPINIFEX_AUTO")
+	setEnv(t, nil)
 	cfg, err := Load()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -15,3 +60,629 @@ func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
 		t.Fatal("expected nil config when SPINIFEX_AUTO is not set")
 	}
 }
+
+func TestLoadBuildsConfigFromEnv(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_AUTO":            "1",
+		"SPINIFEX_PASSWORD":        "s3cret",
+		"SPINIFEX_HOSTNAME":        "node-7",
+		"SPINIFEX_EMAIL":           "  ops@example.com  ",
+		"SPINIFEX_WAN_IFACE":       "eno1",
+		"SPINIFEX_WAN_DNS":         "1.1.1.1,8.8.8.8",
+		"SPINIFEX_LAN_IFACE":       "eno2",
+		"SPINIFEX_LAN_VLAN":        "100",
+		"SPINIFEX_LAN_MTU":         "9000",
+		"SPINIFEX_GPU_PASSTHROUGH": "1",
+	})
+
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.Hostname != "node-7" || cfg.RootPassword != "s3cret" {
+		t.Errorf("identity = %q/%q", cfg.Hostname, cfg.RootPassword)
+	}
+	if cfg.Email != "ops@example.com" {
+		t.Errorf("Email = %q, want trimmed", cfg.Email)
+	}
+	if cfg.Storage.FS != install.FSExt4 || len(cfg.Storage.Disks) != 1 {
+		t.Errorf("Storage = %v / %d disks", cfg.Storage.FS, len(cfg.Storage.Disks))
+	}
+	if cfg.WAN.Interface != "eno1" || !cfg.WAN.DHCPMode {
+		t.Errorf("WAN = %+v, want eno1 on DHCP", cfg.WAN)
+	}
+	if !slices.Equal(cfg.WAN.DNS, []string{"1.1.1.1", "8.8.8.8"}) {
+		t.Errorf("WAN.DNS = %v", cfg.WAN.DNS)
+	}
+	if cfg.LAN.Interface != "eno2" || cfg.LAN.VLAN != 100 || cfg.LAN.MTU != 9000 {
+		t.Errorf("LAN = %+v", cfg.LAN)
+	}
+	// Only the wan plane installs a default route.
+	if cfg.LAN.Gateway != "" {
+		t.Errorf("LAN.Gateway = %q, want cleared", cfg.LAN.Gateway)
+	}
+	if !cfg.VPC.Folded() {
+		t.Errorf("VPC = %+v, want folded when no _IFACE is set", cfg.VPC)
+	}
+	if !cfg.GPUPassthrough {
+		t.Error("GPUPassthrough not enabled")
+	}
+}
+
+func TestLoadDefaultsHostname(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_AUTO":      "1",
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+	})
+
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.Hostname != "spinifex-node" {
+		t.Errorf("Hostname = %q, want spinifex-node", cfg.Hostname)
+	}
+}
+
+func TestLoadRequiresPassword(t *testing.T) {
+	setEnv(t, map[string]string{"SPINIFEX_AUTO": "1"})
+
+	cfg, err := Load()
+	if err == nil {
+		t.Fatalf("expected error, got config %+v", cfg)
+	}
+	if !strings.Contains(err.Error(), "SPINIFEX_PASSWORD") {
+		t.Errorf("err = %v, want it to name SPINIFEX_PASSWORD", err)
+	}
+	if !strings.HasPrefix(err.Error(), "autoinstall config:") {
+		t.Errorf("err = %v, want the autoinstall wrapper", err)
+	}
+}
+
+func TestBuildConfigStaticWAN(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     map[string]string
+		wantErr string
+	}{
+		{
+			name: "full static",
+			env: map[string]string{
+				"SPINIFEX_WAN_MODE": "static",
+				"SPINIFEX_WAN_IP":   "10.0.0.5",
+				"SPINIFEX_WAN_MASK": "255.255.255.0",
+				"SPINIFEX_WAN_GW":   "10.0.0.1",
+			},
+		},
+		{
+			name: "missing gateway",
+			env: map[string]string{
+				"SPINIFEX_WAN_MODE": "static",
+				"SPINIFEX_WAN_IP":   "10.0.0.5",
+				"SPINIFEX_WAN_MASK": "255.255.255.0",
+			},
+			wantErr: "SPINIFEX_WAN_GW required",
+		},
+		{
+			name:    "missing address",
+			env:     map[string]string{"SPINIFEX_WAN_MODE": "static"},
+			wantErr: "SPINIFEX_WAN_IP and SPINIFEX_WAN_MASK required",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+			env := map[string]string{
+				"SPINIFEX_PASSWORD":  "s3cret",
+				"SPINIFEX_WAN_IFACE": "eno1",
+			}
+			maps.Copy(env, tt.env)
+			setEnv(t, env)
+
+			cfg, err := buildConfig()
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("buildConfig: %v", err)
+			}
+			if cfg.WAN.DHCPMode {
+				t.Error("WAN.DHCPMode = true, want static")
+			}
+			if cfg.WAN.Address != "10.0.0.5" || cfg.WAN.Gateway != "10.0.0.1" {
+				t.Errorf("WAN = %+v", cfg.WAN)
+			}
+		})
+	}
+}
+
+// SPINIFEX_ROLE and SPINIFEX_JOIN_ADDR no longer exist: the installer cannot
+// form a multi-node cluster, so a stale cmdline from an older ISO has to be
+// inert rather than producing a half-configured node.
+func TestBuildConfigSkipFormation(t *testing.T) {
+	tests := []struct {
+		name         string
+		env          map[string]string
+		wantSkipForm bool
+	}{
+		{
+			name:         "default initializes a single-node cluster",
+			wantSkipForm: false,
+		},
+		{
+			name:         "formation deferred to the provisioner",
+			env:          map[string]string{"SPINIFEX_SKIP_FORMATION": "1"},
+			wantSkipForm: true,
+		},
+		{
+			name:         "stale join cmdline is ignored",
+			env:          map[string]string{"SPINIFEX_ROLE": "join", "SPINIFEX_JOIN_ADDR": "10.0.0.1:9999"},
+			wantSkipForm: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+			env := map[string]string{
+				"SPINIFEX_PASSWORD":  "s3cret",
+				"SPINIFEX_WAN_IFACE": "eno1",
+			}
+			maps.Copy(env, tt.env)
+			setEnv(t, env)
+
+			cfg, err := buildConfig()
+			if err != nil {
+				t.Fatalf("buildConfig: %v", err)
+			}
+			if cfg.SkipFormation != tt.wantSkipForm {
+				t.Errorf("SkipFormation = %v, want %v", cfg.SkipFormation, tt.wantSkipForm)
+			}
+		})
+	}
+}
+
+func TestBuildConfigRejectsDuplicatePlaneBinding(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+		"SPINIFEX_LAN_IFACE": "eno1",
+	})
+
+	if _, err := buildConfig(); err == nil || !strings.Contains(err.Error(), "network roles") {
+		t.Fatalf("err = %v, want a network roles validation failure", err)
+	}
+}
+
+func TestBuildConfigPropagatesStorageError(t *testing.T) {
+	setEnv(t, map[string]string{
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+		"SPINIFEX_FS":        "btrfs",
+	})
+
+	_, err := buildConfig()
+	if err == nil || !strings.HasPrefix(err.Error(), "storage:") {
+		t.Fatalf("err = %v, want a storage: prefix", err)
+	}
+}
+
+func TestParseRole(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     map[string]string
+		want    install.NetworkRole
+		wantErr string
+	}{
+		{
+			name: "dhcp by default",
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true},
+		},
+		{
+			name: "vlan and mtu",
+			env:  map[string]string{"SPINIFEX_LAN_VLAN": "42", "SPINIFEX_LAN_MTU": "9000"},
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true, VLAN: 42, MTU: 9000},
+		},
+		{
+			name: "dns list is comma separated",
+			env:  map[string]string{"SPINIFEX_LAN_DNS": "1.1.1.1,9.9.9.9"},
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true, DNS: []string{"1.1.1.1", "9.9.9.9"}},
+		},
+		{
+			name: "static",
+			env: map[string]string{
+				"SPINIFEX_LAN_MODE": "STATIC",
+				"SPINIFEX_LAN_IP":   "192.168.1.2",
+				"SPINIFEX_LAN_MASK": "255.255.255.0",
+				"SPINIFEX_LAN_GW":   "192.168.1.1",
+			},
+			want: install.NetworkRole{
+				Interface: "eno1",
+				Address:   "192.168.1.2",
+				Mask:      "255.255.255.0",
+				Gateway:   "192.168.1.1",
+			},
+		},
+		{
+			name:    "static without mask",
+			env:     map[string]string{"SPINIFEX_LAN_MODE": "static", "SPINIFEX_LAN_IP": "192.168.1.2"},
+			wantErr: "SPINIFEX_LAN_IP and SPINIFEX_LAN_MASK required",
+		},
+		{
+			name:    "unparsable vlan",
+			env:     map[string]string{"SPINIFEX_LAN_VLAN": "trunk"},
+			wantErr: "SPINIFEX_LAN_VLAN",
+		},
+		{
+			name:    "unparsable mtu",
+			env:     map[string]string{"SPINIFEX_LAN_MTU": "jumbo"},
+			wantErr: "SPINIFEX_LAN_MTU",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			setEnv(t, tt.env)
+
+			got, err := parseRole("LAN", "eno1")
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("parseRole: %v", err)
+			}
+			if fmt.Sprintf("%+v", got) != fmt.Sprintf("%+v", tt.want) {
+				t.Errorf("role = %+v, want %+v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseZFSOpts(t *testing.T) {
+	t.Run("unset leaves the computed defaults", func(t *testing.T) {
+		setEnv(t, nil)
+		got, err := parseZFSOpts()
+		if err != nil {
+			t.Fatalf("parseZFSOpts: %v", err)
+		}
+		if got != (install.ZFSOpts{}) {
+			t.Errorf("opts = %+v, want zero", got)
+		}
+	})
+
+	t.Run("all tunables", func(t *testing.T) {
+		setEnv(t, map[string]string{
+			"SPINIFEX_ZFS_ASHIFT":      " 12 ",
+			"SPINIFEX_ZFS_COPIES":      "2",
+			"SPINIFEX_ZFS_ARC_MAX_MIB": "4096",
+			"SPINIFEX_ZFS_COMPRESS":    " lz4 ",
+			"SPINIFEX_ZFS_CHECKSUM":    "blake3",
+		})
+		got, err := parseZFSOpts()
+		if err != nil {
+			t.Fatalf("parseZFSOpts: %v", err)
+		}
+		want := install.ZFSOpts{Ashift: 12, Copies: 2, ARCMaxMiB: 4096, Compress: "lz4", Checksum: "blake3"}
+		if got != want {
+			t.Errorf("opts = %+v, want %+v", got, want)
+		}
+	})
+
+	for _, env := range []string{"SPINIFEX_ZFS_ASHIFT", "SPINIFEX_ZFS_COPIES", "SPINIFEX_ZFS_ARC_MAX_MIB"} {
+		t.Run("unparsable "+env, func(t *testing.T) {
+			setEnv(t, map[string]string{env: "auto"})
+			if _, err := parseZFSOpts(); err == nil || !strings.Contains(err.Error(), env) {
+				t.Fatalf("err = %v, want it to name %s", err, env)
+			}
+		})
+	}
+}
+
+func TestResolveStorage(t *testing.T) {
+	four := []install.Disk{
+		mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 40*gib),
+		mkDisk("/dev/vdd", 40*gib), mkDisk("/dev/vde", 40*gib),
+	}
+
+	tests := []struct {
+		name      string
+		disks     []install.Disk
+		env       map[string]string
+		wantFS    install.FSType
+		wantPaths []string
+		wantErr   string
+	}{
+		{
+			name:      "ext4 single disk by default",
+			disks:     four[:1],
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb"},
+		},
+		{
+			name:      "explicit member list preserves order",
+			disks:     four,
+			env:       map[string]string{"SPINIFEX_FS": "zfs-raid10", "SPINIFEX_DISKS": "vde,/dev/vdd,vdc, vdb "},
+			wantFS:    install.FSZFSRAID10,
+			wantPaths: []string{"/dev/vde", "/dev/vdd", "/dev/vdc", "/dev/vdb"},
+		},
+		{
+			name:    "unknown filesystem",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_FS": "btrfs"},
+			wantErr: "unknown filesystem",
+		},
+		{
+			// Choosing which disks to erase is not an unattended decision.
+			name:    "multi-disk pool without an explicit list",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_FS": "zfs-raid1"},
+			wantErr: "set SPINIFEX_DISKS to an explicit comma-separated list",
+		},
+		{
+			name:    "member that is not present",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_DISKS": "vdz"},
+			wantErr: `"vdz" is not an available disk`,
+		},
+		{
+			name:    "member list is for pools, not ext4 roles",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_DISKS": "vdb,vdc"},
+			wantErr: "ext4 takes one OS disk",
+		},
+		{
+			name:  "dedicated role drives, os first",
+			disks: four,
+			env: map[string]string{
+				"SPINIFEX_DISK":            "vdb",
+				"SPINIFEX_DISK_SPINIFEX":   "vdc",
+				"SPINIFEX_DISK_PREDASTORE": "vdd",
+			},
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb", "/dev/vdc", "/dev/vdd"},
+		},
+		{
+			// The two-drive layout: predastore gets its own disk while
+			// /var/lib/spinifex stays on the root filesystem.
+			name:      "predastore alone",
+			disks:     four,
+			env:       map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_PREDASTORE": "vdc"},
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb", "/dev/vdc"},
+		},
+		{
+			name:  "role drive on a pool",
+			disks: four,
+			env: map[string]string{
+				"SPINIFEX_FS": "zfs-raid1", "SPINIFEX_DISKS": "vdb,vdc",
+				"SPINIFEX_DISK_SPINIFEX": "vdd",
+			},
+			wantErr: "SPINIFEX_DISK_SPINIFEX applies to ext4 only",
+		},
+		{
+			name:    "role drive that is not present",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_PREDASTORE": "vdz"},
+			wantErr: "SPINIFEX_DISK_PREDASTORE",
+		},
+		{
+			// Two roles cannot share a drive; the disk is erased once and the
+			// second mount would land on top of the first.
+			name:    "role drive repeats the os disk",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_SPINIFEX": "vdb"},
+			wantErr: "selected more than once",
+		},
+		{
+			name:    "bad zfs tunable",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_ZFS_ASHIFT": "big"},
+			wantErr: "SPINIFEX_ZFS_ASHIFT",
+		},
+		{
+			name:    "scan failure",
+			disks:   nil,
+			env:     map[string]string{"SPINIFEX_DISKS": "vdb"},
+			wantErr: "sysfs unavailable",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var scanErr error
+			if tt.disks == nil {
+				scanErr = errors.New("sysfs unavailable")
+			}
+			withDisks(t, tt.disks, scanErr)
+			setEnv(t, tt.env)
+
+			got, err := resolveStorage()
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveStorage: %v", err)
+			}
+			if got.FS != tt.wantFS {
+				t.Errorf("FS = %v, want %v", got.FS, tt.wantFS)
+			}
+			if !slices.Equal(got.Paths(), tt.wantPaths) {
+				t.Errorf("Paths = %v, want %v", got.Paths(), tt.wantPaths)
+			}
+		})
+	}
+}
+
+func TestResolveDisk(t *testing.T) {
+	small := mkDisk("/dev/vdb", 40*gib)
+	large := mkDisk("/dev/vdc", 400*gib)
+	nvme := mkDisk("/dev/nvme0n1", 200*gib)
+
+	tests := []struct {
+		name     string
+		disks    []install.Disk
+		target   string
+		wantPath string
+		wantErr  string
+	}{
+		{name: "auto with one candidate", disks: []install.Disk{small}, target: "auto", wantPath: "/dev/vdb"},
+		{name: "empty target is auto", disks: []install.Disk{small}, wantPath: "/dev/vdb"},
+		{
+			name:    "auto with several candidates",
+			disks:   []install.Disk{small, large},
+			target:  "auto",
+			wantErr: "expected exactly one disk, found 2",
+		},
+		{name: "largest", disks: []install.Disk{small, large, nvme}, target: "LARGEST", wantPath: "/dev/vdc"},
+		{name: "smallest", disks: []install.Disk{large, small, nvme}, target: " smallest ", wantPath: "/dev/vdb"},
+		{name: "nvme", disks: []install.Disk{small, large, nvme}, target: "nvme", wantPath: "/dev/nvme0n1"},
+		{
+			name:    "nvme with none present",
+			disks:   []install.Disk{small, large},
+			target:  "nvme",
+			wantErr: "expected exactly one NVMe disk, found 0",
+		},
+		{name: "exact path", disks: []install.Disk{small, large}, target: "/dev/vdc", wantPath: "/dev/vdc"},
+		{
+			name:    "path that is not present",
+			disks:   []install.Disk{small},
+			target:  "/dev/vdz",
+			wantErr: "is not an available disk",
+		},
+		{name: "no candidates at all", disks: []install.Disk{}, target: "largest", wantErr: "no non-removable disks found"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, tt.disks, nil)
+
+			got, err := resolveDisk(tt.target)
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveDisk(%q): %v", tt.target, err)
+			}
+			if got.Path != tt.wantPath {
+				t.Errorf("path = %q, want %q", got.Path, tt.wantPath)
+			}
+		})
+	}
+}
+
+func TestCandidateDisksExcludesRemovableAndLiveMedia(t *testing.T) {
+	fixed := mkDisk("/dev/vdb", 40*gib)
+	usb := mkDisk("/dev/sdz", 16*gib)
+	usb.Removable = true
+	boot := mkDisk("/dev/sdy", 16*gib)
+	boot.LiveMedia = true
+	withDisks(t, []install.Disk{usb, fixed, boot}, nil)
+
+	got, err := candidateDisks()
+	if err != nil {
+		t.Fatalf("candidateDisks: %v", err)
+	}
+	if len(got) != 1 || got[0].Path != "/dev/vdb" {
+		t.Fatalf("candidates = %v, want only /dev/vdb", got)
+	}
+}
+
+func TestCandidateDisksPropagatesScanError(t *testing.T) {
+	withDisks(t, nil, errors.New("read /sys/block: denied"))
+	if _, err := candidateDisks(); err == nil {
+		t.Fatal("expected the scan error to propagate")
+	}
+}
+
+func TestPickBySizeRejectsEmptySet(t *testing.T) {
+	if _, err := pickBySize(nil, true); err == nil {
+		t.Fatal("expected an error for an empty disk set")
+	}
+}
+
+func TestPickBySizeLeavesInputOrderIntact(t *testing.T) {
+	disks := []install.Disk{mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 400*gib)}
+	if _, err := pickBySize(disks, false); err != nil {
+		t.Fatalf("pickBySize: %v", err)
+	}
+	if disks[0].Path != "/dev/vdb" {
+		t.Errorf("input reordered: %v", disks)
+	}
+}
+
+func TestDiskList(t *testing.T) {
+	got := diskList([]install.Disk{mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 2048*gib)})
+	want := "  /dev/vdb (40.0G, empty)\n  /dev/vdc (2.0T, empty)"
+	if got != want {
+		t.Errorf("diskList =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestResolveNICExplicitName(t *testing.T) {
+	got, err := resolveNIC("eno1", "")
+	if err != nil {
+		t.Fatalf("resolveNIC: %v", err)
+	}
+	if got != "eno1" {
+		t.Errorf("nic = %q, want eno1", got)
+	}
+}
+
+// The auto path reads the host's real interfaces, so assert the selection rules
+// rather than a specific name.
+func TestResolveNICAutoSkipsVirtualInterfaces(t *testing.T) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		t.Skipf("no interface list on this host: %v", err)
+	}
+
+	got, err := resolveNIC("auto", "")
+	if err != nil {
+		if len(ifaces) > 0 {
+			t.Logf("no physical NIC among %d interfaces: %v", len(ifaces), err)
+		}
+		return
+	}
+
+	idx := slices.IndexFunc(ifaces, func(i net.Interface) bool { return i.Name == got })
+	if idx < 0 {
+		t.Fatalf("resolveNIC returned %q, which is not a host interface", got)
+	}
+	sel := ifaces[idx]
+	if sel.Flags&net.FlagLoopback != 0 || sel.Flags&net.FlagBroadcast == 0 || len(sel.HardwareAddr) == 0 {
+		t.Errorf("selected %q with flags %v and MAC %q", got, sel.Flags, sel.HardwareAddr)
+	}
+	for _, pfx := range virtualNICPrefixes {
+		if strings.HasPrefix(got, pfx) {
+			t.Errorf("selected virtual interface %q (prefix %q)", got, pfx)
+		}
+	}
+}
+
+func TestResolveNICAutoHonoursExclude(t *testing.T) {
+	first, err := resolveNIC("auto", "")
+	if err != nil {
+		t.Skipf("no physical NIC on this host: %v", err)
+	}
+	got, err := resolveNIC("", first)
+	if err == nil && got == first {
+		t.Errorf("resolveNIC returned the excluded interface %q", first)
+	}
+}

cmd/installer/firstboot/firstboot.go

@@ -14,14 +14,23 @@ import (
 // Config holds the values the firstboot service needs to configure the node.
 type Config struct {
 	Hostname string
-	// EncapIP is the Geneve tunnel IP for OVN. Set to the LAN bridge IP when a
-	// dedicated LAN NIC is present, otherwise the WAN bridge IP. Empty when DHCP
-	// is used — setup-ovn.sh auto-detects the IP from the default route in that case.
+	// EncapIP is the Geneve tunnel IP for OVN, taken from the vpc plane after
+	// collapsing (vpc <- lan <- wan). Empty when that plane uses DHCP —
+	// setup-ovn.sh auto-detects the IP from the default route in that case.
 	EncapIP string
-	// ClusterRole is "init" or "join".
-	ClusterRole string
-	// JoinAddr is host:port of the primary node, only used when ClusterRole is "join".
-	JoinAddr string
+	// LANIP is the internal cluster address, taken from the lan plane after
+	// collapsing. Passed as --bind and --cluster-bind so predastore replication,
+	// the NATS mesh and OVN control traffic stay off the public interface.
+	// Empty leaves the 0.0.0.0 wildcard default, which is correct for a
+	// single-NIC node where lan folds onto wan.
+	LANIP string
+	// WANIP is the public address, taken from the wan plane. Passed as
+	// --advertise so northstar's :53 listener, the awsgw registry host and the
+	// dial target recorded for off-host clients stay on the public interface.
+	// Without it a concrete --bind is echoed back as the advertise address,
+	// which would move all three onto the internal plane. Empty when wan uses
+	// DHCP, leaving spx to auto-detect from the default route.
+	WANIP string
 	// Email is the operator email collected by the TUI or SPINIFEX_EMAIL on
 	// the headless path. Passed to `spx admin init --email=<value>` when set;
 	// omitted entirely when empty.
@@ -147,7 +156,8 @@ resolvconf -u
 # Write the banner (first time, spinifex-banner service will do this on reboot)
 /usr/local/bin/spx admin banner --boot-check
 
-# Cluster formation — capture credentials to file for display on console.
+# Cluster formation. Output goes to the journal via the systemd unit; spx
+# admin init writes credentials to disk and never prints them.
 %s 2>&1
 
 # Fix ownership of files spx admin init wrote. spx runs as root under
@@ -197,6 +207,9 @@ fi
 
 # Enable services to start, on reboot
 systemctl enable spinifex.target spinifex-banner.service
+# WantedBy=timers.target only self-activates once enabled — without this the
+# JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart.
+systemctl enable --now spinifex-nats-watchdog.timer
 
 # Mark complete only after every step above has succeeded. Until this point,
 # any failure (set -e) leaves the marker absent and the next reboot retries
@@ -244,14 +257,67 @@ func buildClusterCmd(cfg Config) string {
 	if cfg.GPUPassthrough {
 		gpuFlag = " --gpu-passthrough"
 	}
-	switch cfg.ClusterRole {
-	case "join":
-		return fmt.Sprintf("spx admin join --node %s --host %s%s", cfg.Hostname, cfg.JoinAddr, emailFlag)
-	default:
-		return fmt.Sprintf("spx admin init --node %s --nodes 1%s%s", cfg.Hostname, emailFlag, gpuFlag)
+	// Without these the node takes the 0.0.0.0 wildcard default and every
+	// internal service resolves onto the auto-detected WAN address, which is
+	// exactly what the three-plane model exists to prevent.
+	bindFlags := ""
+	if cfg.LANIP != "" {
+		bindFlags = fmt.Sprintf(" --bind %s --cluster-bind %s", cfg.LANIP, cfg.LANIP)
 	}
+	// --advertise must be explicit whenever --bind is: spx echoes a concrete
+	// bind address straight back as the advertise address and never reaches its
+	// WAN auto-detection, which would silently publish the internal plane as
+	// this node's public dial target.
+	preamble := ""
+	switch {
+	case bindFlags == "":
+		// Nothing pinned, so spx auto-detects both and a guessed advertise
+		// address would only get in the way.
+	case cfg.WANIP != "":
+		bindFlags += " --advertise " + cfg.WANIP
+	default:
+		// A DHCP wan has no address at install time, but it does by the time
+		// firstboot runs, so read it off the bridge instead of shipping the
+		// bind address as the public one.
+		preamble = wanAdvertisePreamble
+		bindFlags += ` $SPX_ADVERTISE`
+	}
+
+	// Always a single-node cluster. The installer cannot form a multi-node one:
+	// cluster membership decides the OVN database topology and the join token
+	// only exists once the primary has booted, so neither is knowable while the
+	// nodes are still being installed. Multi-node is a post-install conversion,
+	// documented in the multi-node install guide.
+	cmd := fmt.Sprintf("spx admin init --node %s --nodes 1%s%s%s", cfg.Hostname, bindFlags, emailFlag, gpuFlag)
+	return preamble + cmd
 }
 
+// wanAdvertisePreamble resolves the wan plane's DHCP lease into $SPX_ADVERTISE
+// just before formation. br-wan is always the wan bridge — that plane is the one
+// role that cannot fold — so its address is the node's public identity.
+//
+// Falling through with SPX_ADVERTISE empty is deliberate: no lease means there
+// is no public address to advertise yet, and spx echoing the bind address is
+// still better than formation failing outright.
+const wanAdvertisePreamble = `# The wan plane leases its address, so it is only knowable at boot. Without this
+# spx would echo --bind back as the advertise address and publish the internal
+# plane as this node's public dial target.
+SPX_ADVERTISE=""
+echo "[firstboot] waiting for the wan plane to acquire an address..."
+for _i in $(seq 1 60); do
+    _wan_ip=$(ip -4 -o addr show br-wan scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1)
+    if [ -n "$_wan_ip" ]; then
+        SPX_ADVERTISE=" --advertise $_wan_ip"
+        echo "[firstboot] wan address $_wan_ip (${_i}s)"
+        break
+    fi
+    sleep 1
+done
+if [ -z "$SPX_ADVERTISE" ]; then
+    echo "[firstboot] warning: br-wan has no address after 60s — this node will advertise its bind address"
+fi
+`
+
 // shellEscapeSingle wraps s in single quotes with any embedded single
 // quotes escaped. Minimal — we only need this because the email value is
 // interpolated into a shell script written by Write().

cmd/installer/firstboot/firstboot_test.go

@@ -2,6 +2,7 @@ package firstboot
 
 import (
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -24,7 +25,7 @@ func TestWriteScriptNoCallbackWhenEmpty(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	cfg := Config{Hostname: "test-node"}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -45,7 +46,6 @@ func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
 	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	cfg := Config{
 		Hostname:        "test-node",
-		ClusterRole:     "init",
 		InstallCallback: callbackURL,
 	}
 	if err := Write(root, cfg); err != nil {
@@ -70,7 +70,7 @@ func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	cfg := Config{Hostname: "test-node"}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -88,7 +88,7 @@ func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
+	cfg := Config{Hostname: "test-node", SkipFormation: true}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -114,6 +114,25 @@ func readScript(t *testing.T, root string) string {
 	return string(b)
 }
 
+// TestWriteScriptEnablesNatsWatchdogTimer pins that firstboot activates the
+// JetStream ENOSPC-latch watchdog timer. The unit file's WantedBy=timers.target
+// only takes effect once enabled — without this line the watchdog is dropped
+// onto disk but never runs, and a full disk needs a manual nats restart forever.
+func TestWriteScriptEnablesNatsWatchdogTimer(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node"}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if !strings.Contains(content, "systemctl enable --now spinifex-nats-watchdog.timer") {
+		t.Error("firstboot script must enable --now spinifex-nats-watchdog.timer")
+	}
+}
+
 func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
@@ -121,7 +140,6 @@ func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	cfg := Config{
 		Hostname:        "node1",
-		ClusterRole:     "init",
 		InstallCallback: callbackURL,
 	}
 	if err := Write(root, cfg); err != nil {
@@ -146,3 +164,121 @@ func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 		t.Error("curl must appear after done marker write")
 	}
 }
+
+// A multi-NIC node must keep cluster traffic on the internal plane while still
+// publishing the public one. --advertise has to be explicit to do that: spx
+// returns a concrete --bind verbatim as the advertise address and never reaches
+// its WAN auto-detection, so omitting it moves northstar's :53 listener and the
+// off-host dial target onto the internal plane.
+func TestBuildClusterCmdSeparatesBindFromAdvertise(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "hydrogen",
+		LANIP:    "10.0.0.3",
+		WANIP:    "216.218.163.99",
+	})
+	for _, want := range []string{
+		"--bind 10.0.0.3",
+		"--cluster-bind 10.0.0.3",
+		"--advertise 216.218.163.99",
+	} {
+		if !strings.Contains(cmd, want) {
+			t.Errorf("missing %q in:\n%s", want, cmd)
+		}
+	}
+}
+
+// Without a bind address there is nothing to correct: spx auto-detects both,
+// and a stray --advertise would pin the node to an address the installer only
+// guessed at. A single-NIC node also collapses wan and lan onto one address,
+// where the split is meaningless.
+func TestBuildClusterCmdOmitsAdvertiseWithoutBind(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "node1",
+		WANIP:    "216.218.163.99",
+	})
+	if strings.Contains(cmd, "--advertise") {
+		t.Errorf("advertise must not be passed without a bind address:\n%s", cmd)
+	}
+}
+
+// A DHCP wan has no address at install time, so the advertise value has to be
+// resolved at boot. Shipping --bind alone here would republish the internal
+// plane as the node's public dial target, which is the case the static path
+// already guards against.
+func TestBuildClusterCmdResolvesDHCPWanAtBoot(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "hydrogen",
+		LANIP:    "10.0.0.3",
+		// WANIP empty — the wan plane is on DHCP.
+	})
+	if !strings.Contains(cmd, "ip -4 -o addr show br-wan") {
+		t.Errorf("must read the wan address off the bridge:\n%s", cmd)
+	}
+	if !strings.Contains(cmd, "$SPX_ADVERTISE") {
+		t.Errorf("resolved address must reach the command:\n%s", cmd)
+	}
+	// The preamble has to run before the command that consumes it.
+	if strings.Index(cmd, "SPX_ADVERTISE=\"\"") > strings.Index(cmd, "spx admin init") {
+		t.Errorf("preamble must precede the spx invocation:\n%s", cmd)
+	}
+	if !strings.Contains(cmd, "--bind 10.0.0.3") {
+		t.Errorf("bind still comes from the lan plane:\n%s", cmd)
+	}
+}
+
+// With no bind address there is nothing to correct, so neither the literal flag
+// nor the boot-time lookup should appear — spx auto-detects both ends.
+func TestBuildClusterCmdSkipsWanLookupWithoutBind(t *testing.T) {
+	cmd := buildClusterCmd(Config{Hostname: "node1"})
+	if strings.Contains(cmd, "SPX_ADVERTISE") || strings.Contains(cmd, "--advertise") {
+		t.Errorf("no bind means no advertise handling at all:\n%s", cmd)
+	}
+}
+
+// The advertise handling injects a shell preamble into the generated script, so
+// the permutations that change its shape are syntax-checked rather than only
+// string-matched — a broken quote here bricks first boot with no installer error.
+func TestGeneratedScriptIsValidShell(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		cfg  Config
+	}{
+		{"dhcp wan resolves at boot", Config{Hostname: "hydrogen", LANIP: "10.0.0.3"}},
+		{"static wan is inlined", Config{Hostname: "hydrogen", LANIP: "10.0.0.3", WANIP: "216.218.163.99"}},
+		{"single nic, nothing pinned", Config{Hostname: "node1"}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			root := t.TempDir()
+			makeRootDirs(t, root)
+			if err := Write(root, tc.cfg); err != nil {
+				t.Fatalf("Write: %v", err)
+			}
+			path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
+			if out, err := exec.Command("bash", "-n", path).CombinedOutput(); err != nil {
+				script, _ := os.ReadFile(path)
+				t.Fatalf("generated script is not valid shell: %v\n%s\n---\n%s", err, out, script)
+			}
+		})
+	}
+}
+
+// The installer cannot form a multi-node cluster: membership decides the OVN
+// database topology, and the join token only exists once the primary has
+// booted. Neither is knowable at install time, so firstboot must only ever
+// initialize a single-node cluster and leave multi-node to a post-install
+// conversion.
+func TestBuildClusterCmdNeverJoins(t *testing.T) {
+	for _, cfg := range []Config{
+		{Hostname: "node1"},
+		{Hostname: "node1", LANIP: "10.0.0.3", WANIP: "216.218.163.99"},
+		{Hostname: "node1", LANIP: "10.0.0.3"},
+	} {
+		cmd := buildClusterCmd(cfg)
+		if strings.Contains(cmd, "spx admin join") {
+			t.Errorf("firstboot must never join a cluster:\n%s", cmd)
+		}
+		if !strings.Contains(cmd, "--nodes 1") {
+			t.Errorf("want a single-node init, got:\n%s", cmd)
+		}
+	}
+}

cmd/installer/install/assets/grub-install-wrapper

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Diversion wrapper for grub-install on a spinifex-boot-tool system.
+#
+# On ZFS root the bootloader lives on per-disk ESPs, not in /boot. A plain
+# grub-install writes a core image that tries to read the pool, which works
+# until the next OpenZFS feature flag lands and then silently does not. Worse,
+# the grub-pc and shim packages run grub-install from their own postinst, so an
+# ordinary `apt upgrade` would undo the layout with no visible error.
+#
+# Mirrors the wrapper Proxmox ships for the same reason.
+
+set -e
+
+REAL="/usr/sbin/grub-install.real"
+TOOL="/usr/sbin/spinifex-boot-tool"
+
+# Not a managed system: behave exactly as the real grub-install.
+if ! "$TOOL" status --quiet 2>/dev/null; then
+	exec "$REAL" "$@"
+fi
+
+# dpkg sets this when a maintainer script is running. The bootloader packages
+# reinstalling themselves is expected and is redirected, not refused.
+case "${DPKG_MAINTSCRIPT_PACKAGE:-}" in
+grub-* | shim-*)
+	echo "This system is managed by spinifex-boot-tool; redirecting grub-install to 'spinifex-boot-tool reinit'." >&2
+	exec "$TOOL" reinit
+	;;
+esac
+
+cat >&2 <<EOF
+E: This system's bootloader is managed by spinifex-boot-tool.
+
+Running grub-install directly would install GRUB somewhere the firmware does
+not read, and would leave the per-disk ESPs holding a stale kernel.
+
+Use instead:
+  spinifex-boot-tool refresh     re-sync kernels and grub.cfg to every ESP
+  spinifex-boot-tool reinit      reinstall GRUB on every ESP
+  spinifex-boot-tool status      list the ESPs under management
+
+To run the real grub-install anyway (you probably do not want this):
+  $REAL $*
+EOF
+exit 1

cmd/installer/install/assets/spinifex-boot-tool

@@ -0,0 +1,412 @@
+#!/bin/bash
+# spinifex-boot-tool — keep a bootloader on every disk of a ZFS root pool.
+#
+# GRUB cannot be trusted to read a ZFS pool: its ZFS reader lags OpenZFS by
+# years, and enabling a new feature flag on the pool can make the machine
+# unbootable with no warning. So /boot on the pool is never read by firmware.
+# Instead each pool member carries a small FAT32 ESP holding its own copy of
+# GRUB, the kernels and the initramfs images, and this tool keeps them in step.
+#
+# Modelled directly on Proxmox's proxmox-boot-tool, which has run this design
+# across their entire installed base for years.
+
+set -u
+set -o pipefail
+
+ESP_LIST="/etc/kernel/spinifex-boot-uuids"
+ESPMOUNTS="/var/tmp/espmounts"
+MARKER="spinifex-boot-uuids"
+
+# Kernels are only ever copied out of the real /boot, which lives on the pool.
+# Once an ESP is bind-mounted over /boot this variable is how we still find it.
+BOOTROOT="/boot"
+
+warn() { echo "W: $*" >&2; }
+die() { echo "E: $*" >&2; exit 1; }
+
+usage() {
+	cat <<EOF
+Usage: $(basename "$0") <command>
+
+  format <partition> [--force]  make <partition> a Spinifex ESP
+  init <partition>              register <partition> and install a bootloader on it
+  clean [--dry-run]             drop ESPs that are no longer present
+  refresh [--hook <name>]       re-sync kernels and grub.cfg to every ESP
+  reinit                        re-run refresh, reinstalling GRUB on every ESP
+  kernel <add|remove> <ver>     pin or unpin a kernel version
+  status [--quiet]              list registered ESPs
+EOF
+}
+
+# reexec_in_mountns re-runs this script inside a private mount namespace. Every
+# code path here bind-mounts an ESP over /boot; if the script is killed at the
+# wrong moment outside a namespace, the running system is left with its real
+# /boot masked and the next kernel upgrade writes into a FAT partition.
+reexec_in_mountns() {
+	if [ "${SPINIFEX_BOOT_TOOL_NS:-0}" = "1" ]; then
+		return 0
+	fi
+	export SPINIFEX_BOOT_TOOL_NS=1
+	# --propagation private stops the bind mounts leaking back to the host.
+	exec unshare --mount --propagation private -- "$0" "$@"
+}
+
+# --- ESP registry -----------------------------------------------------------
+
+# esp_list prints the registered ESP filesystem UUIDs.
+esp_list() {
+	[ -f "$ESP_LIST" ] || return 0
+	grep -v '^[[:space:]]*\(#\|$\)' "$ESP_LIST" || true
+}
+
+# uuid_of prints the FAT filesystem UUID of a partition.
+uuid_of() {
+	blkid "$1" -o value -s UUID 2>/dev/null
+}
+
+# path_for_uuid resolves a registered UUID back to a device, or nothing if the
+# disk has been pulled.
+path_for_uuid() {
+	local dev
+	dev="/dev/disk/by-uuid/$1"
+	[ -e "$dev" ] || return 1
+	readlink -f "$dev"
+}
+
+# mount_esp mounts the ESP for a UUID and echoes its mountpoint.
+mount_esp() {
+	local uuid="$1" dev mp
+	dev=$(path_for_uuid "$uuid") || return 1
+	mp="$ESPMOUNTS/$uuid"
+	mkdir -p "$mp"
+	mountpoint -q "$mp" || mount "$dev" "$mp" || return 1
+	echo "$mp"
+}
+
+umount_esp() {
+	local mp="$ESPMOUNTS/$1"
+	mountpoint -q "$mp" && umount "$mp"
+	rmdir "$mp" 2>/dev/null || true
+}
+
+# --- format / init ----------------------------------------------------------
+
+do_format() {
+	local part="$1" force="${2:-}"
+	[ -b "$part" ] || die "$part is not a block device"
+
+	if [ "$force" != "--force" ]; then
+		local existing
+		existing=$(blkid -o value -s TYPE "$part" 2>/dev/null || true)
+		[ -z "$existing" ] || die "$part already contains a $existing filesystem; pass --force to overwrite"
+	fi
+	mkfs.vfat -F 32 "$part" || die "mkfs.vfat failed on $part"
+}
+
+do_init() {
+	local part="$1"
+	[ -b "$part" ] || die "$part is not a block device"
+
+	local uuid
+	uuid=$(uuid_of "$part") || true
+	[ -n "$uuid" ] || die "$part has no filesystem UUID — run '$(basename "$0") format $part' first"
+
+	mkdir -p "$(dirname "$ESP_LIST")"
+	touch "$ESP_LIST"
+	if ! grep -qx "$uuid" "$ESP_LIST"; then
+		echo "$uuid" >>"$ESP_LIST"
+	fi
+	echo "Registered $part ($uuid)"
+
+	install_hooks
+	# Only the ESP just registered: the others are already current, and
+	# refreshing all of them here makes registering N disks an N-squared
+	# sequence of grub-installs.
+	refresh_one "$uuid" || die "could not make $part bootable"
+}
+
+do_clean() {
+	local dry="${1:-}" uuid keep=()
+	for uuid in $(esp_list); do
+		if path_for_uuid "$uuid" >/dev/null 2>&1; then
+			keep+=("$uuid")
+		else
+			echo "Removing stale ESP $uuid"
+		fi
+	done
+	if [ "$dry" = "--dry-run" ]; then
+		return 0
+	fi
+	printf '%s\n' "${keep[@]+"${keep[@]}"}" >"$ESP_LIST"
+}
+
+# --- refresh ----------------------------------------------------------------
+
+# kernel_versions lists the kernels present in the real /boot, newest first, so
+# the default menu entry is the newest kernel.
+kernel_versions() {
+	local img
+	for img in "$BOOTROOT"/vmlinuz-*; do
+		[ -e "$img" ] || continue
+		basename "$img" | sed 's/^vmlinuz-//'
+	done | sort -Vr
+}
+
+# sync_esp copies the current kernels and initramfs images onto one ESP and
+# removes anything left from a kernel that is no longer installed — an ESP is
+# small and a stale 100MB initramfs is what makes the next upgrade fail.
+sync_esp() {
+	local mp="$1" kver f base
+
+	local -A wanted=()
+	for kver in $(kernel_versions); do
+		wanted["vmlinuz-$kver"]=1
+		wanted["initrd.img-$kver"]=1
+	done
+
+	for f in "$mp"/vmlinuz-* "$mp"/initrd.img-*; do
+		[ -e "$f" ] || continue
+		base=$(basename "$f")
+		[ -n "${wanted[$base]:-}" ] || rm -f "$f"
+	done
+
+	for kver in $(kernel_versions); do
+		for base in "vmlinuz-$kver" "initrd.img-$kver"; do
+			[ -e "$BOOTROOT/$base" ] || continue
+			# Copy to a temp name and rename: a half-written kernel on the ESP
+			# is a machine that does not come back from a reboot.
+			cp -f "$BOOTROOT/$base" "$mp/$base.tmp" || return 1
+			mv -f "$mp/$base.tmp" "$mp/$base" || return 1
+		done
+	done
+
+	# GRUB reads its theme from $prefix, which is the ESP's own grub directory,
+	# so the branding has to be copied out of /boot alongside the kernels.
+	mkdir -p "$mp/grub/fonts"
+	for f in grub/theme.txt grub/splash.png grub/fonts/unicode.pf2; do
+		[ -e "$BOOTROOT/$f" ] && cp -f "$BOOTROOT/$f" "$mp/$f"
+	done
+
+	# Marks the partition as ours, so a later `format` on the wrong disk is
+	# obviously a mistake rather than a silent overwrite.
+	echo "$MARKER" >"$mp/$MARKER"
+	return 0
+}
+
+# grub_install_bin echoes the real grub-install. On a managed system the normal
+# name is a wrapper that refuses to run, and it must not intercept this tool's
+# own calls — that would make the refusal a self-deadlock.
+grub_install_bin() {
+	if [ -x /usr/sbin/grub-install.real ]; then
+		echo /usr/sbin/grub-install.real
+	else
+		echo grub-install
+	fi
+}
+
+# install_grub puts GRUB itself on one ESP, for both firmware types.
+#
+# --boot-directory points at the ESP, not at /boot, so the GRUB core image
+# embeds a FAT reader and a search for the ESP's own UUID. It never needs to
+# read the pool, which is the entire point of this tool.
+install_grub() {
+	local mp="$1" dev="$2" rc=0 grub
+	grub="$(grub_install_bin)"
+
+	if [ ! -d /sys/firmware/efi ]; then
+		"$grub" --target=i386-pc --boot-directory="$mp" --recheck "$dev" || rc=1
+		return $rc
+	fi
+
+	# NVRAM is written at install time only. A refresh runs on every kernel
+	# upgrade, and rewriting boot variables that often has bricked firmware on
+	# more than one vendor's boards.
+	local nvram="--no-nvram"
+	[ "${SPINIFEX_BOOT_TOOL_NVRAM:-0}" = "1" ] && nvram=""
+
+	# Twice: once as a named entry, and once at the removable path, which is all
+	# some firmware will ever look at and is what keeps the disk bootable after
+	# its NVRAM entry is lost or the drive is moved to another machine.
+	# shellcheck disable=SC2086 # $nvram is a single optional flag, not a path
+	"$grub" --target=x86_64-efi \
+		--efi-directory="$mp" --boot-directory="$mp" \
+		--bootloader-id=spinifex $nvram --recheck "$dev" || rc=1
+	"$grub" --target=x86_64-efi \
+		--efi-directory="$mp" --boot-directory="$mp" \
+		--bootloader-id=spinifex --removable --no-nvram --recheck "$dev" || rc=1
+	return $rc
+}
+
+# update_grub_on regenerates grub.cfg for one ESP.
+#
+# The bind mount is the trick that makes this work: grub-mkconfig probes the
+# device behind /boot to decide how to find the kernel at boot. With the ESP
+# mounted there it emits a FAT UUID search and paths relative to the ESP root,
+# so the generated config matches what the embedded core image can actually do.
+update_grub_on() {
+	local mp="$1" rc=0
+	mount --bind "$mp" "$BOOTROOT" || return 1
+	grub-mkconfig -o "$BOOTROOT/grub/grub.cfg" || rc=1
+	umount "$BOOTROOT" || warn "could not unmount the bind mount over $BOOTROOT"
+	return $rc
+}
+
+# refresh_one makes a single registered ESP independently bootable: kernels,
+# bootloader, and a grub.cfg generated against that ESP's own UUID.
+refresh_one() {
+	local uuid="$1" mp dev disk rc=0
+
+	if ! dev=$(path_for_uuid "$uuid"); then
+		warn "ESP $uuid is not present, skipping"
+		return 1
+	fi
+	if ! mp=$(mount_esp "$uuid"); then
+		warn "could not mount ESP $uuid"
+		return 1
+	fi
+
+	# The whole disk, not the partition: BIOS GRUB embeds into the boot
+	# sector and the BIOS boot partition, both of which live on the disk.
+	disk=$(lsblk -no pkname "$dev")
+	disk="/dev/$disk"
+
+	if sync_esp "$mp" && install_grub "$mp" "$disk" && update_grub_on "$mp"; then
+		echo "Synced $dev ($uuid)"
+	else
+		warn "failed to update ESP $uuid on $dev"
+		rc=1
+	fi
+	umount_esp "$uuid"
+	return $rc
+}
+
+do_refresh() {
+	local uuid failed=0 ok=0 total=0
+
+	local uuids
+	uuids=$(esp_list)
+	if [ -z "$uuids" ]; then
+		echo "No ESPs registered, nothing to do."
+		return 0
+	fi
+
+	for uuid in $uuids; do
+		total=$((total + 1))
+		if refresh_one "$uuid"; then
+			ok=$((ok + 1))
+		else
+			failed=$((failed + 1))
+		fi
+	done
+
+	if [ "$ok" -eq 0 ]; then
+		die "no ESP could be updated ($failed of $total failed) — this system may not boot"
+	fi
+	if [ "$failed" -gt 0 ]; then
+		# Strict mode is used by the installer, where a partly-updated set of
+		# ESPs means the install is broken and should fail loudly now rather
+		# than at the first reboot.
+		if [ "${SPINIFEX_BOOT_TOOL_STRICT:-0}" = "1" ]; then
+			die "$failed of $total ESPs failed to update"
+		fi
+		warn "$failed of $total ESPs failed to update; $ok succeeded, so the system will still boot"
+	fi
+	return 0
+}
+
+do_status() {
+	local quiet="${1:-}" uuid dev
+	if [ ! -s "$ESP_LIST" ]; then
+		[ "$quiet" = "--quiet" ] || echo "No ESPs registered — this system does not use spinifex-boot-tool."
+		return 1
+	fi
+	[ "$quiet" = "--quiet" ] && return 0
+	for uuid in $(esp_list); do
+		if dev=$(path_for_uuid "$uuid"); then
+			echo "$uuid  $dev"
+		else
+			echo "$uuid  MISSING"
+		fi
+	done
+}
+
+do_kernel() {
+	local action="$1" kver="${2:-}"
+	local pinfile="/etc/kernel/spinifex-boot-pin"
+	case "$action" in
+	add)
+		[ -n "$kver" ] || die "kernel add needs a version"
+		touch "$pinfile"
+		grep -qx "$kver" "$pinfile" || echo "$kver" >>"$pinfile"
+		;;
+	remove)
+		[ -n "$kver" ] || die "kernel remove needs a version"
+		[ -f "$pinfile" ] && sed -i "\|^$kver\$|d" "$pinfile"
+		;;
+	*) die "unknown kernel action: $action" ;;
+	esac
+	do_refresh
+}
+
+# --- hooks ------------------------------------------------------------------
+
+# install_hooks wires this tool into every package operation that would
+# otherwise write a bootloader to the wrong place.
+install_hooks() {
+	mkdir -p /etc/kernel/postinst.d /etc/kernel/postrm.d /etc/initramfs/post-update.d
+
+	# Debian's own hook runs update-grub against the real /boot, which is on the
+	# pool. Firmware never reads that, so it would produce a config nothing uses
+	# while the ESPs silently kept an old kernel.
+	if [ -e /etc/kernel/postinst.d/zz-update-grub ]; then
+		dpkg-divert --divert /etc/kernel/postinst.d/zz-update-grub.distrib \
+			--rename --add /etc/kernel/postinst.d/zz-update-grub >/dev/null 2>&1 || true
+	fi
+
+	for hook in /etc/kernel/postinst.d/zz-spinifex-boot-tool \
+		/etc/kernel/postrm.d/zz-spinifex-boot-tool \
+		/etc/initramfs/post-update.d/spinifex-boot-tool; do
+		cat >"$hook" <<'HOOK'
+#!/bin/sh
+# Re-sync the ESPs after a kernel or initramfs change.
+set -e
+exec /usr/sbin/spinifex-boot-tool refresh --hook "$(basename "$0")"
+HOOK
+		chmod 0755 "$hook"
+	done
+}
+
+# --- dispatch ---------------------------------------------------------------
+
+cmd="${1:-}"
+[ $# -gt 0 ] && shift
+
+case "$cmd" in
+format) do_format "$@" ;;
+init)
+	reexec_in_mountns init "$@"
+	do_init "$@"
+	;;
+refresh)
+	# --hook <name> is passed by the kernel hooks purely so the reason for a
+	# refresh is visible in the log.
+	if [ "${1:-}" = "--hook" ]; then shift 2 || shift; fi
+	reexec_in_mountns refresh
+	do_refresh
+	;;
+reinit)
+	reexec_in_mountns reinit
+	do_refresh
+	;;
+clean) do_clean "$@" ;;
+kernel)
+	reexec_in_mountns kernel "$@"
+	do_kernel "$@"
+	;;
+status) do_status "$@" ;;
+help | --help | -h | "") usage ;;
+*)
+	usage
+	exit 1
+	;;
+esac

cmd/installer/install/blockdev.go

@@ -0,0 +1,406 @@
+package install
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// Disk is one candidate block device with every attribute the installer needs
+// to validate a selection: sizes for the same-size rule, block sizes for ashift
+// and the 4Kn/BIOS check, rotational for autotrim, and ByID for pool members.
+type Disk struct {
+	Path  string // /dev/sda
+	ByID  string // /dev/disk/by-id/ata-… — stable across controller reordering
+	Bytes int64
+	Model string
+
+	LogicalBlockSize  int
+	PhysicalBlockSize int
+
+	Rotational bool
+	Removable  bool
+
+	// LiveMedia marks the device the installer itself booted from. Selecting it
+	// would erase the running system mid-install.
+	LiveMedia bool
+
+	// Content is a human-readable summary of what is on the disk today
+	// ("ext4 filesystem", "empty"), shown before the operator confirms erasure.
+	Content string
+}
+
+// skipPrefixes are kernel block devices that are never install targets:
+// virtual, mapped, or removable media. Mirrors Proxmox's hd_list filter.
+var skipPrefixes = []string{"ram", "loop", "md", "dm-", "fd", "sr", "zram", "nbd"}
+
+// The kernel-owned trees the scan reads, indirected so tests can point it at a
+// fixture instead of the running machine's hardware.
+var (
+	sysBlockDir      = "/sys/block"
+	sysClassBlockDir = "/sys/class/block"
+	devByIDDir       = "/dev/disk/by-id"
+	procMountsPath   = "/proc/mounts"
+	procSwapsPath    = "/proc/swaps"
+)
+
+// ListDisks enumerates install-candidate disks from /sys/block. It reads sysfs
+// and blkid only — pool detection is separate (ImportablePools) because it is
+// far slower and not every caller needs it.
+func ListDisks() ([]Disk, error) {
+	entries, err := os.ReadDir(sysBlockDir)
+	if err != nil {
+		return nil, fmt.Errorf("read /sys/block: %w", err)
+	}
+
+	byID := buildByIDIndex()
+	live := liveMediaDevices()
+
+	var disks []Disk
+	for _, e := range entries {
+		name := e.Name()
+		if hasAnyPrefix(name, skipPrefixes) {
+			continue
+		}
+		// A zero-sized device is an empty card reader slot, not a disk.
+		sectors := sysfsInt(name, "size")
+		if sectors == 0 {
+			continue
+		}
+		d := Disk{
+			Path:              "/dev/" + name,
+			ByID:              byID[name],
+			Bytes:             sectors * 512,
+			Model:             sysfsString(name, "device/model"),
+			LogicalBlockSize:  sysfsIntDefault(name, "queue/logical_block_size", 512),
+			PhysicalBlockSize: sysfsIntDefault(name, "queue/physical_block_size", 512),
+			Rotational:        sysfsInt(name, "queue/rotational") == 1,
+			Removable:         sysfsInt(name, "removable") == 1,
+			LiveMedia:         live[name],
+		}
+		d.Content = describeContent(d.Path)
+		disks = append(disks, d)
+	}
+	slices.SortFunc(disks, func(a, b Disk) int { return strings.Compare(a.Path, b.Path) })
+	return disks, nil
+}
+
+// SizeHuman renders the disk size for display, e.g. "1.7T".
+func (d Disk) SizeHuman() string {
+	switch {
+	case d.Bytes >= 1<<40:
+		return fmt.Sprintf("%.1fT", float64(d.Bytes)/(1<<40))
+	case d.Bytes >= 1<<30:
+		return fmt.Sprintf("%.1fG", float64(d.Bytes)/(1<<30))
+	case d.Bytes >= 1<<20:
+		return fmt.Sprintf("%.1fM", float64(d.Bytes)/(1<<20))
+	default:
+		return fmt.Sprintf("%dB", d.Bytes)
+	}
+}
+
+// Stable returns the path to use when building a pool: the by-id path when one
+// exists, else the kernel name. Pools referencing /dev/sdX break when a
+// controller enumerates disks in a different order after a reboot.
+func (d Disk) Stable() string {
+	if d.ByID != "" {
+		return d.ByID
+	}
+	return d.Path
+}
+
+// PartitionPath returns the device path for partition n of this disk. NVMe and
+// other devices whose name ends in a digit take a 'p' separator.
+func (d Disk) PartitionPath(n int) string { return partitionPath(d.Path, n) }
+
+// StablePartitionPath is PartitionPath against the by-id path, which udev
+// publishes with a "-partN" suffix rather than the kernel's 'p' convention.
+func (d Disk) StablePartitionPath(n int) string {
+	if d.ByID == "" {
+		return d.PartitionPath(n)
+	}
+	return fmt.Sprintf("%s-part%d", d.ByID, n)
+}
+
+func partitionPath(disk string, n int) string {
+	if len(disk) > 0 && disk[len(disk)-1] >= '0' && disk[len(disk)-1] <= '9' {
+		return fmt.Sprintf("%sp%d", disk, n)
+	}
+	return fmt.Sprintf("%s%d", disk, n)
+}
+
+// readKernelPartitions returns the partition device names the kernel currently
+// publishes for a disk, sorted.
+//
+// It reads sysfs rather than /dev deliberately. A partition device node
+// outlives the table that created it, so a stale node is indistinguishable
+// from a fresh one by existence alone — which is how a disk whose old table
+// the kernel never dropped passes every check and then gets formatted at the
+// previous layout's offsets.
+func readKernelPartitions(disk string) ([]string, error) {
+	name := filepath.Base(disk)
+	entries, err := os.ReadDir(filepath.Join(sysBlockDir, name))
+	if err != nil {
+		return nil, fmt.Errorf("read the kernel's partition list for %s: %w", disk, err)
+	}
+	var out []string
+	for _, e := range entries {
+		// Partitions are the child directories carrying a "partition" attribute.
+		// "queue", "device" and the plain attribute files sit alongside them.
+		if _, err := os.Stat(filepath.Join(sysBlockDir, name, e.Name(), "partition")); err != nil {
+			continue
+		}
+		out = append(out, e.Name())
+	}
+	slices.Sort(out)
+	return out, nil
+}
+
+// kernelPartitions is indirected so tests can drive the partition-table
+// assertions without a real block device.
+var kernelPartitions = readKernelPartitions
+
+// partitionDevice maps a kernel partition name back to its device node,
+// resolved alongside the disk it belongs to rather than assuming /dev.
+func partitionDevice(disk, part string) string {
+	return filepath.Join(filepath.Dir(disk), part)
+}
+
+// diskHolders names the md, LVM or device-mapper devices claiming this disk or
+// one of its partitions. A holder is the usual reason the kernel refuses to
+// re-read a partition table, so it is what the operator needs told when the
+// installer gives up on a disk.
+func diskHolders(disk string) []string {
+	name := filepath.Base(disk)
+	dirs := []string{filepath.Join(sysBlockDir, name, "holders")}
+	parts, _ := kernelPartitions(disk)
+	for _, p := range parts {
+		dirs = append(dirs, filepath.Join(sysBlockDir, name, p, "holders"))
+	}
+
+	seen := map[string]bool{}
+	var out []string
+	for _, dir := range dirs {
+		entries, err := os.ReadDir(dir)
+		if err != nil {
+			continue
+		}
+		for _, e := range entries {
+			if seen[e.Name()] {
+				continue
+			}
+			seen[e.Name()] = true
+			out = append(out, e.Name())
+		}
+	}
+	slices.Sort(out)
+	return out
+}
+
+// diskMount is one filesystem mounted from a disk the installer is about to
+// erase.
+type diskMount struct {
+	Device     string
+	Mountpoint string
+}
+
+// devicesOnDisk is the set of device paths belonging to a disk: the whole disk
+// and every partition the kernel currently publishes. Matching /proc tables
+// against a set rather than a string prefix keeps /dev/nvme0n1 from claiming
+// /dev/nvme0n11.
+func devicesOnDisk(disk string) map[string]bool {
+	out := map[string]bool{disk: true}
+	parts, _ := kernelPartitions(disk)
+	for _, p := range parts {
+		out[partitionDevice(disk, p)] = true
+	}
+	return out
+}
+
+// mountsOnDisk returns the filesystems mounted from a disk, deepest mountpoint
+// first so a nested mount is released before the one it sits inside.
+func mountsOnDisk(disk string) []diskMount {
+	devs := devicesOnDisk(disk)
+	var out []diskMount
+	for _, fields := range procTable(procMountsPath, 0) {
+		if len(fields) >= 2 && devs[fields[0]] {
+			out = append(out, diskMount{Device: fields[0], Mountpoint: fields[1]})
+		}
+	}
+	slices.SortFunc(out, func(a, b diskMount) int {
+		return strings.Count(b.Mountpoint, "/") - strings.Count(a.Mountpoint, "/")
+	})
+	return out
+}
+
+// swapsOnDisk returns the active swap areas backed by a disk.
+func swapsOnDisk(disk string) []string {
+	devs := devicesOnDisk(disk)
+	var out []string
+	// The first line of /proc/swaps is a column header, not an entry.
+	for _, fields := range procTable(procSwapsPath, 1) {
+		if len(fields) >= 1 && devs[fields[0]] {
+			out = append(out, fields[0])
+		}
+	}
+	return out
+}
+
+// procTable splits a whitespace-delimited /proc table into fields per line,
+// dropping the first skip lines and any blank ones.
+func procTable(path string, skip int) [][]string {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+	var out [][]string
+	n := 0
+	for line := range strings.SplitSeq(string(data), "\n") {
+		n++
+		if n <= skip {
+			continue
+		}
+		if fields := strings.Fields(line); len(fields) > 0 {
+			out = append(out, fields)
+		}
+	}
+	return out
+}
+
+// byIDPreference orders the /dev/disk/by-id/ namespaces we prefer to name a
+// pool member by. wwn- ids are last: they are stable but opaque, so a degraded
+// pool reports a number nobody can map to a drive bay.
+var byIDPreference = []string{"nvme-", "ata-", "scsi-", "virtio-", "usb-", "wwn-"}
+
+// buildByIDIndex maps kernel device name to its preferred by-id path.
+func buildByIDIndex() map[string]string {
+	entries, err := os.ReadDir(devByIDDir)
+	if err != nil {
+		return map[string]string{}
+	}
+	// Track the chosen preference rank per device so a later, less-preferred
+	// id cannot displace a better one already found.
+	rank := map[string]int{}
+	out := map[string]string{}
+	for _, e := range entries {
+		id := e.Name()
+		// Partition links describe a slice of the disk, not the disk.
+		if strings.Contains(id, "-part") {
+			continue
+		}
+		target, err := filepath.EvalSymlinks(filepath.Join(devByIDDir, id))
+		if err != nil {
+			continue
+		}
+		dev := filepath.Base(target)
+		r := prefixRank(id)
+		if existing, ok := rank[dev]; ok && existing <= r {
+			continue
+		}
+		rank[dev] = r
+		out[dev] = filepath.Join(devByIDDir, id)
+	}
+	return out
+}
+
+func prefixRank(id string) int {
+	for i, p := range byIDPreference {
+		if strings.HasPrefix(id, p) {
+			return i
+		}
+	}
+	return len(byIDPreference)
+}
+
+// liveMediaDevices returns the set of parent disk names backing the live
+// installer's own media, so they can never be selected as install targets.
+func liveMediaDevices() map[string]bool {
+	out := map[string]bool{}
+	data, err := os.ReadFile(procMountsPath)
+	if err != nil {
+		return out
+	}
+	liveMounts := []string{"/cdrom", "/run/live/medium", "/lib/live/mount/medium"}
+	for line := range strings.SplitSeq(string(data), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) < 2 || !strings.HasPrefix(fields[0], "/dev/") {
+			continue
+		}
+		if !slices.Contains(liveMounts, fields[1]) {
+			continue
+		}
+		if parent := parentDisk(filepath.Base(fields[0])); parent != "" {
+			out[parent] = true
+		}
+	}
+	return out
+}
+
+// parentDisk resolves a partition's kernel name to its whole-disk name by
+// walking /sys/class/block/<part>/.. — the sysfs parent of a partition is its
+// disk. Returns the input unchanged when it is already a whole disk.
+func parentDisk(dev string) string {
+	link, err := filepath.EvalSymlinks(filepath.Join(sysClassBlockDir, dev))
+	if err != nil {
+		return dev
+	}
+	// A whole disk lives directly under a bus path; a partition sits one level
+	// deeper, inside its disk's directory.
+	if _, err := os.Stat(filepath.Join(link, "partition")); err != nil {
+		return dev
+	}
+	return filepath.Base(filepath.Dir(link))
+}
+
+// describeContent summarises what is currently on a disk for the confirmation
+// screen. Best-effort: an unreadable disk is reported as unknown, not empty,
+// because "empty" invites an operator to erase something that is not.
+func describeContent(dev string) string {
+	out, err := exec.Command("blkid", "-p", "-o", "value", "-s", "TYPE", dev).Output()
+	if err == nil {
+		if t := strings.TrimSpace(string(out)); t != "" {
+			return t + " filesystem"
+		}
+	}
+	// No filesystem on the whole device: report the partition table instead,
+	// since that is what tells the operator the disk is in use.
+	out, err = exec.Command("blkid", "-p", "-o", "value", "-s", "PTTYPE", dev).Output()
+	if err != nil {
+		return "unknown"
+	}
+	if t := strings.TrimSpace(string(out)); t != "" {
+		return t + " partition table"
+	}
+	return "empty"
+}
+
+func sysfsString(dev, file string) string {
+	data, err := os.ReadFile(filepath.Join(sysBlockDir, dev, file))
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(data))
+}
+
+func sysfsInt(dev, file string) int64 {
+	n, err := strconv.ParseInt(sysfsString(dev, file), 10, 64)
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+func sysfsIntDefault(dev, file string, def int) int {
+	if n := sysfsInt(dev, file); n > 0 {
+		return int(n)
+	}
+	return def
+}
+
+func hasAnyPrefix(s string, prefixes []string) bool {
+	return slices.ContainsFunc(prefixes, func(p string) bool { return strings.HasPrefix(s, p) })
+}

cmd/installer/install/blockdev_test.go

@@ -0,0 +1,484 @@
+package install
+
+import (
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// fakeDisk describes one device to lay down in the fixture sysfs tree.
+type fakeDisk struct {
+	name       string
+	sectors    int64
+	model      string
+	logical    int
+	physical   int
+	rotational bool
+	removable  bool
+	partitions []string
+}
+
+// fakeSysfs builds a /sys/block, /sys/class/block, /dev/disk/by-id and
+// /proc/mounts fixture and points the scan at it for the duration of the test.
+type fakeSysfs struct {
+	root string
+	t    *testing.T
+}
+
+func newFakeSysfs(t *testing.T, disks ...fakeDisk) *fakeSysfs {
+	t.Helper()
+	f := &fakeSysfs{root: t.TempDir(), t: t}
+
+	swap := func(dst *string, v string) {
+		prev := *dst
+		*dst = v
+		t.Cleanup(func() { *dst = prev })
+	}
+	swap(&sysBlockDir, filepath.Join(f.root, "sys/block"))
+	swap(&sysClassBlockDir, filepath.Join(f.root, "sys/class/block"))
+	swap(&devByIDDir, filepath.Join(f.root, "dev/disk/by-id"))
+	swap(&procMountsPath, filepath.Join(f.root, "proc/mounts"))
+	swap(&procSwapsPath, filepath.Join(f.root, "proc/swaps"))
+
+	f.mkdirAll(sysBlockDir)
+	f.mkdirAll(sysClassBlockDir)
+	for _, d := range disks {
+		f.addDisk(d)
+	}
+	return f
+}
+
+func (f *fakeSysfs) mkdirAll(dir string) {
+	f.t.Helper()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		f.t.Fatalf("mkdir %s: %v", dir, err)
+	}
+}
+
+func (f *fakeSysfs) write(path, content string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Dir(path))
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		f.t.Fatalf("write %s: %v", path, err)
+	}
+}
+
+// addDisk lays out one device the way the kernel does: the attribute files
+// under /sys/block/<dev>, and a /sys/class/block symlink per disk and
+// partition, since parentDisk walks the latter.
+func (f *fakeSysfs) addDisk(d fakeDisk) {
+	f.t.Helper()
+	dir := filepath.Join(sysBlockDir, d.name)
+	f.write(filepath.Join(dir, "size"), strconv.FormatInt(d.sectors, 10)+"\n")
+	if d.model != "" {
+		f.write(filepath.Join(dir, "device/model"), d.model+"\n")
+	}
+	if d.logical > 0 {
+		f.write(filepath.Join(dir, "queue/logical_block_size"), strconv.Itoa(d.logical)+"\n")
+	}
+	if d.physical > 0 {
+		f.write(filepath.Join(dir, "queue/physical_block_size"), strconv.Itoa(d.physical)+"\n")
+	}
+	f.write(filepath.Join(dir, "queue/rotational"), boolAttr(d.rotational))
+	f.write(filepath.Join(dir, "removable"), boolAttr(d.removable))
+
+	f.symlink(dir, filepath.Join(sysClassBlockDir, d.name))
+	for _, part := range d.partitions {
+		partDir := filepath.Join(dir, part)
+		f.write(filepath.Join(partDir, "partition"), "1\n")
+		f.symlink(partDir, filepath.Join(sysClassBlockDir, part))
+	}
+}
+
+func (f *fakeSysfs) symlink(target, link string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Dir(link))
+	if err := os.Symlink(target, link); err != nil {
+		f.t.Fatalf("symlink %s -> %s: %v", link, target, err)
+	}
+}
+
+// addByID creates a by-id alias pointing at a kernel device name. The link
+// target must exist for EvalSymlinks, so a placeholder /dev tree is used.
+func (f *fakeSysfs) addByID(id, dev string) {
+	f.t.Helper()
+	devPath := filepath.Join(f.root, "dev", dev)
+	f.write(devPath, "")
+	f.symlink(devPath, filepath.Join(devByIDDir, id))
+}
+
+func (f *fakeSysfs) setMounts(content string) {
+	f.t.Helper()
+	f.write(procMountsPath, content)
+}
+
+func (f *fakeSysfs) setSwaps(content string) {
+	f.t.Helper()
+	f.write(procSwapsPath, content)
+}
+
+// addHolder marks a partition as claimed by md, LVM or device-mapper — the
+// state that makes the kernel refuse to re-read the disk's partition table.
+func (f *fakeSysfs) addHolder(disk, part, holder string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Join(sysBlockDir, disk, part, "holders", holder))
+}
+
+func boolAttr(b bool) string {
+	if b {
+		return "1\n"
+	}
+	return "0\n"
+}
+
+func TestListDisksReadsSysfsAttributes(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{
+		name: "sdb", sectors: 2 << 20, model: "SAMSUNG MZ7", logical: 512, physical: 4096, rotational: true,
+	})
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	if len(disks) != 1 {
+		t.Fatalf("got %d disks, want 1: %+v", len(disks), disks)
+	}
+	d := disks[0]
+	if d.Path != "/dev/sdb" || d.Bytes != (2<<20)*512 {
+		t.Errorf("path/size = %s/%d", d.Path, d.Bytes)
+	}
+	if d.Model != "SAMSUNG MZ7" {
+		t.Errorf("Model = %q", d.Model)
+	}
+	if d.LogicalBlockSize != 512 || d.PhysicalBlockSize != 4096 {
+		t.Errorf("block sizes = %d/%d", d.LogicalBlockSize, d.PhysicalBlockSize)
+	}
+	if !d.Rotational || d.Removable || d.LiveMedia {
+		t.Errorf("flags = rot %v, rem %v, live %v", d.Rotational, d.Removable, d.LiveMedia)
+	}
+	if d.Content == "" {
+		t.Error("Content is empty; an unreadable device must report something")
+	}
+}
+
+func TestListDisksSkipsVirtualAndEmptyDevices(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "loop0", sectors: 1 << 20},
+		fakeDisk{name: "dm-0", sectors: 1 << 20},
+		fakeDisk{name: "zram0", sectors: 1 << 20},
+		fakeDisk{name: "sr0", sectors: 1 << 20},
+		fakeDisk{name: "nbd0", sectors: 1 << 20},
+		// An empty card-reader slot reports zero sectors.
+		fakeDisk{name: "mmcblk0", sectors: 0},
+	)
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	var paths []string
+	for _, d := range disks {
+		paths = append(paths, d.Path)
+	}
+	if !slices.Equal(paths, []string{"/dev/sda"}) {
+		t.Errorf("paths = %v, want only /dev/sda", paths)
+	}
+}
+
+func TestListDisksSortsByPath(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sdc", sectors: 1 << 20},
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "sdb", sectors: 1 << 20},
+	)
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	var paths []string
+	for _, d := range disks {
+		paths = append(paths, d.Path)
+	}
+	if !slices.Equal(paths, []string{"/dev/sda", "/dev/sdb", "/dev/sdc"}) {
+		t.Errorf("paths = %v, want sorted", paths)
+	}
+}
+
+func TestListDisksDefaultsBlockSizesWhenSysfsIsSilent(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "vda", sectors: 1 << 20})
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	if disks[0].LogicalBlockSize != 512 || disks[0].PhysicalBlockSize != 512 {
+		t.Errorf("block sizes = %d/%d, want the 512 default",
+			disks[0].LogicalBlockSize, disks[0].PhysicalBlockSize)
+	}
+}
+
+func TestListDisksErrorsWhenSysfsIsUnreadable(t *testing.T) {
+	newFakeSysfs(t)
+	sysBlockDir = filepath.Join(t.TempDir(), "does-not-exist")
+
+	if _, err := ListDisks(); err == nil {
+		t.Fatal("expected an error when /sys/block cannot be read")
+	}
+}
+
+func TestListDisksMarksTheLiveMedium(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "sdz", sectors: 1 << 20, removable: true, partitions: []string{"sdz1"}},
+	)
+	// The installer boots from a partition; the whole disk must be excluded.
+	f.setMounts("/dev/sdz1 /run/live/medium iso9660 ro 0 0\n" +
+		"proc /proc proc rw 0 0\n" +
+		"/dev/sda1 / ext4 rw 0 0\n")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	byPath := map[string]Disk{}
+	for _, d := range disks {
+		byPath[d.Path] = d
+	}
+	if !byPath["/dev/sdz"].LiveMedia {
+		t.Error("/dev/sdz backs the live medium and was not marked")
+	}
+	if byPath["/dev/sda"].LiveMedia {
+		t.Error("/dev/sda is not the live medium but was marked")
+	}
+}
+
+func TestLiveMediaDevicesIsEmptyWithoutProcMounts(t *testing.T) {
+	newFakeSysfs(t)
+	if got := liveMediaDevices(); len(got) != 0 {
+		t.Errorf("liveMediaDevices = %v, want empty when /proc/mounts is missing", got)
+	}
+}
+
+func TestParentDiskLeavesWholeDisksAlone(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20, partitions: []string{"sda1"}})
+
+	if got := parentDisk("sda"); got != "sda" {
+		t.Errorf("parentDisk(sda) = %q, want sda", got)
+	}
+	if got := parentDisk("sda1"); got != "sda" {
+		t.Errorf("parentDisk(sda1) = %q, want sda", got)
+	}
+	// An unknown name cannot be resolved and is returned unchanged.
+	if got := parentDisk("sdq9"); got != "sdq9" {
+		t.Errorf("parentDisk(sdq9) = %q, want it unchanged", got)
+	}
+}
+
+func TestBuildByIDIndexPrefersTheMostLegibleAlias(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20})
+	// wwn- is stable but opaque, so ata- must win regardless of scan order.
+	f.addByID("wwn-0x5000c500", "sda")
+	f.addByID("ata-SAMSUNG_MZ7_S1", "sda")
+	f.addByID("ata-SAMSUNG_MZ7_S1-part1", "sda")
+
+	idx := buildByIDIndex()
+	want := filepath.Join(devByIDDir, "ata-SAMSUNG_MZ7_S1")
+	if idx["sda"] != want {
+		t.Errorf("by-id[sda] = %q, want %q", idx["sda"], want)
+	}
+}
+
+func TestBuildByIDIndexIsEmptyWhenTheDirIsAbsent(t *testing.T) {
+	newFakeSysfs(t)
+	if got := buildByIDIndex(); len(got) != 0 {
+		t.Errorf("buildByIDIndex = %v, want empty", got)
+	}
+}
+
+func TestListDisksAttachesByIDPath(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20})
+	f.addByID("ata-TEST_SERIAL0", "sda")
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	want := filepath.Join(devByIDDir, "ata-TEST_SERIAL0")
+	if disks[0].ByID != want {
+		t.Errorf("ByID = %q, want %q", disks[0].ByID, want)
+	}
+	if disks[0].Stable() != want {
+		t.Errorf("Stable() = %q, want the by-id path", disks[0].Stable())
+	}
+}
+
+func TestPrefixRank(t *testing.T) {
+	if prefixRank("nvme-Foo") >= prefixRank("wwn-0x1") {
+		t.Error("nvme- must outrank wwn-")
+	}
+	if prefixRank("md-uuid-x") != len(byIDPreference) {
+		t.Error("an unknown namespace must rank last")
+	}
+}
+
+func TestSizeHuman(t *testing.T) {
+	tests := []struct {
+		bytes int64
+		want  string
+	}{
+		{2 << 40, "2.0T"},
+		{500 << 30, "500.0G"},
+		{64 << 20, "64.0M"},
+		{512, "512B"},
+	}
+	for _, tt := range tests {
+		if got := (Disk{Bytes: tt.bytes}).SizeHuman(); got != tt.want {
+			t.Errorf("SizeHuman(%d) = %q, want %q", tt.bytes, got, tt.want)
+		}
+	}
+}
+
+func TestStableFallsBackToKernelName(t *testing.T) {
+	d := Disk{Path: "/dev/sda"}
+	if d.Stable() != "/dev/sda" {
+		t.Errorf("Stable() = %q", d.Stable())
+	}
+	if d.StablePartitionPath(2) != "/dev/sda2" {
+		t.Errorf("StablePartitionPath(2) = %q", d.StablePartitionPath(2))
+	}
+}
+
+func TestHasAnyPrefix(t *testing.T) {
+	if !hasAnyPrefix("loop3", skipPrefixes) {
+		t.Error("loop3 should match")
+	}
+	if hasAnyPrefix("sda", skipPrefixes) {
+		t.Error("sda should not match")
+	}
+}
+
+func TestReadKernelPartitionsListsOnlyPartitions(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{
+		name: "nvme0n1", sectors: 1 << 22,
+		partitions: []string{"nvme0n1p3", "nvme0n1p1", "nvme0n1p2"},
+	})
+
+	got, err := readKernelPartitions("/dev/nvme0n1")
+	if err != nil {
+		t.Fatalf("readKernelPartitions: %v", err)
+	}
+	// Sorted, and "queue" and "device" alongside them are not partitions.
+	want := []string{"nvme0n1p1", "nvme0n1p2", "nvme0n1p3"}
+	if !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestReadKernelPartitionsIsEmptyForAnUnpartitionedDisk(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "nvme2n1", sectors: 1 << 22})
+
+	got, err := readKernelPartitions("/dev/nvme2n1")
+	if err != nil {
+		t.Fatalf("readKernelPartitions: %v", err)
+	}
+	if len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestReadKernelPartitionsFailsOnAnUnknownDisk(t *testing.T) {
+	newFakeSysfs(t)
+
+	if _, err := readKernelPartitions("/dev/nvme9n1"); err == nil {
+		t.Fatal("expected an error for a disk with no sysfs entry")
+	}
+}
+
+func TestDiskHoldersNamesEveryClaimant(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{
+		name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"},
+	})
+	f.addHolder("sda", "sda1", "md0")
+	f.addHolder("sda", "sda2", "dm-0")
+
+	if got, want := diskHolders("/dev/sda"), []string{"dm-0", "md0"}; !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestDiskHoldersIsEmptyWhenNothingClaimsTheDisk(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
+
+	if got := diskHolders("/dev/sda"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestMountsOnDiskFindsOnlyThisDiskDeepestFirst(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"}},
+		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
+	)
+	f.setMounts(strings.Join([]string{
+		"/dev/sda1 /oldroot ext4 rw 0 0",
+		"/dev/sda2 /oldroot/boot/efi vfat rw 0 0",
+		"/dev/sdb1 /other ext4 rw 0 0",
+		"proc /proc proc rw 0 0",
+	}, "\n") + "\n")
+
+	got := mountsOnDisk("/dev/sda")
+	want := []diskMount{
+		{Device: "/dev/sda2", Mountpoint: "/oldroot/boot/efi"},
+		{Device: "/dev/sda1", Mountpoint: "/oldroot"},
+	}
+	if !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+// /dev/nvme0n1 must not claim /dev/nvme0n11, which a string-prefix match would.
+func TestMountsOnDiskDoesNotMatchASiblingNamespace(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "nvme0n1", sectors: 1 << 22, partitions: []string{"nvme0n1p1"}},
+		fakeDisk{name: "nvme0n11", sectors: 1 << 22, partitions: []string{"nvme0n11p1"}},
+	)
+	f.setMounts("/dev/nvme0n11p1 /elsewhere ext4 rw 0 0\n")
+
+	if got := mountsOnDisk("/dev/nvme0n1"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestSwapsOnDiskSkipsTheHeaderAndOtherDisks(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}},
+		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
+	)
+	f.setSwaps(strings.Join([]string{
+		"Filename\t\t\t\tType\t\tSize\tUsed\tPriority",
+		"/dev/sda1 partition 8388604 0 -2",
+		"/dev/sdb1 partition 8388604 0 -3",
+	}, "\n") + "\n")
+
+	if got, want := swapsOnDisk("/dev/sda"), []string{"/dev/sda1"}; !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestSwapsOnDiskIsEmptyWithoutASwapsTable(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
+
+	if got := swapsOnDisk("/dev/sda"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}

cmd/installer/install/boottool.go

@@ -0,0 +1,202 @@
+package install
+
+import (
+	_ "embed"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+//go:embed assets/spinifex-boot-tool
+var bootToolScript []byte
+
+//go:embed assets/grub-install-wrapper
+var grubInstallWrapper []byte
+
+const (
+	bootToolPath     = "/usr/sbin/spinifex-boot-tool"
+	grubInstallPath  = "/usr/sbin/grub-install"
+	grubInstallReal  = "/usr/sbin/grub-install.real"
+	bootToolUUIDList = "/etc/kernel/spinifex-boot-uuids"
+)
+
+// installBootToolZFS sets up per-disk bootloaders on a ZFS root system.
+//
+// Nothing in the boot path is allowed to read the pool: GRUB's ZFS support
+// trails OpenZFS, so a pool feature flag enabled years from now could make the
+// machine unbootable. Each disk instead carries a self-contained ESP with its
+// own GRUB, kernels and initramfs.
+func installBootToolZFS(cfg DiskConfig) error {
+	if err := writeTargetFile(bootToolPath, bootToolScript, 0o755); err != nil {
+		return err
+	}
+	if err := writeGrubDefaults(cfg); err != nil {
+		return err
+	}
+
+	if err := bindChrootMounts(); err != nil {
+		return err
+	}
+	defer unbindChrootMounts()
+
+	// Diverted before the first init: init runs a refresh, and refresh calls
+	// grub-install. With the wrapper already in place that call has to reach
+	// the real binary, which is what the diversion arranges.
+	if err := divertGrubInstall(); err != nil {
+		return err
+	}
+
+	// A partially-updated set of ESPs is an install that boots today and fails
+	// after the next kernel upgrade, so every disk must succeed here.
+	env := []string{"SPINIFEX_BOOT_TOOL_STRICT=1"}
+	for _, d := range cfg.Disks {
+		esp := d.PartitionPath(espPartNum)
+		if err := runEnv(env, "chroot", mountRoot, bootToolPath, "init", esp); err != nil {
+			return fmt.Errorf("spinifex-boot-tool init %s: %w", esp, err)
+		}
+	}
+
+	// The one moment firmware boot variables are written. Refreshes run on
+	// every kernel upgrade and deliberately leave NVRAM alone: repeatedly
+	// rewriting boot entries has bricked more than one vendor's boards. Losing
+	// the entry is survivable anyway, because each ESP also carries the
+	// removable-path loader that firmware falls back to.
+	if err := runEnv(append(env, "SPINIFEX_BOOT_TOOL_NVRAM=1"),
+		"chroot", mountRoot, bootToolPath, "refresh"); err != nil {
+		return fmt.Errorf("spinifex-boot-tool refresh: %w", err)
+	}
+	return verifyBootConfig(cfg)
+}
+
+// divertGrubInstall moves the real grub-install aside and puts the wrapper in
+// its place. dpkg-divert makes the move survive package upgrades, which is the
+// point: grub-pc's postinst runs grub-install and would otherwise quietly
+// replace a working boot layout with one that reads the pool.
+func divertGrubInstall() error {
+	if err := run("chroot", mountRoot, "dpkg-divert", "--divert", grubInstallReal,
+		"--rename", "--add", grubInstallPath); err != nil {
+		return fmt.Errorf("divert grub-install: %w", err)
+	}
+	return writeTargetFile(grubInstallPath, grubInstallWrapper, 0o755)
+}
+
+// verifyBootConfig asserts that the config the machine will actually boot from
+// names the ZFS root. grub-mkconfig degrades quietly when grub-probe cannot
+// read the pool, emitting a config that gets as far as the GRUB menu and then
+// panics with "unable to mount root fs" — which looks like a hardware fault.
+func verifyBootConfig(cfg DiskConfig) error {
+	uuids, err := os.ReadFile(filepath.Join(mountRoot, strings.TrimPrefix(bootToolUUIDList, "/")))
+	if err != nil {
+		return fmt.Errorf("read ESP list: %w", err)
+	}
+	if len(strings.Fields(string(uuids))) != len(cfg.Disks) {
+		return fmt.Errorf("expected %d registered ESPs, found %d — some disks have no bootloader",
+			len(cfg.Disks), len(strings.Fields(string(uuids))))
+	}
+
+	// One ESP is enough to prove grub-mkconfig produced the right thing; they
+	// are all generated by the same code path in the same run.
+	mp := filepath.Join(mountRoot, "var/tmp/verify-esp")
+	if err := os.MkdirAll(mp, 0o755); err != nil {
+		return err
+	}
+	defer func() { _ = os.Remove(mp) }()
+
+	esp := cfg.Disks[0].PartitionPath(espPartNum)
+	if err := run("mount", esp, mp); err != nil {
+		return fmt.Errorf("mount %s for verification: %w", esp, err)
+	}
+	defer func() { _ = run("umount", mp) }()
+
+	grubCfg, err := os.ReadFile(filepath.Join(mp, "grub/grub.cfg"))
+	if err != nil {
+		return fmt.Errorf("read grub.cfg from %s: %w", esp, err)
+	}
+	want := "root=ZFS=" + ZFSRootDataset
+	if !strings.Contains(string(grubCfg), want) {
+		return fmt.Errorf("grub.cfg on %s does not contain %q — the installed system would not find its root filesystem", esp, want)
+	}
+	if !strings.Contains(string(grubCfg), "vmlinuz-") {
+		return fmt.Errorf("grub.cfg on %s references no kernel", esp)
+	}
+	return nil
+}
+
+// writeTargetFile writes a file into the install target.
+func writeTargetFile(path string, content []byte, mode os.FileMode) error {
+	full := filepath.Join(mountRoot, strings.TrimPrefix(path, "/"))
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		return err
+	}
+	if err := os.WriteFile(full, content, mode); err != nil {
+		return fmt.Errorf("write %s: %w", path, err)
+	}
+	// WriteFile does not apply the mode to a file that already exists, and this
+	// path is re-run on a retried install.
+	return os.Chmod(full, mode)
+}
+
+// writeGrubDefaults writes /etc/default/grub and the branding snippet.
+//
+// Both consoles are listed: tty0 stays primary for a local install while ttyS0
+// mirrors kernel output for headless racks and CI. The last console= on the
+// cmdline becomes the system console, so serial-only hardware still works.
+func writeGrubDefaults(cfg DiskConfig) error {
+	cmdline := "console=tty0 console=ttyS0,115200 systemd.show_status=1"
+	if cfg.FS.IsZFS() {
+		// Stated explicitly rather than relying on grub-mkconfig's probe. It
+		// appears after the root= that 10_linux computes, so it wins if the
+		// probe came back with something wrong.
+		cmdline = fmt.Sprintf("root=ZFS=%s boot=zfs %s", ZFSRootDataset, cmdline)
+	}
+	grubDefault := fmt.Sprintf(`GRUB_DEFAULT=0
+GRUB_TIMEOUT=5
+GRUB_DISTRIBUTOR=Spinifex
+GRUB_CMDLINE_LINUX_DEFAULT=""
+GRUB_CMDLINE_LINUX="%s"
+`, cmdline)
+	if err := os.WriteFile(filepath.Join(mountRoot, "etc/default/grub"), []byte(grubDefault), 0o644); err != nil {
+		return fmt.Errorf("write /etc/default/grub: %w", err)
+	}
+
+	// gfxterm must be activated before serial is appended — background_image
+	// silently does nothing in text mode. $prefix is used rather than a literal
+	// /boot/grub so the same snippet is correct whether GRUB's root is the ext4
+	// filesystem or a per-disk ESP.
+	grubTheme := `#!/bin/sh
+exec tail -n +3 $0
+insmod all_video
+insmod font
+if loadfont $prefix/fonts/unicode.pf2; then
+  set gfxmode=auto
+  insmod gfxterm
+  terminal_output gfxterm
+fi
+insmod serial
+if serial --unit=0 --speed=115200 --timeout=1; then
+  terminal_input  --append serial
+  terminal_output --append serial
+fi
+
+# --- Branding ---
+insmod png
+set theme=$prefix/theme.txt
+export theme
+`
+	if err := os.WriteFile(filepath.Join(mountRoot, "etc/grub.d/06_spinifex"), []byte(grubTheme), 0o755); err != nil {
+		return fmt.Errorf("write /etc/grub.d/06_spinifex: %w", err)
+	}
+	return nil
+}
+
+// refreshBootToolNote logs the command an operator needs after replacing a
+// disk. Nothing in the installer can do this for them later.
+func refreshBootToolNote(cfg DiskConfig) {
+	if cfg.FS.IsZFS() {
+		slog.Info("bootloader installed on every pool member",
+			"disks", len(cfg.Disks),
+			"after_disk_replacement", "spinifex-boot-tool format <esp> && spinifex-boot-tool init <esp>")
+	}
+}

cmd/installer/install/config.go

@@ -2,42 +2,107 @@
 // collected configuration.
 package install
 
+import (
+	"fmt"
+	"net"
+	"strings"
+)
+
+// Plane names the three traffic planes a node presents. Each plane is a role
+// bound to a target, not a fixed NIC: the same three roles collapse onto one,
+// two or three interfaces depending on the hardware.
+type Plane string
+
+const (
+	// PlaneWAN carries public north-south traffic: ingress/egress, EIPs and
+	// SNAT/DNAT. It is the OVN external bridge-mapping.
+	PlaneWAN Plane = "wan"
+
+	// PlaneLAN carries internal cluster traffic: predastore replication, the
+	// NATS cluster mesh, OVN NB/SB control, Raft and S3DB.
+	PlaneLAN Plane = "lan"
+
+	// PlaneVPC carries the tenant overlay: OVN Geneve tunnels between nodes.
+	// It is the ovn-encap-ip endpoint.
+	PlaneVPC Plane = "vpc"
+)
+
+// Bridge returns the Linux bridge that carries this plane.
+func (p Plane) Bridge() string { return "br-" + string(p) }
+
+// NetworkRole binds one plane to a network target. An empty Interface means
+// the role is folded onto the next role up the chain (vpc <- lan <- wan) and
+// no bridge of its own is created.
+//
+// VLAN is what allows two roles to share one physical NIC while staying
+// separate planes: each gets its own NIC.<vlan> subinterface and bridge. Two
+// roles on the same Interface with no VLAN are the same plane, not two.
+type NetworkRole struct {
+	// Interface is the physical NIC this role binds to; empty means folded.
+	Interface string
+
+	// VLAN is the 802.1Q tag, or 0 for untagged.
+	VLAN int
+
+	// DHCPMode leaves addressing to the DHCP client; Address/Mask are then unset.
+	DHCPMode bool
+
+	Address string
+	Mask    string
+
+	// Gateway is only meaningful on the wan role — the other planes are
+	// link-local to the rack and must not install a default route.
+	Gateway string
+
+	DNS []string
+
+	// MTU is written as MTUBytes= on the link; 0 leaves the kernel default.
+	// East-west planes typically run 9000, wan stays at 1500.
+	MTU int
+
+	// WiFiSSID and WiFiPass are set when the selected NIC is wireless.
+	WiFiSSID string
+	WiFiPass string
+}
+
+// Folded reports whether this role has no interface of its own and inherits
+// the plane of the role above it.
+func (r NetworkRole) Folded() bool { return r.Interface == "" }
+
+// Link returns the name of the interface carrying this role's address: the
+// physical NIC, or its VLAN subinterface when tagged. Empty when folded.
+func (r NetworkRole) Link() string {
+	if r.Folded() {
+		return ""
+	}
+	if r.VLAN > 0 {
+		return fmt.Sprintf("%s.%d", r.Interface, r.VLAN)
+	}
+	return r.Interface
+}
+
 // Config holds all values collected by the installer UI.
 type Config struct {
-	// Disk is the block device path to install onto (e.g. /dev/sda).
-	Disk string
+	// Storage is the filesystem choice and the ordered set of disks it is
+	// built from. In ext4 mode it holds exactly one disk.
+	Storage DiskConfig
 
-	// WAN interface — always present. The management IP lives on br-wan (a Linux
-	// bridge over the NIC), not on the physical NIC itself. This allows OVN/OVS
-	// to safely attach to the bridge without disrupting host connectivity.
-	WANInterface string
-	WANDHCPMode  bool
-	WANAddress   string   // empty if WANDHCPMode
-	WANMask      string   // empty if WANDHCPMode
-	WANGateway   string   // empty if WANDHCPMode
-	WANDNS       []string // empty if WANDHCPMode; e.g. ["1.1.1.1", "8.8.8.8"]
-	WANWiFiSSID  string   // non-empty if a WiFi NIC was selected
-	WANWiFiPass  string   // non-empty if a WiFi NIC was selected
+	// WAN is always present and can never be folded — a node needs an uplink.
+	// The address lives on br-wan (a Linux bridge over the NIC), not on the
+	// physical NIC itself, so OVN/OVS can attach to the bridge without
+	// disrupting host connectivity.
+	WAN NetworkRole
 
-	// LAN interface — only set when 2+ NICs are present. The Geneve encap IP
-	// for OVN tunnels lives on br-lan. When empty, the WAN address is used.
-	LANInterface string
-	LANDHCPMode  bool
-	LANAddress   string
-	LANMask      string
-	LANDNS       []string
-	LANWiFiSSID  string
-	LANWiFiPass  string
+	// LAN carries internal cluster traffic. Folded onto WAN on single-NIC nodes.
+	LAN NetworkRole
+
+	// VPC carries Geneve overlay traffic and supplies ovn-encap-ip. Folded onto
+	// LAN unless a node dedicates an interface or VLAN to it.
+	VPC NetworkRole
 
 	// Node identity
 	Hostname string
 
-	// ClusterRole is "init" or "join".
-	ClusterRole string
-
-	// JoinAddr is the primary node address (host:port) when ClusterRole is "join".
-	JoinAddr string
-
 	// SkipFormation skips spx admin init/join in firstboot; the provisioning
 	// controller (e.g. bm-bootstrap.sh) owns cluster formation instead.
 	SkipFormation bool
@@ -60,3 +125,111 @@ type Config struct {
 	// Passes --gpu-passthrough to `spx admin init` during firstboot.
 	GPUPassthrough bool
 }
+
+// Resolve returns the role that actually carries the given plane after
+// collapsing folded roles up the chain vpc <- lan <- wan, along with the plane
+// it landed on. A folded vpc on a single-NIC node resolves to the wan role.
+func (c *Config) Resolve(p Plane) (NetworkRole, Plane) {
+	switch p {
+	case PlaneVPC:
+		if !c.VPC.Folded() {
+			return c.VPC, PlaneVPC
+		}
+		return c.Resolve(PlaneLAN)
+	case PlaneLAN:
+		if !c.LAN.Folded() {
+			return c.LAN, PlaneLAN
+		}
+		return c.WAN, PlaneWAN
+	default:
+		return c.WAN, PlaneWAN
+	}
+}
+
+// PlaneAddress returns the configured address for a plane after collapsing.
+// Empty when the resolved role uses DHCP, since no static address is known
+// until the lease is taken.
+func (c *Config) PlaneAddress(p Plane) string {
+	role, _ := c.Resolve(p)
+	if role.DHCPMode {
+		return ""
+	}
+	return role.Address
+}
+
+// PlaneBridge returns the bridge carrying a plane after collapsing, so callers
+// asking for the vpc bridge on a single-NIC node correctly get br-wan.
+func (c *Config) PlaneBridge(p Plane) string {
+	_, landed := c.Resolve(p)
+	return landed.Bridge()
+}
+
+// RoleBinding pairs a plane with the role that carries it.
+type RoleBinding struct {
+	Plane Plane
+	Role  NetworkRole
+}
+
+// Roles returns the bound roles in bridge-creation order, skipping folded ones.
+// wan comes first because br-lan and br-vpc are activated manually after it.
+func (c *Config) Roles() []RoleBinding {
+	out := []RoleBinding{{PlaneWAN, c.WAN}}
+	if !c.LAN.Folded() {
+		out = append(out, RoleBinding{PlaneLAN, c.LAN})
+	}
+	if !c.VPC.Folded() {
+		out = append(out, RoleBinding{PlaneVPC, c.VPC})
+	}
+	return out
+}
+
+// Validate enforces the sharing rules the UI and the headless path must both
+// obey: wan always needs a target, and two roles may only share an interface
+// when distinct VLAN ids keep them on separate broadcast domains.
+func (c *Config) Validate() error {
+	if c.WAN.Folded() {
+		return fmt.Errorf("wan role must be bound to an interface")
+	}
+
+	seen := map[string]Plane{}
+	for _, r := range c.Roles() {
+		link := r.Role.Link()
+		if prev, dup := seen[link]; dup {
+			return fmt.Errorf("%s and %s both bind %s: give each a distinct VLAN id or fold one onto the other",
+				prev, r.Plane, link)
+		}
+		seen[link] = r.Plane
+		if r.Role.VLAN < 0 || r.Role.VLAN > 4094 {
+			return fmt.Errorf("%s: VLAN id %d out of range (1-4094)", r.Plane, r.Role.VLAN)
+		}
+		if r.Role.MTU != 0 && (r.Role.MTU < 68 || r.Role.MTU > 9216) {
+			return fmt.Errorf("%s: MTU %d out of range (68-9216)", r.Plane, r.Role.MTU)
+		}
+		// Resolvers belong to the plane that holds the default route. Accepting
+		// one here and dropping it during generation would leave an unattended
+		// install believing it had set a resolver that was never written.
+		if r.Plane != PlaneWAN && len(r.Role.DNS) > 0 {
+			return fmt.Errorf("%s: DNS servers belong to the wan plane, which holds the default route", r.Plane)
+		}
+		// Netgen converts the mask with net.ParseIP, so a prefix length reaches
+		// it as garbage and fails the install after the disk is partitioned.
+		// Catch it here, where the operator can still fix it.
+		if m := strings.TrimSpace(r.Role.Mask); m != "" && !validDottedMask(m) {
+			return fmt.Errorf("%s: subnet mask %q must be dotted-decimal, e.g. 255.255.255.0", r.Plane, m)
+		}
+	}
+	return nil
+}
+
+// validDottedMask reports whether s is a contiguous dotted-decimal IPv4 subnet
+// mask. Prefix-length form ("24", "/24") is deliberately not accepted: netgen
+// parses the mask with net.ParseIP, so anything else fails the install itself.
+func validDottedMask(s string) bool {
+	ip := net.ParseIP(strings.TrimSpace(s))
+	if ip == nil || ip.To4() == nil {
+		return false
+	}
+	// Size reports zero bits for a non-contiguous mask, which netgen rejects too.
+	_, bits := net.IPMask(ip.To4()).Size()
+	return bits != 0
+}

cmd/installer/install/config_test.go

@@ -0,0 +1,463 @@
+package install
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// Node layouts drawn from the us-west-1 racks, which between them exercise
+// every collapse permutation the installer has to express.
+func threeNIC() *Config { // hydrogen / radon
+	return &Config{
+		WAN: NetworkRole{Interface: "eno6", Address: "216.218.163.99", Mask: "255.255.255.224", Gateway: "216.218.163.97"},
+		LAN: NetworkRole{Interface: "ens1f0np0", Address: "10.0.0.3", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "ens1f1np1", Address: "10.1.0.3", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func vlanSplit() *Config { // helium — one 100gbe carrying lan and vpc
+	return &Config{
+		WAN: NetworkRole{Interface: "enx6c1ff7c1475a", DHCPMode: true},
+		LAN: NetworkRole{Interface: "enp129s0np0", VLAN: 10, Address: "10.0.0.2", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "enp129s0np0", VLAN: 20, Address: "10.1.0.2", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func dualPortTagged() *Config { // us-west-1-az2 — wan and lan share port 1
+	return &Config{
+		WAN: NetworkRole{Interface: "enp1s0f0", VLAN: 30, Address: "216.218.163.10", Mask: "255.255.255.224", Gateway: "216.218.163.97"},
+		LAN: NetworkRole{Interface: "enp1s0f0", VLAN: 10, Address: "10.0.0.10", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "enp1s0f1", VLAN: 20, Address: "10.1.0.10", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func TestResolveCollapsesUpTheChain(t *testing.T) {
+	singleNIC := &Config{WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"}}
+	twoNIC := &Config{
+		WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"},
+		LAN: NetworkRole{Interface: "eth1", Address: "10.0.0.5", Mask: "255.255.0.0"},
+	}
+
+	tests := []struct {
+		name      string
+		cfg       *Config
+		plane     Plane
+		wantAddr  string
+		wantPlane Plane
+	}{
+		{"1 NIC: lan folds onto wan", singleNIC, PlaneLAN, "192.168.1.10", PlaneWAN},
+		{"1 NIC: vpc folds all the way to wan", singleNIC, PlaneVPC, "192.168.1.10", PlaneWAN},
+		{"2 NIC: vpc folds onto lan", twoNIC, PlaneVPC, "10.0.0.5", PlaneLAN},
+		{"2 NIC: lan stays put", twoNIC, PlaneLAN, "10.0.0.5", PlaneLAN},
+		{"3 NIC: vpc is its own plane", threeNIC(), PlaneVPC, "10.1.0.3", PlaneVPC},
+		{"VLAN split: vpc is its own plane", vlanSplit(), PlaneVPC, "10.1.0.2", PlaneVPC},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.cfg.PlaneAddress(tt.plane); got != tt.wantAddr {
+				t.Errorf("PlaneAddress(%s) = %q, want %q", tt.plane, got, tt.wantAddr)
+			}
+			if _, landed := tt.cfg.Resolve(tt.plane); landed != tt.wantPlane {
+				t.Errorf("Resolve(%s) landed on %s, want %s", tt.plane, landed, tt.wantPlane)
+			}
+		})
+	}
+}
+
+// A DHCP plane has no address known at install time; setup-ovn.sh auto-detects
+// it from the default route at boot instead.
+func TestPlaneAddressEmptyUnderDHCP(t *testing.T) {
+	if got := vlanSplit().PlaneAddress(PlaneWAN); got != "" {
+		t.Errorf("PlaneAddress(wan) = %q, want empty for a DHCP role", got)
+	}
+}
+
+func TestValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     *Config
+		wantErr string
+	}{
+		{"three physical NICs", threeNIC(), ""},
+		{"VLAN split on one NIC", vlanSplit(), ""},
+		{"dual port, all tagged", dualPortTagged(), ""},
+		{
+			"wan cannot be folded",
+			&Config{LAN: NetworkRole{Interface: "eth1"}},
+			"wan role must be bound",
+		},
+		{
+			"two roles share a NIC untagged",
+			&Config{
+				WAN: NetworkRole{Interface: "eth0"},
+				LAN: NetworkRole{Interface: "eth0"},
+			},
+			"both bind eth0",
+		},
+		{
+			"duplicate VLAN id on one NIC",
+			&Config{
+				WAN: NetworkRole{Interface: "eth0", VLAN: 10},
+				LAN: NetworkRole{Interface: "eth0", VLAN: 10},
+			},
+			"both bind eth0.10",
+		},
+		{
+			"VLAN id out of range",
+			&Config{WAN: NetworkRole{Interface: "eth0", VLAN: 5000}},
+			"out of range",
+		},
+		{
+			"MTU out of range",
+			&Config{WAN: NetworkRole{Interface: "eth0", MTU: 70000}},
+			"MTU 70000 out of range",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("Validate() = %v, want nil", err)
+				}
+				return
+			}
+			if err == nil {
+				t.Fatalf("Validate() = nil, want error containing %q", tt.wantErr)
+			}
+			if !strings.Contains(err.Error(), tt.wantErr) {
+				t.Errorf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Two roles sharing one NIC must produce a single parent .network carrying both
+// VLAN declarations — not two files racing to configure the same interface.
+func TestWriteParentNICTaggedSharesOneFile(t *testing.T) {
+	dir := t.TempDir()
+	p := &parentNIC{vlans: []string{"enp129s0np0.10", "enp129s0np0.20"}, mtu: 9000}
+	if err := writeParentNIC(dir, "enp129s0np0", p); err != nil {
+		t.Fatal(err)
+	}
+
+	got := readFile(t, dir, "05-spinifex-nic-enp129s0np0.network")
+	for _, want := range []string{
+		"Name=enp129s0np0",
+		"MTUBytes=9000",
+		"VLAN=enp129s0np0.10",
+		"VLAN=enp129s0np0.20",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("parent .network missing %q:\n%s", want, got)
+		}
+	}
+	// A tagged parent is never itself enslaved — the VLAN devices are.
+	if strings.Contains(got, "Bridge=") {
+		t.Errorf("tagged parent must not be bridged directly:\n%s", got)
+	}
+}
+
+func TestWriteNetworkdBridgeVLAN(t *testing.T) {
+	dir := t.TempDir()
+	cfg := vlanSplit()
+	if err := writeNetworkdBridge(dir, PlaneVPC, cfg.VPC, true); err != nil {
+		t.Fatal(err)
+	}
+
+	netdev := readFile(t, dir, "10-spinifex-vpc-vlan.netdev")
+	if !strings.Contains(netdev, "Kind=vlan") || !strings.Contains(netdev, "Id=20") {
+		t.Errorf("VLAN netdev wrong:\n%s", netdev)
+	}
+	if !strings.Contains(netdev, "Name=enp129s0np0.20") {
+		t.Errorf("VLAN netdev name wrong:\n%s", netdev)
+	}
+
+	vlanNet := readFile(t, dir, "11-spinifex-vpc-vlan.network")
+	if !strings.Contains(vlanNet, "Bridge=br-vpc") {
+		t.Errorf("VLAN device must be enslaved to br-vpc:\n%s", vlanNet)
+	}
+
+	brNet := readFile(t, dir, "20-spinifex-vpc.network")
+	if !strings.Contains(brNet, "Address=10.1.0.2/16") {
+		t.Errorf("bridge address wrong:\n%s", brNet)
+	}
+	// Only wan installs a default route; an east-west plane must not.
+	if strings.Contains(brNet, "Gateway=") {
+		t.Errorf("vpc plane must not install a default route:\n%s", brNet)
+	}
+	if !strings.Contains(brNet, "ActivationPolicy=manual") {
+		t.Errorf("east-west bridge must not auto-activate:\n%s", brNet)
+	}
+}
+
+// The wan plane is the one that carries a default route and auto-activates.
+func TestWriteNetworkdBridgeWANUntagged(t *testing.T) {
+	dir := t.TempDir()
+	cfg := threeNIC()
+	if err := writeNetworkdBridge(dir, PlaneWAN, cfg.WAN, false); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := os.Stat(filepath.Join(dir, "10-spinifex-wan-vlan.netdev")); !os.IsNotExist(err) {
+		t.Error("untagged role must not produce a VLAN netdev")
+	}
+	brNet := readFile(t, dir, "20-spinifex-wan.network")
+	if !strings.Contains(brNet, "Gateway=216.218.163.97") {
+		t.Errorf("wan must install its default route:\n%s", brNet)
+	}
+	if strings.Contains(brNet, "ActivationPolicy=manual") {
+		t.Errorf("wan bridge must auto-activate:\n%s", brNet)
+	}
+}
+
+func TestToFirstbootConfigSourcesPlanes(t *testing.T) {
+	cfg := threeNIC()
+	cfg.Hostname = "hydrogen"
+	fb := cfg.toFirstbootConfig()
+
+	if fb.EncapIP != "10.1.0.3" {
+		t.Errorf("EncapIP = %q, want the vpc plane address", fb.EncapIP)
+	}
+	if fb.LANIP != "10.0.0.3" {
+		t.Errorf("LANIP = %q, want the lan plane address", fb.LANIP)
+	}
+	// The public address has to travel separately: spx echoes a concrete --bind
+	// back as the advertise address, so leaving this empty would publish the
+	// internal plane as the node's public dial target.
+	if fb.WANIP != "216.218.163.99" {
+		t.Errorf("WANIP = %q, want the wan plane address", fb.WANIP)
+	}
+
+	// On a single-NIC node all three collapse onto wan.
+	single := &Config{WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"}}
+	fb = single.toFirstbootConfig()
+	if fb.EncapIP != "192.168.1.10" || fb.LANIP != "192.168.1.10" || fb.WANIP != "192.168.1.10" {
+		t.Errorf("single-NIC collapse wrong: EncapIP=%q LANIP=%q WANIP=%q", fb.EncapIP, fb.LANIP, fb.WANIP)
+	}
+}
+
+func readFile(t *testing.T, dir, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(dir, name))
+	if err != nil {
+		t.Fatalf("reading %s: %v", name, err)
+	}
+	return string(b)
+}
+
+// Resolvers belong to the plane holding the default route. The east-west
+// bridges are link-local and activate manually, so a resolver pinned to one is
+// unreachable while resolved still tries it — a timeout in front of every
+// lookup. The TUI cannot express this; the kernel cmdline can, so it is caught.
+func TestValidateRejectsDNSOffWAN(t *testing.T) {
+	cfg := threeNIC()
+	cfg.LAN.DNS = []string{"10.0.0.1"}
+	if err := cfg.Validate(); err == nil {
+		t.Fatal("lan must not carry DNS servers")
+	}
+
+	cfg = threeNIC()
+	cfg.VPC.DNS = []string{"10.1.0.1"}
+	if err := cfg.Validate(); err == nil {
+		t.Fatal("vpc must not carry DNS servers")
+	}
+
+	cfg = threeNIC()
+	cfg.WAN.DNS = []string{"8.8.8.8", "1.1.1.1"}
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("wan carries the resolvers: %v", err)
+	}
+}
+
+// Even if a role somehow carries DNS, generation must not write it onto an
+// east-west bridge.
+func TestWriteNetworkdBridgeKeepsDNSOnWAN(t *testing.T) {
+	dir := t.TempDir()
+	lan := NetworkRole{Interface: "ens1f0np0", Address: "10.0.0.3", Mask: "255.255.0.0", DNS: []string{"10.0.0.1"}}
+	if err := writeNetworkdBridge(dir, PlaneLAN, lan, true); err != nil {
+		t.Fatalf("writeNetworkdBridge: %v", err)
+	}
+	if got := readFile(t, dir, "20-spinifex-lan.network"); strings.Contains(got, "DNS=") {
+		t.Errorf("lan bridge must carry no resolver:\n%s", got)
+	}
+
+	wan := NetworkRole{Interface: "eno6", Address: "216.218.163.99", Mask: "255.255.255.224", Gateway: "216.218.163.97", DNS: []string{"8.8.8.8"}}
+	if err := writeNetworkdBridge(dir, PlaneWAN, wan, false); err != nil {
+		t.Fatalf("writeNetworkdBridge: %v", err)
+	}
+	if got := readFile(t, dir, "20-spinifex-wan.network"); !strings.Contains(got, "DNS=8.8.8.8") {
+		t.Errorf("wan bridge must carry the resolver:\n%s", got)
+	}
+}
+
+func TestSwapSize(t *testing.T) {
+	dir := t.TempDir()
+
+	// Without an override, RAM drives the size and the disk caps it.
+	ram, err := totalRAM()
+	if err != nil {
+		t.Fatalf("totalRAM: %v", err)
+	}
+	got, err := swapSize(dir)
+	if err != nil {
+		t.Fatalf("swapSize: %v", err)
+	}
+	want := ram / 2
+	if ram >= swapRAMThreshold {
+		want = swapLargeBytes
+	}
+	if got > want {
+		t.Errorf("swapSize = %d, must not exceed what RAM implies (%d)", got, want)
+	}
+	if got != 0 && got < swapMinBytes {
+		t.Errorf("swapSize = %d, must be 0 rather than below the %d floor", got, int64(swapMinBytes))
+	}
+
+	// An explicit size wins, so a large node can be told to take the full 32G
+	// and a constrained one can opt out entirely.
+	for _, tt := range []struct {
+		env  string
+		want int64
+	}{
+		{"32G", 32 << 30},
+		{"512M", 512 << 20},
+		{"0", 0},
+		{"1073741824", 1 << 30},
+	} {
+		t.Setenv("SPINIFEX_SWAP_SIZE", tt.env)
+		got, err := swapSize(dir)
+		if err != nil {
+			t.Fatalf("swapSize(%s): %v", tt.env, err)
+		}
+		if got != tt.want {
+			t.Errorf("swapSize(%s) = %d, want %d", tt.env, got, tt.want)
+		}
+	}
+
+	t.Setenv("SPINIFEX_SWAP_SIZE", "banana")
+	if _, err := swapSize(dir); err == nil {
+		t.Error("an unparseable size must be reported, not silently ignored")
+	}
+}
+
+// The swap line must only appear when the file backing it exists — systemd
+// fails the boot's swap unit on a missing file rather than skipping it.
+func TestFstabSwapLineTracksTheFile(t *testing.T) {
+	if _, err := parseSize("8G"); err != nil {
+		t.Fatalf("parseSize: %v", err)
+	}
+	if n, _ := parseSize("2G"); n != 2<<30 {
+		t.Errorf("parseSize(2G) = %d, want %d", n, int64(2)<<30)
+	}
+	if _, err := parseSize("-5G"); err == nil {
+		t.Error("negative sizes must be rejected")
+	}
+}
+
+// Bridge ports and VLAN trunks never reach the "degraded" state
+// systemd-networkd-wait-online requires — the address sits on the bridge above
+// them. Leaving them required makes that unit fail on every boot and stall
+// until it times out, which is what a three-NIC node was doing.
+func TestSlaveLinksAreNotRequiredForOnline(t *testing.T) {
+	dir := t.TempDir()
+
+	// Untagged bridge port.
+	if err := writeParentNIC(dir, "ens4", &parentNIC{bridge: "br-lan", mtu: 9000}); err != nil {
+		t.Fatal(err)
+	}
+	got := readFile(t, dir, "05-spinifex-nic-ens4.network")
+	if !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("bridge port must not gate network-online:\n%s", got)
+	}
+	if !strings.Contains(got, "MTUBytes=9000") {
+		t.Errorf("MTU must survive alongside it:\n%s", got)
+	}
+
+	// VLAN trunk carrying tagged planes.
+	if err := writeParentNIC(dir, "enp129s0np0", &parentNIC{vlans: []string{"enp129s0np0.10"}}); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "05-spinifex-nic-enp129s0np0.network"); !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("VLAN trunk must not gate network-online:\n%s", got)
+	}
+
+	// The VLAN device itself is enslaved too.
+	cfg := vlanSplit()
+	if err := writeNetworkdBridge(dir, PlaneVPC, cfg.VPC, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "11-spinifex-vpc-vlan.network"); !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("VLAN device must not gate network-online:\n%s", got)
+	}
+
+	// The wan bridge is what should gate it, and still does.
+	three := threeNIC()
+	if err := writeNetworkdBridge(dir, PlaneWAN, three.WAN, false); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "20-spinifex-wan.network"); strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("wan bridge must remain required for network-online:\n%s", got)
+	}
+}
+
+// A prefix length reaches netgen's net.ParseIP as garbage and fails the install
+// after the disk has been partitioned, so it has to be rejected up front. The
+// TUI advertised "255.255.255.0 or 24" while only the first ever worked.
+func TestValidateRejectsPrefixLengthMask(t *testing.T) {
+	for _, bad := range []string{"24", "/24", "16", "255.255.0", "255.0.255.0", "banana"} {
+		cfg := threeNIC()
+		cfg.LAN.Mask = bad
+		if err := cfg.Validate(); err == nil {
+			t.Errorf("mask %q must be rejected", bad)
+		} else if !strings.Contains(err.Error(), "dotted-decimal") {
+			t.Errorf("mask %q: error should name the expected form, got: %v", bad, err)
+		}
+	}
+
+	for _, good := range []string{"255.255.255.0", "255.255.0.0", "255.255.255.252", "255.255.255.255"} {
+		cfg := threeNIC()
+		cfg.LAN.Mask = good
+		if err := cfg.Validate(); err != nil {
+			t.Errorf("mask %q must be accepted: %v", good, err)
+		}
+	}
+
+	// A DHCP role carries no mask, and that must stay valid.
+	cfg := threeNIC()
+	cfg.LAN = NetworkRole{Interface: "ens1f0np0", DHCPMode: true}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("a DHCP role has no mask to check: %v", err)
+	}
+}
+
+// A bridged uplink must present the enslaved NIC's address. Left to its default
+// policy, systemd-networkd generates one for the bridge, so the node appears on
+// the wire as an address belonging to no NIC — DHCP requests carry a source that
+// disagrees with their own chaddr and the unicast reply is never delivered.
+func TestBridgeInheritsPortMAC(t *testing.T) {
+	dir := t.TempDir()
+	cfg := threeNIC()
+
+	for _, tc := range []struct {
+		plane  Plane
+		role   NetworkRole
+		manual bool
+	}{
+		{PlaneWAN, cfg.WAN, false},
+		{PlaneLAN, cfg.LAN, true},
+		{PlaneVPC, cfg.VPC, true},
+	} {
+		if err := writeNetworkdBridge(dir, tc.plane, tc.role, tc.manual); err != nil {
+			t.Fatal(err)
+		}
+		got := readFile(t, dir, "20-spinifex-"+string(tc.plane)+".netdev")
+		if !strings.Contains(got, "MACAddressPolicy=none") {
+			t.Errorf("%s bridge must inherit its port's MAC:\n%s", tc.plane, got)
+		}
+	}
+}

cmd/installer/install/disk.go

@@ -0,0 +1,278 @@
+package install
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+)
+
+// Partition numbers, identical on every disk the installer touches so that a
+// replacement drive can be cloned with `sgdisk -R` and nothing has to be
+// recomputed at 3am.
+const (
+	biosPartNum = 1
+	espPartNum  = 2
+	rootPartNum = 3
+
+	// dataPartNum is the sole partition on a role drive. Data drives carry no
+	// bootloader, so they start at 1 rather than mirroring the boot layout.
+	dataPartNum = 1
+)
+
+// GPT partition type GUIDs, by their sgdisk short code.
+const (
+	typeBIOSBoot = "EF02"
+	typeESP      = "EF00"
+	typeLinuxFS  = "8300"
+	typeZFS      = "BF00"
+)
+
+// Partition geometry. The ESP is sized as Proxmox does: 512MiB is enough for
+// two kernels, but a 1GiB ESP on a real disk costs nothing and keeps room for
+// the extra kernels an operator pins during an upgrade.
+const (
+	espSmallMiB     = 512
+	espLargeMiB     = 1024
+	espLargeDiskGiB = 100
+
+	// biosBootMiB is the BIOS Boot Partition GRUB's i386-pc core.img is embedded
+	// into. GPT leaves no post-MBR gap, so without it there is nowhere to put it.
+	biosBootMiB = 1
+
+	// tailReserveMiB is left unallocated at the end of every disk. A replacement
+	// drive of the "same" size is routinely a few thousand sectors smaller, and
+	// without slack `sgdisk -R` refuses to clone the table onto it — which is
+	// discovered while standing in front of a degraded pool.
+	tailReserveMiB = 1024
+)
+
+// partitionSettleTimeout bounds the wait for the kernel and udev to publish a
+// disk's new partition table. A var so tests do not pay it.
+var partitionSettleTimeout = 15 * time.Second
+
+// releaseDisk detaches whatever the live environment has attached to a disk
+// before it is erased. A mounted filesystem or an active swap area makes
+// BLKRRPART return EBUSY, and the kernel then keeps serving the disk's old
+// partition table while sgdisk writes the new one and reports success.
+func releaseDisk(d Disk) {
+	for _, m := range mountsOnDisk(d.Path) {
+		err := runQuiet("umount", m.Device)
+		if err != nil {
+			// Lazy detach: a busy mount must not stall an install that is about to
+			// erase the filesystem underneath it anyway.
+			err = runQuiet("umount", "-l", m.Device)
+		}
+		if err != nil {
+			slog.Warn("could not unmount before erase", "device", m.Device, "mountpoint", m.Mountpoint, "err", err)
+			continue
+		}
+		slog.Info("released mount before erase", "device", m.Device, "mountpoint", m.Mountpoint)
+	}
+	for _, s := range swapsOnDisk(d.Path) {
+		if err := runQuiet("swapoff", s); err != nil {
+			slog.Warn("could not disable swap before erase", "device", s, "err", err)
+			continue
+		}
+		slog.Info("disabled swap before erase", "device", s)
+	}
+}
+
+// settlePartitions forces the kernel to re-read a disk's partition table and
+// blocks until its view matches want, naming the partitions by number.
+//
+// An empty want asserts the disk has no partitions at all. That is the only
+// state no leftover can satisfy: a disk whose previous table had the same
+// partition numbers as the one being written — an old OS install is p1, p2, p3,
+// exactly what the boot layout writes — passes an "are the expected partitions
+// there" check on stale nodes alone.
+func settlePartitions(d Disk, want ...int) error {
+	if err := runQuiet("partprobe", d.Path); err != nil {
+		// partprobe rescans the whole disk and fails if it cannot open any part of
+		// it; the ioctl on its own sometimes still gets through.
+		if err := runQuiet("blockdev", "--rereadpt", d.Path); err != nil {
+			slog.Warn("could not force a partition table re-read", "disk", d.Path, "err", err)
+		}
+	}
+	_ = runQuiet("udevadm", "settle", "--timeout=10")
+
+	wanted := make([]string, 0, len(want))
+	for _, n := range want {
+		wanted = append(wanted, filepath.Base(d.PartitionPath(n)))
+	}
+	slices.Sort(wanted)
+
+	deadline := time.Now().Add(partitionSettleTimeout)
+	for {
+		got, err := kernelPartitions(d.Path)
+		if err != nil {
+			return err
+		}
+		if slices.Equal(got, wanted) {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return staleTableError(d, wanted, got)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// staleTableError explains a kernel partition view that never caught up. The
+// erase case gets its own wording because it is the dangerous one — proceeding
+// would format the previous layout — and because the holder is usually the
+// whole diagnosis.
+func staleTableError(d Disk, wanted, got []string) error {
+	if len(wanted) == 0 {
+		msg := fmt.Sprintf("%s still shows partitions [%s] after its table was erased — the kernel is using the old table, so formatting would write at the previous layout's offsets",
+			d.Path, strings.Join(got, ", "))
+		if h := diskHolders(d.Path); len(h) > 0 {
+			return fmt.Errorf("%s; it is held by %s, which must be stopped before this disk can be installed to",
+				msg, strings.Join(h, ", "))
+		}
+		return fmt.Errorf("%s; something in the live environment is holding it open", msg)
+	}
+	return fmt.Errorf("%s: the kernel shows partitions [%s] but the new table has [%s] — the table was written but never picked up",
+		d.Path, strings.Join(got, ", "), strings.Join(wanted, ", "))
+}
+
+// espSizeMiB picks the ESP size for a disk.
+func espSizeMiB(d Disk) int {
+	if d.Bytes >= espLargeDiskGiB<<30 {
+		return espLargeMiB
+	}
+	return espSmallMiB
+}
+
+// partitionDisks lays down an identical GPT on every selected disk. Each disk
+// gets a BIOS boot partition and an ESP whether or not this machine booted in
+// EFI mode: the cost is 1GiB, and it means a board swap or a firmware change
+// from BIOS to UEFI does not leave an unbootable pool.
+func partitionDisks(cfg DiskConfig) error {
+	ptype := typeLinuxFS
+	if cfg.FS.IsZFS() {
+		ptype = typeZFS
+	}
+	for _, d := range cfg.bootDisks() {
+		if err := partitionOne(d, ptype); err != nil {
+			return fmt.Errorf("partition %s: %w", d.Path, err)
+		}
+	}
+	for _, rm := range cfg.DataMounts() {
+		if err := partitionDataDisk(rm); err != nil {
+			return fmt.Errorf("partition %s (%s): %w", rm.Disk.Path, rm.Role, err)
+		}
+	}
+	return waitForPartitions(cfg)
+}
+
+// partitionDataDisk writes the single-partition GPT for a role drive. It gets
+// no bios_boot and no ESP: nothing installs a bootloader there, and a formatted
+// ESP no firmware entry points at only misleads whoever reads lsblk in an
+// incident.
+func partitionDataDisk(rm RoleMount) error {
+	if err := clearDisk(rm.Disk); err != nil {
+		return fmt.Errorf("clear: %w", err)
+	}
+	// Same tail reserve as a boot drive, for the same reason: a replacement of
+	// the "same" size is routinely a few thousand sectors smaller.
+	if err := run("sgdisk",
+		"-n", fmt.Sprintf("%d:1M:-%dM", dataPartNum, tailReserveMiB),
+		"-t", fmt.Sprintf("%d:%s", dataPartNum, typeLinuxFS),
+		"-c", fmt.Sprintf("%d:%s", dataPartNum, rm.Role.Label()),
+		rm.Disk.Path,
+	); err != nil {
+		return err
+	}
+	if err := run("sgdisk", "-G", rm.Disk.Path); err != nil {
+		slog.Warn("could not randomise GPT GUIDs", "disk", rm.Disk.Path, "err", err)
+	}
+	return settlePartitions(rm.Disk, dataPartNum)
+}
+
+// partitionOne writes the GPT for a single disk.
+func partitionOne(d Disk, rootType string) error {
+	// Stale ZFS labels and filesystem signatures survive a new partition table
+	// and make a retried install fail with errors that read like hardware faults.
+	if err := clearDisk(d); err != nil {
+		return fmt.Errorf("clear: %w", err)
+	}
+
+	esp := espSizeMiB(d)
+	if err := run("sgdisk",
+		"-n", fmt.Sprintf("%d:1M:+%dM", biosPartNum, biosBootMiB),
+		"-t", fmt.Sprintf("%d:%s", biosPartNum, typeBIOSBoot),
+		"-c", fmt.Sprintf("%d:bios_boot", biosPartNum),
+		"-n", fmt.Sprintf("%d:0:+%dM", espPartNum, esp),
+		"-t", fmt.Sprintf("%d:%s", espPartNum, typeESP),
+		"-c", fmt.Sprintf("%d:ESP", espPartNum),
+		"-n", fmt.Sprintf("%d:0:-%dM", rootPartNum, tailReserveMiB),
+		"-t", fmt.Sprintf("%d:%s", rootPartNum, rootType),
+		"-c", fmt.Sprintf("%d:root", rootPartNum),
+		d.Path,
+	); err != nil {
+		return err
+	}
+	// A fresh disk-level GUID: cloning a table with `sgdisk -R` copies it, and
+	// two disks sharing one leaves the pool ambiguous to udev.
+	if err := run("sgdisk", "-G", d.Path); err != nil {
+		slog.Warn("could not randomise GPT GUIDs", "disk", d.Path, "err", err)
+	}
+	return settlePartitions(d, biosPartNum, espPartNum, rootPartNum)
+}
+
+// waitForPartitions ensures every partition device node the installer is about
+// to use exists. The kernel's own view was already asserted per disk as each
+// table was written; udev publishes the /dev nodes and by-id symlinks after
+// that, and Trixie is slow enough about it that mkfs races them.
+func waitForPartitions(cfg DiskConfig) error {
+	deadline := time.Now().Add(partitionSettleTimeout)
+	for _, d := range cfg.bootDisks() {
+		wanted := []string{d.PartitionPath(espPartNum), d.PartitionPath(rootPartNum)}
+		// The pool is built from by-id paths, so their symlinks have to exist
+		// too — udev publishes them later than the kernel device node.
+		if d.ByID != "" {
+			wanted = append(wanted, d.StablePartitionPath(rootPartNum))
+		}
+		for _, part := range wanted {
+			if err := waitForPath(part, deadline); err != nil {
+				return err
+			}
+		}
+	}
+	for _, rm := range cfg.DataMounts() {
+		if err := waitForPath(rm.Disk.PartitionPath(dataPartNum), deadline); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func waitForPath(path string, deadline time.Time) error {
+	for {
+		if _, err := os.Stat(path); err == nil {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("device %s did not appear within timeout — the kernel or udev did not pick up the new partition table", path)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// formatESPs makes a FAT32 filesystem on every disk's ESP. Each one is
+// independent — there is no RAID here, they are kept in step by content.
+func formatESPs(cfg DiskConfig) error {
+	for _, d := range cfg.bootDisks() {
+		esp := d.PartitionPath(espPartNum)
+		// A distinct label per member so `lsblk -f` on a degraded machine tells
+		// the operator which disk each ESP belongs to.
+		if err := run("mkfs.fat", "-F", "32", "-n", "EFI", esp); err != nil {
+			return fmt.Errorf("format ESP %s: %w", esp, err)
+		}
+	}
+	return nil
+}

cmd/installer/install/diskconfig.go

@@ -0,0 +1,483 @@
+package install
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"strings"
+)
+
+// Pool and dataset names. rpool matches Proxmox, which is what every ZFS
+// recovery guide and forum answer assumes — the name an operator will be
+// reading off a serial console at 3am.
+const (
+	ZFSPoolName    = "rpool"
+	ZFSRootDataset = ZFSPoolName + "/ROOT/spinifex"
+)
+
+// FSType selects the root filesystem and, for ZFS, the vdev topology.
+type FSType string
+
+const (
+	FSExt4      FSType = "ext4"
+	FSZFSRAID0  FSType = "zfs-raid0"
+	FSZFSRAID1  FSType = "zfs-raid1"
+	FSZFSRAID10 FSType = "zfs-raid10"
+	FSZFSRAIDZ1 FSType = "zfs-raidz1"
+	FSZFSRAIDZ2 FSType = "zfs-raidz2"
+	FSZFSRAIDZ3 FSType = "zfs-raidz3"
+)
+
+// AllFSTypes is the selector order presented by the UI. ext4 is first and is
+// the default, so ZFS — which holds the kernel at its shipped version and
+// reserves RAM for the ARC — is always an explicit choice.
+var AllFSTypes = []FSType{
+	FSExt4, FSZFSRAID0, FSZFSRAID1, FSZFSRAID10, FSZFSRAIDZ1, FSZFSRAIDZ2, FSZFSRAIDZ3,
+}
+
+// fsMeta carries the per-topology facts that drive display and validation.
+var fsMeta = map[FSType]struct {
+	label     string
+	minDisks  int
+	redundant bool   // subject to the same-size rule
+	vdev      string // zpool create keyword; empty means a plain stripe
+}{
+	FSExt4:      {"ext4", 1, false, ""},
+	FSZFSRAID0:  {"zfs (RAID0)", 1, false, ""},
+	FSZFSRAID1:  {"zfs (RAID1)", 2, true, "mirror"},
+	FSZFSRAID10: {"zfs (RAID10)", 4, true, "mirror"},
+	FSZFSRAIDZ1: {"zfs (RAIDZ-1)", 3, true, "raidz1"},
+	FSZFSRAIDZ2: {"zfs (RAIDZ-2)", 4, true, "raidz2"},
+	FSZFSRAIDZ3: {"zfs (RAIDZ-3)", 5, true, "raidz3"},
+}
+
+// IsZFS reports whether this mode creates a pool rather than a plain filesystem.
+func (f FSType) IsZFS() bool { return strings.HasPrefix(string(f), "zfs-") }
+
+// Label is the display name, matching proxinstall's wording so the screen is
+// recognisable to anyone who has installed Proxmox.
+func (f FSType) Label() string { return fsMeta[f].label }
+
+// MinDisks is the smallest member count the topology can be built from.
+func (f FSType) MinDisks() int { return fsMeta[f].minDisks }
+
+// Redundant reports whether the topology stores parity or mirrors, and so
+// requires members of matching size. A RAID0 stripe over unequal disks is
+// legitimate and is deliberately not covered.
+func (f FSType) Redundant() bool { return fsMeta[f].redundant }
+
+// vdevKeyword is the zpool create keyword introducing each vdev.
+func (f FSType) vdevKeyword() string { return fsMeta[f].vdev }
+
+// ParseFSType maps a SPINIFEX_FS value to an FSType.
+func ParseFSType(s string) (FSType, error) {
+	f := FSType(strings.ToLower(strings.TrimSpace(s)))
+	if _, ok := fsMeta[f]; !ok {
+		names := make([]string, 0, len(AllFSTypes))
+		for _, t := range AllFSTypes {
+			names = append(names, string(t))
+		}
+		return "", fmt.Errorf("unknown filesystem %q — must be one of: %s", s, strings.Join(names, ", "))
+	}
+	return f, nil
+}
+
+// DiskRole is the purpose a selected drive serves on an ext4 install. A ZFS
+// pool carries every workload on one set of members, so roles do not apply
+// there — the topology already spans all the disks.
+type DiskRole string
+
+const (
+	RoleOS         DiskRole = "os"
+	RoleSpinifex   DiskRole = "spinifex"
+	RolePredastore DiskRole = "predastore"
+)
+
+// AllDiskRoles is canonical order, outermost mountpoint first. Mounting, fstab
+// emission and rsync protect filters all walk it forwards; teardown walks it
+// back, because predastore nests inside spinifex.
+var AllDiskRoles = []DiskRole{RoleOS, RoleSpinifex, RolePredastore}
+
+// roleMountpoints is where each role lands on the installed system. A role left
+// unassigned simply stays on the filesystem above it.
+var roleMountpoints = map[DiskRole]string{
+	RoleOS:         "/",
+	RoleSpinifex:   "/var/lib/spinifex",
+	RolePredastore: "/var/lib/spinifex/predastore",
+}
+
+// Label is the role name as shown in the UI and written to partition labels.
+func (r DiskRole) Label() string { return string(r) }
+
+// Mountpoint is the role's path on the installed system.
+func (r DiskRole) Mountpoint() string { return roleMountpoints[r] }
+
+// RoleMount binds one drive to one role.
+type RoleMount struct {
+	Role DiskRole
+	Disk Disk
+}
+
+// Mountpoint is where this drive's filesystem is mounted on the target.
+func (r RoleMount) Mountpoint() string { return r.Role.Mountpoint() }
+
+// ZFSOpts are the advanced tunables. A zero value means "use the computed
+// default", which is why Ashift and ARCMaxMiB are plain ints — 0 is not legal
+// for either, so it is unambiguous as a sentinel.
+type ZFSOpts struct {
+	Ashift    int
+	Compress  string
+	Checksum  string
+	Copies    int
+	ARCMaxMiB int
+}
+
+// DiskConfig is the storage half of the installer's configuration. Disks is
+// ordered and the order is significant: RAID10 pairs members two at a time, and
+// on ext4 the first entry is the OS drive.
+type DiskConfig struct {
+	FS    FSType
+	Disks []Disk
+	ZFS   ZFSOpts
+
+	// Roles assigns ext4 drives to mountpoints, in AllDiskRoles order. Empty
+	// means a single-disk install with everything on the root filesystem.
+	Roles []RoleMount
+}
+
+// WithRoles returns a copy with ext4 drive roles assigned and Disks rebuilt
+// from them in canonical order, so the list of drives to erase and the list of
+// roles cannot drift apart — there is one place a drive can be named, not two.
+func (d DiskConfig) WithRoles(roles []RoleMount) DiskConfig {
+	d.Roles, d.Disks = nil, nil
+	add := func(rm RoleMount) {
+		d.Roles = append(d.Roles, rm)
+		d.Disks = append(d.Disks, rm.Disk)
+	}
+	for _, want := range AllDiskRoles {
+		for _, rm := range roles {
+			if rm.Role == want {
+				add(rm)
+			}
+		}
+	}
+	// Anything with an unknown or duplicate role is kept so Validate can name
+	// it, rather than being dropped into a silently smaller selection.
+	for _, rm := range roles {
+		if _, known := roleMountpoints[rm.Role]; !known {
+			add(rm)
+		}
+	}
+	return d
+}
+
+// RolesForDisks assigns roles positionally, which is the default the UI offers:
+// first disk picked is the OS, second is spinifex, third is predastore.
+func RolesForDisks(disks []Disk) []RoleMount {
+	out := make([]RoleMount, 0, len(disks))
+	for i, disk := range disks {
+		var role DiskRole
+		if i < len(AllDiskRoles) {
+			role = AllDiskRoles[i]
+		}
+		out = append(out, RoleMount{Role: role, Disk: disk})
+	}
+	return out
+}
+
+// DataMounts is every role except the OS drive, shallowest mountpoint first.
+// This is the traversal every stage of the install shares.
+func (d DiskConfig) DataMounts() []RoleMount {
+	var out []RoleMount
+	for _, rm := range d.Roles {
+		if rm.Role != RoleOS {
+			out = append(out, rm)
+		}
+	}
+	return out
+}
+
+// bootDisks are the drives a bootloader is installed to. Every ZFS member gets
+// its own ESP; on ext4 only the OS drive does, because a data drive holding a
+// bootloader that nothing points at is a trap during recovery.
+func (d DiskConfig) bootDisks() []Disk {
+	if d.FS.IsZFS() || len(d.Disks) == 0 {
+		return d.Disks
+	}
+	return d.Disks[:1]
+}
+
+// Primary is the first selected disk — the OS drive in ext4 mode, and the
+// disk shown in single-disk summaries.
+func (d DiskConfig) Primary() Disk {
+	if len(d.Disks) == 0 {
+		return Disk{}
+	}
+	return d.Disks[0]
+}
+
+// Paths lists the selected device paths, for logs and confirmation screens.
+func (d DiskConfig) Paths() []string {
+	out := make([]string, len(d.Disks))
+	for i, disk := range d.Disks {
+		out[i] = disk.Path
+	}
+	return out
+}
+
+// Buildable reports whether the selection has enough members, of the right
+// count, to form the topology at all. Capacity and fault tolerance are
+// meaningless until it does — a 3-disk RAIDZ-3 has no geometry to describe.
+func (d DiskConfig) Buildable() bool {
+	if _, ok := fsMeta[d.FS]; !ok {
+		return false
+	}
+	if len(d.Disks) < d.FS.MinDisks() {
+		return false
+	}
+	return d.FS != FSZFSRAID10 || len(d.Disks)%2 == 0
+}
+
+// Requirement describes what the topology still needs, for the preview line
+// shown in place of a capacity that cannot be computed yet.
+func (d DiskConfig) Requirement() string {
+	req := fmt.Sprintf("needs %d disks", d.FS.MinDisks())
+	if d.FS == FSZFSRAID10 {
+		req = "needs an even number of disks, at least 4"
+	}
+	if d.FS.Redundant() {
+		req += " of the same size"
+	}
+	return req
+}
+
+// UsableBytes estimates post-parity capacity, for the geometry preview. It is
+// an operator-facing approximation and deliberately ignores metadata overhead.
+// Returns 0 for a selection that cannot form the topology, since the parity
+// arithmetic below goes to zero and then negative as members run out.
+func (d DiskConfig) UsableBytes() int64 {
+	n := int64(len(d.Disks))
+	if n == 0 || !d.Buildable() {
+		return 0
+	}
+	smallest := d.Disks[0].Bytes
+	for _, disk := range d.Disks {
+		smallest = min(smallest, disk.Bytes)
+	}
+	switch d.FS {
+	case FSExt4, FSZFSRAID0:
+		var total int64
+		for _, disk := range d.Disks {
+			total += disk.Bytes
+		}
+		return total
+	case FSZFSRAID1:
+		return smallest
+	case FSZFSRAID10:
+		return smallest * (n / 2)
+	case FSZFSRAIDZ1:
+		return smallest * (n - 1)
+	case FSZFSRAIDZ2:
+		return smallest * (n - 2)
+	case FSZFSRAIDZ3:
+		return smallest * (n - 3)
+	}
+	return 0
+}
+
+// Tolerated returns the number of simultaneous disk failures the topology
+// survives, for the geometry preview.
+func (d DiskConfig) Tolerated() int {
+	if !d.Buildable() {
+		return 0
+	}
+	switch d.FS {
+	case FSZFSRAID1:
+		return max(len(d.Disks)-1, 0)
+	case FSZFSRAID10:
+		return 1 // one per mirror pair; the honest worst case is one
+	case FSZFSRAIDZ1:
+		return 1
+	case FSZFSRAIDZ2:
+		return 2
+	case FSZFSRAIDZ3:
+		return 3
+	}
+	return 0
+}
+
+// sizeTolerance is the fraction by which pool members may differ in size.
+// Matches Proxmox's zfs_mirror_size_check: anything beyond 10% and the pool
+// silently truncates every member to the smallest.
+const sizeTolerance = 0.10
+
+// bootedEFI reports whether firmware booted this machine in EFI mode. GRUB's
+// i386-pc target cannot read a 4Kn drive, so legacy BIOS constrains selection.
+func bootedEFI() bool {
+	_, err := os.Stat("/sys/firmware/efi")
+	return err == nil
+}
+
+// Validate checks a disk selection against every rule that must hold before any
+// destructive command runs. The UI calls it on each keystroke so failures block
+// Continue, rather than aborting an install that has already repartitioned.
+func (d DiskConfig) Validate() error {
+	if _, ok := fsMeta[d.FS]; !ok {
+		return fmt.Errorf("no filesystem selected")
+	}
+	if len(d.Disks) == 0 {
+		return fmt.Errorf("%s: select at least one disk", d.FS.Label())
+	}
+	if err := d.validateRoles(); err != nil {
+		return err
+	}
+	if n := len(d.Disks); n < d.FS.MinDisks() {
+		return fmt.Errorf("%s: needs at least %d disks, %d selected", d.FS.Label(), d.FS.MinDisks(), n)
+	}
+	if d.FS == FSZFSRAID10 && len(d.Disks)%2 != 0 {
+		return fmt.Errorf("%s: needs an even number of disks, %d selected", d.FS.Label(), len(d.Disks))
+	}
+
+	seen := map[string]bool{}
+	for _, disk := range d.Disks {
+		if seen[disk.Path] {
+			return fmt.Errorf("%s is selected more than once", disk.Path)
+		}
+		seen[disk.Path] = true
+
+		if disk.LiveMedia {
+			return fmt.Errorf("%s is the installer's own boot media and cannot be erased", disk.Path)
+		}
+		if disk.Bytes < minRootBytes {
+			return fmt.Errorf("%s is too small (%s, need at least %dGiB)",
+				disk.Path, disk.SizeHuman(), minRootBytes>>30)
+		}
+	}
+
+	// GRUB's i386-pc target cannot read a 4Kn drive, so on legacy BIOS a drive
+	// holding a bootloader would leave the installed system unreachable. Only
+	// boot drives are constrained: a data role is never booted from.
+	for _, disk := range d.bootDisks() {
+		if !bootedEFI() && disk.LogicalBlockSize == 4096 {
+			return fmt.Errorf("%s is a 4Kn drive (4096-byte sectors) and this machine booted in legacy BIOS mode — booting from it is not supported; enable UEFI in firmware",
+				disk.Path)
+		}
+	}
+
+	return d.validateSizes()
+}
+
+// validateRoles checks the ext4 role assignment. Roles are the only way to use
+// more than one disk without ZFS, so a multi-disk selection without them is
+// rejected rather than quietly installed onto the first drive.
+func (d DiskConfig) validateRoles() error {
+	if d.FS.IsZFS() {
+		if len(d.Roles) > 0 {
+			return fmt.Errorf("%s: drive roles apply to ext4 only — a pool spans every member already", d.FS.Label())
+		}
+		return nil
+	}
+	if len(d.Roles) == 0 {
+		if len(d.Disks) > 1 {
+			return fmt.Errorf("%s: %d disks selected but no roles assigned — assign os, spinifex and predastore, or choose a zfs mode",
+				d.FS.Label(), len(d.Disks))
+		}
+		return nil
+	}
+	if len(d.Roles) > len(AllDiskRoles) {
+		return fmt.Errorf("%s: at most %d disks, %d selected — the roles are os, spinifex and predastore",
+			d.FS.Label(), len(AllDiskRoles), len(d.Roles))
+	}
+
+	seen := map[DiskRole]bool{}
+	for _, rm := range d.Roles {
+		if _, ok := roleMountpoints[rm.Role]; !ok {
+			return fmt.Errorf("%s has no role assigned — every selected disk needs one", rm.Disk.Path)
+		}
+		if seen[rm.Role] {
+			return fmt.Errorf("two disks are assigned the %s role", rm.Role)
+		}
+		seen[rm.Role] = true
+	}
+	if !seen[RoleOS] {
+		return fmt.Errorf("%s: no disk assigned the os role — one disk must hold the operating system", d.FS.Label())
+	}
+	if len(d.Roles) != len(d.Disks) {
+		return fmt.Errorf("internal: %d roles for %d disks", len(d.Roles), len(d.Disks))
+	}
+	return nil
+}
+
+// minRootBytes matches Proxmox's hard floor for a root disk.
+const minRootBytes = 2 << 30
+
+// validateSizes enforces the same-size rule where the topology needs it:
+// across all members for mirrors and raidz, and per-pair for RAID10 — a
+// striped mirror only requires each pair to match, not every disk in the pool.
+func (d DiskConfig) validateSizes() error {
+	if !d.FS.Redundant() {
+		return nil
+	}
+	if d.FS == FSZFSRAID10 {
+		for i := 0; i+1 < len(d.Disks); i += 2 {
+			if err := sizesMatch(d.Disks[i], d.Disks[i+1]); err != nil {
+				return fmt.Errorf("%s mirror pair %d: %w", d.FS.Label(), i/2+1, err)
+			}
+		}
+		return nil
+	}
+	for _, disk := range d.Disks[1:] {
+		if err := sizesMatch(d.Disks[0], disk); err != nil {
+			return fmt.Errorf("%s: %w", d.FS.Label(), err)
+		}
+	}
+	return nil
+}
+
+// SizesWithinTolerance reports whether two disks could be members of the same
+// redundant pool. The UI uses it to decide whether suggesting ZFS is honest.
+func SizesWithinTolerance(a, b Disk) bool { return sizesMatch(a, b) == nil }
+
+// sizesMatch reports whether two members are within the tolerance, naming both
+// disks and their sizes — that message is the operator's only diagnostic.
+func sizesMatch(a, b Disk) error {
+	if math.Abs(float64(a.Bytes-b.Bytes)) > float64(a.Bytes)*sizeTolerance {
+		return fmt.Errorf("disks must be the same size within %.0f%%: %s is %s, %s is %s",
+			sizeTolerance*100, a.Path, a.SizeHuman(), b.Path, b.SizeHuman())
+	}
+	return nil
+}
+
+// Warnings returns non-blocking advice for a valid selection. A RAID0 stripe
+// over mismatched disks is legal but almost always a mistake, so it is
+// surfaced without preventing the install — which is what Proxmox does.
+func (d DiskConfig) Warnings() []string {
+	var out []string
+	if d.FS == FSZFSRAID0 && len(d.Disks) > 1 {
+		for _, disk := range d.Disks[1:] {
+			if sizesMatch(d.Disks[0], disk) != nil {
+				out = append(out, "striping disks of different sizes — capacity is usable but a failure of any disk loses the pool")
+				break
+			}
+		}
+	}
+	if d.FS.IsZFS() && d.FS != FSZFSRAID0 && d.Buildable() && d.Tolerated() == 0 {
+		out = append(out, "this topology stores no redundancy")
+	}
+	// A three-drive role layout looks like an array and is not one: each drive
+	// is an independent point of failure, so the node loses whatever sat on it.
+	if len(d.DataMounts()) > 0 {
+		out = append(out, "role-assigned drives store no redundancy — losing any one drive loses what it held")
+	}
+	for _, rm := range d.DataMounts() {
+		if rm.Role == RolePredastore && rm.Disk.Bytes < smallPredastoreBytes {
+			out = append(out, fmt.Sprintf("%s is small for object storage (%s) — predastore will fill quickly",
+				rm.Disk.Path, rm.Disk.SizeHuman()))
+		}
+	}
+	return out
+}
+
+// smallPredastoreBytes is where a dedicated object-store drive stops being
+// worth the drive. Advisory only: the install is legal below it.
+const smallPredastoreBytes = 64 << 30

cmd/installer/install/diskconfig_test.go

@@ -0,0 +1,267 @@
+package install
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDiskConfigValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     DiskConfig
+		wantErr string
+	}{
+		{"ext4 on one disk", DiskConfig{FS: FSExt4, Disks: disks(1, 40)}, ""},
+		{"raidz1 on three matching disks", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)}, ""},
+		{"raid10 on four matching disks", DiskConfig{FS: FSZFSRAID10, Disks: disks(4, 40)}, ""},
+		// The single-drive ZFS case: no redundancy, but checksums, compression
+		// and snapshots still apply.
+		{"raid0 on one disk", DiskConfig{FS: FSZFSRAID0, Disks: disks(1, 40)}, ""},
+
+		{"no filesystem chosen", DiskConfig{Disks: disks(1, 40)}, "no filesystem selected"},
+		{"no disks", DiskConfig{FS: FSZFSRAIDZ1}, "select at least one disk"},
+		{"ext4 spans disks only with roles", DiskConfig{FS: FSExt4, Disks: disks(2, 40)}, "no roles assigned"},
+		{"raidz1 needs three", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(2, 40)}, "at least 3 disks"},
+		{"raidz2 needs four", DiskConfig{FS: FSZFSRAIDZ2, Disks: disks(3, 40)}, "at least 4 disks"},
+		{"raidz3 needs five", DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(4, 40)}, "at least 5 disks"},
+		{"raid1 needs two", DiskConfig{FS: FSZFSRAID1, Disks: disks(1, 40)}, "at least 2 disks"},
+		{"raid10 rejects an odd count", DiskConfig{FS: FSZFSRAID10, Disks: disks(5, 40)}, "even number"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			switch {
+			case tt.wantErr == "" && err != nil:
+				t.Fatalf("Validate() = %v, want nil", err)
+			case tt.wantErr != "" && err == nil:
+				t.Fatalf("Validate() = nil, want error containing %q", tt.wantErr)
+			case tt.wantErr != "" && !strings.Contains(err.Error(), tt.wantErr):
+				t.Fatalf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestValidateRejectsDuplicateDisk(t *testing.T) {
+	d := disk("a", 40)
+	err := DiskConfig{FS: FSZFSRAID1, Disks: []Disk{d, d}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "more than once") {
+		t.Fatalf("Validate() = %v, want a duplicate-selection error", err)
+	}
+}
+
+func TestValidateRejectsLiveMedia(t *testing.T) {
+	d := disk("a", 40)
+	d.LiveMedia = true
+	err := DiskConfig{FS: FSExt4, Disks: []Disk{d}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "boot media") {
+		t.Fatalf("Validate() = %v, want the installer's own media to be refused", err)
+	}
+}
+
+func TestValidateRejectsTooSmallDisk(t *testing.T) {
+	err := DiskConfig{FS: FSExt4, Disks: []Disk{disk("a", 1)}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "too small") {
+		t.Fatalf("Validate() = %v, want a minimum-size error", err)
+	}
+}
+
+// A pool truncates every member to the smallest, so mismatched disks in a
+// redundant topology silently throw capacity away.
+func TestValidateSizeTolerance(t *testing.T) {
+	tests := []struct {
+		name    string
+		fs      FSType
+		sizes   []int64
+		wantErr bool
+	}{
+		{"identical disks", FSZFSRAIDZ1, []int64{40, 40, 40}, false},
+		{"within 10%", FSZFSRAIDZ1, []int64{100, 95, 92}, false},
+		{"beyond 10%", FSZFSRAIDZ1, []int64{100, 100, 80}, true},
+		{"mirror with a mismatched member", FSZFSRAID1, []int64{100, 50}, true},
+		// A stripe has no parity to waste, so mismatched sizes are a warning
+		// rather than a refusal — matching Proxmox.
+		{"raid0 permits mismatched disks", FSZFSRAID0, []int64{100, 40}, false},
+		{"raid10 checks each pair, not the whole pool", FSZFSRAID10, []int64{40, 40, 100, 100}, false},
+		{"raid10 rejects a mismatched pair", FSZFSRAID10, []int64{40, 100, 100, 100}, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			members := make([]Disk, len(tt.sizes))
+			for i, gib := range tt.sizes {
+				members[i] = disk(string(rune('a'+i)), gib)
+			}
+			err := DiskConfig{FS: tt.fs, Disks: members}.Validate()
+			if tt.wantErr && err == nil {
+				t.Fatalf("Validate() = nil, want a size-mismatch error")
+			}
+			if !tt.wantErr && err != nil {
+				t.Fatalf("Validate() = %v, want nil", err)
+			}
+		})
+	}
+}
+
+func TestWarningsFlagMismatchedStripe(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAID0, Disks: []Disk{disk("a", 100), disk("b", 40)}}
+	warnings := cfg.Warnings()
+	if len(warnings) == 0 {
+		t.Fatal("a stripe over mismatched disks should warn")
+	}
+}
+
+func TestUsableBytes(t *testing.T) {
+	tests := []struct {
+		name    string
+		fs      FSType
+		n       int
+		wantGiB int64
+	}{
+		{"stripe adds every disk", FSZFSRAID0, 3, 120},
+		{"mirror gives one disk", FSZFSRAID1, 2, 40},
+		{"raid10 gives half", FSZFSRAID10, 4, 80},
+		{"raidz1 loses one to parity", FSZFSRAIDZ1, 3, 80},
+		{"raidz2 loses two", FSZFSRAIDZ2, 4, 80},
+		{"raidz3 loses three", FSZFSRAIDZ3, 5, 80},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if got := cfg.UsableBytes() >> 30; got != tt.wantGiB {
+				t.Errorf("UsableBytes() = %dG, want %dG", got, tt.wantGiB)
+			}
+		})
+	}
+}
+
+// The parity arithmetic goes to zero and then negative as members run out, and
+// a preview reading "-40GiB usable" is worse than no figure at all.
+func TestUsableBytesIsZeroWhenTopologyCannotBeBuilt(t *testing.T) {
+	tests := []struct {
+		name string
+		fs   FSType
+		n    int
+	}{
+		{"raidz3 with exactly its parity count", FSZFSRAIDZ3, 3},
+		{"raidz3 with fewer disks than parity", FSZFSRAIDZ3, 2},
+		{"raidz2 with two disks", FSZFSRAIDZ2, 2},
+		{"raidz1 with one disk", FSZFSRAIDZ1, 1},
+		{"mirror with one disk", FSZFSRAID1, 1},
+		{"raid10 with an odd count", FSZFSRAID10, 3},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if cfg.Buildable() {
+				t.Fatalf("Buildable() = true for %s across %d disks", tt.fs, tt.n)
+			}
+			if got := cfg.UsableBytes(); got != 0 {
+				t.Errorf("UsableBytes() = %d, want 0", got)
+			}
+			if got := cfg.Tolerated(); got != 0 {
+				t.Errorf("Tolerated() = %d, want 0", got)
+			}
+		})
+	}
+}
+
+// A selection that cannot be built must not also be described as unredundant:
+// the operator's problem is the disk count, not the topology.
+func TestWarningsStaySilentUntilBuildable(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(3, 40)}
+	if got := cfg.Warnings(); len(got) != 0 {
+		t.Errorf("Warnings() = %v, want none", got)
+	}
+}
+
+func TestRequirementNamesTheSameSizeRule(t *testing.T) {
+	if got := (DiskConfig{FS: FSZFSRAIDZ2}).Requirement(); !strings.Contains(got, "4 disks") ||
+		!strings.Contains(got, "same size") {
+		t.Errorf("Requirement() = %q, want the disk count and the same-size rule", got)
+	}
+	if got := (DiskConfig{FS: FSZFSRAID10}).Requirement(); !strings.Contains(got, "even") {
+		t.Errorf("Requirement() = %q, want the even-count rule", got)
+	}
+	// A stripe has no matching requirement, so claiming one would be wrong.
+	if got := (DiskConfig{FS: FSZFSRAID0}).Requirement(); strings.Contains(got, "same size") {
+		t.Errorf("Requirement() = %q, want no same-size rule for a stripe", got)
+	}
+}
+
+func TestTolerated(t *testing.T) {
+	tests := []struct {
+		fs   FSType
+		n    int
+		want int
+	}{
+		{FSExt4, 1, 0},
+		{FSZFSRAID0, 3, 0},
+		{FSZFSRAID1, 2, 1},
+		{FSZFSRAID1, 3, 2},
+		{FSZFSRAID10, 4, 1},
+		{FSZFSRAIDZ1, 3, 1},
+		{FSZFSRAIDZ2, 4, 2},
+		{FSZFSRAIDZ3, 5, 3},
+	}
+	for _, tt := range tests {
+		t.Run(string(tt.fs), func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if got := cfg.Tolerated(); got != tt.want {
+				t.Errorf("Tolerated() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseFSType(t *testing.T) {
+	for _, fs := range AllFSTypes {
+		got, err := ParseFSType(strings.ToUpper(string(fs)))
+		if err != nil {
+			t.Errorf("ParseFSType(%q): %v", fs, err)
+			continue
+		}
+		if got != fs {
+			t.Errorf("ParseFSType(%q) = %q", fs, got)
+		}
+	}
+	// The error has to list the valid values: it is read off a serial console
+	// by someone whose unattended install just stopped.
+	_, err := ParseFSType("btrfs")
+	if err == nil || !strings.Contains(err.Error(), "zfs-raidz1") {
+		t.Fatalf("ParseFSType(btrfs) = %v, want an error listing the accepted values", err)
+	}
+}
+
+func TestPartitionPathSeparator(t *testing.T) {
+	tests := []struct{ dev, want string }{
+		{"/dev/sda", "/dev/sda3"},
+		// A trailing digit needs a 'p' or the name is ambiguous.
+		{"/dev/nvme0n1", "/dev/nvme0n1p3"},
+		{"/dev/vda", "/dev/vda3"},
+	}
+	for _, tt := range tests {
+		if got := (Disk{Path: tt.dev}).PartitionPath(3); got != tt.want {
+			t.Errorf("PartitionPath(%s) = %s, want %s", tt.dev, got, tt.want)
+		}
+	}
+}
+
+// udev names partition links with a -partN suffix regardless of the kernel's
+// separator, so the by-id form must not reuse the 'p' rule.
+func TestStablePartitionPathUsesPartSuffix(t *testing.T) {
+	d := Disk{Path: "/dev/nvme0n1", ByID: "/dev/disk/by-id/nvme-SAMSUNG_1234"}
+	if got := d.StablePartitionPath(3); got != "/dev/disk/by-id/nvme-SAMSUNG_1234-part3" {
+		t.Errorf("StablePartitionPath() = %s", got)
+	}
+}
+
+func TestESPSizeMiB(t *testing.T) {
+	if got := espSizeMiB(disk("a", 40)); got != espSmallMiB {
+		t.Errorf("espSizeMiB(40G) = %d, want %d", got, espSmallMiB)
+	}
+	if got := espSizeMiB(disk("a", 500)); got != espLargeMiB {
+		t.Errorf("espSizeMiB(500G) = %d, want %d", got, espLargeMiB)
+	}
+}

cmd/installer/install/install_test.go

@@ -9,7 +9,7 @@ func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
 	const url = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	t.Setenv("SPINIFEX_INSTALL_CALLBACK", url)
 
-	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	cfg := &Config{Hostname: "node1"}
 	fb := cfg.toFirstbootConfig()
 	if fb.InstallCallback != url {
 		t.Errorf("InstallCallback = %q, want %q", fb.InstallCallback, url)
@@ -19,7 +19,7 @@ func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
 func TestToFirstbootConfigEmptyCallbackWhenEnvUnset(t *testing.T) {
 	os.Unsetenv("SPINIFEX_INSTALL_CALLBACK")
 
-	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	cfg := &Config{Hostname: "node1"}
 	fb := cfg.toFirstbootConfig()
 	if fb.InstallCallback != "" {
 		t.Errorf("InstallCallback = %q, want empty", fb.InstallCallback)

cmd/installer/install/roles_test.go

@@ -0,0 +1,391 @@
+package install
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// roleCfg builds an ext4 configuration from role/disk-name pairs, in whatever
+// order they are given — WithRoles is responsible for normalising them.
+func roleCfg(pairs ...any) DiskConfig {
+	var roles []RoleMount
+	for i := 0; i+1 < len(pairs); i += 2 {
+		roles = append(roles, RoleMount{
+			Role: pairs[i].(DiskRole),
+			Disk: disk(pairs[i+1].(string), 100),
+		})
+	}
+	return DiskConfig{FS: FSExt4}.WithRoles(roles)
+}
+
+// fakeUUIDs makes partUUID return a value derived from the device path, so a
+// generated fstab can be checked without touching a real disk.
+func fakeUUIDs(t *testing.T) {
+	t.Helper()
+	prev := partUUID
+	partUUID = func(dev string) (string, error) { return "uuid" + strings.ReplaceAll(dev, "/", "-"), nil }
+	t.Cleanup(func() { partUUID = prev })
+}
+
+func TestWithRolesNormalisesOrderAndDerivesDisks(t *testing.T) {
+	// Given deepest-first, which is the order that breaks mounting.
+	cfg := roleCfg(RolePredastore, "c", RoleSpinifex, "b", RoleOS, "a")
+
+	wantRoles := []DiskRole{RoleOS, RoleSpinifex, RolePredastore}
+	for i, rm := range cfg.Roles {
+		if rm.Role != wantRoles[i] {
+			t.Fatalf("role %d = %s, want %s", i, rm.Role, wantRoles[i])
+		}
+	}
+	// Disks is derived, so the OS drive is first and every role drive appears
+	// exactly once in the list the installer erases.
+	if got := cfg.Paths(); !slices.Equal(got, []string{"/dev/a", "/dev/b", "/dev/c"}) {
+		t.Fatalf("Paths() = %v, want the roles in canonical order", got)
+	}
+	if cfg.Primary().Path != "/dev/a" {
+		t.Errorf("Primary() = %s, want the os drive", cfg.Primary().Path)
+	}
+}
+
+func TestDataMountsExcludeTheOSDriveAndNestShallowestFirst(t *testing.T) {
+	got := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c").DataMounts()
+	want := []string{"/var/lib/spinifex", "/var/lib/spinifex/predastore"}
+	for i, rm := range got {
+		if rm.Mountpoint() != want[i] {
+			t.Fatalf("mount %d = %s, want %s", i, rm.Mountpoint(), want[i])
+		}
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d data mounts, want %d", len(got), len(want))
+	}
+}
+
+// A predastore drive without a spinifex drive is the two-drive layout, and it
+// nests a mount inside the root filesystem rather than inside another mount.
+func TestPredastoreRoleWithoutSpinifexIsValid(t *testing.T) {
+	cfg := roleCfg(RoleOS, "a", RolePredastore, "b")
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("Validate() = %v, want nil", err)
+	}
+	if got := cfg.DataMounts(); len(got) != 1 || got[0].Mountpoint() != "/var/lib/spinifex/predastore" {
+		t.Fatalf("DataMounts() = %v, want predastore only", got)
+	}
+}
+
+func TestValidateRoles(t *testing.T) {
+	twoRoles := func(a, b DiskRole) DiskConfig { return roleCfg(a, "a", b, "b") }
+
+	tests := []struct {
+		name    string
+		cfg     DiskConfig
+		wantErr string
+	}{
+		{"single disk needs no roles", DiskConfig{FS: FSExt4, Disks: disks(1, 40)}, ""},
+		{"os alone", roleCfg(RoleOS, "a"), ""},
+		{"all three", roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"), ""},
+		{"no os role", twoRoles(RoleSpinifex, RolePredastore), "no disk assigned the os role"},
+		{"duplicate role", twoRoles(RoleOS, RoleOS), "two disks are assigned the os role"},
+		{"unknown role", roleCfg(RoleOS, "a", DiskRole("cache"), "b"), "no role assigned"},
+		{"more disks than roles", roleCfg(
+			RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c", DiskRole("cache"), "d"), "at most 3 disks"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			switch {
+			case tt.wantErr == "" && err != nil:
+				t.Fatalf("Validate() = %v, want nil", err)
+			case tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)):
+				t.Fatalf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Roles describe where a workload lives on a plain filesystem. A pool already
+// spans every member, so accepting them there would silently do nothing.
+func TestValidateRejectsRolesOnZFS(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	cfg.Roles = []RoleMount{{Role: RoleOS, Disk: cfg.Disks[0]}}
+	if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "ext4 only") {
+		t.Fatalf("Validate() = %v, want a roles-are-ext4-only error", err)
+	}
+}
+
+func TestBootDisksAreOnlyTheOSDriveOnExt4(t *testing.T) {
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if got := cfg.bootDisks(); len(got) != 1 || got[0].Path != "/dev/a" {
+		t.Fatalf("bootDisks() = %v, want the os drive only", got)
+	}
+	// Every ZFS member carries its own ESP, so all of them stay boot disks.
+	zfs := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	if got := zfs.bootDisks(); len(got) != 2 {
+		t.Fatalf("bootDisks() = %v, want every pool member", got)
+	}
+}
+
+func TestWriteFstabEmitsRoleMountsShallowestFirst(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if err := writeFstab(cfg); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	got := readFile(t, root, "etc/fstab")
+
+	// Order matters: `mount -a` in a rescue shell walks the file top-down, and
+	// predastore cannot be mounted before the filesystem it sits inside.
+	wantOrder := []string{" / ", "/boot/efi", " /var/lib/spinifex ", " /var/lib/spinifex/predastore "}
+	at := -1
+	for _, want := range wantOrder {
+		i := strings.Index(got, want)
+		if i < 0 {
+			t.Fatalf("fstab is missing %q:\n%s", want, got)
+		}
+		if i < at {
+			t.Fatalf("%q is out of order:\n%s", want, got)
+		}
+		at = i
+	}
+
+	for _, dev := range []string{"/dev/b1", "/dev/c1"} {
+		if want := "UUID=uuid-dev-" + strings.TrimPrefix(dev, "/dev/"); !strings.Contains(got, want) {
+			t.Errorf("fstab does not mount %s by UUID:\n%s", dev, got)
+		}
+	}
+	// Data mounts are fsck'd after root, never alongside it.
+	for line := range strings.SplitSeq(got, "\n") {
+		if strings.Contains(line, "/var/lib/spinifex") && !strings.HasSuffix(line, "0 2") {
+			t.Errorf("data mount should be fsck pass 2: %q", line)
+		}
+	}
+}
+
+// Fail-closed by design: a missing data drive must stop the boot for an admin,
+// not let predastore write objects onto the OS drive while everything reports
+// healthy. nofail would silently produce exactly that.
+func TestWriteFstabNeverMarksRoleMountsNofail(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	if err := writeFstab(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	if got := readFile(t, root, "etc/fstab"); strings.Contains(got, "nofail") {
+		t.Errorf("role mounts must not be nofail:\n%s", got)
+	}
+}
+
+func TestWriteFstabSingleDiskIsUnchanged(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	if err := writeFstab(roleCfg(RoleOS, "a")); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	if got := readFile(t, root, "etc/fstab"); strings.Contains(got, "/var/lib/spinifex") {
+		t.Errorf("a single-disk install should mount nothing extra:\n%s", got)
+	}
+}
+
+// rsync runs with --delete, so every mountpoint that exists before the copy
+// needs protecting or the copy can delete across a mount boundary on a retry.
+func TestMountProtectFiltersCoverEveryRoleMount(t *testing.T) {
+	got := mountProtectFilters(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"))
+	want := []string{"--filter=P /var/lib/spinifex", "--filter=P /var/lib/spinifex/predastore"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("mountProtectFilters() = %v, want %v", got, want)
+	}
+	if got := mountProtectFilters(roleCfg(RoleOS, "a")); len(got) != 0 {
+		t.Errorf("a single-disk install has no mounts to protect, got %v", got)
+	}
+	// ZFS keeps its dataset filters — the union is by mode, not additive.
+	if got := mountProtectFilters(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}); len(got) == 0 {
+		t.Error("zfs mode should still protect its dataset mountpoints")
+	}
+}
+
+func TestMountPartitionsMountsRolesAfterRootAndBeforeTheCopy(t *testing.T) {
+	f := fakeCommands(t)
+	root := withMountRoot(t)
+
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if err := mountPartitions(cfg); err != nil {
+		t.Fatalf("mountPartitions: %v", err)
+	}
+
+	// The nested mount cannot precede the one it sits inside, or it is masked.
+	want := []string{
+		"mount /dev/a3 " + root,
+		"mount /dev/a2 " + root + "/boot/efi",
+		"mount /dev/b1 " + root + "/var/lib/spinifex",
+		"mount /dev/c1 " + root + "/var/lib/spinifex/predastore",
+	}
+	var mounts []string
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "mount ") {
+			mounts = append(mounts, c)
+		}
+	}
+	if !slices.Equal(mounts, want) {
+		t.Fatalf("mount order:\n got %v\nwant %v", mounts, want)
+	}
+}
+
+func TestCleanupTargetUnmountsDeepestFirst(t *testing.T) {
+	f := fakeCommands(t)
+	root := withMountRoot(t)
+
+	cleanupTarget(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"))
+
+	// Unmounting the outer filesystem first fails with EBUSY and leaves a
+	// half-mounted target behind for the next attempt.
+	want := []string{
+		"umount " + root + "/var/lib/spinifex/predastore",
+		"umount " + root + "/var/lib/spinifex",
+		"umount " + root + "/boot/efi",
+		"umount " + root,
+	}
+	// The chroot binds are released first and are not part of the target order.
+	var got []string
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "umount ") && slices.Contains(want, c) {
+			got = append(got, c)
+		}
+	}
+	if !slices.Equal(got, want) {
+		t.Fatalf("unmount order:\n got %v\nwant %v", got, want)
+	}
+}
+
+func TestPartitionDisksGivesDataDrivesOnePartitionAndNoESP(t *testing.T) {
+	f := fakeCommands(t)
+	nodes := tempDisks(t, 2, 200)
+	cfg := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: nodes[0]}, {Role: RoleSpinifex, Disk: nodes[1]}})
+	osDisk, data := nodes[0].Path, nodes[1].Path
+
+	if err := partitionDisks(cfg); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+
+	f.mustRun(t, "sgdisk", osDisk, "-n 3:0:-1024M", "-t 3:8300")
+	f.mustRun(t, "sgdisk", osDisk, "-t 2:EF00")
+	// A data drive is never booted from, so a bios_boot or an ESP there would
+	// only mislead whoever reads lsblk during an incident.
+	for _, c := range f.calls {
+		if strings.Contains(c, data) && (strings.Contains(c, "EF02") || strings.Contains(c, "EF00")) {
+			t.Errorf("data drive must get no bios_boot or ESP: %q", c)
+		}
+	}
+	f.mustRun(t, "sgdisk", data, "-n 1:1M:-1024M", "-t 1:8300", "-c 1:spinifex")
+}
+
+func TestFormatPartitionsLabelsEachRoleFilesystem(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+
+	if err := formatPartitions(cfg); err != nil {
+		t.Fatalf("formatPartitions: %v", err)
+	}
+	// One ESP only: a data drive holding a bootloader nothing points at would
+	// mislead whoever reads lsblk during an incident.
+	esps := 0
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "mkfs.fat") {
+			esps++
+		}
+	}
+	if esps != 1 {
+		t.Errorf("want one ESP, got %d:\n%v", esps, f.calls)
+	}
+	f.mustRun(t, "mkfs.ext4", "-L", "spinifex", "/dev/b1")
+	f.mustRun(t, "mkfs.ext4", "-L", "predastore", "/dev/c1")
+}
+
+func TestWarningsFlagTheAbsenceOfRedundancy(t *testing.T) {
+	got := strings.Join(roleCfg(RoleOS, "a", RoleSpinifex, "b").Warnings(), "\n")
+	if !strings.Contains(got, "no redundancy") {
+		t.Errorf("a multi-drive role layout must warn it is not an array, got %q", got)
+	}
+	if got := roleCfg(RoleOS, "a").Warnings(); len(got) != 0 {
+		t.Errorf("a single-disk install has nothing to warn about, got %v", got)
+	}
+}
+
+func TestWarningsFlagASmallPredastoreDrive(t *testing.T) {
+	cfg := DiskConfig{FS: FSExt4}.WithRoles([]RoleMount{
+		{Role: RoleOS, Disk: disk("a", 500)},
+		{Role: RolePredastore, Disk: disk("b", 8)},
+	})
+	got := strings.Join(cfg.Warnings(), "\n")
+	if !strings.Contains(got, "small for object storage") {
+		t.Errorf("an 8GiB predastore drive should be flagged, got %q", got)
+	}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("the warning must not block the install: %v", err)
+	}
+}
+
+func TestRolesForDisksAssignsPositionally(t *testing.T) {
+	got := RolesForDisks(disks(3, 40))
+	for i, rm := range got {
+		if want := AllDiskRoles[i]; rm.Role != want {
+			t.Errorf("disk %d got role %s, want %s", i, rm.Role, want)
+		}
+	}
+	// A fourth disk has no role, which Validate is left to reject by name.
+	if extra := RolesForDisks(disks(4, 40)); extra[3].Role != "" {
+		t.Errorf("disk 4 got role %q, want none", extra[3].Role)
+	}
+}
+
+func TestFourKnDataDriveIsAllowedWhereABootDriveIsNot(t *testing.T) {
+	if bootedEFI() {
+		t.Skip("machine booted in EFI mode; the 4Kn rule does not apply")
+	}
+	fourKn := func(name string) Disk {
+		d := disk(name, 100)
+		d.LogicalBlockSize = 4096
+		return d
+	}
+	// GRUB's i386-pc target cannot read a 4Kn drive, but nothing boots from a
+	// data drive, so only the OS role is constrained.
+	ok := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: disk("a", 100)}, {Role: RoleSpinifex, Disk: fourKn("b")}})
+	if err := ok.Validate(); err != nil {
+		t.Errorf("a 4Kn data drive should be accepted: %v", err)
+	}
+
+	bad := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: fourKn("a")}, {Role: RoleSpinifex, Disk: disk("b", 100)}})
+	if err := bad.Validate(); err == nil || !strings.Contains(err.Error(), "4Kn") {
+		t.Errorf("Validate() = %v, want a 4Kn rejection for the os drive", err)
+	}
+}
+
+func TestVerifyRoleLayoutCatchesAMountThatNeverHappened(t *testing.T) {
+	root := withMountRoot(t)
+	// Plain directories on one filesystem: exactly what a mount performed after
+	// the rootfs copy, or not at all, leaves behind.
+	if err := os.MkdirAll(filepath.Join(root, "var/lib/spinifex/predastore"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	err := verifyRoleLayout(roleCfg(RoleOS, "a", RoleSpinifex, "b"))
+	if err == nil || !strings.Contains(err.Error(), "not a separate filesystem") {
+		t.Fatalf("verifyRoleLayout() = %v, want a missing-mount error", err)
+	}
+	if !strings.Contains(fmt.Sprint(err), "/dev/b") {
+		t.Errorf("the error must name the drive that was not mounted: %v", err)
+	}
+}

cmd/installer/install/zfs.go

@@ -0,0 +1,374 @@
+package install
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math"
+	"math/bits"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// ARC bounds. Spinifex RAM is guest RAM: Proxmox defaults non-PVE products to
+// half of system memory, which on a 256GB compute node would remove 128GB of
+// schedulable instance capacity. 10% capped at 16GiB is their PVE behaviour and
+// the right trade for a hypervisor.
+const (
+	arcMinMiB     = 64
+	arcMaxCapMiB  = 16 * 1024
+	arcFraction   = 0.10
+	arcHeadroomMi = 1024 // always leave the OS a GiB above the ARC ceiling
+)
+
+// zfsDataset describes one dataset created at install time. Order matters:
+// parents must precede children so mountpoints nest correctly.
+type zfsDataset struct {
+	name       string
+	mountpoint string
+	props      []string
+}
+
+// datasets is the layout described in the plan. Recordsizes are matched to each
+// service's access pattern; 16K on the predastore index keeps RAIDZ-1 parity
+// overhead at 33% rather than the ~50% a 4-8K record would cost.
+var datasets = []zfsDataset{
+	{ZFSPoolName + "/ROOT", "none", []string{"canmount=off"}},
+	{ZFSRootDataset, "/", []string{"acltype=posix"}},
+	{ZFSPoolName + "/data", "/var/lib/spinifex", []string{"canmount=off"}},
+	{ZFSPoolName + "/data/viperblock", "/var/lib/spinifex/viperblock",
+		[]string{"recordsize=128K", "logbias=throughput"}},
+	{ZFSPoolName + "/data/predastore", "/var/lib/spinifex/predastore",
+		[]string{"recordsize=1M"}},
+	// Broken out so the later move of object data onto dedicated drives is a
+	// zfs send | zfs recv rather than an rsync with downtime.
+	{ZFSPoolName + "/data/predastore-nodes", "/var/lib/spinifex/predastore/distributed/nodes", nil},
+	// BadgerDB index plus the Raft BoltDB: small random, fsync-heavy.
+	{ZFSPoolName + "/data/predastore-db", "/var/lib/spinifex/predastore/distributed/db",
+		[]string{"recordsize=16K", "logbias=latency"}},
+	{ZFSPoolName + "/data/nats", "/var/lib/spinifex/nats", []string{"recordsize=16K"}},
+	{ZFSPoolName + "/log", "/var/log/spinifex", []string{"compression=zstd"}},
+}
+
+// buildVdev renders the vdev specification for zpool create. Pure: it is the
+// function that decides pool shape, so it is exhaustively unit-tested rather
+// than discovered to be wrong on hardware.
+func buildVdev(cfg DiskConfig) (string, error) {
+	if err := cfg.Validate(); err != nil {
+		return "", err
+	}
+	if !cfg.FS.IsZFS() {
+		return "", fmt.Errorf("buildVdev: %s is not a zfs topology", cfg.FS)
+	}
+
+	// Members are named by their partition-3 by-id path: a pool built on
+	// /dev/sdX breaks the first time a controller enumerates in a new order.
+	members := make([]string, len(cfg.Disks))
+	for i, d := range cfg.Disks {
+		members[i] = d.StablePartitionPath(rootPartNum)
+	}
+
+	keyword := cfg.FS.vdevKeyword()
+	if keyword == "" {
+		return strings.Join(members, " "), nil // plain stripe
+	}
+	// RAID10 is a stripe of two-disk mirrors, paired in selection order — which
+	// is why the UI numbers selected disks instead of just ticking them.
+	if cfg.FS == FSZFSRAID10 {
+		var parts []string
+		for i := 0; i+1 < len(members); i += 2 {
+			parts = append(parts, fmt.Sprintf("mirror %s %s", members[i], members[i+1]))
+		}
+		return strings.Join(parts, " "), nil
+	}
+	return keyword + " " + strings.Join(members, " "), nil
+}
+
+// detectAshift derives the pool sector size from the widest sector any member
+// reports. ashift cannot be changed after creation, and one that is too small
+// costs read-modify-write on every block for the life of the pool, so the floor
+// is 12 (4K) even when every disk claims 512.
+func detectAshift(disks []Disk) int {
+	widest := 4096
+	for _, d := range disks {
+		widest = max(widest, d.PhysicalBlockSize, d.LogicalBlockSize)
+	}
+	// ashift is the base-2 exponent of the sector size.
+	return bits.Len(uint(widest)) - 1
+}
+
+// defaultARCMaxMiB computes the ARC ceiling from total system memory.
+func defaultARCMaxMiB(totalMiB int) int {
+	want := int(math.Round(float64(totalMiB) * arcFraction))
+	want = min(want, arcMaxCapMiB)
+	// Never let the ARC crowd the OS out of its last GiB on a small node.
+	if ceiling := totalMiB - arcHeadroomMi; ceiling > 0 {
+		want = min(want, ceiling)
+	}
+	return max(want, arcMinMiB)
+}
+
+// totalMemoryMiB reads MemTotal from /proc/meminfo.
+func totalMemoryMiB() int {
+	data, err := os.ReadFile("/proc/meminfo")
+	if err != nil {
+		return 0
+	}
+	for line := range strings.SplitSeq(string(data), "\n") {
+		if !strings.HasPrefix(line, "MemTotal:") {
+			continue
+		}
+		var kb int
+		if _, err := fmt.Sscanf(line, "MemTotal: %d kB", &kb); err == nil {
+			return kb / 1024
+		}
+	}
+	return 0
+}
+
+// resolveZFSOpts fills unset advanced options with their computed defaults, so
+// every value written to the pool is explicit and auditable afterwards.
+func resolveZFSOpts(cfg DiskConfig) ZFSOpts {
+	o := cfg.ZFS
+	if o.Ashift == 0 {
+		o.Ashift = detectAshift(cfg.Disks)
+	}
+	if o.Compress == "" {
+		// Named explicitly rather than "on" so a future OpenZFS default change
+		// cannot silently alter what this pool does.
+		o.Compress = "lz4"
+	}
+	if o.Checksum == "" {
+		o.Checksum = "on"
+	}
+	if o.Copies == 0 {
+		o.Copies = 1
+	}
+	if o.ARCMaxMiB == 0 {
+		o.ARCMaxMiB = defaultARCMaxMiB(totalMemoryMiB())
+	}
+	return o
+}
+
+// loadZFSModule brings up the ZFS kernel module in the live environment and
+// waits for /dev/zfs. The module is prebuilt against the shipped kernel at ISO
+// build time, so failure here means the ISO is broken, not the hardware.
+func loadZFSModule() error {
+	if err := run("modprobe", "zfs"); err != nil {
+		slog.Warn("modprobe zfs failed, checking for /dev/zfs anyway", "err", err)
+	}
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		if _, err := os.Stat("/dev/zfs"); err == nil {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("/dev/zfs did not appear after modprobe zfs — the ZFS module is missing or does not match the running kernel")
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// ImportablePools returns the names of pools visible to `zpool import`. There
+// is still no machine-readable output in OpenZFS 2.3, so the human format is
+// parsed — the same approach Proxmox::Sys::ZFS takes.
+func ImportablePools() ([]string, error) {
+	out, err := exec.Command("zpool", "import").Output()
+	if err != nil {
+		// zpool exits non-zero when there is nothing to import, which is the
+		// normal case. Anything else — no binary, no /dev/zfs — must surface, or
+		// a machine with an existing pool looks empty to the caller.
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("zpool import: %w", err)
+	}
+	return parseImportablePools(string(out)), nil
+}
+
+// parseImportablePools extracts pool names from `zpool import` output. Split
+// out from the command call so it can be tested against captured fixtures.
+func parseImportablePools(out string) []string {
+	var pools []string
+	scanner := bufio.NewScanner(strings.NewReader(out))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		// Only the "pool:" key introduces a pool; "id:", "state:" and the
+		// config tree that follows are attributes of the one before it.
+		if name, ok := strings.CutPrefix(line, "pool:"); ok {
+			if n := strings.TrimSpace(name); n != "" {
+				pools = append(pools, n)
+			}
+		}
+	}
+	return pools
+}
+
+// createPool builds the root pool across the selected disks. altroot keeps
+// every dataset mounted under the install target instead of over the live
+// filesystem, and cachefile=none stops the live environment from claiming it.
+func createPool(cfg DiskConfig, opts ZFSOpts) error {
+	vdev, err := buildVdev(cfg)
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"create", "-f",
+		"-o", "cachefile=none",
+		"-o", fmt.Sprintf("ashift=%d", opts.Ashift),
+		"-O", "atime=on", "-O", "relatime=on",
+		"-O", "xattr=sa",
+		"-O", fmt.Sprintf("compression=%s", opts.Compress),
+		"-O", fmt.Sprintf("checksum=%s", opts.Checksum),
+		"-O", fmt.Sprintf("copies=%d", opts.Copies),
+		"-O", "mountpoint=none",
+		"-R", mountRoot,
+		ZFSPoolName,
+	}
+	args = append(args, strings.Fields(vdev)...)
+	if err := run("zpool", args...); err != nil {
+		return fmt.Errorf("zpool create: %w", err)
+	}
+
+	// TRIM matters on a pool that churns EBS volumes: without it the drives'
+	// garbage collection degrades write latency over months, and diagnosing
+	// that after the fact is miserable. Detected, never prompted.
+	if allFlash(cfg.Disks) {
+		if err := run("zpool", "set", "autotrim=on", ZFSPoolName); err != nil {
+			slog.Warn("could not enable autotrim", "err", err)
+		}
+	}
+	return nil
+}
+
+// allFlash reports whether every member is non-rotational.
+func allFlash(disks []Disk) bool {
+	for _, d := range disks {
+		if d.Rotational {
+			return false
+		}
+	}
+	return len(disks) > 0
+}
+
+// createDatasets lays out the service datasets. Mountpoints are set relative to
+// the pool's altroot, so they land under the install target now and at their
+// real paths once the installed system imports the pool.
+func createDatasets() error {
+	for _, ds := range datasets {
+		args := []string{"create", "-o", "mountpoint=" + ds.mountpoint}
+		for _, p := range ds.props {
+			args = append(args, "-o", p)
+		}
+		args = append(args, ds.name)
+		if err := run("zfs", args...); err != nil {
+			return fmt.Errorf("create dataset %s: %w", ds.name, err)
+		}
+	}
+	return nil
+}
+
+// datasetProtectFilters keeps rsync --delete away from the dataset mountpoints.
+//
+// The datasets are mounted before the copy so files land on the right one, but
+// the live source has nothing at those paths, so --delete tries to rmdir a busy
+// mountpoint and fails the whole transfer. Protect rules exempt them.
+func datasetProtectFilters(cfg DiskConfig) []string {
+	if !cfg.FS.IsZFS() {
+		return nil
+	}
+	var out []string
+	for _, ds := range datasets {
+		if ds.mountpoint == "/" || ds.mountpoint == "none" {
+			continue
+		}
+		out = append(out, "--filter=P "+ds.mountpoint)
+	}
+	return out
+}
+
+// writeZFSSystemConfig writes the target's ZFS tunables: the ARC ceiling, and
+// the matching host reserve so the daemon's admission gate stops handing out
+// memory the ARC is holding.
+func writeZFSSystemConfig(opts ZFSOpts) error {
+	modprobeDir := filepath.Join(mountRoot, "etc/modprobe.d")
+	if err := os.MkdirAll(modprobeDir, 0o755); err != nil {
+		return err
+	}
+	arcBytes := int64(opts.ARCMaxMiB) << 20
+	conf := fmt.Sprintf("# Generated by the Spinifex installer.\n"+
+		"# Guest RAM is the scarce resource on a hypervisor, so the ARC is\n"+
+		"# capped well below the ZFS default of half of system memory.\n"+
+		"options zfs zfs_arc_max=%d\n", arcBytes)
+	if err := os.WriteFile(filepath.Join(modprobeDir, "zfs.conf"), []byte(conf), 0o644); err != nil {
+		return fmt.Errorf("write zfs.conf: %w", err)
+	}
+
+	// The daemon computes schedulable memory as host total minus this reserve
+	// (defaultHostReserve in daemon/resource.go). ARC is invisible to it, so
+	// the ceiling has to be declared or instances get admitted into memory the
+	// ARC already owns.
+	reserve := defaultHostReserveGB + float64(opts.ARCMaxMiB)/1024.0
+	env := fmt.Sprintf("# Generated by the Spinifex installer.\n"+
+		"# %.1fGB base reserve + %.1fGB ZFS ARC ceiling.\n"+
+		"SPINIFEX_RESERVED_MEM_GB=%.1f\n",
+		defaultHostReserveGB, float64(opts.ARCMaxMiB)/1024.0, reserve)
+	confDir := filepath.Join(mountRoot, "etc/spinifex")
+	if err := os.MkdirAll(confDir, 0o755); err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(confDir, "host.env"), []byte(env), 0o644)
+}
+
+// defaultHostReserveGB mirrors defaultHostReserve.memGB in
+// spinifex/daemon/resource.go. Changing one without the other silently
+// misreports how much memory a ZFS node can schedule.
+const defaultHostReserveGB = 2.0
+
+// exportPool releases the pool so a retry after a failed install is not blocked
+// by "pool is busy". The live environment runs spinifex-init as PID 1, so
+// nothing else will ever clean this up.
+func exportPool() {
+	if err := run("zpool", "export", ZFSPoolName); err != nil {
+		slog.Warn("zpool export failed", "pool", ZFSPoolName, "err", err)
+	}
+}
+
+// clearDisk takes a disk over unconditionally, whatever it holds: everything
+// attached to it is released, every signature on it is wiped, its partition
+// table is erased, and the kernel is made to agree the disk is now empty.
+//
+// The last part is the point. sgdisk writes to the platters and exits 0 even
+// when the kernel refuses to drop the old table, so without the assertion an
+// install proceeds against partition nodes describing the previous layout.
+//
+// The probes are quiet because a blank disk makes all of them fail, and their
+// stderr on the install console reads like the install is going wrong.
+func clearDisk(d Disk) error {
+	releaseDisk(d)
+
+	// Every partition, not just the root slice. ZFS labels live at both ends of
+	// a device and wipefs does not know about them, so a data drive that was
+	// once a pool member keeps its label and fails a later zpool create.
+	parts, err := kernelPartitions(d.Path)
+	if err != nil {
+		return err
+	}
+	for _, p := range parts {
+		dev := partitionDevice(d.Path, p)
+		_ = runQuiet("zpool", "labelclear", "-f", dev)
+		_ = runQuiet("wipefs", "-a", dev)
+	}
+
+	_ = runQuiet("wipefs", "-a", d.Path)
+	if err := runQuiet("sgdisk", "-Z", d.Path); err != nil {
+		return fmt.Errorf("erase partition table: %w", err)
+	}
+	return settlePartitions(d)
+}

cmd/installer/install/zfs_test.go

@@ -0,0 +1,311 @@
+package install
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+// disk builds a test disk with a by-id path, since that is what pool members
+// are actually named by.
+func disk(name string, gib int64) Disk {
+	return Disk{
+		Path:              "/dev/" + name,
+		ByID:              "/dev/disk/by-id/virtio-" + name,
+		Bytes:             gib << 30,
+		LogicalBlockSize:  512,
+		PhysicalBlockSize: 512,
+	}
+}
+
+func disks(n int, gib int64) []Disk {
+	out := make([]Disk, n)
+	for i := range out {
+		out[i] = disk(string(rune('a'+i)), gib)
+	}
+	return out
+}
+
+func TestBuildVdev(t *testing.T) {
+	tests := []struct {
+		name  string
+		fs    FSType
+		disks []Disk
+		want  string
+	}{
+		{
+			"raid0 is a bare stripe with no keyword",
+			FSZFSRAID0, disks(2, 40),
+			"/dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3",
+		},
+		{
+			"single-disk raid0 is the one-drive case",
+			FSZFSRAID0, disks(1, 40),
+			"/dev/disk/by-id/virtio-a-part3",
+		},
+		{
+			"raid1 is one mirror vdev",
+			FSZFSRAID1, disks(2, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3",
+		},
+		{
+			"raid1 across three disks is a three-way mirror, not a stripe",
+			FSZFSRAID1, disks(3, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 /dev/disk/by-id/virtio-c-part3",
+		},
+		{
+			"raid10 pairs members in selection order",
+			FSZFSRAID10, disks(4, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 " +
+				"mirror /dev/disk/by-id/virtio-c-part3 /dev/disk/by-id/virtio-d-part3",
+		},
+		{
+			"raidz1 across three disks",
+			FSZFSRAIDZ1, disks(3, 40),
+			"raidz1 /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 /dev/disk/by-id/virtio-c-part3",
+		},
+		{
+			"raidz2 needs four",
+			FSZFSRAIDZ2, disks(4, 40),
+			"raidz2 /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 " +
+				"/dev/disk/by-id/virtio-c-part3 /dev/disk/by-id/virtio-d-part3",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := buildVdev(DiskConfig{FS: tt.fs, Disks: tt.disks})
+			if err != nil {
+				t.Fatalf("buildVdev: %v", err)
+			}
+			if got != tt.want {
+				t.Errorf("buildVdev()\n got: %s\nwant: %s", got, tt.want)
+			}
+		})
+	}
+}
+
+// A pool built on /dev/sdX silently breaks when the controller enumerates in a
+// different order, so the fallback only applies when there is no by-id path.
+func TestBuildVdevFallsBackToKernelNameWithoutByID(t *testing.T) {
+	d := disk("a", 40)
+	d.ByID = ""
+	got, err := buildVdev(DiskConfig{FS: FSZFSRAID0, Disks: []Disk{d}})
+	if err != nil {
+		t.Fatalf("buildVdev: %v", err)
+	}
+	if got != "/dev/a3" {
+		t.Errorf("buildVdev() = %q, want /dev/a3", got)
+	}
+}
+
+func TestBuildVdevRejectsExt4(t *testing.T) {
+	if _, err := buildVdev(DiskConfig{FS: FSExt4, Disks: disks(1, 40)}); err == nil {
+		t.Fatal("buildVdev accepted ext4")
+	}
+}
+
+func TestBuildVdevValidatesFirst(t *testing.T) {
+	// Three disks cannot form striped mirrors, and producing a two-disk mirror
+	// plus a silently dropped disk would be worse than an error.
+	if _, err := buildVdev(DiskConfig{FS: FSZFSRAID10, Disks: disks(3, 40)}); err == nil {
+		t.Fatal("buildVdev accepted an odd disk count for RAID10")
+	}
+}
+
+func TestDetectAshift(t *testing.T) {
+	tests := []struct {
+		name     string
+		logical  int
+		physical int
+		want     int
+	}{
+		// ashift cannot be changed after creation and one that is too small
+		// costs a read-modify-write per block forever, so 512e drives are
+		// deliberately treated as 4K.
+		{"512n drive is floored at 4K", 512, 512, 12},
+		{"512e drive reports 4K physical", 512, 4096, 12},
+		{"4Kn drive", 4096, 4096, 12},
+		{"8K native drive", 8192, 8192, 13},
+		{"16K native drive", 16384, 16384, 14},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			d := Disk{LogicalBlockSize: tt.logical, PhysicalBlockSize: tt.physical}
+			if got := detectAshift([]Disk{d}); got != tt.want {
+				t.Errorf("detectAshift() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+// A pool cannot have two sector sizes, so the widest member wins — the reverse
+// would make every write to the large-sector disk a read-modify-write.
+func TestDetectAshiftTakesWidestMember(t *testing.T) {
+	got := detectAshift([]Disk{
+		{LogicalBlockSize: 512, PhysicalBlockSize: 512},
+		{LogicalBlockSize: 512, PhysicalBlockSize: 8192},
+	})
+	if got != 13 {
+		t.Errorf("detectAshift() = %d, want 13", got)
+	}
+}
+
+func TestDefaultARCMaxMiB(t *testing.T) {
+	tests := []struct {
+		name    string
+		totalMi int
+		want    int
+	}{
+		{"tiny VM gets 10% of very little", 2048, 205},
+		{"8GB node gets 10%", 8192, 819},
+		{"64GB node gets 10%", 65536, 6554},
+		// The point of the cap: guest RAM is what this machine sells, and half
+		// of a 256GB node is 128GB of instance capacity the ARC would swallow.
+		{"256GB node is capped at 16GiB", 262144, 16384},
+		{"1TB node is still capped at 16GiB", 1048576, 16384},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := defaultARCMaxMiB(tt.totalMi); got != tt.want {
+				t.Errorf("defaultARCMaxMiB(%d) = %d, want %d", tt.totalMi, got, tt.want)
+			}
+		})
+	}
+}
+
+// An unreadable /proc/meminfo must not produce a zero or negative ceiling,
+// which ZFS would either reject or read as "no limit".
+func TestDefaultARCMaxMiBNeverGoesBelowFloor(t *testing.T) {
+	for _, total := range []int{0, 128, 512} {
+		if got := defaultARCMaxMiB(total); got < arcMinMiB {
+			t.Errorf("defaultARCMaxMiB(%d) = %d, below the floor of %d", total, got, arcMinMiB)
+		}
+	}
+}
+
+func TestResolveZFSOptsFillsDefaults(t *testing.T) {
+	got := resolveZFSOpts(DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)})
+	if got.Ashift != 12 {
+		t.Errorf("Ashift = %d, want 12", got.Ashift)
+	}
+	if got.Compress != "lz4" {
+		t.Errorf("Compress = %q, want lz4", got.Compress)
+	}
+	if got.Checksum != "on" {
+		t.Errorf("Checksum = %q, want on", got.Checksum)
+	}
+	if got.Copies != 1 {
+		t.Errorf("Copies = %d, want 1", got.Copies)
+	}
+	if got.ARCMaxMiB <= 0 {
+		t.Errorf("ARCMaxMiB = %d, want a positive ceiling", got.ARCMaxMiB)
+	}
+}
+
+func TestResolveZFSOptsKeepsOperatorChoices(t *testing.T) {
+	in := DiskConfig{
+		FS:    FSZFSRAIDZ1,
+		Disks: disks(3, 40),
+		ZFS:   ZFSOpts{Ashift: 13, Compress: "zstd", Checksum: "blake3", Copies: 2, ARCMaxMiB: 2048},
+	}
+	got := resolveZFSOpts(in)
+	if got != in.ZFS {
+		t.Errorf("resolveZFSOpts overrode explicit options: got %+v, want %+v", got, in.ZFS)
+	}
+}
+
+func TestParseImportablePools(t *testing.T) {
+	// Captured from `zpool import` on a machine with a pool from a previous
+	// install: the config tree below each header must not be mistaken for one.
+	const out = `   pool: rpool
+     id: 8543219876543210987
+  state: ONLINE
+ action: The pool can be imported using its name or numeric identifier.
+ config:
+
+	rpool                     ONLINE
+	  raidz1-0                ONLINE
+	    virtio-spinifex-disk1-part3  ONLINE
+	    virtio-spinifex-disk2-part3  ONLINE
+
+   pool: tank
+     id: 1111111111111111111
+  state: ONLINE
+ config:
+
+	tank                      ONLINE
+`
+	got := parseImportablePools(out)
+	want := []string{"rpool", "tank"}
+	if len(got) != len(want) {
+		t.Fatalf("parseImportablePools() = %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("pool %d = %q, want %q", i, got[i], want[i])
+		}
+	}
+}
+
+func TestParseImportablePoolsEmpty(t *testing.T) {
+	if got := parseImportablePools("no pools available to import\n"); len(got) != 0 {
+		t.Errorf("parseImportablePools() = %v, want none", got)
+	}
+}
+
+// Every dataset must be creatable in order: a child cannot be created before
+// its parent, and getting this wrong only shows up on real hardware.
+func TestDatasetsAreOrderedParentFirst(t *testing.T) {
+	seen := map[string]bool{ZFSPoolName: true}
+	for _, ds := range datasets {
+		parent := ds.name[:strings.LastIndex(ds.name, "/")]
+		if !seen[parent] {
+			t.Errorf("dataset %s is created before its parent %s", ds.name, parent)
+		}
+		seen[ds.name] = true
+	}
+}
+
+// The root dataset has to be the one the bootloader names, or the machine
+// mounts an empty filesystem and panics.
+func TestRootDatasetIsCreated(t *testing.T) {
+	for _, ds := range datasets {
+		if ds.name == ZFSRootDataset {
+			if ds.mountpoint != "/" {
+				t.Fatalf("%s mountpoint = %q, want /", ZFSRootDataset, ds.mountpoint)
+			}
+			return
+		}
+	}
+	t.Fatalf("%s is never created", ZFSRootDataset)
+}
+
+// rsync --delete tries to rmdir a dataset mountpoint the live source does not
+// have, which fails on a busy mount and aborts the whole copy. Every dataset
+// with a real mountpoint must be protected.
+func TestDatasetProtectFilters(t *testing.T) {
+	got := datasetProtectFilters(DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)})
+	for _, ds := range datasets {
+		if ds.mountpoint == "/" || ds.mountpoint == "none" {
+			continue
+		}
+		want := "--filter=P " + ds.mountpoint
+		if !slices.Contains(got, want) {
+			t.Errorf("missing protect rule for %s (%s)", ds.name, ds.mountpoint)
+		}
+	}
+	// The root dataset is the destination itself, and "none" datasets are never
+	// mounted, so protecting either would be meaningless.
+	for _, f := range got {
+		if f == "--filter=P /" || f == "--filter=P none" {
+			t.Errorf("unexpected protect rule %q", f)
+		}
+	}
+}
+
+func TestDatasetProtectFiltersSkipsExt4(t *testing.T) {
+	if got := datasetProtectFilters(DiskConfig{FS: FSExt4, Disks: disks(1, 40)}); got != nil {
+		t.Errorf("datasetProtectFilters() = %v on ext4, want none", got)
+	}
+}

cmd/installer/netprobe/netprobe.go

@@ -0,0 +1,426 @@
+// Package netprobe identifies physical network interfaces by hardware rather
+// than kernel name, so an operator choosing between eno1, ens1f0np0 and
+// ens1f1np1 can tell the 1gbe Broadcom from the 25gbe Mellanox.
+package netprobe
+
+import (
+	"cmp"
+	"fmt"
+	"log/slog"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// sysClassNet is the kernel's per-interface attribute tree. A variable, not a
+// constant, so tests can point the probe at a fixture instead of the running
+// machine's hardware.
+var sysClassNet = "/sys/class/net"
+
+// The external helpers the probe leans on, indirected so tests can exercise it
+// without the host's real tooling.
+var (
+	listInterfaces = net.Interfaces
+	commandOutput  = func(name string, args ...string) ([]byte, error) {
+		return exec.Command(name, args...).Output()
+	}
+)
+
+// NIC describes one physical interface as presented in the installer's
+// selection table.
+type NIC struct {
+	Name string
+
+	// Vendor and Model come from the udev hardware database, e.g. "Mellanox
+	// Technologies" and "MT27710 Family [ConnectX-4 Lx]". Empty when the
+	// database carries no entry for the card.
+	Vendor string
+	Model  string
+
+	// Driver is the kernel module that claimed the port, e.g. "mlx5_core". It
+	// comes straight from sysfs, so it still identifies a card the hardware
+	// database misses.
+	Driver string
+
+	// DeviceID is the PCI or USB "vendor:device" pair, the last-resort identity
+	// for hardware neither the database nor a driver name pins down.
+	DeviceID string
+
+	// AltName is the first alternative name systemd assigns, which is what
+	// operators often see in switch and cabling documentation.
+	AltName string
+
+	// Speed is the negotiated link speed rendered for display ("25 Gbps") and
+	// SpeedMbps the raw figure behind it. Both are zero on a port with no
+	// carrier, which is why Probe brings links up before reading.
+	Speed     string
+	SpeedMbps int
+
+	// Carrier reports a cable with link. It orders the picker so the port the
+	// operator has just plugged in is the one offered first.
+	Carrier bool
+
+	// State is the link summary shown in the table: "online", "no cable" or
+	// "down".
+	State string
+
+	MTU    int
+	IsWiFi bool
+}
+
+// Label renders the NIC's identity line: kernel name, negotiated speed and link
+// state. The hardware detail is deliberately absent — it goes on its own line
+// via Hardware, where there is room to read it.
+func (n NIC) Label() string {
+	speed := n.Speed
+	if speed == "" {
+		speed = "—"
+	}
+	state := n.State
+	if state == "" {
+		state = "unknown"
+	}
+	return fmt.Sprintf("%s  %s  %s", n.Name, speed, state)
+}
+
+// Hardware renders the card's identity for the detail line, degrading from the
+// database name to the driver and device id rather than to "unknown". An
+// operator needs something to match against the card in their hand.
+func (n NIC) Hardware() string {
+	if hw := strings.TrimSpace(n.Vendor + " " + n.Model); hw != "" {
+		return hw
+	}
+	switch {
+	case n.Driver != "" && n.DeviceID != "":
+		return n.Driver + " [" + n.DeviceID + "]"
+	case n.Driver != "":
+		return n.Driver
+	case n.DeviceID != "":
+		return n.DeviceID
+	}
+	return "no hardware detail"
+}
+
+// linkUpTimeout bounds how long Probe waits after bringing links up for speed
+// and carrier to settle. A 25gbe port negotiating against a cold switch can
+// take a second or two to report a speed.
+var linkUpTimeout = 2 * time.Second
+
+// Probe enumerates non-loopback physical interfaces and enriches each from
+// sysfs and the udev hardware database, ordered so the ports worth assigning
+// come first. Links are brought up first: a down port reports no speed and no
+// carrier, which would leave every fast port unidentified at exactly the moment
+// the operator needs to pick one.
+//
+// Nothing the probe depends on needs a running systemd. The live installer
+// boots straight into spinifex-init with only udevd started, so networkctl —
+// which has to reach systemd-networkd — enriches gaps and is never the source.
+func Probe() ([]NIC, error) {
+	ifaces, err := listInterfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	var names, physical []string
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagLoopback != 0 {
+			continue
+		}
+		if len(iface.HardwareAddr) == 0 {
+			continue
+		}
+		names = append(names, iface.Name)
+		if hasDevice(iface.Name) {
+			physical = append(physical, iface.Name)
+		}
+	}
+
+	// Bridges, bonds and veth pairs carry a MAC like any port but have no
+	// device behind them, and offering them would let a plane be bound to
+	// something that does not exist until after the install. Keep them only if
+	// the filter found nothing, so an unusual platform still gets a list.
+	if len(physical) > 0 {
+		names = physical
+	}
+
+	brought := false
+	for _, name := range names {
+		if _, err := commandOutput("ip", "link", "set", "dev", name, "up"); err != nil {
+			slog.Debug("netprobe: could not bring link up", "iface", name, "err", err)
+			continue
+		}
+		brought = true
+	}
+	if brought {
+		time.Sleep(linkUpTimeout)
+	}
+
+	nics := make([]NIC, 0, len(names))
+	for _, name := range names {
+		nics = append(nics, describe(name))
+	}
+	slices.SortFunc(nics, byUsefulness)
+	return nics, nil
+}
+
+// describe reads everything about one interface that needs no daemon, then lets
+// networkctl fill the gaps if it happens to be answering.
+func describe(name string) NIC {
+	nic := NIC{
+		Name:     name,
+		Driver:   sysLinkBase(name, "device/driver"),
+		DeviceID: deviceID(name),
+		MTU:      sysInt(name, "mtu"),
+		IsWiFi:   isWiFi(name),
+	}
+
+	// The kernel reports -1 for a port that cannot state its speed; treat that
+	// as unknown rather than letting it sort below every real figure.
+	nic.SpeedMbps = max(sysInt(name, "speed"), 0)
+	nic.Speed = formatSpeed(nic.SpeedMbps)
+	nic.Carrier = sysAttr(name, "carrier") == "1"
+	nic.State = linkState(name, nic.Carrier)
+
+	props := udevProperties(name)
+	nic.Vendor = props["ID_VENDOR_FROM_DATABASE"]
+	nic.Model = props["ID_MODEL_FROM_DATABASE"]
+
+	// udev's predictable names are the ones that appear in cabling notes; the
+	// path-based name is the one networkctl lists first.
+	for _, key := range []string{"ID_NET_NAME_PATH", "ID_NET_NAME_ONBOARD", "ID_NET_NAME_SLOT"} {
+		if alt := props[key]; alt != "" && alt != name {
+			nic.AltName = alt
+			break
+		}
+	}
+
+	if out, err := commandOutput("networkctl", "status", "--no-pager", name); err == nil {
+		fill(&nic, ParseStatus(string(out)))
+	} else {
+		slog.Debug("netprobe: networkctl unavailable, using sysfs only", "iface", name, "err", err)
+	}
+	return nic
+}
+
+// fill copies parsed hardware detail onto a NIC without overwriting anything
+// sysfs already established. State is never taken: the carrier bit is a truer
+// answer to "is this cable live" than networkd's configuration status.
+func fill(dst *NIC, src NIC) {
+	if dst.Vendor == "" {
+		dst.Vendor = src.Vendor
+	}
+	if dst.Model == "" {
+		dst.Model = src.Model
+	}
+	if dst.AltName == "" {
+		dst.AltName = src.AltName
+	}
+	if dst.Speed == "" {
+		dst.Speed = src.Speed
+	}
+	if dst.MTU == 0 {
+		dst.MTU = src.MTU
+	}
+}
+
+// byUsefulness orders the picker so the interfaces an operator would actually
+// assign come first: a live cable, then wired over wireless, then the fastest
+// port. The rest still appear, just below the plausible choices.
+func byUsefulness(a, b NIC) int {
+	if a.Carrier != b.Carrier {
+		return boolOrder(a.Carrier)
+	}
+	if a.IsWiFi != b.IsWiFi {
+		return boolOrder(b.IsWiFi)
+	}
+	if a.SpeedMbps != b.SpeedMbps {
+		return cmp.Compare(b.SpeedMbps, a.SpeedMbps)
+	}
+	return cmp.Compare(a.Name, b.Name)
+}
+
+// boolOrder sorts true ahead of false.
+func boolOrder(first bool) int {
+	if first {
+		return -1
+	}
+	return 1
+}
+
+// formatSpeed renders the kernel's Mbit/s figure the way a datasheet does.
+func formatSpeed(mbps int) string {
+	switch {
+	case mbps <= 0:
+		return ""
+	case mbps%1000 == 0:
+		return fmt.Sprintf("%d Gbps", mbps/1000)
+	default:
+		return fmt.Sprintf("%d Mbps", mbps)
+	}
+}
+
+// linkState collapses the kernel's flags into the three outcomes an operator
+// acts on: the port is live, the port is enabled but unplugged, or the port
+// never came up at all. operstate reports "down" for the latter two alike.
+func linkState(name string, carrier bool) string {
+	switch {
+	case carrier:
+		return "online"
+	case adminUp(name):
+		return "no cable"
+	default:
+		return "down"
+	}
+}
+
+// adminUp reads the IFF_UP bit out of the interface flags.
+func adminUp(name string) bool {
+	flags, err := strconv.ParseUint(strings.TrimPrefix(sysAttr(name, "flags"), "0x"), 16, 64)
+	return err == nil && flags&0x1 != 0
+}
+
+// deviceID reads the PCI or USB "vendor:device" pair. A NIC's device link
+// points at the PCI function directly; on USB it points at the interface, whose
+// parent carries the ids.
+func deviceID(name string) string {
+	dev := filepath.Join(sysClassNet, name, "device")
+	if id := idPair(dev, "vendor", "device"); id != "" {
+		return id
+	}
+	return idPair(filepath.Join(dev, ".."), "idVendor", "idProduct")
+}
+
+func idPair(dir, vendorAttr, productAttr string) string {
+	vendor := readTrimmed(filepath.Join(dir, vendorAttr))
+	product := readTrimmed(filepath.Join(dir, productAttr))
+	if vendor == "" || product == "" {
+		return ""
+	}
+	return strings.TrimPrefix(vendor, "0x") + ":" + strings.TrimPrefix(product, "0x")
+}
+
+// udevProperties reads the interface's udev database entry, which is where the
+// hardware-database vendor and model names live. udevd populates it during the
+// installer's device enumeration, so this works with no systemd running.
+func udevProperties(name string) map[string]string {
+	out, err := commandOutput("udevadm", "info", "--query=property", "--path="+filepath.Join(sysClassNet, name))
+	if err != nil {
+		slog.Debug("netprobe: udevadm info failed", "iface", name, "err", err)
+		return nil
+	}
+
+	props := make(map[string]string)
+	for line := range strings.SplitSeq(string(out), "\n") {
+		if key, value, ok := strings.Cut(strings.TrimSpace(line), "="); ok {
+			props[key] = value
+		}
+	}
+	return props
+}
+
+// ParseStatus extracts the hardware fields from `networkctl status <iface>`
+// output. The format is a set of right-aligned "Key: value" lines, with
+// Alternative Names continuing across subsequent indented lines.
+func ParseStatus(out string) NIC {
+	var nic NIC
+	var inAltNames bool
+
+	for line := range strings.SplitSeq(out, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" {
+			inAltNames = false
+			continue
+		}
+
+		key, value, found := strings.Cut(trimmed, ": ")
+		if !found {
+			// A continuation line under "Alternative Names:" — take the first
+			// only, which is the one operators see in cabling notes.
+			if inAltNames && nic.AltName == "" {
+				nic.AltName = trimmed
+			}
+			continue
+		}
+		inAltNames = false
+
+		key = strings.TrimSpace(key)
+		value = strings.TrimSpace(value)
+		switch key {
+		case "Vendor":
+			nic.Vendor = value
+		case "Model":
+			nic.Model = value
+		case "Speed":
+			nic.Speed = value
+		case "Online state":
+			// Rendered as "online" or "online (configured)".
+			if f := strings.Fields(value); len(f) > 0 {
+				nic.State = f[0]
+			}
+		case "MTU":
+			// Rendered as "9000 (min: 68, max: 9978)".
+			if f := strings.Fields(value); len(f) > 0 {
+				if mtu, err := strconv.Atoi(f[0]); err == nil {
+					nic.MTU = mtu
+				}
+			}
+		case "Alternative Names":
+			inAltNames = true
+			if f := strings.Fields(value); len(f) > 0 && nic.AltName == "" {
+				nic.AltName = f[0]
+			}
+		}
+	}
+	return nic
+}
+
+// hasDevice reports whether the interface is backed by real hardware, which
+// sysfs shows as a device link to the PCI, USB or platform node behind it.
+func hasDevice(name string) bool {
+	_, err := os.Stat(filepath.Join(sysClassNet, name, "device"))
+	return err == nil
+}
+
+// isWiFi reports whether the interface has a wireless subdirectory in sysfs.
+func isWiFi(name string) bool {
+	_, err := os.Stat(filepath.Join(sysClassNet, name, "wireless"))
+	return err == nil
+}
+
+// sysAttr reads one sysfs attribute. A missing or unreadable attribute is not
+// an error worth surfacing: speed and carrier both fail on a port with no link,
+// which the state column already conveys.
+func sysAttr(name, attr string) string {
+	return readTrimmed(filepath.Join(sysClassNet, name, attr))
+}
+
+func sysInt(name, attr string) int {
+	n, err := strconv.Atoi(sysAttr(name, attr))
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+// sysLinkBase resolves a sysfs symlink and returns its final element, which is
+// how the driver name is exposed.
+func sysLinkBase(name, attr string) string {
+	target, err := os.Readlink(filepath.Join(sysClassNet, name, attr))
+	if err != nil {
+		return ""
+	}
+	return filepath.Base(target)
+}
+
+func readTrimmed(path string) string {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(b))
+}

cmd/installer/netprobe/netprobe_test.go

@@ -0,0 +1,292 @@
+package netprobe
+
+import (
+	"errors"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Captured from hydrogen (HP DL325 Gen10, dual 25gbe Mellanox) in us-west-1-az1.
+const mellanox25g = `● 6: ens1f0np0
+                   Link File: /usr/lib/systemd/network/99-default.link
+                Network File: /etc/systemd/network/12-spinifex-lan-nic.network
+                       State: enslaved (configured)
+                Online state: online
+                        Type: ether
+                        Path: pci-0000:84:00.0
+                      Driver: mlx5_core
+                      Vendor: Mellanox Technologies
+                       Model: MT27710 Family [ConnectX-4 Lx]
+           Alternative Names: enp132s0f0np0
+                              enx040973e5ce60
+            Hardware Address: 04:09:73:e5:ce:60 (Hewlett Packard Enterprise)
+                         MTU: 9000 (min: 68, max: 9978)
+                       QDisc: mq
+                      Master: br-lan
+IPv6 Address Generation Mode: none
+    Number of Queues (Tx/Rx): 384/48
+            Auto negotiation: yes
+                       Speed: 25Gbps
+                      Duplex: full
+           Activation Policy: up
+         Required For Online: yes
+                Connected To: MikroTik (MikroTik RouterOS 7.11.3 (stable) Sep/27/2023 13:09:44 CRS504-4XQ) on port bridge/qsfp28-3-3
+`
+
+// A link with no carrier reports neither Speed nor an online state, which is
+// the case Probe brings links up to avoid.
+const linkDown = `● 3: eno6
+                   Link File: /usr/lib/systemd/network/99-default.link
+                       State: off
+                Online state: off
+                        Type: ether
+                      Driver: bnxt_en
+                      Vendor: Broadcom Inc. and subsidiaries
+                       Model: NetXtreme BCM5720 Gigabit Ethernet PCIe
+            Hardware Address: 04:09:73:e5:ce:5f
+                         MTU: 1500 (min: 60, max: 9000)
+`
+
+func TestParseStatusMellanox(t *testing.T) {
+	nic := ParseStatus(mellanox25g)
+
+	if nic.Vendor != "Mellanox Technologies" {
+		t.Errorf("Vendor = %q, want %q", nic.Vendor, "Mellanox Technologies")
+	}
+	if nic.Model != "MT27710 Family [ConnectX-4 Lx]" {
+		t.Errorf("Model = %q, want %q", nic.Model, "MT27710 Family [ConnectX-4 Lx]")
+	}
+	if nic.Speed != "25Gbps" {
+		t.Errorf("Speed = %q, want %q", nic.Speed, "25Gbps")
+	}
+	if nic.State != "online" {
+		t.Errorf("State = %q, want %q", nic.State, "online")
+	}
+	if nic.MTU != 9000 {
+		t.Errorf("MTU = %d, want 9000", nic.MTU)
+	}
+	// Only the first alternative name is taken; enx040973e5ce60 is ignored.
+	if nic.AltName != "enp132s0f0np0" {
+		t.Errorf("AltName = %q, want %q", nic.AltName, "enp132s0f0np0")
+	}
+}
+
+func TestParseStatusLinkDown(t *testing.T) {
+	nic := ParseStatus(linkDown)
+
+	if nic.Speed != "" {
+		t.Errorf("Speed = %q, want empty for a down link", nic.Speed)
+	}
+	if nic.State != "off" {
+		t.Errorf("State = %q, want %q", nic.State, "off")
+	}
+	if nic.Model != "NetXtreme BCM5720 Gigabit Ethernet PCIe" {
+		t.Errorf("Model = %q, want the Broadcom model", nic.Model)
+	}
+	if nic.MTU != 1500 {
+		t.Errorf("MTU = %d, want 1500", nic.MTU)
+	}
+}
+
+func TestParseStatusEmpty(t *testing.T) {
+	// A truncated or unparseable status must not panic or invent values.
+	nic := ParseStatus("● 2: eth0\n     Online state:\n     Alternative Names:\n")
+	if nic.State != "" || nic.AltName != "" {
+		t.Errorf("expected empty fields from valueless keys, got State=%q AltName=%q", nic.State, nic.AltName)
+	}
+}
+
+func TestLabelFallsBackWhenSpeedUnknown(t *testing.T) {
+	got := NIC{Name: "eth0", State: "no cable"}.Label()
+	want := "eth0  —  no cable"
+	if got != want {
+		t.Errorf("Label() = %q, want %q", got, want)
+	}
+}
+
+// Hardware must never render "unknown": an operator standing at the console
+// needs something to match against the card in their hand, and sysfs always has
+// at least a driver or a device id.
+func TestHardwareDegradesRatherThanUnknown(t *testing.T) {
+	tests := []struct {
+		name string
+		nic  NIC
+		want string
+	}{
+		{
+			name: "database name wins",
+			nic:  NIC{Vendor: "Mellanox Technologies", Model: "MT27710", Driver: "mlx5_core", DeviceID: "15b3:1015"},
+			want: "Mellanox Technologies MT27710",
+		},
+		{"vendor only", NIC{Vendor: "Intel Corporation"}, "Intel Corporation"},
+		{"driver and id", NIC{Driver: "igb", DeviceID: "8086:1533"}, "igb [8086:1533]"},
+		{"driver only", NIC{Driver: "virtio_net"}, "virtio_net"},
+		{"id only", NIC{DeviceID: "0bda:8153"}, "0bda:8153"},
+		{"nothing at all", NIC{Name: "eth0"}, "no hardware detail"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.nic.Hardware(); got != tt.want {
+				t.Errorf("Hardware() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestFormatSpeed(t *testing.T) {
+	tests := []struct {
+		mbps int
+		want string
+	}{
+		{25000, "25 Gbps"},
+		{1000, "1 Gbps"},
+		{100, "100 Mbps"},
+		{2500, "2500 Mbps"},
+		{0, ""},
+		{-1, ""},
+	}
+
+	for _, tt := range tests {
+		if got := formatSpeed(tt.mbps); got != tt.want {
+			t.Errorf("formatSpeed(%d) = %q, want %q", tt.mbps, got, tt.want)
+		}
+	}
+}
+
+// A live cable outranks everything, then wired over wireless, then speed. This
+// is what puts the ports an operator would actually assign at indices 0..2,
+// where newModel pre-fills wan, lan and vpc from.
+func TestByUsefulnessOrdersPluggedPortsFirst(t *testing.T) {
+	nics := []NIC{
+		{Name: "eno1", SpeedMbps: 1000},
+		{Name: "wlp2s0", Carrier: true, IsWiFi: true, SpeedMbps: 866},
+		{Name: "ens1f1np1", SpeedMbps: 25000},
+		{Name: "ens1f0np0", Carrier: true, SpeedMbps: 25000},
+		{Name: "eno2", Carrier: true, SpeedMbps: 1000},
+	}
+	slices.SortFunc(nics, byUsefulness)
+
+	want := []string{"ens1f0np0", "eno2", "wlp2s0", "ens1f1np1", "eno1"}
+	for i, name := range want {
+		if nics[i].Name != name {
+			t.Fatalf("order = %v, want %v", names(nics), want)
+		}
+	}
+}
+
+func names(nics []NIC) []string {
+	out := make([]string, len(nics))
+	for i, n := range nics {
+		out[i] = n.Name
+	}
+	return out
+}
+
+func TestProbeReadsSysfsWithoutSystemd(t *testing.T) {
+	root := t.TempDir()
+	sysClassNet = root
+	linkUpTimeout = 0
+	t.Cleanup(func() {
+		sysClassNet = "/sys/class/net"
+		linkUpTimeout = 2 * time.Second
+	})
+
+	// ens1f0np0: a 25gbe Mellanox with a live cable, identified by the hardware
+	// database. eno1: a 1gbe port that is up but unplugged, and absent from the
+	// database, so only its driver and PCI id identify it.
+	writeIface(t, root, "ens1f0np0", map[string]string{
+		"speed": "25000", "carrier": "1", "mtu": "9000", "flags": "0x1003",
+		"device/vendor": "0x15b3", "device/device": "0x1015",
+	}, "mlx5_core")
+	writeIface(t, root, "eno1", map[string]string{
+		"speed": "1000", "carrier": "0", "mtu": "1500", "flags": "0x1003",
+		"device/vendor": "0x8086", "device/device": "0x1533",
+	}, "igb")
+
+	listInterfaces = func() ([]net.Interface, error) {
+		return []net.Interface{
+			{Name: "lo", Flags: net.FlagLoopback},
+			{Name: "eno1", HardwareAddr: net.HardwareAddr{4, 9, 0x73, 0xe5, 0xce, 0x5f}},
+			{Name: "ens1f0np0", HardwareAddr: net.HardwareAddr{4, 9, 0x73, 0xe5, 0xce, 0x60}},
+			{Name: "veth0"}, // no hardware address, so not a physical port
+			{Name: "br-lan", HardwareAddr: net.HardwareAddr{2, 0, 0, 0, 0, 1}}, // a bridge, so no device behind it
+		}, nil
+	}
+	t.Cleanup(func() { listInterfaces = net.Interfaces })
+
+	// Only udevadm answers. networkctl fails the way it does in the live
+	// installer, where there is no systemd-networkd for it to reach.
+	commandOutput = func(name string, args ...string) ([]byte, error) {
+		switch {
+		case name == "ip":
+			return nil, nil
+		case name == "udevadm" && strings.HasSuffix(args[len(args)-1], "ens1f0np0"):
+			return []byte("ID_VENDOR_FROM_DATABASE=Mellanox Technologies\n" +
+				"ID_MODEL_FROM_DATABASE=MT27710 Family [ConnectX-4 Lx]\n" +
+				"ID_NET_NAME_PATH=enp132s0f0np0\n"), nil
+		case name == "udevadm":
+			return []byte("ID_NET_NAME_PATH=enp1s0f0\n"), nil
+		default:
+			return nil, errors.New("Failed to connect to bus: No such file or directory")
+		}
+	}
+	t.Cleanup(func() {
+		commandOutput = func(name string, args ...string) ([]byte, error) {
+			return exec.Command(name, args...).Output()
+		}
+	})
+
+	nics, err := Probe()
+	if err != nil {
+		t.Fatalf("Probe: %v", err)
+	}
+	if got := names(nics); !slices.Equal(got, []string{"ens1f0np0", "eno1"}) {
+		t.Fatalf("probed %v, want the two physical ports with the plugged one first", got)
+	}
+
+	mlx := nics[0]
+	if mlx.Label() != "ens1f0np0  25 Gbps  online" {
+		t.Errorf("Label() = %q", mlx.Label())
+	}
+	if want := "Mellanox Technologies MT27710 Family [ConnectX-4 Lx]"; mlx.Hardware() != want {
+		t.Errorf("Hardware() = %q, want %q", mlx.Hardware(), want)
+	}
+	if mlx.AltName != "enp132s0f0np0" || mlx.MTU != 9000 || !mlx.Carrier {
+		t.Errorf("unexpected detail: %+v", mlx)
+	}
+
+	// Up but unplugged, and unknown to the database: the driver and PCI id are
+	// all the operator gets, which is still enough to identify the card.
+	igb := nics[1]
+	if igb.Label() != "eno1  1 Gbps  no cable" {
+		t.Errorf("Label() = %q", igb.Label())
+	}
+	if igb.Hardware() != "igb [8086:1533]" {
+		t.Errorf("Hardware() = %q", igb.Hardware())
+	}
+}
+
+// writeIface lays out the sysfs attributes Probe reads for one interface,
+// including the driver symlink it resolves the module name from.
+func writeIface(t *testing.T, root, name string, attrs map[string]string, driver string) {
+	t.Helper()
+	for attr, value := range attrs {
+		path := filepath.Join(root, name, attr)
+		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(path, []byte(value+"\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := os.Symlink("../../../bus/pci/drivers/"+driver, filepath.Join(root, name, "device/driver")); err != nil {
+		t.Fatal(err)
+	}
+}

cmd/installer/systemd/systemd.go

@@ -6,44 +6,38 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
-// WriteLANBridgeUnit installs a non-critical oneshot service that activates
-// br-lan via networkctl *after* network-online.target. br-lan has
-// ActivationPolicy=manual in its networkd .network file, so networkd creates
-// the bridge at boot but does not bring it up automatically. This service
-// triggers the activation once the WAN path is confirmed online, ensuring a
-// missing LAN cable or DHCP timeout on the secondary bridge never stalls
+// WriteBridgeUnit installs a non-critical oneshot service that activates an
+// east-west bridge (br-lan, br-vpc) via networkctl *after* network-online.target.
+// Those bridges carry ActivationPolicy=manual in their networkd .network file,
+// so networkd creates them at boot but does not bring them up automatically.
+// This service triggers the activation once the WAN path is confirmed online,
+// ensuring a missing cable or DHCP timeout on a secondary bridge never stalls
 // the management interface or firstboot.
-func WriteLANBridgeUnit(root string) error {
-	unit := `[Unit]
-Description=Spinifex LAN bridge (non-critical)
+func WriteBridgeUnit(root, bridge string) error {
+	name := "spinifex-" + strings.TrimPrefix(bridge, "br-") + "-bridge.service"
+	unit := fmt.Sprintf(`[Unit]
+Description=Spinifex %s bridge (non-critical)
 After=network-online.target
 Wants=network-online.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/networkctl up br-lan
+ExecStart=/usr/bin/networkctl up %s
 RemainAfterExit=yes
 # Failure is non-critical — cable unplugged or switch not ready at boot.
 SuccessExitStatus=0 1
 
 [Install]
 WantedBy=multi-user.target
-`
-	unitPath := filepath.Join(root, "etc/systemd/system/spinifex-lan-bridge.service")
+`, bridge, bridge)
+	unitPath := filepath.Join(root, "etc/systemd/system", name)
 	if err := os.WriteFile(unitPath, []byte(unit), 0o644); err != nil {
 		return err
 	}
-	wantsDir := filepath.Join(root, "etc/systemd/system/multi-user.target.wants")
-	if err := os.MkdirAll(wantsDir, 0o755); err != nil {
-		return err
-	}
-	link := filepath.Join(wantsDir, "spinifex-lan-bridge.service")
-	if err := os.Remove(link); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("remove stale symlink %s: %w", link, err)
-	}
-	return os.Symlink("/etc/systemd/system/spinifex-lan-bridge.service", link)
+	return EnableUnit(root, name)
 }
 
 // WriteFirstbootUnit writes the spinifex-firstboot.service oneshot unit that
@@ -74,6 +68,8 @@ ExecStopPost=/bin/sh -c 'if [ "$EXIT_STATUS" != "0" ]; then printf "\n*** Spinif
 # ovn-central startup cannot wedge multi-user.target and keep getty from
 # ever reaching the login prompt.
 TimeoutStartSec=300s
+# Install diagnostics land in the journal, which persists for the life of the
+# node — nothing this script runs may print secrets to stdout or stderr.
 StandardOutput=journal
 StandardError=journal
 

cmd/installer/ui/network.go

@@ -0,0 +1,696 @@
+package ui
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	"github.com/charmbracelet/bubbles/textinput"
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/charmbracelet/lipgloss"
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+	"github.com/mulgadc/spinifex/cmd/installer/netprobe"
+)
+
+// foldedNIC marks a role with no interface of its own, which collapses onto
+// the plane above it (vpc <- lan <- wan).
+const foldedNIC = -1
+
+// roleField identifies one editable element on the per-role screen. Which of
+// these are actually shown depends on the plane, the addressing mode, whether
+// the NIC is wireless, and whether Advanced is on.
+type roleField int
+
+const (
+	roleFieldNIC roleField = iota
+	roleFieldMethod
+	roleFieldIP
+	roleFieldMask
+	roleFieldGateway
+	roleFieldDNS
+	roleFieldVLAN
+	roleFieldMTU
+	roleFieldSSID
+	roleFieldWiFiPass
+)
+
+// roleForm is the editable state for one plane. The three forms are the whole
+// of the installer's network configuration.
+type roleForm struct {
+	plane install.Plane
+
+	// nic indexes into model.nics, or foldedNIC when the role is folded.
+	nic   int
+	dhcp  bool
+	focus roleField
+
+	ip       textinput.Model
+	mask     textinput.Model
+	gateway  textinput.Model
+	dns      textinput.Model
+	vlan     textinput.Model
+	mtu      textinput.Model
+	ssid     textinput.Model
+	wifiPass textinput.Model
+}
+
+func newRoleForm(plane install.Plane, nic int) roleForm {
+	text := func(placeholder string, limit int) textinput.Model {
+		in := textinput.New()
+		in.Prompt = ""
+		in.Placeholder = placeholder
+		if limit > 0 {
+			in.CharLimit = limit
+		}
+		return in
+	}
+
+	f := roleForm{
+		plane:    plane,
+		nic:      nic,
+		dhcp:     plane == install.PlaneWAN,
+		ip:       text(defaultIPPlaceholder(plane), 0),
+		mask:     text("255.255.255.0", 0),
+		gateway:  text("192.168.1.1", 0),
+		dns:      text("1.1.1.1, 8.8.8.8", 0),
+		vlan:     text("untagged", 4),
+		mtu:      text("1500", 5),
+		ssid:     text("Network SSID", 64),
+		wifiPass: text("WiFi password", 128),
+	}
+	f.wifiPass.EchoMode = textinput.EchoPassword
+	return f
+}
+
+// defaultIPPlaceholder hints the addressing convention for each plane so an
+// operator filling in a rack does not have to remember it.
+func defaultIPPlaceholder(plane install.Plane) string {
+	switch plane {
+	case install.PlaneLAN:
+		return "10.0.0.3"
+	case install.PlaneVPC:
+		return "10.1.0.3"
+	default:
+		return "192.168.1.10"
+	}
+}
+
+func (f *roleForm) folded() bool { return f.nic == foldedNIC }
+
+// canFold reports whether this role may be left unbound. wan always needs an
+// uplink, so only lan and vpc can fold.
+func (f *roleForm) canFold() bool { return f.plane != install.PlaneWAN }
+
+// visibleFields returns the fields shown for this form, in tab order. Hiding
+// rather than disabling keeps the default screen narrow: a DHCP role shows two
+// lines, and VLAN/MTU only appear once Advanced is on.
+func (f *roleForm) visibleFields(isWiFi, advanced bool) []roleField {
+	if f.folded() {
+		return []roleField{roleFieldNIC}
+	}
+	fields := []roleField{roleFieldNIC, roleFieldMethod}
+	if !f.dhcp {
+		fields = append(fields, roleFieldIP, roleFieldMask)
+		// Default route and resolvers are properties of the node, not of each
+		// plane, and wan is the only one that carries either: it holds the
+		// default route, so it is the only plane that can reach an upstream
+		// resolver. On DHCP the lease supplies both.
+		if f.plane == install.PlaneWAN {
+			fields = append(fields, roleFieldGateway, roleFieldDNS)
+		}
+	}
+	if advanced {
+		fields = append(fields, roleFieldVLAN, roleFieldMTU)
+	}
+	if isWiFi {
+		fields = append(fields, roleFieldSSID, roleFieldWiFiPass)
+	}
+	return fields
+}
+
+// input returns the textinput backing a field, or nil for the non-text fields.
+func (f *roleForm) input(field roleField) *textinput.Model {
+	switch field {
+	case roleFieldIP:
+		return &f.ip
+	case roleFieldMask:
+		return &f.mask
+	case roleFieldGateway:
+		return &f.gateway
+	case roleFieldDNS:
+		return &f.dns
+	case roleFieldVLAN:
+		return &f.vlan
+	case roleFieldMTU:
+		return &f.mtu
+	case roleFieldSSID:
+		return &f.ssid
+	case roleFieldWiFiPass:
+		return &f.wifiPass
+	default:
+		return nil
+	}
+}
+
+// blurAll drops focus from every input so only the focused field has a cursor.
+func (f *roleForm) blurAll() {
+	for _, field := range []roleField{
+		roleFieldIP, roleFieldMask, roleFieldGateway, roleFieldDNS,
+		roleFieldVLAN, roleFieldMTU, roleFieldSSID, roleFieldWiFiPass,
+	} {
+		f.input(field).Blur()
+	}
+}
+
+// focusCurrent gives the cursor to the focused field, if it is a text field.
+func (f *roleForm) focusCurrent() {
+	f.blurAll()
+	if in := f.input(f.focus); in != nil {
+		in.Focus()
+	}
+}
+
+// toRole converts the form into the install package's role model.
+func (f *roleForm) toRole(nics []netprobe.NIC) install.NetworkRole {
+	if f.folded() || f.nic >= len(nics) {
+		return install.NetworkRole{}
+	}
+	role := install.NetworkRole{
+		Interface: nics[f.nic].Name,
+		DHCPMode:  f.dhcp,
+		VLAN:      atoiOr(f.vlan.Value(), 0),
+		MTU:       atoiOr(f.mtu.Value(), 0),
+	}
+	if !f.dhcp {
+		role.Address = strings.TrimSpace(f.ip.Value())
+		role.Mask = strings.TrimSpace(f.mask.Value())
+		role.DNS = parseDNS(f.dns.Value())
+		if f.plane == install.PlaneWAN {
+			role.Gateway = strings.TrimSpace(f.gateway.Value())
+		}
+	}
+	if nics[f.nic].IsWiFi {
+		role.WiFiSSID = strings.TrimSpace(f.ssid.Value())
+		role.WiFiPass = f.wifiPass.Value()
+	}
+	return role
+}
+
+func atoiOr(s string, fallback int) int {
+	n, err := strconv.Atoi(strings.TrimSpace(s))
+	if err != nil {
+		return fallback
+	}
+	return n
+}
+
+// ── Roles overview screen ─────────────────────────────────────────────────────
+
+// continueRow is the cursor position below the three role rows.
+const continueRow = 3
+
+func (m model) handleRolesKey(key string) (tea.Model, tea.Cmd) {
+	switch key {
+	case "up", "k":
+		if m.roleCursor > 0 {
+			m.roleCursor--
+		}
+	case "down", "j":
+		if m.roleCursor < continueRow {
+			m.roleCursor++
+		}
+	case "a":
+		// Advanced reveals VLAN and MTU on every role at once, rather than
+		// per-role, so the operator flips it on for the whole node.
+		m.advanced = !m.advanced
+		m.validationErr = ""
+	case "enter":
+		if m.roleCursor == continueRow {
+			if err := m.buildConfig().Validate(); err != nil {
+				m.validationErr = capitalise(err.Error())
+				return m, nil
+			}
+			m.validationErr = ""
+			m.screen = screenIdentity
+			m.hostnameInput.Focus()
+			return m, nil
+		}
+		m.validationErr = ""
+		m.screen = screenNetworkRole
+		m.roles[m.roleCursor].focus = roleFieldNIC
+		m.roles[m.roleCursor].blurAll()
+	case "esc":
+		m.screen = screenDiskConfirm
+		m.eraseInput.Focus()
+	}
+	return m, nil
+}
+
+func (m model) viewNetworkRoles(w int) string {
+	title := styleTitle.Render("Network Planes")
+	subtitle := styleMuted.Render("A role left unbound folds onto the plane above it.")
+
+	adv := styleMuted.Render("Advanced: off")
+	if m.advanced {
+		adv = styleSuccess.Render("Advanced: on")
+	}
+
+	header := fmt.Sprintf("  %-*s%-*s", colRole, "ROLE", colIface, "INTERFACE")
+	if m.advanced {
+		header += fmt.Sprintf("%-*s%-*s", colVLAN, "VLAN", colMTU, "MTU")
+	}
+	header += "ADDRESS"
+
+	lines := []string{title, subtitle, "", adv, "", styleMuted.Render(header)}
+
+	for i := range m.roles {
+		row := m.renderRoleRow(i)
+		if i == m.roleCursor {
+			lines = append(lines, styleSelected.Render("> "+row))
+		} else {
+			lines = append(lines, "  "+row)
+		}
+		if detail := m.roleHardware(i, bodyWidth(w)); detail != "" {
+			lines = append(lines, detail)
+		}
+	}
+
+	lines = append(lines, "")
+	cont := "  Continue"
+	if m.roleCursor == continueRow {
+		cont = styleSelected.Render("> Continue")
+	}
+	lines = append(lines, cont)
+
+	if m.validationErr != "" {
+		lines = append(lines, "", styleError.Render(m.validationErr))
+	}
+	lines = append(lines, styleHelp.Render("↑/↓ move • Enter edit • a advanced • Esc back"))
+
+	body := lipgloss.JoinVertical(lipgloss.Left, lines...)
+	return lipgloss.Place(w, m.height, lipgloss.Center, lipgloss.Center,
+		styleBox.Width(min(w-4, 78)).Render(body),
+	)
+}
+
+// renderRoleRow renders one line of the overview table. A folded role names
+// the plane it collapsed onto rather than showing a blank, so the operator can
+// always see where the traffic will actually go.
+func (m model) renderRoleRow(i int) string {
+	f := &m.roles[i]
+	name := fmt.Sprintf("%-5s", f.plane)
+
+	if f.folded() {
+		_, landed := m.buildConfig().Resolve(f.plane)
+		return fmt.Sprintf("%-*s%s", colRole, name, styleMuted.Render("folds onto "+string(landed)))
+	}
+
+	iface := styleMuted.Render("—")
+	if f.nic < len(m.nics) {
+		iface = nicSummary(m.nics[f.nic])
+	}
+
+	addr := "DHCP"
+	if !f.dhcp {
+		addr = formatCIDR(f.ip.Value(), f.mask.Value())
+		if addr == "" {
+			addr = styleWarning.Render("not set")
+		}
+	}
+
+	row := padCell(name, colRole) + padCell(iface, colIface)
+	if m.advanced {
+		row += padCell(valueOr(f.vlan.Value(), "—"), colVLAN) + padCell(valueOr(f.mtu.Value(), "—"), colMTU)
+	}
+	return row + addr
+}
+
+// roleHardware is the vendor/model line beneath a role row, indented under the
+// interface column. It is what tells an operator that wan really is the 25gbe
+// Mellanox and not the onboard 1gbe.
+func (m model) roleHardware(i, width int) string {
+	f := &m.roles[i]
+	if f.folded() || f.nic >= len(m.nics) {
+		return ""
+	}
+	indent := 2 + colRole
+	return strings.Repeat(" ", indent) + styleMuted.Render(nicHardware(m.nics[f.nic], width-indent))
+}
+
+// nicSummary renders name, speed and link state, colouring the state so a port
+// with no cable stands out rather than having to be read for.
+func nicSummary(n netprobe.NIC) string {
+	state := styleWarning.Render(n.State)
+	if n.Carrier {
+		state = styleSuccess.Render(n.State)
+	}
+	speed := n.Speed
+	if speed == "" {
+		speed = "—"
+	}
+	return fmt.Sprintf("%s  %s  %s", n.Name, speed, state)
+}
+
+// nicHardware is the detail line: what the card is, plus the predictable name
+// it also answers to, which is the one written on cabling notes. The alt name
+// is dropped rather than truncated when the two will not fit together.
+func nicHardware(n netprobe.NIC, width int) string {
+	hw := n.Hardware()
+	const separator = "  ·  "
+	if n.AltName != "" && len([]rune(hw))+len(separator)+len([]rune(n.AltName)) <= width {
+		return hw + separator + n.AltName
+	}
+	return truncate(hw, width)
+}
+
+// bodyWidth is the text width inside the framed body. Detail lines are trimmed
+// to it rather than left to wrap, which would push the table out of alignment.
+func bodyWidth(w int) int {
+	return min(w-4, 78) - 4
+}
+
+// padCell pads a cell to width by display width, which the %-*s verb cannot do
+// once a cell carries the ANSI escapes of a style.
+func padCell(s string, width int) string {
+	return lipgloss.PlaceHorizontal(width, lipgloss.Left, s)
+}
+
+// Column widths for the overview table, shared by the header and the rows.
+const (
+	colRole  = 6
+	colIface = 32
+	colVLAN  = 7
+	colMTU   = 7
+)
+
+// focusMark gives the focused text field a visible caret without every field
+// rendering its own prompt.
+func focusMark(focused bool) string {
+	if focused {
+		return styleLabel.Render("> ")
+	}
+	return "  "
+}
+
+// formatCIDR renders an address and mask the way an operator writes them,
+// falling back to the raw mask when it is not dotted-decimal.
+func formatCIDR(addr, mask string) string {
+	addr = strings.TrimSpace(addr)
+	mask = strings.TrimSpace(mask)
+	if addr == "" {
+		return ""
+	}
+	if mask == "" {
+		return addr
+	}
+	if ip := net.ParseIP(mask); ip != nil && ip.To4() != nil {
+		if ones, bits := net.IPMask(ip.To4()).Size(); bits != 0 {
+			return fmt.Sprintf("%s/%d", addr, ones)
+		}
+	}
+	return addr + "/" + strings.TrimPrefix(mask, "/")
+}
+
+func valueOr(v, fallback string) string {
+	if v = strings.TrimSpace(v); v != "" {
+		return v
+	}
+	return fallback
+}
+
+// ── Per-role editor screen ────────────────────────────────────────────────────
+
+func (m model) handleRoleKey(key string, msg tea.KeyMsg) (tea.Model, tea.Cmd) {
+	f := &m.roles[m.roleCursor]
+	fields := f.visibleFields(m.nicIsWiFi(f.nic), m.advanced)
+	pos := fieldIndex(fields, f.focus)
+
+	switch key {
+	case "esc":
+		f.blurAll()
+		m.screen = screenNetworkRoles
+		m.validationErr = ""
+		return m, nil
+
+	case "enter":
+		if err := m.validateRole(m.roleCursor); err != "" {
+			m.validationErr = err
+			return m, nil
+		}
+		f.blurAll()
+		m.screen = screenNetworkRoles
+		m.validationErr = ""
+		return m, nil
+
+	case "tab", "down":
+		pos = (pos + 1) % len(fields)
+		f.focus = fields[pos]
+		f.focusCurrent()
+		return m, nil
+
+	case "shift+tab", "up":
+		pos = (pos - 1 + len(fields)) % len(fields)
+		f.focus = fields[pos]
+		f.focusCurrent()
+		return m, nil
+
+	case "left", "right":
+		// Only the two selector fields consume arrows; text fields need them
+		// for cursor movement.
+		switch f.focus {
+		case roleFieldNIC:
+			return m.cycleNIC(m.roleCursor, key == "right"), nil
+		case roleFieldMethod:
+			f.dhcp = key == "left"
+			f.focusCurrent()
+			return m, nil
+		}
+	}
+
+	if in := f.input(f.focus); in != nil {
+		var cmd tea.Cmd
+		*in, cmd = in.Update(msg)
+		return m, cmd
+	}
+	return m, nil
+}
+
+// cycleNIC steps the interface selection, including the fold option for lan
+// and vpc. Wrapping keeps a single key usable on a node with one NIC.
+func (m model) cycleNIC(idx int, forward bool) model {
+	f := &m.roles[idx]
+	low := 0
+	if f.canFold() {
+		low = foldedNIC
+	}
+	high := len(m.nics) - 1
+
+	if forward {
+		if f.nic >= high {
+			f.nic = low
+		} else {
+			f.nic++
+		}
+		return m
+	}
+	if f.nic <= low {
+		f.nic = high
+	} else {
+		f.nic--
+	}
+	return m
+}
+
+func (m model) nicIsWiFi(idx int) bool {
+	return idx >= 0 && idx < len(m.nics) && m.nics[idx].IsWiFi
+}
+
+func (m model) viewNetworkRole(w int) string {
+	f := &m.roles[m.roleCursor]
+	title := styleTitle.Render(fmt.Sprintf("Configure %s plane", strings.ToUpper(string(f.plane))))
+	subtitle := styleMuted.Render(planeDescription(f.plane))
+
+	lines := []string{title, subtitle, ""}
+	for _, field := range f.visibleFields(m.nicIsWiFi(f.nic), m.advanced) {
+		lines = append(lines, m.renderRoleField(f, field, bodyWidth(w)), "")
+	}
+
+	if m.validationErr != "" {
+		lines = append(lines, styleError.Render(m.validationErr), "")
+	}
+	lines = append(lines, styleHelp.Render("Tab/↑↓ move • ←/→ change • Enter done • Esc cancel"))
+
+	body := lipgloss.JoinVertical(lipgloss.Left, lines...)
+	return lipgloss.Place(w, m.height, lipgloss.Center, lipgloss.Center,
+		styleBox.Width(min(w-4, 78)).Render(body),
+	)
+}
+
+// planeDescription explains what rides each plane, so the choice is not just a
+// three-letter name.
+func planeDescription(p install.Plane) string {
+	switch p {
+	case install.PlaneWAN:
+		return "Public ingress and egress, EIPs and NAT (br-wan)."
+	case install.PlaneLAN:
+		return "Storage replication, NATS mesh and OVN control (br-lan)."
+	default:
+		return "Geneve tunnels carrying EC2 traffic between nodes (br-vpc)."
+	}
+}
+
+// roleLabelWidth is the field-label column on the editor screen, shared so a
+// wrapped value can indent to line up underneath it.
+const roleLabelWidth = 14
+
+func (m model) renderRoleField(f *roleForm, field roleField, width int) string {
+	focused := f.focus == field
+	label := styleLabel.Render(fmt.Sprintf("%-*s", roleLabelWidth, roleFieldLabel(field)))
+
+	switch field {
+	case roleFieldNIC:
+		return label + m.renderNICChoice(f, focused, width)
+
+	case roleFieldMethod:
+		return label + renderToggle([]string{"DHCP (automatic)", "Static"}, boolToIdx(!f.dhcp), focused)
+
+	default:
+		if in := f.input(field); in != nil {
+			return label + focusMark(focused) + in.View()
+		}
+		return label
+	}
+}
+
+func roleFieldLabel(field roleField) string {
+	switch field {
+	case roleFieldNIC:
+		return "Interface"
+	case roleFieldMethod:
+		return "IP method"
+	case roleFieldIP:
+		return "IP address"
+	case roleFieldMask:
+		return "Subnet mask"
+	case roleFieldGateway:
+		return "Gateway"
+	case roleFieldDNS:
+		return "DNS"
+	case roleFieldVLAN:
+		return "VLAN id"
+	case roleFieldMTU:
+		return "MTU"
+	case roleFieldSSID:
+		return "WiFi SSID"
+	default:
+		return "WiFi password"
+	}
+}
+
+// renderNICChoice shows the selected interface on the field line and its
+// hardware identity on a line of its own beneath. Vendor and model are what
+// make a port identifiable, and they do not fit alongside the selector.
+func (m model) renderNICChoice(f *roleForm, focused bool, width int) string {
+	var text, detail string
+	switch {
+	case f.folded():
+		_, landed := m.buildConfig().Resolve(f.plane)
+		text = fmt.Sprintf("(fold onto %s)", landed)
+	case f.nic < len(m.nics):
+		text = nicSummary(m.nics[f.nic])
+		detail = nicHardware(m.nics[f.nic], width-roleLabelWidth-2)
+	default:
+		text = "(none detected)"
+	}
+
+	if focused {
+		text = styleSelected.Render(" ← " + text + " → ")
+	} else {
+		text = styleLabel.Render("[ " + text + " ]")
+	}
+	if detail == "" {
+		return text
+	}
+	return text + "\n" + strings.Repeat(" ", roleLabelWidth+2) + styleMuted.Render(detail)
+}
+
+func renderToggle(options []string, selected int, focused bool) string {
+	var parts []string
+	for i, opt := range options {
+		switch {
+		case i == selected && focused:
+			parts = append(parts, styleSelected.Render(" "+opt+" "))
+		case i == selected:
+			parts = append(parts, styleLabel.Render("["+opt+"]"))
+		default:
+			parts = append(parts, styleMuted.Render(opt))
+		}
+	}
+	return strings.Join(parts, "  ")
+}
+
+func boolToIdx(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+func fieldIndex(fields []roleField, want roleField) int {
+	for i, f := range fields {
+		if f == want {
+			return i
+		}
+	}
+	return 0
+}
+
+// validateRole checks one plane's entries. Cross-role rules (a shared NIC
+// without distinct VLAN ids, an unbound wan) belong to install.Config.Validate
+// and run when the operator continues.
+func (m model) validateRole(idx int) string {
+	f := &m.roles[idx]
+	if f.folded() {
+		return ""
+	}
+	if len(m.nics) == 0 {
+		return "No network interfaces detected"
+	}
+
+	if !f.dhcp {
+		if net.ParseIP(strings.TrimSpace(f.ip.Value())) == nil {
+			return "Enter a valid IP address"
+		}
+		if !validSubnetMask(f.mask.Value()) {
+			return "Enter a dotted-decimal subnet mask, e.g. 255.255.255.0 or 255.255.0.0"
+		}
+		if f.plane == install.PlaneWAN && net.ParseIP(strings.TrimSpace(f.gateway.Value())) == nil {
+			return "Enter a valid gateway address"
+		}
+	}
+
+	if v := strings.TrimSpace(f.vlan.Value()); v != "" {
+		id, err := strconv.Atoi(v)
+		if err != nil || id < 1 || id > 4094 {
+			return "VLAN id must be between 1 and 4094"
+		}
+	}
+	if v := strings.TrimSpace(f.mtu.Value()); v != "" {
+		mtu, err := strconv.Atoi(v)
+		if err != nil || mtu < 68 || mtu > 9216 {
+			return "MTU must be between 68 and 9216"
+		}
+	}
+	if m.nicIsWiFi(f.nic) && strings.TrimSpace(f.ssid.Value()) == "" {
+		return "Enter the WiFi network SSID"
+	}
+	return ""
+}
+
+func capitalise(s string) string {
+	if s == "" {
+		return s
+	}
+	return strings.ToUpper(s[:1]) + s[1:]
+}

cmd/installer/ui/network_test.go

@@ -0,0 +1,357 @@
+package ui
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+	"github.com/mulgadc/spinifex/cmd/installer/netprobe"
+)
+
+func nics(n int) []netprobe.NIC {
+	out := make([]netprobe.NIC, 0, n)
+	names := []string{"eno1", "ens1f0np0", "ens1f1np1", "ens2f0np0"}
+	for i := range n {
+		out = append(out, netprobe.NIC{Name: names[i], Speed: "25Gbps", Vendor: "Mellanox", State: "online"})
+	}
+	return out
+}
+
+// The pre-fill is the whole of the simple path: an operator with a
+// conventional node should be able to press Continue without editing anything.
+func TestNewModelPrefillsRolesByNICCount(t *testing.T) {
+	tests := []struct {
+		name             string
+		count            int
+		wantLAN, wantVPC int
+	}{
+		{"one NIC folds everything onto wan", 1, foldedNIC, foldedNIC},
+		{"two NICs dedicate the second to lan, vpc folds", 2, 1, foldedNIC},
+		{"three NICs give each plane its own", 3, 1, 2},
+		{"four NICs still only bind three", 4, 1, 2},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(tt.count))
+
+			if got := m.roles[0].nic; got != 0 {
+				t.Errorf("wan nic = %d, want 0", got)
+			}
+			if got := m.roles[1].nic; got != tt.wantLAN {
+				t.Errorf("lan nic = %d, want %d", got, tt.wantLAN)
+			}
+			if got := m.roles[2].nic; got != tt.wantVPC {
+				t.Errorf("vpc nic = %d, want %d", got, tt.wantVPC)
+			}
+		})
+	}
+}
+
+// Hiding rather than disabling is what keeps the default screen narrow.
+func TestVisibleFields(t *testing.T) {
+	tests := []struct {
+		name     string
+		mutate   func(*roleForm)
+		isWiFi   bool
+		advanced bool
+		want     []roleField
+	}{
+		{
+			name:   "wan on DHCP asks two questions",
+			mutate: func(f *roleForm) { f.dhcp = true },
+			want:   []roleField{roleFieldNIC, roleFieldMethod},
+		},
+		{
+			name:   "wan static adds a gateway",
+			mutate: func(f *roleForm) { f.dhcp = false },
+			want:   []roleField{roleFieldNIC, roleFieldMethod, roleFieldIP, roleFieldMask, roleFieldGateway, roleFieldDNS},
+		},
+		{
+			name:     "advanced reveals VLAN and MTU",
+			mutate:   func(f *roleForm) { f.dhcp = true },
+			advanced: true,
+			want:     []roleField{roleFieldNIC, roleFieldMethod, roleFieldVLAN, roleFieldMTU},
+		},
+		{
+			name:   "wireless adds credentials",
+			mutate: func(f *roleForm) { f.dhcp = true },
+			isWiFi: true,
+			want:   []roleField{roleFieldNIC, roleFieldMethod, roleFieldSSID, roleFieldWiFiPass},
+		},
+		{
+			name:   "a folded role has nothing to configure",
+			mutate: func(f *roleForm) { f.nic = foldedNIC },
+			want:   []roleField{roleFieldNIC},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newRoleForm(install.PlaneWAN, 0)
+			tt.mutate(&f)
+			got := f.visibleFields(tt.isWiFi, tt.advanced)
+
+			if len(got) != len(tt.want) {
+				t.Fatalf("visibleFields() = %v, want %v", got, tt.want)
+			}
+			for i := range got {
+				if got[i] != tt.want[i] {
+					t.Fatalf("visibleFields() = %v, want %v", got, tt.want)
+				}
+			}
+		})
+	}
+}
+
+// A non-wan role must be able to reach the folded state, and wan must not.
+func TestCycleNICFoldReachableOnlyOffWAN(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(2))
+
+	// lan: 1 -> 0 -> folded -> wraps to the last NIC.
+	m.roles[1].nic = 1
+	for _, want := range []int{0, foldedNIC, 1} {
+		m = m.cycleNIC(1, false)
+		if m.roles[1].nic != want {
+			t.Fatalf("lan nic = %d, want %d", m.roles[1].nic, want)
+		}
+	}
+
+	// wan cycles only through real interfaces.
+	m.roles[0].nic = 0
+	for range 4 {
+		m = m.cycleNIC(0, false)
+		if m.roles[0].nic == foldedNIC {
+			t.Fatal("wan reached the folded state, but it always needs an uplink")
+		}
+	}
+}
+
+func TestToRoleReadsVLANAndMTU(t *testing.T) {
+	f := newRoleForm(install.PlaneLAN, 0)
+	f.dhcp = false
+	f.ip.SetValue("10.0.0.2")
+	f.mask.SetValue("255.255.0.0")
+	f.vlan.SetValue("10")
+	f.mtu.SetValue("9000")
+
+	role := f.toRole(nics(1))
+
+	if role.VLAN != 10 || role.MTU != 9000 {
+		t.Errorf("VLAN/MTU = %d/%d, want 10/9000", role.VLAN, role.MTU)
+	}
+	if role.Link() != "eno1.10" {
+		t.Errorf("Link() = %q, want eno1.10", role.Link())
+	}
+	// Only wan carries a default route, even if a gateway was typed earlier.
+	f.gateway.SetValue("10.0.0.1")
+	if got := f.toRole(nics(1)).Gateway; got != "" {
+		t.Errorf("lan Gateway = %q, want empty", got)
+	}
+}
+
+func TestToRoleFoldedIsEmpty(t *testing.T) {
+	f := newRoleForm(install.PlaneVPC, foldedNIC)
+	if !f.toRole(nics(2)).Folded() {
+		t.Error("a folded form must produce a folded role")
+	}
+}
+
+func TestValidateRole(t *testing.T) {
+	tests := []struct {
+		name    string
+		plane   install.Plane
+		mutate  func(*roleForm)
+		wantErr string
+	}{
+		{"valid static lan", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("10.0.0.2")
+			f.mask.SetValue("255.255.0.0")
+		}, ""},
+		{"bad ip", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("not-an-ip")
+			f.mask.SetValue("255.255.0.0")
+		}, "valid IP"},
+		{"wan needs a gateway", install.PlaneWAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("216.218.163.99")
+			f.mask.SetValue("255.255.255.224")
+		}, "gateway"},
+		{"VLAN out of range", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = true
+			f.vlan.SetValue("9000")
+		}, "VLAN id must be"},
+		{"MTU out of range", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = true
+			f.mtu.SetValue("70000")
+		}, "MTU must be"},
+		{"folded roles skip validation", install.PlaneVPC, func(f *roleForm) {
+			f.nic = foldedNIC
+			f.ip.SetValue("nonsense")
+		}, ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(3))
+			m.roles[0] = newRoleForm(tt.plane, 0)
+			tt.mutate(&m.roles[0])
+
+			got := m.validateRole(0)
+			if tt.wantErr == "" {
+				if got != "" {
+					t.Fatalf("validateRole() = %q, want no error", got)
+				}
+				return
+			}
+			if !strings.Contains(got, tt.wantErr) {
+				t.Fatalf("validateRole() = %q, want it to mention %q", got, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Continue must refuse a layout install.Config.Validate would reject, rather
+// than failing later during the install itself.
+func TestContinueBlocksOnSharedNICWithoutVLANs(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(2))
+	m.screen = screenNetworkRoles
+	m.roleCursor = continueRow
+	m.roles[1].nic = 0 // lan onto the wan NIC, both untagged
+
+	next, _ := m.handleRolesKey("enter")
+	got := next.(model)
+
+	if got.screen != screenNetworkRoles {
+		t.Error("expected to stay on the roles screen when the layout is invalid")
+	}
+	if !strings.Contains(got.validationErr, "both bind eno1") {
+		t.Errorf("validationErr = %q, want it to name the shared interface", got.validationErr)
+	}
+
+	// Distinct VLAN ids make the same pairing legal.
+	got.roles[0].vlan.SetValue("30")
+	got.roles[1].vlan.SetValue("10")
+	next, _ = got.handleRolesKey("enter")
+	if next.(model).screen != screenIdentity {
+		t.Error("distinct VLAN ids should allow two roles to share one NIC")
+	}
+}
+
+func TestAdvancedToggle(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(3))
+	m.screen = screenNetworkRoles
+
+	next, _ := m.handleRolesKey("a")
+	if !next.(model).advanced {
+		t.Fatal("'a' should turn Advanced on")
+	}
+	next, _ = next.(model).handleRolesKey("a")
+	if next.(model).advanced {
+		t.Fatal("'a' should turn Advanced back off")
+	}
+}
+
+// The default route and the resolvers are node-level, and wan is the only plane
+// that carries either, so the east-west editors must not ask for them. Their
+// bridges are link-local and activate manually — a resolver pinned there is
+// unreachable, and a gateway would fight the real default route.
+func TestVisibleFieldsOmitsRoutingOffWAN(t *testing.T) {
+	for _, plane := range []install.Plane{install.PlaneLAN, install.PlaneVPC} {
+		t.Run(string(plane), func(t *testing.T) {
+			f := newRoleForm(plane, 1)
+			f.dhcp = false
+			got := f.visibleFields(false, false)
+
+			want := []roleField{roleFieldNIC, roleFieldMethod, roleFieldIP, roleFieldMask}
+			if len(got) != len(want) {
+				t.Fatalf("visibleFields() = %v, want %v", got, want)
+			}
+			for i := range got {
+				if got[i] != want[i] {
+					t.Fatalf("visibleFields() = %v, want %v", got, want)
+				}
+			}
+		})
+	}
+}
+
+// The mask field must accept only what netgen can consume. It previously
+// advertised and accepted a prefix length, which passed the TUI and then failed
+// the install itself with "invalid mask: 24".
+func TestValidSubnetMaskRejectsPrefixLength(t *testing.T) {
+	for _, bad := range []string{"24", "/24", "8", "", "255.255.0", "255.0.255.0", "::1"} {
+		if validSubnetMask(bad) {
+			t.Errorf("validSubnetMask(%q) = true, want false", bad)
+		}
+	}
+	for _, good := range []string{"255.255.255.0", "255.255.0.0", "255.0.0.0", "255.255.255.252"} {
+		if !validSubnetMask(good) {
+			t.Errorf("validSubnetMask(%q) = false, want true", good)
+		}
+	}
+}
+
+// The hardware detail is the point of the NIC table: an operator has to be able
+// to tell which physical port each plane landed on. It gets a line of its own,
+// under the interface column, rather than being squeezed into the row.
+func TestRoleHardwareRendersOnItsOwnLine(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, []netprobe.NIC{{
+		Name: "ens1f0np0", Vendor: "Mellanox Technologies", Model: "MT27710",
+		AltName: "enp132s0f0np0", Speed: "25 Gbps", Carrier: true, State: "online",
+	}})
+
+	view := m.viewNetworkRoles(100)
+	if !strings.Contains(view, "ens1f0np0  25 Gbps  online") {
+		t.Errorf("expected the identity line in:\n%s", view)
+	}
+	detail := m.roleHardware(0, bodyWidth(100))
+	if !strings.Contains(detail, "Mellanox Technologies MT27710") {
+		t.Errorf("roleHardware = %q, want the vendor and model", detail)
+	}
+	if !strings.Contains(detail, "enp132s0f0np0") {
+		t.Errorf("roleHardware = %q, want the alternative name", detail)
+	}
+	// Every line has to fit the box, or lipgloss wraps it and the columns shear.
+	for line := range strings.SplitSeq(view, "\n") {
+		if len([]rune(line)) > 100 {
+			t.Errorf("line exceeds the terminal width: %q", line)
+		}
+	}
+}
+
+// A narrow box drops the alternative name before it truncates the model, since
+// half a model name identifies nothing.
+func TestNICHardwareDropsAltNameBeforeTruncating(t *testing.T) {
+	n := netprobe.NIC{Vendor: "Broadcom Inc. and subsidiaries", Model: "NetXtreme BCM5720", AltName: "enp1s0f1"}
+
+	if got := nicHardware(n, 60); got != "Broadcom Inc. and subsidiaries NetXtreme BCM5720" {
+		t.Errorf("nicHardware(60) = %q, want the name without the alt name", got)
+	}
+	if got := nicHardware(n, 20); got != "Broadcom Inc. and s…" {
+		t.Errorf("nicHardware(20) = %q, want a marked truncation", got)
+	}
+}
+
+// A role bound to a port with no hardware database entry still gets something
+// to identify the card by, never the bare word "unknown".
+func TestRoleHardwareFallsBackToDriver(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, []netprobe.NIC{{
+		Name: "eno1", Driver: "bnxt_en", DeviceID: "14e4:165f", State: "no cable",
+	}})
+
+	if got := m.roleHardware(0, bodyWidth(100)); !strings.Contains(got, "bnxt_en [14e4:165f]") {
+		t.Errorf("roleHardware = %q, want the driver and device id", got)
+	}
+}
+
+// A folded role has no interface, so it must not render an orphan detail line
+// under the plane it collapsed onto.
+func TestRoleHardwareEmptyWhenFolded(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(1))
+	if got := m.roleHardware(1, bodyWidth(100)); got != "" {
+		t.Errorf("roleHardware for a folded role = %q, want empty", got)
+	}
+}

cmd/installer/ui/storage_test.go

@@ -0,0 +1,139 @@
+package ui
+
+import (
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+)
+
+// diskModel is a storage screen with n unselected disks and ext4 chosen, which
+// is where the wizard starts.
+func diskModel(n int) model {
+	m := model{fsCursor: slices.Index(install.AllFSTypes, install.FSExt4)}
+	for i := range n {
+		m.disks = append(m.disks, install.Disk{
+			Path:              string(rune('a'+i)) + "-disk",
+			Bytes:             200 << 30,
+			LogicalBlockSize:  512,
+			PhysicalBlockSize: 512,
+		})
+	}
+	return m
+}
+
+func rolePaths(cfg install.DiskConfig) map[install.DiskRole]string {
+	out := map[install.DiskRole]string{}
+	for _, rm := range cfg.Roles {
+		out[rm.Role] = rm.Disk.Path
+	}
+	return out
+}
+
+// A single-disk machine should need no storage input at all, so the disk that
+// newModel preselects has to arrive already holding the os role.
+func TestPreselectedDiskIsAValidConfiguration(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda", Bytes: 200 << 30, LogicalBlockSize: 512}}, nil)
+	cfg := m.storage()
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("the preselected disk must be installable as-is: %v", err)
+	}
+	if cfg.Primary().Path != "/dev/sda" {
+		t.Errorf("Primary() = %q, want the preselected disk", cfg.Primary().Path)
+	}
+}
+
+// Picking disks in order gives the layout an operator almost always wants, so
+// the role key is only needed for the exceptions.
+func TestSelectingDisksAssignsRolesInOrder(t *testing.T) {
+	m := diskModel(3)
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	got := rolePaths(m.storage())
+	for role, want := range map[install.DiskRole]string{
+		install.RoleOS: "a-disk", install.RoleSpinifex: "b-disk", install.RolePredastore: "c-disk",
+	} {
+		if got[role] != want {
+			t.Errorf("%s = %q, want %q", role, got[role], want)
+		}
+	}
+	if err := m.storage().Validate(); err != nil {
+		t.Errorf("the default assignment must be valid: %v", err)
+	}
+}
+
+// Deselecting has to drop the role with the disk, or every later role shifts
+// onto the wrong drive.
+func TestDeselectingADiskReleasesItsRole(t *testing.T) {
+	m := diskModel(3)
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	m = m.toggleDisk(1) // drop the spinifex drive
+
+	got := rolePaths(m.storage())
+	if got[install.RoleOS] != "a-disk" || got[install.RolePredastore] != "c-disk" {
+		t.Fatalf("remaining roles moved: %v", got)
+	}
+	if _, ok := got[install.RoleSpinifex]; ok {
+		t.Errorf("the spinifex role should be free again: %v", got)
+	}
+}
+
+// The two-drive predastore layout is only reachable by reassigning, so cycling
+// swaps rather than refusing a role another disk already holds.
+func TestCycleRoleSwapsWithTheDiskThatHoldsIt(t *testing.T) {
+	m := diskModel(2)
+	m = m.toggleDisk(0).toggleDisk(1)
+	// b-disk: spinifex → predastore, which no disk holds, so no swap.
+	m = m.cycleRole(1)
+
+	got := rolePaths(m.storage())
+	if got[install.RoleOS] != "a-disk" || got[install.RolePredastore] != "b-disk" {
+		t.Fatalf("roles = %v, want os on a-disk and predastore on b-disk", got)
+	}
+
+	// Cycling past the end wraps onto os, which a-disk holds, so they trade.
+	m = m.cycleRole(1)
+	got = rolePaths(m.storage())
+	if got[install.RoleOS] != "b-disk" || got[install.RolePredastore] != "a-disk" {
+		t.Fatalf("roles = %v, want the two disks to have swapped", got)
+	}
+	if err := m.storage().Validate(); err != nil {
+		t.Errorf("a swap must always leave a valid assignment: %v", err)
+	}
+}
+
+// ZFS pairs mirrors by selection order and has no roles to assign.
+func TestStorageLeavesRolesUnsetOnZFS(t *testing.T) {
+	m := diskModel(2)
+	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1)
+	m = m.toggleDisk(0).toggleDisk(1)
+
+	cfg := m.storage()
+	if len(cfg.Roles) != 0 {
+		t.Errorf("Roles = %v, want none on a pool", cfg.Roles)
+	}
+	if !slices.Equal(cfg.Paths(), []string{"a-disk", "b-disk"}) {
+		t.Errorf("Paths() = %v, want the selection order preserved", cfg.Paths())
+	}
+}
+
+// A three-drive selection looks like an array and is not one, so the mapping
+// and the absence of redundancy both have to be on screen before the erase.
+func TestConfirmScreenStatesTheLayoutAndTheRisk(t *testing.T) {
+	m := diskModel(3)
+	m.height = 40
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	got := m.viewDiskConfirm(100)
+
+	for _, want := range []string{"/var/lib/spinifex/predastore", "c-disk", "losing one loses"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("confirm screen is missing %q:\n%s", want, got)
+		}
+	}
+}

cmd/lb-agent/main.go

@@ -19,11 +19,17 @@ import (
 	"syscall"
 
 	"github.com/mulgadc/spinifex/spinifex/lbagent"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 
 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 func main() {
+	// Without this the agent falls back to slog's default text handler, which folds
+	// attributes into the message so the console scraper recovers a level and nothing
+	// else. No OTLP export: the agent's only egress is the gateway it is watching.
+	otelsetup.SetDefaultJSONLogger(slog.LevelInfo)
+
 	var (
 		lbID       string
 		gatewayURL string

cmd/rds-agent/acknowledge.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mulgadc/spinifex/internal/rdsgw"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+)
+
+// The completion receipt rds-init writes once the master role is durably
+// applied. Deliberately outside PGDATA: clear_unfinished_datadir does an rm -rf
+// of PGDATA, so keeping the receipt out of it means no initdb-adjacent path can
+// touch it and the datadir stays a byte-for-byte stock one.
+const (
+	receiptRelDir  = ".spinifex-rds/bootstrap"
+	receiptFile    = "receipt.env"
+	receiptPayload = "RDS_RECEIPT_PAYLOAD_ID"
+	receiptDBID    = "RDS_RECEIPT_DB_INSTANCE_IDENTIFIER"
+)
+
+func (a *Agent) receiptPath() string {
+	return filepath.Join(a.cfg.DataMount, receiptRelDir, receiptFile)
+}
+
+// Confirms the initial bootstrap, which is what destroys the staged ciphertext.
+// A no-op on the attach path, and deliberately synchronous: it inherits the
+// cancellation semantics register, bootstrap and the data-mount wait already
+// share, and blocking the stateful loops behind it is correct because on a first
+// boot the engine is not bootstrapped until rds-init has finished anyway.
+//
+// It does not wait for the engine to be healthy. The receipt already proves the
+// master role was durably applied, and acknowledging early shortens the window
+// in which the ciphertext exists.
+func (a *Agent) acknowledgeBootstrap(ctx context.Context) error {
+	pending := a.pending
+	if pending == nil {
+		return nil
+	}
+
+	// Polled rather than waited on: rds-init runs after this agent registered,
+	// so the file appears some time into the boot. There is no give-up deadline
+	// here — the control plane's bootstrap timeout already fails the instance,
+	// and a second one in the guest would only give the two a way to disagree.
+	if err := retryObserved(ctx, "bootstrap receipt", func(context.Context) error {
+		return a.checkBootstrapReceipt(pending)
+	}, func(err error) {
+		// An absent receipt is the expected state for the whole initdb window,
+		// so reporting it would put a failure on every healthy create's beat and
+		// then into the reason a timed-out create records.
+		if errors.Is(err, fs.ErrNotExist) {
+			return
+		}
+		a.hb.setBootstrapFailure("bootstrap receipt", err)
+	}); err != nil {
+		return err
+	}
+	a.hb.clearBootstrapFailure()
+
+	if err := retryObserved(ctx, "bootstrap acknowledgement", func(ctx context.Context) error {
+		return terminalOnDenial(a.cp.AcknowledgeBootstrap(ctx, a.id, pending))
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap acknowledgement", err)
+	}); err != nil {
+		if !errors.Is(err, errRetryTerminal) {
+			return err
+		}
+		// The staged payload will never be delivered to this VM, and blocking the
+		// command poller and the parameter guard on that would leave a serving
+		// database no one can administer. The reason rides the heartbeat instead.
+		slog.ErrorContext(ctx, "rds-agent: the initial bootstrap cannot be acknowledged",
+			"payloadId", pending.payloadID, "err", err)
+		return nil
+	}
+	a.hb.clearBootstrapFailure()
+
+	// AcknowledgedAt on the record is the durable audit trail, so the receipt is
+	// removed rather than archived: keeping it would only grow the surface of
+	// stale receipts riding along in data-volume snapshots. A removal failure is
+	// harmless, since a later boot fetches attach and never consults one.
+	if err := os.Remove(a.receiptPath()); err != nil && !os.IsNotExist(err) {
+		slog.WarnContext(ctx, "rds-agent: could not remove the bootstrap receipt",
+			"receipt", a.receiptPath(), "err", err)
+	}
+	slog.InfoContext(ctx, "rds-agent: initial bootstrap acknowledged", "payloadId", pending.payloadID)
+	return nil
+}
+
+// The control plane resolved these from its own record — a superseded
+// generation, a payload it never staged, a data volume that is not this
+// instance's. Asking again cannot change any of them.
+func terminalOnDenial(err error) error {
+	var apiErr *rdsgw.APIError
+	if !errors.As(err, &apiErr) {
+		return err
+	}
+	switch apiErr.Code {
+	case awserrors.ErrorAccessDenied, awserrors.ErrorInvalidParameterValue:
+		return fmt.Errorf("%w: %w", errRetryTerminal, err)
+	}
+	return err
+}
+
+// A receipt naming another payload or another DB instance is treated as absent.
+// The receipt lives on the data volume, so it rides along in every snapshot of
+// it and a restored instance's volume carries the source instance's receipt.
+func (a *Agent) checkBootstrapReceipt(pending *pendingBootstrap) error {
+	path := a.receiptPath()
+	values, err := readReceipt(path)
+	if err != nil {
+		return err
+	}
+	if got := values[receiptPayload]; got != pending.payloadID {
+		return fmt.Errorf("%s names bootstrap payload %q, not %q", path, got, pending.payloadID)
+	}
+	if got := values[receiptDBID]; got != a.id.DBInstanceIdentifier {
+		return fmt.Errorf("%s names DB instance %q, not %q", path, got, a.id.DBInstanceIdentifier)
+	}
+	return nil
+}
+
+// KEY=value, the one format both readers can parse: rds-init in POSIX shell
+// with no guaranteed jq, and this.
+func readReceipt(path string) (map[string]string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	values := make(map[string]string)
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, value, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		values[strings.TrimSpace(key)] = strings.TrimSpace(value)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	return values, nil
+}

cmd/rds-agent/agent.go

@@ -0,0 +1,391 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/gwsign"
+	"github.com/mulgadc/spinifex/internal/rdsgw"
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Overridable via -ldflags "-X main.version=...".
+var version = "dev"
+
+// The gateway resolves the authoritative DB instance from the request's
+// credentials, so DBInstanceIdentifier is an assertion to be checked, not
+// trusted.
+type identity struct {
+	DBInstanceIdentifier string
+	AgentVersion         string
+	EngineVersion        string
+}
+
+// Every method is a SigV4-signed Query request, so the NATS bus stays
+// host-internal.
+type controlPlane interface {
+	Register(ctx context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error)
+	SubmitState(ctx context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error)
+	// Returns Mode=initialize with the master password for as long as a payload
+	// staged for this VM generation is unacknowledged, and Mode=attach after.
+	GetBootstrapConfig(ctx context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error)
+	// Reports that PostgreSQL durably applied the staged password, which is what
+	// lets the control plane destroy it.
+	AcknowledgeBootstrap(ctx context.Context, id identity, pending *pendingBootstrap) error
+	PollCommands(ctx context.Context, id identity, replies []handlers_rds.CommandReply, wait time.Duration) ([]handlers_rds.Command, error)
+}
+
+// Bounds a register, heartbeat or bootstrap request. The long poll sets its own
+// deadline from the wait it asked the gateway to hold.
+const callTimeout = 15 * time.Second
+
+// Keeps the poll's deadline past the requested wait, so a gateway answering at
+// the end of its window is not cut off by the client.
+const pollSlack = 10 * time.Second
+
+type gatewayControlPlane struct {
+	client *rdsgw.Client
+}
+
+var _ controlPlane = (*gatewayControlPlane)(nil)
+
+// Signs with the instance-role credentials the SDK chain resolves from IMDS,
+// against the pinned gateway CA.
+func newGatewayControlPlane(cfg config) (*gatewayControlPlane, error) {
+	signer, err := gwsign.NewIMDS(context.Background(), cfg.Region)
+	if err != nil {
+		return nil, fmt.Errorf("build IMDS signer: %w", err)
+	}
+	// The client timeout is the ceiling for the longest call, the long poll;
+	// shorter calls carry their own deadline.
+	client, err := rdsgw.New(cfg.GatewayURL, cfg.GatewayCA, signer, cfg.Region, cfg.PollWait+pollSlack)
+	if err != nil {
+		return nil, err
+	}
+	return &gatewayControlPlane{client: client}, nil
+}
+
+func (g *gatewayControlPlane) Register(ctx context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error) {
+	params := url.Values{}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	setIfPresent(params, "AgentVersion", id.AgentVersion)
+	setIfPresent(params, "EngineVersion", id.EngineVersion)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.RegisterDBInstanceOutput
+	if err := g.client.Call(ctx, "RegisterDBInstance", params, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func (g *gatewayControlPlane) SubmitState(ctx context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error) {
+	params := url.Values{"EngineHealth": {string(health)}}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	setIfPresent(params, "EngineVersion", id.EngineVersion)
+	setIfPresent(params, "Message", message)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.SubmitDBStateChangeOutput
+	if err := g.client.Call(ctx, "SubmitDBStateChange", params, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func (g *gatewayControlPlane) GetBootstrapConfig(ctx context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error) {
+	params := url.Values{}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.GetDBBootstrapConfigOutput
+	if err := g.client.Call(ctx, "GetDBBootstrapConfig", params, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func (g *gatewayControlPlane) AcknowledgeBootstrap(ctx context.Context, id identity, pending *pendingBootstrap) error {
+	params := url.Values{
+		"PayloadId":    {pending.payloadID},
+		"VMGeneration": {strconv.FormatInt(pending.vmGeneration, 10)},
+	}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	setIfPresent(params, "DataVolumeId", pending.dataVolumeID)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.AcknowledgeDBBootstrapOutput
+	if err := g.client.Call(ctx, "AcknowledgeDBBootstrap", params, &out); err != nil {
+		return err
+	}
+	if !out.Acknowledged {
+		return fmt.Errorf("the control plane did not acknowledge bootstrap payload %s", pending.payloadID)
+	}
+	return nil
+}
+
+// Declared here so the in-guest binary does not link the server side of the API.
+type pollDBCommandsOutput struct {
+	Commands []handlers_rds.Command `xml:"Commands>member"`
+}
+
+func (g *gatewayControlPlane) PollCommands(ctx context.Context, id identity, replies []handlers_rds.CommandReply, wait time.Duration) ([]handlers_rds.Command, error) {
+	params := url.Values{"WaitTimeSeconds": {strconv.FormatInt(int64(wait.Seconds()), 10)}}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	// Replies ride the poll rather than a separate action, so replying and
+	// asking for the next command are one round trip.
+	for i, reply := range replies {
+		member := fmt.Sprintf("Replies.member.%d.", i+1)
+		params.Set(member+"CommandId", reply.CommandID)
+		params.Set(member+"Status", reply.Status)
+		setIfPresent(params, member+"Message", reply.Message)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, wait+pollSlack)
+	defer cancel()
+
+	var out pollDBCommandsOutput
+	if err := g.client.Call(ctx, "PollDBCommands", params, &out); err != nil {
+		return nil, err
+	}
+	return out.Commands, nil
+}
+
+// Only sets a parameter that has a value, so the gateway can tell "no
+// assertion" from "asserted as blank".
+func setIfPresent(params url.Values, key, value string) {
+	if value != "" {
+		params.Set(key, value)
+	}
+}
+
+type Agent struct {
+	cfg             config
+	id              identity
+	cp              controlPlane
+	probe           *engineProbe
+	engine          *postgresEngine
+	handoffWriter   func(string, *handlers_rds.GetDBBootstrapConfigOutput) error
+	dataMountWaiter func(context.Context, string, string) error
+
+	// Set by bootstrap when the fetch replayed a staged payload; nil leaves the
+	// acknowledgement step a no-op, which is every attach boot.
+	pending *pendingBootstrap
+
+	hb    *heartbeater
+	cmd   *commander
+	guard *paramGuard
+}
+
+// Assembles from already-built parts, so tests can pass fakes; New builds the
+// production control plane and delegates here.
+func newAgent(cfg config, cp controlPlane, probe *engineProbe) *Agent {
+	if cfg.DataMount == "" {
+		cfg.DataMount = defaultDataMount
+	}
+	if cfg.MountsFile == "" {
+		cfg.MountsFile = defaultMountsFile
+	}
+	id := identity{
+		DBInstanceIdentifier: cfg.DBInstanceIdentifier,
+		AgentVersion:         version,
+		EngineVersion:        cfg.EngineVersion,
+	}
+	a := &Agent{
+		cfg: cfg, id: id, cp: cp, probe: probe,
+		handoffWriter: writeHandoff, dataMountWaiter: waitForDataMount,
+	}
+	a.engine = newPostgresEngine(cfg, execCommandRunner, execSessionRunner, probe)
+	a.hb = newHeartbeater(cp, probe, a.engine, handlers_rds.HeartbeatInterval)
+	a.cmd = newCommander(cp, newCommandRegistry(a.engine, newGuestStorage(cfg, execCommandRunner)), cfg.PollWait)
+	a.guard = newParamGuard(a.engine, probe, cp)
+	return a
+}
+
+// Does not wait for IMDS: the register loop rides out a datapath still coming up.
+func New(cfg config) (*Agent, error) {
+	if cfg.GatewayURL == "" {
+		return nil, fmt.Errorf("no gateway URL configured (RDS_GATEWAY_URL)")
+	}
+	cp, err := newGatewayControlPlane(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return newAgent(cfg, cp, newEngineProbe(cfg, execProbeRunner)), nil
+}
+
+func (a *Agent) Run(ctx context.Context) error {
+	// Named before the first dial. When register cannot reach the gateway the
+	// retry loop is all that survives, and the address it is dialing is what
+	// separates a mgmt NIC that came up without an address from a control plane
+	// that is genuinely down.
+	slog.Info("rds-agent: starting",
+		"gatewayURL", a.cfg.GatewayURL, "dbInstanceIdentifier", a.id.DBInstanceIdentifier,
+		"agentVersion", a.id.AgentVersion)
+
+	// Registration comes first: its response is the agent's identity.
+	if err := a.register(ctx); err != nil {
+		return err
+	}
+	a.hb.id, a.cmd.id, a.guard.id = a.id, a.id, a.id
+
+	// Beating before the bootstrap keeps a stuck boot visible as a live VM with
+	// a down engine rather than as silence.
+	go a.hb.Run(ctx)
+
+	if err := a.bootstrap(ctx); err != nil {
+		return err
+	}
+
+	// The handoff must land before this wait: rds-datadir needs it to identify
+	// and authorize the volume. Commands and the parameter guard wait because
+	// their durable state belongs on that mount, never on the disposable boot disk.
+	if err := a.dataMountWaiter(ctx, a.cfg.MountsFile, a.cfg.DataMount); err != nil {
+		// A shutdown cancels this wait; main treats the wrapped context.Canceled
+		// as a clean stop, exactly as it does for register and bootstrap above.
+		return fmt.Errorf("wait for data mount %s: %w", a.cfg.DataMount, err)
+	}
+
+	// The receipt it waits on lives on that mount, so this cannot run any earlier.
+	// A cancelled wait is a clean stop here too.
+	if err := a.acknowledgeBootstrap(ctx); err != nil {
+		return err
+	}
+
+	// Directives are only meaningful against a bootstrapped, mounted engine.
+	go a.cmd.Run(ctx)
+
+	// Started after the handoff, so the window it measures covers the engine's
+	// own start rather than the time rds-init spent waiting for this agent.
+	go a.guard.Run(ctx)
+
+	<-ctx.Done()
+	return nil
+}
+
+const dataMountPollInterval = time.Second
+
+// waitForDataMount blocks the stateful agent loops until the kernel reports the
+// configured data mount. Registration, heartbeat, and bootstrap run before it.
+func waitForDataMount(ctx context.Context, mountsFile, target string) error {
+	ticker := time.NewTicker(dataMountPollInterval)
+	defer ticker.Stop()
+
+	for {
+		mounted, err := mountTableContains(mountsFile, target)
+		if err != nil {
+			slog.WarnContext(ctx, "rds-agent: could not read mount table while waiting for data volume",
+				"mountsFile", mountsFile, "dataMount", target, "err", err)
+		} else if mounted {
+			slog.InfoContext(ctx, "rds-agent: data volume mount is ready", "dataMount", target)
+			return nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+	}
+}
+
+func mountTableContains(path, target string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		if len(fields) >= 2 && unescapeMountField(fields[1]) == target {
+			return true, nil
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return false, err
+	}
+	return false, nil
+}
+
+// Linux mount tables encode whitespace and backslashes as octal escapes.
+func unescapeMountField(field string) string {
+	replacer := strings.NewReplacer(`\134`, `\`, `\040`, " ", `\011`, "\t", `\012`, "\n")
+	return replacer.Replace(field)
+}
+
+// Boot-retry bounds. Tight to start, since the usual cause is a control plane
+// still creating this instance's record, and capped so an outage does not turn
+// a booting fleet into a retry storm.
+const (
+	retryMin = 1 * time.Second
+	retryMax = 30 * time.Second
+
+	// With the production backoff, attempt five is about 15 seconds into the
+	// failure and still precedes the first 30-second heartbeat.
+	retryErrorAttempt = 5
+)
+
+// Wrapping an error in this stops retryObserved. For the failures the control
+// plane decides from its own record, no retry can change the answer, and looping
+// on one only keeps the caller blocked.
+var errRetryTerminal = errors.New("terminal failure")
+
+// Never gives up on its own: an agent that stopped trying would leave the
+// control plane with no signal at all.
+func retry(ctx context.Context, what string, fn func(context.Context) error) error {
+	return retryObserved(ctx, what, fn, nil)
+}
+
+// onFailure lets boot-critical callers publish the latest error while the retry
+// remains blocked. It must not block because it runs on the retry path itself.
+func retryObserved(ctx context.Context, what string, fn func(context.Context) error, onFailure func(error)) error {
+	delay := retryMin
+	for attempt := 1; ; attempt++ {
+		err := fn(ctx)
+		if err == nil {
+			return nil
+		}
+		if ctx.Err() != nil {
+			return fmt.Errorf("%s: %w", what, ctx.Err())
+		}
+		if onFailure != nil {
+			onFailure(err)
+		}
+		if errors.Is(err, errRetryTerminal) {
+			slog.ErrorContext(ctx, "rds-agent: "+what+" failed terminally, not retrying", "err", err)
+			return fmt.Errorf("%s: %w", what, err)
+		}
+		if attempt >= retryErrorAttempt {
+			slog.ErrorContext(ctx, "rds-agent: "+what+" still failing, retrying",
+				"attempt", attempt, "retryIn", delay, "err", err)
+		} else {
+			slog.WarnContext(ctx, "rds-agent: "+what+" failed, retrying",
+				"attempt", attempt, "retryIn", delay, "err", err)
+		}
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("%s: %w", what, ctx.Err())
+		case <-time.After(delay):
+		}
+		delay = min(delay*2, retryMax)
+	}
+}

cmd/rds-agent/agent_test.go

@@ -0,0 +1,807 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/gwsign"
+	"github.com/mulgadc/spinifex/internal/rdsgw"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	gateway_rds "github.com/mulgadc/spinifex/spinifex/gateway/rds"
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeControlPlane is a scriptable controlPlane. Every call records what the
+// agent sent, so a test asserts on the request as well as the reaction.
+type fakeControlPlane struct {
+	mu sync.Mutex
+
+	registerOut  *handlers_rds.RegisterDBInstanceOutput
+	registerErrs []error
+	registerReqs []identity
+
+	bootstrapOut  *handlers_rds.GetDBBootstrapConfigOutput
+	bootstrapErrs []error
+	bootstrapReqs []identity
+	// Scripted responses ahead of bootstrapOut, so a test can answer the first
+	// fetch differently from the ones the agent retries with.
+	bootstrapQueue []*handlers_rds.GetDBBootstrapConfigOutput
+
+	ackErrs []error
+	ackReqs []pendingBootstrap
+
+	states []submittedState
+
+	pollReplies [][]handlers_rds.CommandReply
+	pollQueue   [][]handlers_rds.Command
+	pollErr     error
+	polled      chan struct{}
+}
+
+type submittedState struct {
+	id      identity
+	health  handlers_rds.EngineHealth
+	message string
+}
+
+var _ controlPlane = (*fakeControlPlane)(nil)
+
+func newFakeControlPlane() *fakeControlPlane {
+	return &fakeControlPlane{
+		registerOut: &handlers_rds.RegisterDBInstanceOutput{
+			DBInstanceIdentifier: "db-resolved", HeartbeatIntervalSeconds: 30,
+		},
+		polled: make(chan struct{}, 64),
+	}
+}
+
+// nextErr pops the next scripted error, so a test can fail a call n times and
+// then let it succeed.
+func nextErr(errs *[]error) error {
+	if len(*errs) == 0 {
+		return nil
+	}
+	err := (*errs)[0]
+	*errs = (*errs)[1:]
+	return err
+}
+
+func (f *fakeControlPlane) Register(_ context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.registerReqs = append(f.registerReqs, id)
+	if err := nextErr(&f.registerErrs); err != nil {
+		return nil, err
+	}
+	return f.registerOut, nil
+}
+
+func (f *fakeControlPlane) SubmitState(_ context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.states = append(f.states, submittedState{id: id, health: health, message: message})
+	// Zero means "keep the cadence you have", so a test's fast interval is not
+	// reset by every beat.
+	return &handlers_rds.SubmitDBStateChangeOutput{Acknowledged: true}, nil
+}
+
+func (f *fakeControlPlane) GetBootstrapConfig(_ context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.bootstrapReqs = append(f.bootstrapReqs, id)
+	if err := nextErr(&f.bootstrapErrs); err != nil {
+		return nil, err
+	}
+	if len(f.bootstrapQueue) > 0 {
+		out := f.bootstrapQueue[0]
+		f.bootstrapQueue = f.bootstrapQueue[1:]
+		return out, nil
+	}
+	return f.bootstrapOut, nil
+}
+
+func (f *fakeControlPlane) AcknowledgeBootstrap(_ context.Context, _ identity, pending *pendingBootstrap) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.ackReqs = append(f.ackReqs, *pending)
+	return nextErr(&f.ackErrs)
+}
+
+func (f *fakeControlPlane) PollCommands(ctx context.Context, _ identity, replies []handlers_rds.CommandReply, _ time.Duration) ([]handlers_rds.Command, error) {
+	f.mu.Lock()
+	f.pollReplies = append(f.pollReplies, slices.Clone(replies))
+	err := f.pollErr
+	var out []handlers_rds.Command
+	if len(f.pollQueue) > 0 {
+		out, f.pollQueue = f.pollQueue[0], f.pollQueue[1:]
+	}
+	f.mu.Unlock()
+
+	select {
+	case f.polled <- struct{}{}:
+	default:
+	}
+	if err != nil {
+		return nil, err
+	}
+	if len(out) > 0 {
+		return out, nil
+	}
+	// An idle window: block until the agent hangs up, as the gateway's long poll
+	// does, rather than spinning the caller's loop.
+	<-ctx.Done()
+	return nil, ctx.Err()
+}
+
+func (f *fakeControlPlane) ackRequests() []pendingBootstrap {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return slices.Clone(f.ackReqs)
+}
+
+func (f *fakeControlPlane) snapshotStates() []submittedState {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]submittedState(nil), f.states...)
+}
+
+func (f *fakeControlPlane) snapshotReplies() [][]handlers_rds.CommandReply {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([][]handlers_rds.CommandReply, len(f.pollReplies))
+	for i := range f.pollReplies {
+		out[i] = slices.Clone(f.pollReplies[i])
+	}
+	return out
+}
+
+// testConfig points the agent's file output at a temp dir and its probe at a
+// name no real binary answers to.
+func testConfig(t *testing.T) config {
+	t.Helper()
+	return config{
+		GatewayURL: "https://gw.test",
+		HandoffDir: filepath.Join(t.TempDir(), "spinifex-rds"),
+		EngineHost: defaultEngineHost,
+		EnginePort: defaultEnginePort,
+		PGIsReady:  "pg_isready",
+		PollWait:   time.Second,
+	}
+}
+
+// staticProbe is a probeRunner that always reports the same exit code.
+func staticProbe(code int) probeRunner {
+	return func(context.Context, string, ...string) (int, error) { return code, nil }
+}
+
+func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput {
+	mode := handlers_rds.BootstrapModeAttach
+	if password != nil {
+		mode = handlers_rds.BootstrapModeInitialize
+	}
+	return &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:                 mode,
+		DBInstanceIdentifier: "db-resolved",
+		Engine:               "postgres",
+		MasterUsername:       "master",
+		MasterUserPassword:   password,
+		Port:                 5432,
+		DataVolumeID:         "vol-data-01",
+		DataVolumeSerial:     "voldata01",
+		VMGeneration:         1,
+		PayloadID:            testPayloadID,
+		BootstrapPending:     password != nil,
+	}
+}
+
+const testPayloadID = "bp-0123456789abcdef"
+
+// Writes the completion receipt rds-init leaves behind, so the acknowledgement
+// step finds what a finished initialization would have produced.
+func writeReceipt(t *testing.T, a *Agent, payloadID, dbInstanceIdentifier string) {
+	t.Helper()
+	path := a.receiptPath()
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		t.Fatalf("mkdir receipt dir: %v", err)
+	}
+	body := fmt.Sprintf("%s=%s\n%s=%s\n", receiptPayload, payloadID, receiptDBID, dbInstanceIdentifier)
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatalf("write receipt: %v", err)
+	}
+}
+
+// runAgent runs the agent until its bootstrap has landed, then cancels. It
+// returns once Run has exited so no goroutine outlives the test.
+func runAgent(t *testing.T, a *Agent) error {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool {
+		_, err := os.Stat(filepath.Join(a.cfg.HandoffDir, handoffEnvFile))
+		return err == nil
+	}, "handoff file to be written")
+
+	cancel()
+	select {
+	case err := <-done:
+		// The cancellation above is this helper's own, so Run reporting it is the
+		// expected clean stop rather than a failure, exactly as main treats it.
+		if errors.Is(err, context.Canceled) {
+			return nil
+		}
+		return err
+	case <-time.After(5 * time.Second):
+		t.Fatal("Run did not exit after cancellation")
+		return nil
+	}
+}
+
+// waitFor polls cond until it holds or the test times out.
+func waitFor(t *testing.T, cond func() bool, what string) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatalf("timed out waiting for %s", what)
+}
+
+func TestRun_RegistersThenDeliversBootstrap(t *testing.T) {
+	password := "s3cret"
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(&password)
+
+	cfg := testConfig(t)
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	if err := runAgent(t, a); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	if len(cp.registerReqs) != 1 {
+		t.Fatalf("registered %d times, want 1", len(cp.registerReqs))
+	}
+	// The agent had no configured identifier, so the gateway's resolution is
+	// what every later call must carry.
+	if got := cp.bootstrapReqs[0].DBInstanceIdentifier; got != "db-resolved" {
+		t.Errorf("bootstrap sent identifier %q, want the registered db-resolved", got)
+	}
+
+	env := readFile(t, filepath.Join(cfg.HandoffDir, handoffEnvFile))
+	if !strings.Contains(env, "RDS_MODE='initialize'") || !strings.Contains(env, "RDS_MASTER_PASSWORD='s3cret'") {
+		t.Errorf("handoff env = %q, want initialize mode carrying the password", env)
+	}
+	// The probe follows the port the control plane assigned, not the default.
+	if got := a.probe.port.Load(); got != 5432 {
+		t.Errorf("probe port = %d, want 5432", got)
+	}
+}
+
+// A reboot re-fetches in attach mode. The agent must write that payload as it
+// stands rather than refusing it or inventing a password.
+func TestRun_AttachModeNeedsNoPassword(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	env := readFile(t, filepath.Join(cfg.HandoffDir, handoffEnvFile))
+	if !strings.Contains(env, "RDS_MODE='attach'") {
+		t.Errorf("handoff env = %q, want attach mode", env)
+	}
+	if strings.Contains(env, "RDS_MASTER_PASSWORD") {
+		t.Errorf("handoff env = %q, want no password key at all in attach mode", env)
+	}
+}
+
+// The record may not exist yet when a freshly launched VM first calls, so both
+// boot-critical calls have to ride out failures instead of exiting.
+func TestRun_RetriesRegisterAndBootstrap(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cp.registerErrs = []error{errors.New("DBInstanceNotFound")}
+	cp.bootstrapErrs = []error{errors.New("ServerInternal")}
+
+	cfg := testConfig(t)
+	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	if len(cp.registerReqs) != 2 {
+		t.Errorf("register attempts = %d, want 2 (one failure, one success)", len(cp.registerReqs))
+	}
+	if len(cp.bootstrapReqs) != 2 {
+		t.Errorf("bootstrap attempts = %d, want 2 (one failure, one success)", len(cp.bootstrapReqs))
+	}
+}
+
+// The fetch mutates nothing, so a handoff that cannot be written is retried
+// against the payload already in hand rather than costing a second fetch.
+func TestBootstrap_RetriesHandoffWithoutRefetching(t *testing.T) {
+	cp := newFakeControlPlane()
+	password := "staged-secret"
+	cp.bootstrapOut = bootstrapOutput(&password)
+	cfg := testConfig(t)
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+
+	writes := 0
+	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
+		writes++
+		if got.MasterUserPassword == nil || *got.MasterUserPassword != password {
+			t.Fatalf("handoff attempt %d lost the initialize password", writes)
+		}
+		if writes == 1 {
+			return errors.New("temporary tmpfs write error")
+		}
+		return nil
+	}
+
+	if err := a.bootstrap(t.Context()); err != nil {
+		t.Fatalf("bootstrap: %v", err)
+	}
+	if len(cp.bootstrapReqs) != 1 {
+		t.Errorf("bootstrap fetches = %d, want 1", len(cp.bootstrapReqs))
+	}
+	if writes != 2 {
+		t.Errorf("handoff writes = %d, want 2", writes)
+	}
+	if a.pending == nil || a.pending.payloadID != testPayloadID {
+		t.Errorf("pending bootstrap = %+v, want the staged payload %s", a.pending, testPayloadID)
+	}
+}
+
+// A daemon that predates the encrypted payload answers initialize with nothing
+// to initialise with. Writing that handoff would let rds-init run initdb with an
+// empty master password, so it is retried until an upgraded node answers.
+func TestBootstrap_RetriesInitializeWithNoPassword(t *testing.T) {
+	cp := newFakeControlPlane()
+	empty := ""
+	password := "staged-secret"
+	cp.bootstrapQueue = []*handlers_rds.GetDBBootstrapConfigOutput{{
+		Mode: handlers_rds.BootstrapModeInitialize, DBInstanceIdentifier: "db-resolved",
+		MasterUsername: "master", MasterUserPassword: &empty,
+	}}
+	cp.bootstrapOut = bootstrapOutput(&password)
+	cfg := testConfig(t)
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+
+	var handed []*handlers_rds.GetDBBootstrapConfigOutput
+	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
+		handed = append(handed, got)
+		return nil
+	}
+
+	if err := a.bootstrap(t.Context()); err != nil {
+		t.Fatalf("bootstrap: %v", err)
+	}
+	if len(handed) != 1 {
+		t.Fatalf("handoff writes = %d, want 1 once a usable payload arrived", len(handed))
+	}
+	if handed[0].MasterUserPassword == nil || *handed[0].MasterUserPassword != password {
+		t.Error("the agent wrote a handoff for a fetch that carried no master password")
+	}
+}
+
+func TestAcknowledgeBootstrap_WaitsForTheReceiptThenRetiresIt(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1, dataVolumeID: "vol-data-01"}
+
+	// rds-init runs after the agent registers, so the receipt is absent when the
+	// acknowledgement step first looks and appears some way into the boot.
+	if err := a.checkBootstrapReceipt(a.pending); err == nil {
+		t.Fatal("a missing receipt must not read as a completed bootstrap")
+	}
+	writeReceipt(t, a, testPayloadID, "db-resolved")
+
+	if err := a.acknowledgeBootstrap(t.Context()); err != nil {
+		t.Fatalf("acknowledgeBootstrap: %v", err)
+	}
+	if len(cp.ackRequests()) != 1 {
+		t.Fatalf("acknowledgements = %d, want 1", len(cp.ackRequests()))
+	}
+	if got := cp.ackRequests()[0]; got != *a.pending {
+		t.Errorf("acknowledged %+v, want %+v", got, *a.pending)
+	}
+	if _, err := os.Stat(a.receiptPath()); !os.IsNotExist(err) {
+		t.Error("the receipt must be removed once the record holds the audit trail")
+	}
+}
+
+// The receipt rides along in every snapshot of the data volume, so a restored
+// instance's volume carries the source instance's receipt. Acknowledging on one
+// would destroy the ciphertext this instance still needs.
+func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
+	tests := map[string]struct{ payloadID, dbID string }{
+		"another payload":  {payloadID: "bp-somebody-else", dbID: "db-resolved"},
+		"another instance": {payloadID: testPayloadID, dbID: "db-the-snapshot-source"},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			cp := newFakeControlPlane()
+			cfg := testConfig(t)
+			cfg.DataMount = t.TempDir()
+			a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+			a.id = identity{DBInstanceIdentifier: "db-resolved"}
+			a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+			writeReceipt(t, a, tc.payloadID, tc.dbID)
+
+			ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
+			defer cancel()
+			if err := a.acknowledgeBootstrap(ctx); err == nil {
+				t.Fatal("acknowledgeBootstrap returned nil, want it still waiting on a real receipt")
+			}
+			if n := len(cp.ackRequests()); n != 0 {
+				t.Errorf("acknowledgements = %d, want 0", n)
+			}
+		})
+	}
+}
+
+// An attach boot has nothing staged, so it must not go looking for a receipt the
+// datadir will never grow.
+func TestAcknowledgeBootstrap_IsANoOpOnAttach(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+
+	if err := a.acknowledgeBootstrap(t.Context()); err != nil {
+		t.Fatalf("acknowledgeBootstrap: %v", err)
+	}
+	if n := len(cp.ackRequests()); n != 0 {
+		t.Errorf("acknowledgements = %d, want 0 when nothing was staged", n)
+	}
+}
+
+// A configured identifier is asserted on the wire so a mis-provisioned VM is
+// rejected by the gateway instead of adopting whatever it resolves to.
+func TestRun_SendsConfiguredIdentifier(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.DBInstanceIdentifier = "db-configured"
+	cfg.EngineVersion = "18.1"
+	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	sent := cp.registerReqs[0]
+	if sent.DBInstanceIdentifier != "db-configured" || sent.EngineVersion != "18.1" {
+		t.Errorf("register sent %+v, want the configured identifier and engine version", sent)
+	}
+}
+
+func TestRun_WaitsForDataMountBeforePollingCommands(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cfg := testConfig(t)
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(2)))
+
+	waiting := make(chan struct{})
+	release := make(chan struct{})
+	a.dataMountWaiter = func(ctx context.Context, mountsFile, target string) error {
+		if mountsFile != defaultMountsFile || target != defaultDataMount {
+			t.Errorf("mount waiter got %q/%q, want configured defaults", mountsFile, target)
+		}
+		close(waiting)
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-release:
+			return nil
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	select {
+	case <-waiting:
+	case <-ctx.Done():
+		t.Fatal("agent did not reach the data-mount wait after writing the handoff")
+	}
+	if _, err := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); err != nil {
+		t.Fatalf("handoff was not present before mount wait: %v", err)
+	}
+	select {
+	case <-cp.polled:
+		t.Fatal("command poll started before the data volume was mounted")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	close(release)
+	select {
+	case <-cp.polled:
+	case <-ctx.Done():
+		t.Fatal("command poll did not start after the data mount became ready")
+	}
+	cancel()
+	if err := <-done; err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}
+
+func TestMountTableContains(t *testing.T) {
+	mounts := filepath.Join(t.TempDir(), "mounts")
+	require.NoError(t, os.WriteFile(mounts, []byte(
+		"/dev/vda1 / ext4 rw 0 0\n/dev/vdb /var/lib/postgresql ext4 rw 0 0\n"+
+			"/dev/vdc /path\\040with\\040spaces xfs rw 0 0\nmalformed\n"), 0o600))
+
+	tests := []struct {
+		target string
+		want   bool
+	}{
+		{target: "/var/lib/postgresql", want: true},
+		{target: "/path with spaces", want: true},
+		{target: "/missing", want: false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.target, func(t *testing.T) {
+			got, err := mountTableContains(mounts, tc.target)
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestRun_HeartbeatIncludesBootstrapFailure(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapErrs = []error{errors.New("403 AccessDenied: instance profile is not authorised")}
+	cp.registerOut.HeartbeatIntervalSeconds = 0
+
+	cfg := testConfig(t)
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(2)))
+	a.hb.interval = 10 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool {
+		for _, state := range cp.snapshotStates() {
+			if strings.Contains(state.message, "bootstrap fetch failed") &&
+				strings.Contains(state.message, "403 AccessDenied") {
+				return true
+			}
+		}
+		return false
+	}, "the bootstrap failure to reach a heartbeat")
+	cancel()
+	<-done
+}
+
+func TestRun_HeartbeatsEngineHealth(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	// Declining to set a cadence leaves the agent's own, which the test shortens
+	// rather than waiting out the 30s production interval.
+	cp.registerOut.HeartbeatIntervalSeconds = 0
+
+	cfg := testConfig(t)
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a.hb.interval = 10 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool { return len(cp.snapshotStates()) > 0 }, "a heartbeat")
+	cancel()
+	<-done
+
+	beat := cp.snapshotStates()[0]
+	if beat.health != handlers_rds.EngineHealthHealthy {
+		t.Errorf("heartbeat health = %q, want healthy", beat.health)
+	}
+	if beat.id.DBInstanceIdentifier != "db-resolved" {
+		t.Errorf("heartbeat identifier = %q, want db-resolved", beat.id.DBInstanceIdentifier)
+	}
+}
+
+// The cadence is the control plane's to set: it feeds the staleness window the
+// recovery reconciler judges an instance against, so the guest adopts it rather
+// than keeping its own.
+func TestRegister_AdoptsControlPlaneCadence(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cp.registerOut.HeartbeatIntervalSeconds = 45
+
+	cfg := testConfig(t)
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	if err := runAgent(t, a); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if a.hb.interval != 45*time.Second {
+		t.Errorf("heartbeat interval = %v, want the 45s the control plane returned", a.hb.interval)
+	}
+}
+
+// The agent being up says nothing about the engine. A down engine has to reach
+// the control plane as such, or auto-recovery has nothing to act on.
+func TestRun_HeartbeatReportsDownEngine(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cp.registerOut.HeartbeatIntervalSeconds = 0
+
+	cfg := testConfig(t)
+	probe := newEngineProbe(cfg, staticProbe(2))
+	// The engine has served before, so a silent one now is a failure rather
+	// than a boot still in progress.
+	probe.seenHealthy = true
+
+	a := newAgent(cfg, cp, probe)
+	a.hb.interval = 10 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool { return len(cp.snapshotStates()) > 0 }, "a heartbeat")
+	cancel()
+	<-done
+
+	beat := cp.snapshotStates()[0]
+	if beat.health != handlers_rds.EngineHealthUnhealthy {
+		t.Errorf("heartbeat health = %q, want unhealthy while the agent is up", beat.health)
+	}
+	if beat.message == "" {
+		t.Error("unhealthy heartbeat carried no message explaining the probe result")
+	}
+}
+
+// The mirror the agent decodes poll responses into has to stay aligned with the
+// gateway type that produces them; nothing else in the build ties the two.
+func TestPollDBCommandsOutput_MatchesGatewayShape(t *testing.T) {
+	issued := time.Date(2026, 7, 28, 1, 2, 3, 0, time.UTC)
+	// Rendered exactly as the gateway's typed adapter renders it, so a change to
+	// either side's tags fails here.
+	payload := utils.GenerateIAMXMLPayload("PollDBCommands", &gateway_rds.PollDBCommandsOutput{
+		Commands: []handlers_rds.Command{{
+			CommandID:  "cmd-1",
+			Type:       "reload-parameters",
+			Parameters: []handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}},
+			IssuedAt:   &issued,
+		}},
+	})
+	body, err := utils.MarshalToXML(payload)
+	if err != nil {
+		t.Fatalf("marshal poll result: %v", err)
+	}
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/xml")
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	client, err := rdsgw.New(srv.URL, "", gwsign.NewStatic("AKIATEST", "secret"), "us-east-1", time.Second)
+	if err != nil {
+		t.Fatalf("rdsgw.New: %v", err)
+	}
+
+	var got pollDBCommandsOutput
+	if err := client.Call(context.Background(), "PollDBCommands", nil, &got); err != nil {
+		t.Fatalf("Call: %v", err)
+	}
+	if len(got.Commands) != 1 {
+		t.Fatalf("decoded %d commands, want 1", len(got.Commands))
+	}
+	cmd := got.Commands[0]
+	if cmd.CommandID != "cmd-1" || cmd.Type != "reload-parameters" {
+		t.Errorf("command = %+v, want cmd-1/reload-parameters", cmd)
+	}
+	if len(cmd.Parameters) != 1 || cmd.Parameters[0].Value != "8MB" {
+		t.Errorf("Parameters = %+v, want the seeded member", cmd.Parameters)
+	}
+	if cmd.IssuedAt == nil || !cmd.IssuedAt.Equal(issued) {
+		t.Errorf("IssuedAt = %v, want %v", cmd.IssuedAt, issued)
+	}
+}
+
+func readFile(t *testing.T, path string) string {
+	t.Helper()
+	body, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	return string(body)
+}
+
+// AccessDenied is decided by the control plane's own record, so no retry can
+// change it. Looping on one would leave the command poller and the parameter
+// guard unstarted for the life of a VM whose engine is serving normally.
+func TestAcknowledgeBootstrap_StopsOnATerminalDenial(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.ackErrs = []error{&rdsgw.APIError{
+		Action: "AcknowledgeDBBootstrap", StatusCode: 403,
+		Code:    awserrors.ErrorAccessDenied,
+		Message: "bootstrap payload bp-x at generation 1 does not match the current state",
+	}}
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+	writeReceipt(t, a, testPayloadID, "db-resolved")
+
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+	defer cancel()
+	if err := a.acknowledgeBootstrap(ctx); err != nil {
+		t.Fatalf("acknowledgeBootstrap = %v, want nil so the stateful loops still start", err)
+	}
+	if n := len(cp.ackRequests()); n != 1 {
+		t.Errorf("acknowledgements = %d, want 1 attempt and no retry", n)
+	}
+	if a.hb.bootstrapFailure.Load() == nil {
+		t.Error("a terminal denial must still reach the operator through the heartbeat")
+	}
+	if _, err := os.Stat(a.receiptPath()); err != nil {
+		t.Error("a denied acknowledgement must leave the receipt for a later attempt")
+	}
+}
+
+// rds-init runs after this agent, so the receipt is absent for the whole initdb
+// window. Reporting that would put a failure on every healthy create's beat, and
+// then into the reason a create that timed out for some other cause records.
+func TestAcknowledgeBootstrap_DoesNotReportAnAbsentReceiptAsAFailure(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+
+	ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
+	defer cancel()
+	if err := a.acknowledgeBootstrap(ctx); err == nil {
+		t.Fatal("acknowledgeBootstrap returned nil, want it still waiting for rds-init")
+	}
+	if got := a.hb.bootstrapFailure.Load(); got != nil {
+		t.Errorf("bootstrap failure = %q, want none while the receipt has simply not been written", *got)
+	}
+
+	// A receipt that exists but names another instance is a real mismatch and is
+	// reported, which is what separates the two cases.
+	writeReceipt(t, a, testPayloadID, "db-the-snapshot-source")
+	mismatch, cancelMismatch := context.WithTimeout(t.Context(), 50*time.Millisecond)
+	defer cancelMismatch()
+	if err := a.acknowledgeBootstrap(mismatch); err == nil {
+		t.Fatal("a foreign receipt must not be acknowledged")
+	}
+	if a.hb.bootstrapFailure.Load() == nil {
+		t.Error("a receipt naming another instance must be reported")
+	}
+}

cmd/rds-agent/bootstrap.go

@@ -0,0 +1,219 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The staged payload this boot has to acknowledge once PostgreSQL has applied
+// it. Nil on an attach boot, which makes the acknowledgement step a no-op.
+type pendingBootstrap struct {
+	payloadID    string
+	vmGeneration int64
+	dataVolumeID string
+}
+
+// Fetches the boot material and writes the handoff rds-init is blocked waiting
+// for, then points the health probe at the assigned port.
+func (a *Agent) bootstrap(ctx context.Context) error {
+	var cfg *handlers_rds.GetDBBootstrapConfigOutput
+	if err := retryObserved(ctx, "bootstrap fetch", func(ctx context.Context) error {
+		fetched, err := a.cp.GetBootstrapConfig(ctx, a.id)
+		if err != nil {
+			return err
+		}
+		if fetched == nil {
+			return fmt.Errorf("bootstrap fetch returned no config")
+		}
+		// A control plane that predates the encrypted payload answers initialize
+		// with nothing to initialise with. Retrying re-dials the worker queue
+		// group until an upgraded node answers, so a mixed-version fleet heals
+		// within one boot rather than needing a reboot.
+		if fetched.Mode == handlers_rds.BootstrapModeInitialize &&
+			(fetched.MasterUserPassword == nil || *fetched.MasterUserPassword == "") {
+			return fmt.Errorf("bootstrap fetch returned mode=%s with no master password", fetched.Mode)
+		}
+		cfg = fetched
+		return nil
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap fetch", err)
+	}); err != nil {
+		return err
+	}
+	a.hb.clearBootstrapFailure()
+
+	// The fetch mutates nothing, so a handoff that cannot be written is retried
+	// against the same staged payload rather than against an attach response.
+	if err := retryObserved(ctx, "bootstrap handoff", func(context.Context) error {
+		return a.handoffWriter(a.cfg.HandoffDir, cfg)
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap handoff", err)
+	}); err != nil {
+		return err
+	}
+	a.hb.clearBootstrapFailure()
+	if cfg.Port > 0 {
+		a.probe.setPort(int(cfg.Port))
+		a.engine.setPort(int(cfg.Port))
+	}
+	if cfg.BootstrapPending && cfg.PayloadID != "" {
+		a.pending = &pendingBootstrap{
+			payloadID:    cfg.PayloadID,
+			vmGeneration: cfg.VMGeneration,
+			dataVolumeID: cfg.DataVolumeID,
+		}
+	}
+	// Mode is logged but never branched on: rds-init decides whether to initdb
+	// from the state of the datadir.
+	slog.Info("rds-agent: bootstrap config delivered",
+		"mode", cfg.Mode, "port", cfg.Port, "parameters", len(cfg.Parameters),
+		"bootstrapPending", cfg.BootstrapPending)
+	return nil
+}
+
+// The handoff files rds-init reads. They live on tmpfs, so nothing survives a
+// reboot and the next boot re-fetches rather than reusing a stale password or a
+// re-minted cert.
+const (
+	handoffEnvFile    = "bootstrap.env"
+	handoffParamsFile = "parameters.conf"
+	handoffCertFile   = "server.crt"
+	handoffKeyFile    = "server.key"
+
+	// Root-only: only rds-init reads these, and one holds the master password.
+	handoffMode    = 0o600
+	handoffDirMode = 0o700
+)
+
+// bootstrap.env is written last and renamed into place, so its appearance —
+// which rds-init waits on — means the whole handoff is complete.
+func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) error {
+	if cfg.MasterUsername == "" {
+		return fmt.Errorf("bootstrap config carries no master username")
+	}
+	if err := os.MkdirAll(dir, handoffDirMode); err != nil {
+		return fmt.Errorf("create handoff dir %s: %w", dir, err)
+	}
+	// MkdirAll leaves an existing directory's mode alone, and this one may have
+	// been created more permissively earlier in the boot.
+	if err := os.Chmod(dir, handoffDirMode); err != nil {
+		return fmt.Errorf("secure handoff dir %s: %w", dir, err)
+	}
+
+	if err := writeHandoffFile(dir, handoffParamsFile, renderParameters(cfg.Parameters)); err != nil {
+		return err
+	}
+	// Cert and key are written as a pair; half of one would start the engine
+	// with TLS configured against a key it does not have.
+	if cfg.ServingCertificate != "" && cfg.ServingPrivateKey != "" {
+		if err := writeHandoffFile(dir, handoffCertFile, cfg.ServingCertificate); err != nil {
+			return err
+		}
+		if err := writeHandoffFile(dir, handoffKeyFile, cfg.ServingPrivateKey); err != nil {
+			return err
+		}
+	}
+
+	return writeHandoffFile(dir, handoffEnvFile, renderBootstrapEnv(cfg))
+}
+
+// Every value is single-quoted because rds-init sources this with `.`: an
+// unquoted password holding a space, `$` or `;` would be word-split or executed.
+func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
+	var b strings.Builder
+	b.WriteString("# Written by rds-agent. Regenerated on every boot; edits are lost.\n")
+	writeEnvLine(&b, "RDS_MODE", cfg.Mode)
+	writeEnvLine(&b, "RDS_DB_INSTANCE_IDENTIFIER", cfg.DBInstanceIdentifier)
+	writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
+	writeEnvLine(&b, "RDS_DATA_VOLUME_ID", cfg.DataVolumeID)
+	writeEnvLine(&b, "RDS_DATA_VOLUME_SERIAL", cfg.DataVolumeSerial)
+	writeEnvLine(&b, "RDS_VM_GENERATION", strconv.FormatInt(cfg.VMGeneration, 10))
+	writeEnvLine(&b, "RDS_FORMAT_AUTHORIZED", strconv.FormatBool(cfg.FormatAuthorized))
+	// Neither needs volume access, which is why the assertion is made here: the
+	// agent runs before rds-datadir mounts the volume, so rds-init is the first
+	// component that can compare it against the completion receipt on disk.
+	writeEnvLine(&b, "RDS_BOOTSTRAP_PENDING", boolFlag(cfg.BootstrapPending))
+	writeEnvLine(&b, "RDS_PAYLOAD_ID", cfg.PayloadID)
+	// Present only in initialize mode: an attach fetch has no password to write
+	// and rds-init must not find a stale one.
+	if cfg.MasterUserPassword != nil {
+		writeEnvLine(&b, "RDS_MASTER_PASSWORD", *cfg.MasterUserPassword)
+	}
+	if cfg.DBName != "" {
+		writeEnvLine(&b, "RDS_DB_NAME", cfg.DBName)
+	}
+	if cfg.Port > 0 {
+		writeEnvLine(&b, "RDS_PORT", strconv.FormatInt(cfg.Port, 10))
+	}
+	return b.String()
+}
+
+// rds-init compares against 1 rather than sourcing a Go boolean, so the two
+// halves of the guard cannot drift on spelling.
+func boolFlag(v bool) string {
+	if v {
+		return "1"
+	}
+	return "0"
+}
+
+func writeEnvLine(b *strings.Builder, key, value string) {
+	b.WriteString(key)
+	b.WriteString("=")
+	b.WriteString(shellQuote(value))
+	b.WriteString("\n")
+}
+
+// Single quotes suppress every shell expansion; an embedded quote is closed,
+// escaped and reopened.
+func shellQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+// postgresql.conf syntax. Values are quoted so a setting with a space or unit
+// suffix survives; the engine accepts quoted numerics and booleans too.
+func renderParameters(params []handlers_rds.Parameter) string {
+	var b strings.Builder
+	b.WriteString("# Resolved parameter group, written by rds-agent.\n")
+	for _, p := range params {
+		if p.Name == "" {
+			continue
+		}
+		fmt.Fprintf(&b, "%s = '%s'\n", p.Name, strings.ReplaceAll(p.Value, "'", "''"))
+	}
+	return b.String()
+}
+
+// Atomic: a temp file in the same directory, renamed over the target. The mode
+// is set at creation, so content is never briefly readable at the process umask.
+func writeHandoffFile(dir, name, content string) error {
+	path := filepath.Join(dir, name)
+	tmp := path + ".new"
+
+	f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, handoffMode)
+	if err != nil {
+		return fmt.Errorf("create %s: %w", tmp, err)
+	}
+	// A failed write must not leave the temp file behind holding a password.
+	if _, err := f.WriteString(content); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return fmt.Errorf("write %s: %w", tmp, err)
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("close %s: %w", tmp, err)
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("install %s: %w", path, err)
+	}
+	return nil
+}

cmd/rds-agent/bootstrap_test.go

@@ -0,0 +1,221 @@
+package main
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	password := "s3cret"
+
+	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:               handlers_rds.BootstrapModeInitialize,
+		MasterUsername:     "master",
+		MasterUserPassword: &password,
+		DBName:             "appdb",
+		Port:               5433,
+		DataVolumeID:       "vol-data-01",
+		DataVolumeSerial:   "voldata01",
+		VMGeneration:       4,
+		FormatAuthorized:   true,
+		Parameters: []handlers_rds.Parameter{
+			{Name: "max_connections", Value: "200"},
+			{Name: "shared_buffers", Value: "256MB"},
+		},
+		ServingCertificate: "CERT",
+		ServingPrivateKey:  "KEY",
+	})
+	if err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	env := readFile(t, filepath.Join(dir, handoffEnvFile))
+	for _, want := range []string{
+		"RDS_MODE='initialize'",
+		"RDS_MASTER_USERNAME='master'",
+		"RDS_MASTER_PASSWORD='s3cret'",
+		"RDS_DB_NAME='appdb'",
+		"RDS_PORT='5433'",
+		"RDS_DATA_VOLUME_ID='vol-data-01'",
+		"RDS_DATA_VOLUME_SERIAL='voldata01'",
+		"RDS_VM_GENERATION='4'",
+		"RDS_FORMAT_AUTHORIZED='true'",
+	} {
+		if !strings.Contains(env, want) {
+			t.Errorf("handoff env missing %s:\n%s", want, env)
+		}
+	}
+
+	params := readFile(t, filepath.Join(dir, handoffParamsFile))
+	if !strings.Contains(params, "max_connections = '200'") || !strings.Contains(params, "shared_buffers = '256MB'") {
+		t.Errorf("parameters.conf = %q, want both resolved settings", params)
+	}
+	if got := readFile(t, filepath.Join(dir, handoffCertFile)); got != "CERT" {
+		t.Errorf("server.crt = %q, want CERT", got)
+	}
+	if got := readFile(t, filepath.Join(dir, handoffKeyFile)); got != "KEY" {
+		t.Errorf("server.key = %q, want KEY", got)
+	}
+
+	// The handoff holds a master password and a private key; anything readable
+	// beyond root is a disclosure to every process in the guest.
+	if mode := statMode(t, dir); mode != handoffDirMode {
+		t.Errorf("handoff dir mode = %#o, want %#o", mode, handoffDirMode)
+	}
+	for _, name := range []string{handoffEnvFile, handoffParamsFile, handoffCertFile, handoffKeyFile} {
+		if mode := statMode(t, filepath.Join(dir, name)); mode != handoffMode {
+			t.Errorf("%s mode = %#o, want %#o", name, mode, handoffMode)
+		}
+	}
+}
+
+// Attach mode carries no password, and the key must be absent rather than
+// present and empty — rds-init reads an empty value as "no password" only by
+// accident, and a later boot would inherit it.
+func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:           handlers_rds.BootstrapModeAttach,
+		MasterUsername: "master",
+		Port:           5432,
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	env := readFile(t, filepath.Join(dir, handoffEnvFile))
+	if strings.Contains(env, "RDS_MASTER_PASSWORD") {
+		t.Errorf("handoff env = %q, want no RDS_MASTER_PASSWORD key", env)
+	}
+	for _, want := range []string{
+		"RDS_DATA_VOLUME_ID=''",
+		"RDS_DATA_VOLUME_SERIAL=''",
+		"RDS_VM_GENERATION='0'",
+		"RDS_FORMAT_AUTHORIZED='false'",
+	} {
+		if !strings.Contains(env, want) {
+			t.Errorf("handoff env missing fail-closed field %s: %q", want, env)
+		}
+	}
+}
+
+// The env file is sourced by a shell, so a password holding shell metacharacters
+// must survive as data rather than being split, expanded or executed.
+func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	password := `a b$(id);'"\` + "`whoami`"
+
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:               handlers_rds.BootstrapModeInitialize,
+		MasterUsername:     "master",
+		MasterUserPassword: &password,
+		Port:               5432,
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	env := readFile(t, filepath.Join(dir, handoffEnvFile))
+	line := findLine(t, env, "RDS_MASTER_PASSWORD=")
+	if got := shellSource(t, line, "RDS_MASTER_PASSWORD"); got != password {
+		t.Errorf("sourced password = %q, want %q (line was %s)", got, password, line)
+	}
+}
+
+// A serving cert with no key would have rds-init configure TLS against a key
+// that is not there; the pair is delivered together or not at all.
+func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:               handlers_rds.BootstrapModeAttach,
+		MasterUsername:     "master",
+		Port:               5432,
+		ServingCertificate: "CERT",
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	if _, err := os.Stat(filepath.Join(dir, handoffCertFile)); !os.IsNotExist(err) {
+		t.Errorf("server.crt exists without a key; stat err = %v", err)
+	}
+}
+
+// A config with no master username cannot bootstrap anything, and writing it
+// would only move the failure into rds-init after the boot has committed to it.
+func TestWriteHandoff_RejectsConfigWithNoMasterUser(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode: handlers_rds.BootstrapModeAttach,
+		Port: 5432,
+	})
+	if err == nil {
+		t.Fatal("writeHandoff accepted a config with no master username, want an error")
+	}
+	if _, statErr := os.Stat(filepath.Join(dir, handoffEnvFile)); !os.IsNotExist(statErr) {
+		t.Errorf("handoff env was written despite the rejection; stat err = %v", statErr)
+	}
+}
+
+// A directory left over from an earlier boot at a looser mode must be tightened,
+// not accepted as it is found.
+func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatalf("seed dir: %v", err)
+	}
+
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:           handlers_rds.BootstrapModeAttach,
+		MasterUsername: "master",
+		Port:           5432,
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+	if mode := statMode(t, dir); mode != handoffDirMode {
+		t.Errorf("handoff dir mode = %#o, want it tightened to %#o", mode, handoffDirMode)
+	}
+}
+
+func TestShellQuote_EscapesEmbeddedQuotes(t *testing.T) {
+	for _, in := range []string{"", "plain", "it's", `''`, "new\nline", `$HOME`} {
+		if got := shellSource(t, "V="+shellQuote(in), "V"); got != in {
+			t.Errorf("shellQuote(%q) sourced back as %q", in, got)
+		}
+	}
+}
+
+// shellSource evaluates one KEY=value assignment in a real shell and returns
+// what the variable ends up holding. The quoting is only correct if a shell
+// agrees, so the test asks one instead of re-implementing its rules.
+func shellSource(t *testing.T, line, name string) string {
+	t.Helper()
+	out, err := exec.Command("/bin/sh", "-c", line+"\nprintf %s \"$"+name+"\"\n").Output()
+	if err != nil {
+		t.Fatalf("sourcing %q: %v", line, err)
+	}
+	return string(out)
+}
+
+func statMode(t *testing.T, path string) os.FileMode {
+	t.Helper()
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatalf("stat %s: %v", path, err)
+	}
+	return info.Mode().Perm()
+}
+
+func findLine(t *testing.T, body, prefix string) string {
+	t.Helper()
+	for line := range strings.SplitSeq(body, "\n") {
+		if strings.HasPrefix(line, prefix) {
+			return line
+		}
+	}
+	t.Fatalf("no line starting %q in:\n%s", prefix, body)
+	return ""
+}

cmd/rds-agent/command.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strconv"
+	"strings"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Executes one control-plane directive inside the guest and returns the message
+// reported back to the issuer.
+type commandHandler func(ctx context.Context, cmd handlers_rds.Command) (string, error)
+
+// A registry rather than a dispatch switch, so guest operations — password
+// apply, parameter reload, storage grow, snapshot quiesce — add an entry each.
+type commandRegistry map[string]commandHandler
+
+// An unregistered type is replied to as unsupported, so a control plane ahead
+// of the guest gets a clear answer rather than a timeout.
+func newCommandRegistry(engine engineOps, storage storageOps) commandRegistry {
+	return commandRegistry{
+		handlers_rds.CommandSetPassword: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			params := commandParams(cmd)
+			return "", engine.SetPassword(ctx,
+				params[handlers_rds.CommandParamMasterUsername],
+				params[handlers_rds.CommandParamMasterUserPassword])
+		},
+		handlers_rds.CommandApplyParams: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			pending, err := engine.ApplyParameters(ctx, cmd.Parameters)
+			if err != nil {
+				return "", err
+			}
+			// The reply carries no structured payload, so the settings still
+			// awaiting a restart come back as the message itself.
+			return strings.Join(pending, ","), nil
+		},
+		handlers_rds.CommandStopEngine: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			return "", engine.Stop(ctx)
+		},
+		handlers_rds.CommandGrowFilesystem: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			return storage.GrowFilesystem(ctx)
+		},
+		handlers_rds.CommandQuiesce: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			params := commandParams(cmd)
+			hold, err := quiesceHoldFrom(params[handlers_rds.CommandParamQuiesceDeadlineSeconds])
+			if err != nil {
+				return "", err
+			}
+			return "", engine.Quiesce(ctx, params[handlers_rds.CommandParamQuiesceLabel], hold)
+		},
+		handlers_rds.CommandUnquiesce: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			return "", engine.Unquiesce(ctx)
+		},
+	}
+}
+
+// The control plane's hold deadline. A missing or unreadable one is refused
+// rather than defaulted: a hold the guest chose the length of would not be the
+// bound the control plane is relying on.
+func quiesceHoldFrom(value string) (time.Duration, error) {
+	seconds, err := strconv.Atoi(strings.TrimSpace(value))
+	if err != nil || seconds <= 0 {
+		return 0, fmt.Errorf("%s must be a positive number of seconds, got %q",
+			handlers_rds.CommandParamQuiesceDeadlineSeconds, value)
+	}
+	return time.Duration(seconds) * time.Second, nil
+}
+
+// A later value wins, so a malformed command carrying a parameter twice cannot
+// leave the handler acting on the first of two conflicting values.
+func commandParams(cmd handlers_rds.Command) map[string]string {
+	params := make(map[string]string, len(cmd.Parameters))
+	for _, p := range cmd.Parameters {
+		params[p.Name] = p.Value
+	}
+	return params
+}
+
+// Spaces retries after a failed poll, so a broken channel is not re-polled at
+// line rate for as long as it stays broken.
+const pollErrorBackoff = 5 * time.Second
+
+// The agent's command channel: a long poll that carries back the replies for
+// whatever the previous poll delivered.
+type commander struct {
+	cp       controlPlane
+	id       identity
+	registry commandRegistry
+	wait     time.Duration
+	// Cleared only once a poll has carried them, so a failed poll re-delivers
+	// rather than dropping the result of work the guest actually did.
+	pending []handlers_rds.CommandReply
+}
+
+func newCommander(cp controlPlane, registry commandRegistry, wait time.Duration) *commander {
+	if wait <= 0 {
+		wait = defaultPollWait
+	}
+	return &commander{cp: cp, registry: registry, wait: wait}
+}
+
+func (c *commander) Run(ctx context.Context) {
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+		commands, err := c.cp.PollCommands(ctx, c.id, c.pending, c.wait)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			slog.Warn("rds-agent: command poll failed", "err", err)
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(pollErrorBackoff):
+			}
+			continue
+		}
+		c.pending = nil
+
+		for _, cmd := range commands {
+			c.pending = append(c.pending, c.execute(ctx, cmd))
+		}
+	}
+}
+
+// Every command gets a reply: the issuer is blocked on this command ID, so a
+// silent drop costs it a full timeout.
+func (c *commander) execute(ctx context.Context, cmd handlers_rds.Command) handlers_rds.CommandReply {
+	handler, ok := c.registry[cmd.Type]
+	if !ok {
+		slog.Warn("rds-agent: unsupported command", "commandID", cmd.CommandID, "type", cmd.Type)
+		return failedReply(cmd, fmt.Sprintf("unsupported command type %q", cmd.Type))
+	}
+
+	slog.Info("rds-agent: executing command", "commandID", cmd.CommandID, "type", cmd.Type)
+	message, err := handler(ctx, cmd)
+	if err != nil {
+		slog.Error("rds-agent: command failed", "commandID", cmd.CommandID, "type", cmd.Type, "err", err)
+		return failedReply(cmd, err.Error())
+	}
+	return handlers_rds.CommandReply{
+		CommandID: cmd.CommandID,
+		Status:    handlers_rds.CommandStatusSucceeded,
+		Message:   message,
+	}
+}
+
+func failedReply(cmd handlers_rds.Command, message string) handlers_rds.CommandReply {
+	return handlers_rds.CommandReply{
+		CommandID: cmd.CommandID,
+		Status:    handlers_rds.CommandStatusFailed,
+		Message:   message,
+	}
+}

cmd/rds-agent/command_test.go

@@ -0,0 +1,173 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCommander_DispatchesByNameAndReplies(t *testing.T) {
+	var ran handlers_rds.Command
+	registry := commandRegistry{
+		"reload-parameters": func(_ context.Context, cmd handlers_rds.Command) (string, error) {
+			ran = cmd
+			return "reloaded", nil
+		},
+	}
+
+	reply := newCommander(nil, registry, time.Second).execute(context.Background(), handlers_rds.Command{
+		CommandID:  "cmd-1",
+		Type:       "reload-parameters",
+		Parameters: []handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}},
+	})
+
+	if ran.CommandID != "cmd-1" || len(ran.Parameters) != 1 {
+		t.Errorf("handler received %+v, want the command with its parameters", ran)
+	}
+	if reply.Status != handlers_rds.CommandStatusSucceeded || reply.Message != "reloaded" {
+		t.Errorf("reply = %+v, want succeeded/reloaded", reply)
+	}
+	if reply.CommandID != "cmd-1" {
+		t.Errorf("reply command ID = %q, want cmd-1", reply.CommandID)
+	}
+}
+
+func TestCommander_FailedHandlerRepliesWithItsError(t *testing.T) {
+	registry := commandRegistry{
+		"grow-storage": func(context.Context, handlers_rds.Command) (string, error) {
+			return "", errors.New("no space left on device")
+		},
+	}
+
+	reply := newCommander(nil, registry, time.Second).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-2", Type: "grow-storage"})
+
+	if reply.Status != handlers_rds.CommandStatusFailed {
+		t.Errorf("reply status = %q, want failed", reply.Status)
+	}
+	if reply.Message != "no space left on device" {
+		t.Errorf("reply message = %q, want the handler's error", reply.Message)
+	}
+}
+
+// A control plane ahead of the guest issues types this build has no handler
+// for. The issuer is blocked on the command ID, so it gets an answer rather
+// than a timeout.
+func TestCommander_UnknownTypeRepliesFailedNotDropped(t *testing.T) {
+	reply := newCommander(nil, newCommandRegistry(&fakeEngine{}, &fakeStorage{}), time.Second).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-3", Type: "quiesce"})
+
+	if reply.CommandID != "cmd-3" || reply.Status != handlers_rds.CommandStatusFailed {
+		t.Errorf("reply = %+v, want a failed reply for cmd-3", reply)
+	}
+	if reply.Message == "" {
+		t.Error("unsupported-command reply carried no message")
+	}
+}
+
+// The reply for work the guest actually did rides the next poll. A poll that
+// fails must not consume it, or the issuer never learns the outcome.
+func TestCommander_RunCarriesRepliesOnTheNextPoll(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollQueue = [][]handlers_rds.Command{
+		{{CommandID: "cmd-1", Type: "reload-parameters"}},
+	}
+	registry := commandRegistry{
+		"reload-parameters": func(context.Context, handlers_rds.Command) (string, error) { return "done", nil },
+	}
+
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	waitFor(t, func() bool {
+		replies := cp.snapshotReplies()
+		return len(replies) >= 2 && len(replies[1]) == 1
+	}, "the reply to ride the second poll")
+	cancel()
+	<-done
+
+	replies := cp.snapshotReplies()
+	if len(replies[0]) != 0 {
+		t.Errorf("first poll carried replies %+v, want none", replies[0])
+	}
+	if got := replies[1][0]; got.CommandID != "cmd-1" || got.Status != handlers_rds.CommandStatusSucceeded {
+		t.Errorf("second poll carried %+v, want the succeeded reply for cmd-1", got)
+	}
+}
+
+func TestCommander_RetainsAndRetriesTheCompletePendingBatch(t *testing.T) {
+	cp := newFakeControlPlane()
+	pending := []handlers_rds.CommandReply{
+		{CommandID: "cmd-1", Status: handlers_rds.CommandStatusSucceeded},
+		{CommandID: "cmd-2", Status: handlers_rds.CommandStatusFailed, Message: "disk full"},
+	}
+	c := newCommander(cp, commandRegistry{
+		"reload-parameters": func(context.Context, handlers_rds.Command) (string, error) {
+			return "reloaded", nil
+		},
+	}, 10*time.Millisecond)
+	c.pending = append([]handlers_rds.CommandReply(nil), pending...)
+	cp.pollErr = errors.New("reply publication failed")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+	<-cp.polled
+	cancel()
+	<-done
+
+	assert.Equal(t, pending, c.pending)
+	replies := cp.snapshotReplies()
+	require.Len(t, replies, 1)
+	assert.Equal(t, pending, replies[0])
+
+	cp.mu.Lock()
+	cp.pollErr = nil
+	cp.pollQueue = [][]handlers_rds.Command{{{
+		CommandID: "cmd-3",
+		Type:      "reload-parameters",
+	}}}
+	cp.mu.Unlock()
+
+	ctx, cancel = context.WithCancel(context.Background())
+	done = make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+	waitFor(t, func() bool {
+		return len(cp.snapshotReplies()) >= 3
+	}, "the retained batch to succeed and the new reply to reach the following poll")
+	cancel()
+	<-done
+
+	replies = cp.snapshotReplies()
+	assert.Equal(t, pending, replies[1], "the complete failed batch must be retried in order")
+	require.Len(t, replies[2], 1)
+	assert.Equal(t, "cmd-3", replies[2][0].CommandID)
+	assert.Equal(t, handlers_rds.CommandStatusSucceeded, replies[2][0].Status)
+}
+
+// A poll error must not spin the loop: without a backoff a gateway that is down
+// would be re-polled at line rate for as long as it stayed down.
+func TestCommander_BacksOffAfterAFailedPoll(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollErr = errors.New("gateway unreachable")
+
+	c := newCommander(cp, newCommandRegistry(&fakeEngine{}, &fakeStorage{}), 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+	<-done
+
+	// pollErrorBackoff is 5s, so the window admits the first poll and no more.
+	if polls := len(cp.snapshotReplies()); polls > 1 {
+		t.Errorf("polled %d times in 200ms against a broken channel, want 1", polls)
+	}
+}

cmd/rds-agent/config.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/guestenv"
+)
+
+const (
+	defaultEnvFile    = "/etc/spinifex-rds/agent.env"
+	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
+	defaultHandoffDir = "/run/spinifex-rds"
+	defaultEngineHost = "127.0.0.1"
+	defaultEnginePort = 5432
+	defaultPGIsReady  = "pg_isready"
+
+	// The guest layout rds-init lays down. Overridable for the same reason it is
+	// there: a sibling engine preset points at its own paths.
+	defaultPGBin         = "/usr/libexec/postgresql18"
+	defaultPGData        = "/var/lib/postgresql/18/data"
+	defaultSocketDir     = "/run/postgresql"
+	defaultPGUser        = "postgres"
+	defaultRCService     = "rc-service"
+	defaultEngineSvcName = "postgresql"
+	// The long-poll window the agent asks the gateway to hold a request open
+	// for. The gateway caps it at 20s.
+	defaultPollWait = 20 * time.Second
+)
+
+// Static settings delivered per-instance by cloud-init. It carries no secrets:
+// IMDS is readable by anything in the guest, so the master password only
+// arrives via GetDBBootstrapConfig.
+type config struct {
+	GatewayURL string
+	GatewayCA  string
+	Region     string
+	// Optional — the gateway resolves the instance from the caller's
+	// credentials. When set it is sent, so a mis-provisioned VM is rejected.
+	DBInstanceIdentifier string
+	// What this image actually ships. Empty leaves the control plane's recorded
+	// version alone rather than clearing it.
+	EngineVersion string
+	HandoffDir    string
+	EngineHost    string
+	EnginePort    int
+	// Overridable so a test or a sibling engine preset can point at its own.
+	PGIsReady string
+	PollWait  time.Duration
+
+	// Where the command handlers reach the engine: the client binaries, the
+	// datadir the parameter file is installed into, the socket they connect
+	// over, the OS user they drop to, and the service the stop goes through.
+	PGBin         string
+	PGData        string
+	SocketDir     string
+	PGUser        string
+	RCService     string
+	EngineService string
+
+	// Where the data volume is mounted, and the two kernel surfaces a storage
+	// grow resolves its device from. Overridable so a test can point them at
+	// fixtures rather than at the running host's.
+	DataMount  string
+	MountsFile string
+	SysBlock   string
+}
+
+// Reads the cloud-init env file, then lets real env vars override.
+func loadConfig(envFile string) config {
+	get := guestenv.Load(envFile).Get
+
+	cfg := config{
+		GatewayURL:           get("RDS_GATEWAY_URL"),
+		GatewayCA:            get("RDS_GATEWAY_CA"),
+		Region:               get("RDS_REGION"),
+		DBInstanceIdentifier: get("RDS_DB_INSTANCE_IDENTIFIER"),
+		EngineVersion:        get("RDS_ENGINE_VERSION"),
+		HandoffDir:           get("RDS_HANDOFF_DIR"),
+		EngineHost:           get("RDS_ENGINE_HOST"),
+		PGIsReady:            get("RDS_PG_ISREADY"),
+		PGBin:                get("RDS_PG_BIN"),
+		PGData:               get("RDS_PGDATA"),
+		SocketDir:            get("RDS_SOCKET_DIR"),
+		PGUser:               get("RDS_PG_USER"),
+		RCService:            get("RDS_RC_SERVICE"),
+		EngineService:        get("RDS_ENGINE_SERVICE"),
+		DataMount:            get("RDS_DATA_MOUNT"),
+		EnginePort:           defaultEnginePort,
+		PollWait:             defaultPollWait,
+	}
+	if cfg.GatewayCA == "" {
+		cfg.GatewayCA = defaultGatewayCA
+	}
+	if cfg.HandoffDir == "" {
+		cfg.HandoffDir = defaultHandoffDir
+	}
+	if cfg.EngineHost == "" {
+		cfg.EngineHost = defaultEngineHost
+	}
+	if cfg.PGIsReady == "" {
+		cfg.PGIsReady = defaultPGIsReady
+	}
+	if cfg.PGBin == "" {
+		cfg.PGBin = defaultPGBin
+	}
+	if cfg.PGData == "" {
+		cfg.PGData = defaultPGData
+	}
+	if cfg.SocketDir == "" {
+		cfg.SocketDir = defaultSocketDir
+	}
+	if cfg.PGUser == "" {
+		cfg.PGUser = defaultPGUser
+	}
+	if cfg.RCService == "" {
+		cfg.RCService = defaultRCService
+	}
+	if cfg.EngineService == "" {
+		cfg.EngineService = defaultEngineSvcName
+	}
+	// rds-datadir reads the same RDS_DATA_MOUNT, so the two agree on where the
+	// volume landed without either asserting it to the other.
+	if cfg.DataMount == "" {
+		cfg.DataMount = defaultDataMount
+	}
+	cfg.MountsFile = defaultMountsFile
+	cfg.SysBlock = defaultSysBlock
+	// The authoritative port comes from the bootstrap config; this is only what
+	// the probe uses until that first fetch lands.
+	if v := get("RDS_ENGINE_PORT"); v != "" {
+		if p, err := strconv.Atoi(v); err == nil && p > 0 {
+			cfg.EnginePort = p
+		}
+	}
+	if v := get("RDS_POLL_WAIT"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.PollWait = d
+		}
+	}
+	return cfg
+}

cmd/rds-agent/config_test.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestLoadConfig_Defaults(t *testing.T) {
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+
+	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
+		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
+	}
+	if cfg.EngineHost != defaultEngineHost || cfg.EnginePort != defaultEnginePort {
+		t.Errorf("engine endpoint = %s:%d, want the local default", cfg.EngineHost, cfg.EnginePort)
+	}
+	if cfg.PGIsReady != defaultPGIsReady || cfg.PollWait != defaultPollWait {
+		t.Errorf("probe/pollWait = %q/%v, want the built-in defaults", cfg.PGIsReady, cfg.PollWait)
+	}
+	// Identity is optional: the gateway resolves it from the caller's
+	// credentials, so an agent with no configured identifier is normal.
+	if cfg.DBInstanceIdentifier != "" {
+		t.Errorf("DBInstanceIdentifier = %q, want empty when unconfigured", cfg.DBInstanceIdentifier)
+	}
+}
+
+func TestLoadConfig_ReadsEnvFile(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "agent.env")
+	body := "RDS_GATEWAY_URL=https://gw.internal:9999\n" +
+		"RDS_REGION=ap-southeast-2\n" +
+		"RDS_DB_INSTANCE_IDENTIFIER=db-1\n" +
+		"RDS_ENGINE_VERSION=18.1\n" +
+		"RDS_ENGINE_PORT=6543\n" +
+		"RDS_POLL_WAIT=5s\n"
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatalf("write env file: %v", err)
+	}
+
+	cfg := loadConfig(path)
+	if cfg.GatewayURL != "https://gw.internal:9999" || cfg.Region != "ap-southeast-2" {
+		t.Errorf("gateway/region = %q/%q, want the delivered values", cfg.GatewayURL, cfg.Region)
+	}
+	if cfg.DBInstanceIdentifier != "db-1" || cfg.EngineVersion != "18.1" {
+		t.Errorf("identity = %q/%q, want db-1/18.1", cfg.DBInstanceIdentifier, cfg.EngineVersion)
+	}
+	if cfg.EnginePort != 6543 || cfg.PollWait != 5*time.Second {
+		t.Errorf("port/pollWait = %d/%v, want 6543/5s", cfg.EnginePort, cfg.PollWait)
+	}
+}
+
+// An unparseable override falls back to the default rather than leaving the
+// agent with a zero port or a zero-length long poll.
+func TestLoadConfig_IgnoresUnusableOverrides(t *testing.T) {
+	t.Setenv("RDS_ENGINE_PORT", "not-a-port")
+	t.Setenv("RDS_POLL_WAIT", "forever")
+
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	if cfg.EnginePort != defaultEnginePort || cfg.PollWait != defaultPollWait {
+		t.Errorf("port/pollWait = %d/%v, want the defaults", cfg.EnginePort, cfg.PollWait)
+	}
+}

cmd/rds-agent/engine.go

@@ -0,0 +1,532 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The in-guest half of the engine seam. Everything PostgreSQL-shaped that a
+// control-plane command needs lives behind this, so the command registry above
+// stays engine-agnostic.
+type engineOps interface {
+	// Rotates the master role's password live. Never persisted anywhere in the
+	// guest: it exists only for the length of this call.
+	SetPassword(ctx context.Context, username, password string) error
+	// Installs the resolved parameter set and reloads. Returns the settings the
+	// engine accepted but will not honour until it restarts.
+	ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error)
+	// Shuts the engine down cleanly, so the data volume is checkpointed before
+	// the VM is stopped or snapshotted.
+	Stop(ctx context.Context) error
+	// Puts the engine into backup mode so the datadir can be snapshotted at a
+	// checkpoint. Releases itself after hold, whatever happens to the caller.
+	Quiesce(ctx context.Context, label string, hold time.Duration) error
+	// Takes the engine back out of backup mode. Fails when no hold is active,
+	// which is how a hold that expired mid-snapshot becomes visible.
+	Unquiesce(ctx context.Context) error
+}
+
+// The rollback half of the parameter path, kept off engineOps because it is not
+// a control-plane directive: nothing issues it, the agent runs it when the
+// engine fails to come up after a parameter change.
+type parameterRecovery interface {
+	// Reports whether the installed set differed and was replaced.
+	RestoreLastKnownGoodParameters(ctx context.Context) (bool, error)
+	Restart(ctx context.Context) error
+}
+
+// Records the parameter file only after the probe observes the postmaster
+// serving it. This is separate from apply, where static values are installed
+// but not adopted until a later restart.
+type servingParameterRecorder interface {
+	RecordServingParameters(ctx context.Context) error
+}
+
+// One child process. Env replaces the agent's own environment rather than
+// extending it, so a secret placed here reaches only the process that needs it.
+type command struct {
+	Name  string
+	Args  []string
+	Env   []string
+	Stdin string
+	// The OS user to drop to before exec. Empty runs as the agent itself.
+	User string
+}
+
+// Returns stdout. stderr is folded into the error, because psql reports the
+// actual SQL failure there while exiting with a bare status.
+type commandRunner func(ctx context.Context, c command) (string, error)
+
+func execCommandRunner(ctx context.Context, c command) (string, error) {
+	cmd := exec.CommandContext(ctx, c.Name, c.Args...)
+	cmd.Env = c.Env
+	if c.Stdin != "" {
+		cmd.Stdin = strings.NewReader(c.Stdin)
+	}
+	if c.User != "" {
+		credential, err := lookupCredential(c.User)
+		if err != nil {
+			return "", err
+		}
+		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: credential}
+	}
+
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout, cmd.Stderr = &stdout, &stderr
+	if err := cmd.Run(); err != nil {
+		message := strings.TrimSpace(stderr.String())
+		if message == "" {
+			message = err.Error()
+		}
+		return stdout.String(), fmt.Errorf("%s: %s", filepath.Base(c.Name), message)
+	}
+	return stdout.String(), nil
+}
+
+func lookupCredential(username string) (*syscall.Credential, error) {
+	u, err := user.Lookup(username)
+	if err != nil {
+		return nil, fmt.Errorf("resolve user %s: %w", username, err)
+	}
+	uid, err := strconv.ParseUint(u.Uid, 10, 32)
+	if err != nil {
+		return nil, fmt.Errorf("parse uid of %s: %w", username, err)
+	}
+	gid, err := strconv.ParseUint(u.Gid, 10, 32)
+	if err != nil {
+		return nil, fmt.Errorf("parse gid of %s: %w", username, err)
+	}
+	return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil
+}
+
+// The engine is reached over its unix socket under peer authentication, so the
+// agent drops to the postgres OS user rather than holding a password of its own.
+type postgresEngine struct {
+	run       commandRunner
+	startSess sessionRunner
+	psql      string
+	rcService string
+	service   string
+	pgData    string
+	socketDir string
+	osUser    string
+	probe     *engineProbe
+	// Set from the bootstrap config, on a different goroutine than the commands
+	// that read it.
+	port atomic.Int64
+
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
+
+	// The backup mode currently held, if any. Guarded because the expiry timer
+	// releases it on its own goroutine.
+	mu   sync.Mutex
+	held *quiesceHold
+}
+
+var (
+	_ engineOps                = (*postgresEngine)(nil)
+	_ parameterRecovery        = (*postgresEngine)(nil)
+	_ servingParameterRecorder = (*postgresEngine)(nil)
+)
+
+const (
+	parameterRepairTimeout = 90 * time.Second
+	parameterRepairPoll    = time.Second
+)
+
+func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
+	e := &postgresEngine{
+		run:           run,
+		startSess:     startSess,
+		psql:          filepath.Join(cfg.PGBin, "psql"),
+		rcService:     cfg.RCService,
+		service:       cfg.EngineService,
+		pgData:        cfg.PGData,
+		socketDir:     cfg.SocketDir,
+		osUser:        cfg.PGUser,
+		probe:         probe,
+		repairTimeout: parameterRepairTimeout,
+		repairPoll:    parameterRepairPoll,
+	}
+	e.port.Store(int64(cfg.EnginePort))
+	return e
+}
+
+func (e *postgresEngine) setPort(port int) {
+	e.port.Store(int64(port))
+}
+
+// The role name and the password ride the environment and are re-quoted by
+// psql, so neither reaches a shell word or an argv another process can read.
+func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// The statement below runs as the cluster superuser over the socket under
+	// peer auth, so a reserved name in the command payload would hand the
+	// customer the bootstrap superuser rather than rotate their own role.
+	if err := handlers_rds.EnginePostgres().ValidateUsernameNotReserved(username); err != nil {
+		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
+	}
+	// psql interpolates the password into the ALTER ROLE before the server sees
+	// it, and these three are what would write it to the log — the last on any
+	// failure at its own default. SUSET, so the parameter group cannot win.
+	const sql = `SET log_statement = 'none';
+SET log_min_duration_statement = -1;
+SET log_min_error_statement = 'panic';
+\getenv master RDS_MASTER_USERNAME
+\getenv password RDS_MASTER_PASSWORD
+ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
+`
+	_, err := e.psqlRun(ctx, sql,
+		"RDS_MASTER_USERNAME="+username,
+		"RDS_MASTER_PASSWORD="+password,
+	)
+	if err != nil {
+		// psql echoes the failing statement, which here would carry the new
+		// password back to the control plane and into the event ring.
+		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
+	}
+	return nil
+}
+
+// The include the resolved parameter set is rendered to, and the copy of the
+// last one the engine accepted. Both live beside the data rather than in /etc,
+// matching rds-init: a class change boots a fresh root volume, which would
+// otherwise revert them when a fresh root volume is used.
+const (
+	parametersFileName = "10-rds-parameters.conf"
+	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
+	// copy must not be read as a second set of settings.
+	lastGoodFileName = "10-rds-parameters.last-good"
+)
+
+// Installs the resolved set, validates it with the engine's own config parser
+// before the engine ever adopts it, and reloads. A value the engine refuses is
+// rolled back here rather than left on the data volume, where it would survive
+// every VM replace and turn the next restart into a boot loop.
+func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	// The check has to run against the file in place, because the engine parses
+	// the datadir's own include_dir. The window that leaves is closed from both
+	// ends: the rollback below, and the last-known-good restore the agent runs at
+	// boot when the engine does not come up.
+	_, restore, err := e.installParameters(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := e.checkConfig(ctx); err != nil {
+		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restore())
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
+	}
+
+	if _, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n"); err != nil {
+		// The engine may have gone down after the first probe. In that case start
+		// it on the new, parser-checked repair set rather than restoring the set
+		// that already left it unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(fmt.Errorf("reload the engine configuration: %w", err), restore())
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+func (e *postgresEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	repairCtx, cancel := context.WithTimeout(ctx, e.repairTimeout)
+	defer cancel()
+
+	if err := e.Restart(repairCtx); err != nil {
+		return nil, fmt.Errorf("start the engine on the repaired parameter set: %w", err)
+	}
+
+	timer := time.NewTimer(0)
+	defer timer.Stop()
+	lastMessage := "engine did not respond"
+	for {
+		select {
+		case <-repairCtx.Done():
+			if ctx.Err() != nil {
+				return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %w", ctx.Err())
+			}
+			return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %s: %w", lastMessage, repairCtx.Err())
+		case <-timer.C:
+		}
+
+		state, message := e.probe.state(repairCtx)
+		if state == engineServing {
+			pending, err := e.pendingRestartParameters(repairCtx)
+			if err == nil {
+				return pending, nil
+			}
+			lastMessage = err.Error()
+		} else if message != "" {
+			lastMessage = message
+		}
+		timer.Reset(e.repairPoll)
+	}
+}
+
+func (e *postgresEngine) pendingRestartParameters(ctx context.Context) ([]string, error) {
+	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
+	if err != nil {
+		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
+	}
+	var pending []string
+	for line := range strings.SplitSeq(out, "\n") {
+		if name := strings.TrimSpace(line); name != "" {
+			pending = append(pending, name)
+		}
+	}
+	return pending, nil
+}
+
+func (e *postgresEngine) parametersPath() string {
+	return filepath.Join(e.pgData, "conf.d", parametersFileName)
+}
+
+func (e *postgresEngine) lastGoodPath() string {
+	return filepath.Join(e.pgData, "conf.d", lastGoodFileName)
+}
+
+// Written as root and handed to the engine's user, at the same path and mode
+// rds-init installs it at, so a later boot overwrites rather than shadows it.
+// Returns the content that was there and a func putting it back, so the caller
+// can undo the install without re-rendering anything.
+func (e *postgresEngine) installParameters(params []handlers_rds.Parameter) (previous []byte, restore func() error, err error) {
+	path := e.parametersPath()
+	// A missing file is the first-ever apply, and its rollback is a removal.
+	previous, readErr := os.ReadFile(path)
+	if readErr != nil && !os.IsNotExist(readErr) {
+		return nil, nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
+	}
+	existed := readErr == nil
+
+	if err := e.writeEngineFile(path, []byte(renderParameters(params))); err != nil {
+		return nil, nil, err
+	}
+	return previous, func() error {
+		if !existed {
+			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
+			}
+			return nil
+		}
+		return e.writeEngineFile(path, previous)
+	}, nil
+}
+
+// Snapshots the include only when the running postmaster has no settings still
+// pending a restart. The parameter mutex keeps an apply from replacing the file
+// between that check and the copy.
+func (e *postgresEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *postgresEngine) recordServingParametersLocked(ctx context.Context) error {
+	if _, err := os.Stat(e.parametersPath()); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("inspect the serving parameters: %w", err)
+	}
+	pending, err := e.pendingRestartParameters(ctx)
+	if err != nil {
+		return err
+	}
+	if len(pending) > 0 {
+		return nil
+	}
+
+	content, err := os.ReadFile(e.parametersPath())
+	if err != nil {
+		return fmt.Errorf("read the serving parameters: %w", err)
+	}
+	lastGood, err := os.ReadFile(e.lastGoodPath())
+	if err == nil && bytes.Equal(content, lastGood) {
+		return nil
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	return e.writeEngineFile(e.lastGoodPath(), content)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change. The probe is checked again under the parameter lock
+// so a repair that just brought the engine back cannot be reversed.
+func (e *postgresEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	if state, _ := e.probe.state(ctx); state != engineAbsent {
+		return false, nil
+	}
+	lastGood, err := os.ReadFile(e.lastGoodPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	current, err := os.ReadFile(e.parametersPath())
+	if err != nil && !os.IsNotExist(err) {
+		return false, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	if bytes.Equal(current, lastGood) {
+		return false, nil
+	}
+	if err := e.writeEngineFile(e.parametersPath(), lastGood); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// Atomic: a temp file in the same directory, renamed over the target. The temp
+// name deliberately does not end in .conf, so a crash between write and rename
+// cannot leave the engine reading a half-written include.
+func (e *postgresEngine) writeEngineFile(path string, content []byte) error {
+	tmp := path + ".new"
+	if err := os.WriteFile(tmp, content, 0o600); err != nil {
+		return fmt.Errorf("write %s: %w", tmp, err)
+	}
+	if err := e.chownToEngine(tmp); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("install %s: %w", path, err)
+	}
+	return nil
+}
+
+// The engine's own parser, run offline against the datadir. Reading one setting
+// back is enough: postgres parses postgresql.conf and every include first, and
+// exits non-zero naming the file and line of an unknown parameter or a value
+// outside its range.
+func (e *postgresEngine) checkConfig(ctx context.Context) error {
+	if _, err := e.run(ctx, command{
+		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
+		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
+		Env:  []string{"PATH=" + defaultGuestPath},
+		User: e.osUser,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+// The engine reads its own config, so a root-owned file would be unreadable to
+// it. A guest with no such user is a broken image, not something to work around.
+func (e *postgresEngine) chownToEngine(path string) error {
+	credential, err := lookupCredential(e.osUser)
+	if err != nil {
+		return err
+	}
+	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
+		return fmt.Errorf("hand %s to %s: %w", path, e.osUser, err)
+	}
+	return nil
+}
+
+// Through the service manager rather than pg_ctl, so the supervisor records the
+// engine as stopped and does not restart it underneath a VM that is going down.
+func (e *postgresEngine) Stop(ctx context.Context) error {
+	return e.serviceAction(ctx, "stop")
+}
+
+// Only the parameter rollback calls this: a restart the control plane wants goes
+// through RebootDBInstance, which cycles the VM.
+func (e *postgresEngine) Restart(ctx context.Context) error {
+	return e.serviceAction(ctx, "restart")
+}
+
+func (e *postgresEngine) serviceAction(ctx context.Context, action string) error {
+	if _, err := e.run(ctx, command{
+		Name: e.rcService,
+		Args: []string{e.service, action},
+		Env:  []string{"PATH=" + defaultGuestPath},
+	}); err != nil {
+		return fmt.Errorf("%s the %s service: %w", action, e.service, err)
+	}
+	return nil
+}
+
+// A minimal environment for a child that is not carrying secrets.
+const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+// Feeds sql on stdin rather than as an argument, so a statement is never
+// visible in the process table.
+func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.psql,
+		Args:  e.psqlArgs(),
+		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
+		Stdin: sql,
+		User:  e.osUser,
+	})
+}
+
+// Reading the script from stdin is what lets one invocation serve both a
+// one-shot run and a session held open across several statements.
+func (e *postgresEngine) psqlArgs() []string {
+	return []string{
+		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
+		"-v", "ON_ERROR_STOP=1",
+		"-h", e.socketDir,
+		"-p", strconv.FormatInt(e.port.Load(), 10),
+		"-U", e.osUser,
+		"-d", "postgres",
+		"-f", "-",
+	}
+}
+
+func redact(s, secret string) string {
+	if secret == "" {
+		return s
+	}
+	return strings.ReplaceAll(s, secret, "[REDACTED]")
+}

cmd/rds-agent/engine_test.go

@@ -0,0 +1,423 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"os/user"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Stands in for the engine in the registry tests, where what matters is that a
+// command reaches the right method with the right arguments.
+type fakeEngine struct {
+	username string
+	password string
+	params   []handlers_rds.Parameter
+	pending  []string
+	stopped  bool
+	label    string
+	hold     time.Duration
+	released bool
+	err      error
+}
+
+var _ engineOps = (*fakeEngine)(nil)
+
+func (f *fakeEngine) SetPassword(_ context.Context, username, password string) error {
+	f.username, f.password = username, password
+	return f.err
+}
+
+func (f *fakeEngine) ApplyParameters(_ context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	f.params = params
+	return f.pending, f.err
+}
+
+func (f *fakeEngine) Stop(context.Context) error {
+	f.stopped = true
+	return f.err
+}
+
+func (f *fakeEngine) Quiesce(_ context.Context, label string, hold time.Duration) error {
+	f.label, f.hold = label, hold
+	return f.err
+}
+
+func (f *fakeEngine) Unquiesce(context.Context) error {
+	f.released = true
+	return f.err
+}
+
+func TestCommandRegistry_SetPasswordCarriesBothParameters(t *testing.T) {
+	engine := &fakeEngine{}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(), handlers_rds.Command{
+		CommandID: "cmd-1",
+		Type:      handlers_rds.CommandSetPassword,
+		Parameters: []handlers_rds.Parameter{
+			{Name: handlers_rds.CommandParamMasterUsername, Value: "mulgamaster"},
+			{Name: handlers_rds.CommandParamMasterUserPassword, Value: "n3w-pw"},
+		},
+	})
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if engine.username != "mulgamaster" || engine.password != "n3w-pw" {
+		t.Errorf("engine got %q/%q, want mulgamaster/n3w-pw", engine.username, engine.password)
+	}
+	// The reply is recorded by the control plane; the new password must not be
+	// in it.
+	if strings.Contains(reply.Message, "n3w-pw") {
+		t.Errorf("reply message %q carries the new password", reply.Message)
+	}
+}
+
+// CommandReply has no structured payload, so the settings still awaiting a
+// restart come back as the message the issuer parses.
+func TestCommandRegistry_ApplyParamsReportsPendingRestart(t *testing.T) {
+	engine := &fakeEngine{pending: []string{"shared_buffers", "max_connections"}}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(), handlers_rds.Command{
+		CommandID:  "cmd-2",
+		Type:       handlers_rds.CommandApplyParams,
+		Parameters: []handlers_rds.Parameter{{Name: "shared_buffers", Value: "256MB"}},
+	})
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if reply.Message != "shared_buffers,max_connections" {
+		t.Errorf("reply message = %q, want the pending-restart list", reply.Message)
+	}
+	if len(engine.params) != 1 || engine.params[0].Name != "shared_buffers" {
+		t.Errorf("engine got %+v, want the command's parameters", engine.params)
+	}
+}
+
+func TestCommandRegistry_StopEngineReachesTheEngine(t *testing.T) {
+	engine := &fakeEngine{}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-3", Type: handlers_rds.CommandStopEngine})
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if !engine.stopped {
+		t.Error("stop-engine did not reach the engine")
+	}
+}
+
+func TestCommandRegistry_FailureRepliesFailed(t *testing.T) {
+	engine := &fakeEngine{err: errors.New("the engine did not stop")}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-4", Type: handlers_rds.CommandStopEngine})
+
+	if reply.Status != handlers_rds.CommandStatusFailed {
+		t.Fatalf("reply = %+v, want failed", reply)
+	}
+	if !strings.Contains(reply.Message, "the engine did not stop") {
+		t.Errorf("reply message = %q, want the engine's error", reply.Message)
+	}
+}
+
+// Records what a postgresEngine would have run, so the SQL and the argv are
+// asserted without a live cluster.
+type recordingRunner struct {
+	calls []command
+	out   string
+	err   error
+}
+
+func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
+	r.calls = append(r.calls, c)
+	return r.out, r.err
+}
+
+func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
+	t.Helper()
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	cfg.PGData = t.TempDir()
+	if err := os.MkdirAll(filepath.Join(cfg.PGData, "conf.d"), 0o700); err != nil {
+		t.Fatalf("create conf.d: %v", err)
+	}
+	// The guest's postgres user does not exist here, and the chown of the
+	// installed parameter file has to resolve to something real.
+	current, err := user.Current()
+	if err != nil {
+		t.Fatalf("resolve the current user: %v", err)
+	}
+	cfg.PGUser = current.Username
+	return newPostgresEngine(cfg, run, nil, newEngineProbe(cfg, staticProbe(0)))
+}
+
+// The password reaches psql through the environment and is re-quoted there,
+// so it is never a shell word or an argv another process can read.
+func TestPostgresEngine_SetPasswordKeepsTheSecretOutOfArgv(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	call := runner.calls[0]
+	for _, arg := range call.Args {
+		if strings.Contains(arg, "n3w-pw") {
+			t.Fatalf("argv %v carries the password", call.Args)
+		}
+	}
+	if strings.Contains(call.Stdin, "n3w-pw") {
+		t.Errorf("SQL %q carries the password literally, want it read from the environment", call.Stdin)
+	}
+	if !slices.Contains(call.Env, "RDS_MASTER_PASSWORD=n3w-pw") {
+		t.Errorf("env %v does not carry the password", call.Env)
+	}
+	if !strings.Contains(call.Stdin, "ALTER ROLE") {
+		t.Errorf("SQL %q does not alter the role", call.Stdin)
+	}
+}
+
+// psql interpolates the password into the ALTER ROLE before the server sees it,
+// so a parameter group that turns statement logging on would write the rotated
+// password into a postmaster.log every snapshot then carries.
+func TestPostgresEngine_SetPasswordSilencesStatementLogging(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	sql := runner.calls[0].Stdin
+	alter := strings.Index(sql, "ALTER ROLE")
+	if alter < 0 {
+		t.Fatalf("SQL %q does not alter the role", sql)
+	}
+	for _, guard := range []string{
+		"SET log_statement = 'none';",
+		"SET log_min_duration_statement = -1;",
+		"SET log_min_error_statement = 'panic';",
+	} {
+		at := strings.Index(sql, guard)
+		if at < 0 {
+			t.Errorf("SQL %q is missing %q", sql, guard)
+			continue
+		}
+		if at > alter {
+			t.Errorf("SQL %q sets %q only after the ALTER ROLE has already been logged", sql, guard)
+		}
+	}
+}
+
+// psql echoes the failing statement, which here would carry the new password
+// back into a reply the control plane records.
+func TestPostgresEngine_SetPasswordRedactsTheSecretFromAFailure(t *testing.T) {
+	runner := &recordingRunner{err: errors.New(`ERROR: syntax error at or near "n3w-pw"`)}
+	engine := newTestEngine(t, runner.run)
+
+	err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw")
+	if err == nil {
+		t.Fatal("SetPassword succeeded against a failing psql")
+	}
+	if strings.Contains(err.Error(), "n3w-pw") {
+		t.Errorf("error %q leaks the password", err)
+	}
+	if !strings.Contains(err.Error(), "[REDACTED]") {
+		t.Errorf("error %q does not mark the redaction", err)
+	}
+}
+
+func TestPostgresEngine_SetPasswordRejectsAnIncompleteCommand(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", ""); err == nil {
+		t.Error("SetPassword accepted an empty password")
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands for an incomplete request, want 0", len(runner.calls))
+	}
+}
+
+// The apply runs as the cluster superuser under peer auth, so a reserved
+// name arriving in a command payload would rotate the bootstrap superuser's
+// password rather than the customer's own role.
+func TestPostgresEngine_SetPasswordRefusesReservedRoles(t *testing.T) {
+	for _, username := range []string{"postgres", "rds_superuser", "rdsadmin", "pg_toast_owner", "PostGres"} {
+		t.Run(username, func(t *testing.T) {
+			runner := &recordingRunner{}
+			engine := newTestEngine(t, runner.run)
+
+			err := engine.SetPassword(context.Background(), username, "n3w-pw")
+			if err == nil {
+				t.Fatalf("SetPassword accepted the reserved role %q", username)
+			}
+			if len(runner.calls) != 0 {
+				t.Errorf("ran %d commands for a reserved role, want 0", len(runner.calls))
+			}
+		})
+	}
+}
+
+// The parameters land where rds-init installs them, so the next boot overwrites
+// the file rather than shadowing it with a second copy.
+func TestPostgresEngine_ApplyParametersInstallsAndReloads(t *testing.T) {
+	runner := &recordingRunner{out: "shared_buffers\nmax_connections\n"}
+	engine := newTestEngine(t, runner.run)
+
+	pending, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "shared_buffers", Value: "256MB"}})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+
+	installed, err := os.ReadFile(filepath.Join(engine.pgData, "conf.d", "10-rds-parameters.conf"))
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "shared_buffers = '256MB'") {
+		t.Errorf("installed parameters = %q, want the resolved value", installed)
+	}
+	if len(runner.calls) != 3 {
+		t.Fatalf("ran %d commands, want a config check, a reload and a pending-restart read", len(runner.calls))
+	}
+	// The check runs before the reload, so a value the engine would refuse never
+	// reaches a running cluster.
+	if !slices.Contains(runner.calls[0].Args, "-C") {
+		t.Errorf("first command = %v, want the offline config check", runner.calls[0].Args)
+	}
+	if !strings.Contains(runner.calls[1].Stdin, "pg_reload_conf") {
+		t.Errorf("second statement = %q, want the reload", runner.calls[1].Stdin)
+	}
+	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
+		t.Errorf("third statement = %q, want the pending-restart read", runner.calls[2].Stdin)
+	}
+	if len(pending) != 2 || pending[0] != "shared_buffers" || pending[1] != "max_connections" {
+		t.Errorf("pending = %v, want both settings the engine reported", pending)
+	}
+}
+
+// A reload that failed leaves the engine on its old values, so reporting the
+// apply as successful would tell the customer a change took effect that did not.
+func TestPostgresEngine_ApplyParametersSurfacesAFailedReload(t *testing.T) {
+	runner := &reloadFailingRunner{reloadErr: errors.New("reload was rejected")}
+	engine := newTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}}); err == nil {
+		t.Fatal("ApplyParameters succeeded against a failing reload")
+	}
+}
+
+type reloadFailingRunner struct {
+	recordingRunner
+
+	reloadErr error
+}
+
+func (r *reloadFailingRunner) run(ctx context.Context, c command) (string, error) {
+	if strings.Contains(c.Stdin, "pg_reload_conf") {
+		r.calls = append(r.calls, c)
+		return "", r.reloadErr
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+// A failed instance cannot reload through psql. Its corrected, parser-checked
+// include must start the service and remain installed as the serving set.
+func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	codes := []int{2, 2, 0}
+	probeCall := 0
+	cfg := testProbeConfig()
+	engine.probe = newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
+		code := codes[min(probeCall, len(codes)-1)]
+		probeCall++
+		return code, nil
+	})
+	engine.repairTimeout = 100 * time.Millisecond
+	engine.repairPoll = time.Millisecond
+	params := []handlers_rds.Parameter{
+		{Name: "shared_buffers", Value: "32768"},
+		{Name: "work_mem", Value: "8192"},
+	}
+
+	pending, err := engine.ApplyParameters(t.Context(), params)
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if len(pending) != 0 {
+		t.Errorf("pending = %v, want the restarted engine's actual pending set", pending)
+	}
+	if len(runner.calls) != 3 {
+		t.Fatalf("calls = %+v, want config check, service restart and pending read", runner.calls)
+	}
+	if !slices.Contains(runner.calls[0].Args, "-C") {
+		t.Errorf("first call = %+v, want the offline config check", runner.calls[0])
+	}
+	if !slices.Equal(runner.calls[1].Args, []string{"postgresql", "restart"}) {
+		t.Errorf("second call args = %v, want the service restart", runner.calls[1].Args)
+	}
+	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
+		t.Errorf("third call = %+v, want the pending-restart read", runner.calls[2])
+	}
+	installed, err := os.ReadFile(engine.parametersPath())
+	if err != nil {
+		t.Fatalf("read installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "work_mem = '8192'") {
+		t.Errorf("installed parameters = %q, want the repair set retained", installed)
+	}
+}
+
+func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	engine.probe = stubProbe(t, 2, nil)
+	engine.repairTimeout = 10 * time.Millisecond
+	engine.repairPoll = time.Millisecond
+
+	_, err := engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{{Name: "work_mem", Value: "8192"}})
+	if err == nil || !strings.Contains(err.Error(), "wait for the engine") {
+		t.Fatalf("ApplyParameters error = %v, want the repair wait failure", err)
+	}
+	installed, readErr := os.ReadFile(engine.parametersPath())
+	if readErr != nil {
+		t.Fatalf("read installed parameters: %v", readErr)
+	}
+	if !strings.Contains(string(installed), "work_mem = '8192'") {
+		t.Errorf("installed parameters = %q, want the checked repair set retained", installed)
+	}
+}
+
+// Through the service manager, so the supervisor records the engine as stopped
+// and does not restart it underneath a VM that is going down.
+func TestPostgresEngine_StopGoesThroughTheServiceManager(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.Stop(context.Background()); err != nil {
+		t.Fatalf("Stop: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	call := runner.calls[0]
+	if filepath.Base(call.Name) != "rc-service" {
+		t.Errorf("ran %q, want rc-service", call.Name)
+	}
+	if len(call.Args) != 2 || call.Args[0] != "postgresql" || call.Args[1] != "stop" {
+		t.Errorf("args = %v, want [postgresql stop]", call.Args)
+	}
+}

cmd/rds-agent/health.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"sync/atomic"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// A non-zero exit is a result, not a fault, so it comes back as a code; err is
+// reserved for the probe failing to run at all.
+type probeRunner func(ctx context.Context, name string, args ...string) (int, error)
+
+func execProbeRunner(ctx context.Context, name string, args ...string) (int, error) {
+	err := exec.CommandContext(ctx, name, args...).Run()
+	if err == nil {
+		return 0, nil
+	}
+	var exitErr *exec.ExitError
+	if errors.As(err, &exitErr) {
+		return exitErr.ExitCode(), nil
+	}
+	return -1, err
+}
+
+// Shells out to pg_isready rather than dialling the port, since only a startup
+// exchange separates an engine that is serving from one still in recovery.
+type engineProbe struct {
+	run    probeRunner
+	binary string
+	host   string
+	// Set from the bootstrap config once it lands, from a different goroutine
+	// than the heartbeat that reads it.
+	port atomic.Int64
+	// Latches once the engine has answered. The agent is up well before initdb
+	// finishes, so a down engine is "starting" until it has served once.
+	seenHealthy bool
+}
+
+func newEngineProbe(cfg config, run probeRunner) *engineProbe {
+	p := &engineProbe{run: run, binary: cfg.PGIsReady, host: cfg.EngineHost}
+	p.port.Store(int64(cfg.EnginePort))
+	return p
+}
+
+func (p *engineProbe) setPort(port int) {
+	p.port.Store(int64(port))
+}
+
+// What the engine is doing, before the seenHealthy latch collapses "never up"
+// and "was up and went away" into one health. The parameter rollback needs them
+// apart: a postmaster that is up and replaying WAL will come back on its own, one
+// that is not running at all after a parameter change will not.
+type engineState int
+
+const (
+	// Not answering at all, or the probe could not run.
+	engineAbsent engineState = iota
+	// The postmaster is up and rejecting connections: startup or crash recovery.
+	engineRecovering
+	engineServing
+)
+
+// Only the heartbeat calls this, so the seenHealthy latch is single-goroutine
+// state.
+func (p *engineProbe) Check(ctx context.Context) (handlers_rds.EngineHealth, string) {
+	state, message := p.state(ctx)
+	switch state {
+	case engineServing:
+		p.seenHealthy = true
+		return handlers_rds.EngineHealthHealthy, ""
+	case engineRecovering:
+		return handlers_rds.EngineHealthStarting, message
+	default:
+		return p.degraded(), message
+	}
+}
+
+// The raw probe result. Safe to call from a goroutine other than the
+// heartbeat's: it touches no latched state.
+func (p *engineProbe) state(ctx context.Context) (engineState, string) {
+	port := strconv.FormatInt(p.port.Load(), 10)
+	code, err := p.run(ctx, p.binary, "-h", p.host, "-p", port, "-q")
+	switch {
+	case err != nil:
+		// A missing binary or broken image. Reporting healthy on the strength of
+		// nothing would hide it, so this reads as absent like an engine that did
+		// not answer.
+		return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
+	case code == 0:
+		return engineServing, ""
+	case code == 1:
+		return engineRecovering, "engine is rejecting connections (startup or recovery)"
+	default:
+		return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", p.host, port)
+	}
+}
+
+// A non-answering engine: starting until it has answered once, unhealthy after.
+func (p *engineProbe) degraded() handlers_rds.EngineHealth {
+	if p.seenHealthy {
+		return handlers_rds.EngineHealthUnhealthy
+	}
+	return handlers_rds.EngineHealthStarting
+}

cmd/rds-agent/health_test.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
+	tests := []struct {
+		name        string
+		code        int
+		runErr      error
+		seenHealthy bool
+		want        handlers_rds.EngineHealth
+	}{
+		{name: "accepting connections", code: 0, want: handlers_rds.EngineHealthHealthy},
+		{name: "rejecting during startup", code: 1, want: handlers_rds.EngineHealthStarting},
+		{name: "silent before first success", code: 2, want: handlers_rds.EngineHealthStarting},
+		{name: "silent after first success", code: 2, seenHealthy: true, want: handlers_rds.EngineHealthUnhealthy},
+		{name: "no attempt made", code: 3, want: handlers_rds.EngineHealthStarting},
+		{name: "probe missing before first success", runErr: errors.New("executable not found"), want: handlers_rds.EngineHealthStarting},
+		{name: "probe missing after first success", runErr: errors.New("executable not found"), seenHealthy: true, want: handlers_rds.EngineHealthUnhealthy},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
+				return tt.code, tt.runErr
+			})
+			probe.seenHealthy = tt.seenHealthy
+
+			got, message := probe.Check(context.Background())
+			if got != tt.want {
+				t.Errorf("health = %q, want %q", got, tt.want)
+			}
+			if got != handlers_rds.EngineHealthHealthy && message == "" {
+				t.Error("a non-healthy result carried no message explaining it")
+			}
+			if got == handlers_rds.EngineHealthHealthy && message != "" {
+				t.Errorf("a healthy result carried a message: %q", message)
+			}
+		})
+	}
+}
+
+// The latch is what separates a boot still in progress from a failure, and it
+// has to flip the first time the engine answers.
+func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
+	code := 0
+	probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
+		return code, nil
+	})
+
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthHealthy {
+		t.Fatalf("first check = %q, want healthy", got)
+	}
+	code = 2
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthUnhealthy {
+		t.Errorf("check after the engine went away = %q, want unhealthy", got)
+	}
+}
+
+// The port the engine listens on comes from the bootstrap config, so the probe
+// must follow it rather than keep asking the default.
+func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
+	var gotArgs []string
+	probe := newEngineProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, error) {
+		gotArgs = args
+		return 0, nil
+	})
+
+	probe.setPort(6543)
+	probe.Check(context.Background())
+
+	if !strings.Contains(strings.Join(gotArgs, " "), "-p 6543") {
+		t.Errorf("probe args = %v, want the assigned port 6543", gotArgs)
+	}
+}
+
+func testProbeConfig() config {
+	return config{EngineHost: defaultEngineHost, EnginePort: defaultEnginePort, PGIsReady: defaultPGIsReady}
+}

cmd/rds-agent/heartbeat.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"sync/atomic"
+	"time"
+	"unicode/utf8"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+const (
+	servingParameterRecordTimeout = 15 * time.Second
+	maxBootstrapFailureBytes      = 512
+)
+
+// The beat carries the probe's result, not the agent's own liveness: an agent
+// up while the engine is down is what the recovery reconciler must see.
+type heartbeater struct {
+	cp       controlPlane
+	probe    *engineProbe
+	recorder servingParameterRecorder
+	id       identity
+	interval time.Duration
+
+	// Bootstrap and heartbeat run concurrently. Keeping the latest failure here
+	// makes a VM stuck before engine startup explain why through its next beat.
+	bootstrapFailure atomic.Pointer[string]
+}
+
+func newHeartbeater(cp controlPlane, probe *engineProbe, recorder servingParameterRecorder, interval time.Duration) *heartbeater {
+	if interval <= 0 {
+		interval = handlers_rds.HeartbeatInterval
+	}
+	return &heartbeater{cp: cp, probe: probe, recorder: recorder, interval: interval}
+}
+
+// Called before Run starts and from Run's own goroutine after, so the interval
+// is never written concurrently with the loop reading it.
+func (h *heartbeater) setInterval(d time.Duration) {
+	if d > 0 {
+		h.interval = d
+	}
+}
+
+func (h *heartbeater) setBootstrapFailure(what string, err error) {
+	message := what + " failed: " + err.Error()
+	if len(message) > maxBootstrapFailureBytes {
+		limit := maxBootstrapFailureBytes - len("...")
+		for limit > 0 && !utf8.RuneStart(message[limit]) {
+			limit--
+		}
+		message = message[:limit] + "..."
+	}
+	h.bootstrapFailure.Store(&message)
+}
+
+func (h *heartbeater) clearBootstrapFailure() {
+	h.bootstrapFailure.Store(nil)
+}
+
+func (h *heartbeater) beat(ctx context.Context) {
+	health, message := h.probe.Check(ctx)
+	if failure := h.bootstrapFailure.Load(); failure != nil {
+		if message != "" {
+			message += "; "
+		}
+		message += *failure
+	}
+	if health == handlers_rds.EngineHealthHealthy && h.recorder != nil {
+		// Checking every healthy beat also observes a restart that completed
+		// between probes. The recorder skips unchanged and pending-restart sets.
+		recordCtx, cancel := context.WithTimeout(ctx, servingParameterRecordTimeout)
+		err := h.recorder.RecordServingParameters(recordCtx)
+		cancel()
+		if err != nil {
+			slog.Warn("rds-agent: recording the serving parameters failed", "err", err)
+		}
+	}
+
+	out, err := h.cp.SubmitState(ctx, h.id, health, message)
+	if err != nil {
+		// Not escalated: the control plane already treats a missing heartbeat as
+		// staleness, and the next tick retries.
+		slog.Warn("rds-agent: heartbeat failed", "health", health, "err", err)
+		return
+	}
+	slog.Debug("rds-agent: heartbeat", "health", health, "persisted", out.Persisted)
+	h.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
+}
+
+// A timer rather than a ticker, so a cadence change takes effect on the next
+// beat, not at a restart.
+func (h *heartbeater) Run(ctx context.Context) {
+	timer := time.NewTimer(h.interval)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			h.beat(ctx)
+			timer.Reset(h.interval)
+		}
+	}
+}

cmd/rds-agent/heartbeat_test.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The first healthy probe is the first proof that the installed include can
+// start PostgreSQL, so it must seed recovery without an apply command.
+func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	content := []byte("work_mem = '4096'\n")
+	if err := os.WriteFile(engine.parametersPath(), content, 0o600); err != nil {
+		t.Fatalf("write installed parameters: %v", err)
+	}
+
+	cp := newFakeControlPlane()
+	h := newHeartbeater(cp, engine.probe, engine, 0)
+	h.beat(context.Background())
+
+	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read last known good: %v", err)
+	}
+	if string(lastGood) != string(content) {
+		t.Errorf("last known good = %q, want %q", lastGood, content)
+	}
+	states := cp.snapshotStates()
+	if len(states) != 1 || states[0].health != handlers_rds.EngineHealthHealthy {
+		t.Errorf("states = %+v, want one healthy heartbeat", states)
+	}
+}
+
+func TestHeartbeater_BoundsBootstrapFailure(t *testing.T) {
+	h := newHeartbeater(newFakeControlPlane(), newEngineProbe(testProbeConfig(), staticProbe(2)), nil, 0)
+	h.setBootstrapFailure("bootstrap fetch", errors.New(strings.Repeat("界", 1000)))
+
+	failure := h.bootstrapFailure.Load()
+	if failure == nil {
+		t.Fatal("bootstrap failure was not stored")
+	}
+	if len(*failure) > maxBootstrapFailureBytes {
+		t.Errorf("bootstrap failure is %d bytes, want at most %d", len(*failure), maxBootstrapFailureBytes)
+	}
+	if !utf8.ValidString(*failure) {
+		t.Error("bootstrap failure was truncated inside a UTF-8 encoding")
+	}
+	if !strings.HasSuffix(*failure, "...") {
+		t.Errorf("bootstrap failure %q does not show that it was truncated", *failure)
+	}
+}
+
+func TestHeartbeater_ChecksServingParametersOnEveryHealthyProbe(t *testing.T) {
+	code := 0
+	cfg := testProbeConfig()
+	probe := newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
+		return code, nil
+	})
+	recorder := &countingServingRecorder{}
+	h := newHeartbeater(newFakeControlPlane(), probe, recorder, 0)
+
+	h.beat(context.Background())
+	h.beat(context.Background())
+	code = 2
+	h.beat(context.Background())
+	code = 0
+	h.beat(context.Background())
+
+	if recorder.calls != 3 {
+		t.Errorf("record calls = %d, want one per healthy probe", recorder.calls)
+	}
+}
+
+type countingServingRecorder struct {
+	calls int
+}
+
+var _ servingParameterRecorder = (*countingServingRecorder)(nil)
+
+func (r *countingServingRecorder) RecordServingParameters(context.Context) error {
+	r.calls++
+	return nil
+}