fix: write MariaDB's startup spelling for time_zone into the option file

mariadbd has no time_zone startup option — the name only exists for SET — so
the resolved parameter group aborted the server while it parsed defaults, before
it opened the datadir. No pidfile was ever written and the reconciler failed
every instance at the 20-minute engine-health deadline.

ParameterSpec gains an option-file spelling, used only when rendering the
include. time_zone is the one setting that needs it, as default_time_zone, which
resolves @@global.time_zone to the same value. The customer-facing name is
unchanged: the API still reports time_zone and a live apply still issues
SET GLOBAL time_zone. Names read back out of a generated file return to the
catalog's namespace first, so a startup spelling cannot read as an unknown name
and report a pending restart nothing could clear.

setup.sh now feeds the catalog's resolved defaults to the image's own mariadbd
at build time, so a name the server refuses fails the build rather than every
instance built from it.

.github/actions/e2e-analyze/README.md

@@ -70,7 +70,7 @@ When the report format changes intentionally, regenerate the snapshots:
 go test ./.github/actions/e2e-analyze/ -update
 ```
 
-## Per-failure bundles (Stage 2)
+## Per-failure bundles
 
 After parsing, the action materialises a bundle directory per failure
 under `<log-dir>/analysis/NN-<TestName>/`. The report links to it from

.github/actions/e2e-analyze/analyze.go

@@ -1,6 +1,6 @@
 // Package main implements the e2e-analyze GitHub Action.
 //
-// Stage 1: cluster failures from go-junit-report's junit-*.xml by error signature,
+// Cluster failures from go-junit-report's junit-*.xml by error signature,
 // pick the earliest non-cascade failure per suite as the likely root cause,
 // and render a human-readable report.
 //
@@ -73,8 +73,14 @@ type Failure struct {
 	// `test-<suite>.start` file written by the workflow before the suite
 	// runs) can recompute StartAt without re-parsing the XML.
 	OffsetFromSuiteStart time.Duration
+	// SuiteSpan is the suite's own wall-clock span, set only when the suite
+	// ran its tests in parallel. Overlapping tests make the cumulative
+	// offset above pure fiction — it lands minutes past the real failure —
+	// so consumers widen the journal window to the suite instead of
+	// slicing a plausible-looking window around the wrong minute.
+	SuiteSpan time.Duration
 	// BundlePath is the relative path (from log-dir) to the per-failure
-	// bundle written by Stage 2's WriteBundles. Empty when bundles were
+	// bundle written by WriteBundles. Empty when bundles were
 	// not generated (e.g. unit tests that call Render directly).
 	BundlePath string
 }
@@ -88,6 +94,12 @@ type SuiteReport struct {
 	Root      *Failure    // nil if no failures
 	Cascades  []Failure   // everything else that failed in this suite
 	Buckets   [][]Failure // failures grouped by signature (root's bucket first)
+	// EndedAt is go-junit-report's <testsuite timestamp>, which records when
+	// the XML was produced — i.e. when the suite finished, not when it began.
+	EndedAt time.Time
+	// TopLevelTotal is the summed duration of the parentless tests. Longer than
+	// the suite's wall clock means they overlapped.
+	TopLevelTotal time.Duration
 }
 
 // Report is the full analysis across every junit file.
@@ -119,6 +131,28 @@ var cascadeMarkers = []string{
 	"Expected value not to be nil",
 }
 
+// teardownPatterns match the content a test logs from t.Cleanup or from the
+// harness's on-failure diagnostics — i.e. after it has already failed. They are
+// the LAST file:line lines in the failure body, so extractErrorLine's
+// keep-the-last rule picks them over the assertion unless they are skipped.
+// Anchored deliberately: a fragment loose enough to match an assertion would
+// blank out the one line the report exists to show.
+var teardownPatterns = []*regexp.Regexp{
+	regexp.MustCompile(`^DB instance \S+ is gone$`),
+	regexp.MustCompile(`^db diagnostics \S+:`),
+}
+
+// isTeardownLine reports whether a candidate error line was emitted after the
+// failure it would otherwise be mistaken for.
+func isTeardownLine(content string) bool {
+	for _, re := range teardownPatterns {
+		if re.MatchString(content) {
+			return true
+		}
+	}
+	return false
+}
+
 // noiseReplacers normalise dynamic identifiers so that "ten failures, same
 // bug" bucket together. Pattern ordering matters: longer/more-specific
 // patterns must precede the catch-alls.
@@ -206,11 +240,14 @@ func extractErrorLine(body string) string {
 		if m := goFileLineRe.FindStringSubmatch(l); m != nil {
 			// Keep the LAST occurrence: t.Log / harness traces emit
 			// progress lines from the same file:line pattern before the
-			// failing assertion finally fires.
-			goFileLine = strings.TrimSpace(m[1])
+			// failing assertion finally fires. Cleanup runs after it, though,
+			// so its lines are last of all and have to be skipped.
+			if content := strings.TrimSpace(m[1]); !isTeardownLine(content) {
+				goFileLine = content
+			}
 			continue
 		}
-		if fileHintRe.MatchString(l) {
+		if fileHintRe.MatchString(l) && !isTeardownLine(l) {
 			lastTestLine = l
 		}
 	}
@@ -254,11 +291,19 @@ func extractFileHint(body string) string {
 	if m := errorTraceRe.FindStringSubmatch(body); m != nil {
 		return m[1]
 	}
-	all := fileHintRe.FindAllString(body, -1)
-	if len(all) == 0 {
-		return ""
+	// Scanned line by line rather than across the body so a cleanup line's own
+	// file:line does not become the site the report points a reader at.
+	hint := ""
+	for _, raw := range strings.Split(body, "\n") {
+		l := strings.TrimSpace(raw)
+		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isTeardownLine(strings.TrimSpace(m[1])) {
+			continue
+		}
+		if all := fileHintRe.FindAllString(l, -1); len(all) > 0 {
+			hint = all[len(all)-1]
+		}
 	}
-	return all[len(all)-1]
+	return hint
 }
 
 // signature normalises an error line so different runs of the same root
@@ -313,6 +358,47 @@ func computeLeafSet(cases []junitTC) map[string]bool {
 	return leaf
 }
 
+// computeRolledUpSet returns the testcase names whose failure is already
+// reported by a failing descendant. go-junit-report gives such a parent a
+// testcase of its own, and its body is the parent's t.Log output rather than an
+// assertion — a cleanup line like "DB instance … is gone", which passing tests
+// emit too, reads to extractErrorLine exactly like a failure site.
+func computeRolledUpSet(cases []junitTC) map[string]bool {
+	rolledUp := make(map[string]bool, len(cases))
+	for _, tc := range cases {
+		if tc.Failure == nil && tc.Error == nil {
+			continue
+		}
+		// Mark every ancestor: the failure belongs to the deepest case that
+		// carries it, not to the tests it is nested under.
+		name := tc.Name
+		for {
+			i := strings.LastIndex(name, "/")
+			if i <= 0 {
+				break
+			}
+			name = name[:i]
+			rolledUp[name] = true
+		}
+	}
+	return rolledUp
+}
+
+// topLevelTotal sums the durations of the tests with no parent, which are the
+// units t.Parallel overlaps. Compared against the suite's wall clock it says
+// whether the suite ran its tests concurrently — see ApplySuiteStartFiles.
+// suite.Time cannot answer that: go-junit-report sets it to the sum of every
+// testcase, parents included, so it is never less than any subset of itself.
+func topLevelTotal(cases []junitTC) time.Duration {
+	total := 0.0
+	for _, tc := range cases {
+		if !strings.Contains(tc.Name, "/") {
+			total += tc.Time
+		}
+	}
+	return time.Duration(total * float64(time.Second))
+}
+
 // parseStartTime turns a junit timestamp attr (RFC3339) into a time.Time;
 // zero value if unparseable.
 func parseStartTime(s string) time.Time {
@@ -359,6 +445,9 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 		// summing every testcase double-counts. Pre-compute the leaf set so
 		// only leaves contribute to the suite's running offset.
 		isLeaf := computeLeafSet(suite.Cases)
+		rolledUp := computeRolledUpSet(suite.Cases)
+		rep.EndedAt = start
+		rep.TopLevelTotal += topLevelTotal(suite.Cases)
 		cumul := 0.0
 		for _, tc := range suite.Cases {
 			rep.Total++
@@ -376,11 +465,12 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 				continue
 			}
 
-			// Skip the synthetic parent-test "Failed" stub that go-junit-report
-			// emits for any Go test whose subtests failed. It has an empty body
-			// and would otherwise outrank real failures by XML order.
+			// Skip the parent-test entry go-junit-report emits for any Go test
+			// whose subtests failed: it is a rollup of failures already counted
+			// below it, and it would outrank them by XML order. An empty body is
+			// the same stub in the case where the parent logged nothing.
 			trimmed := strings.TrimSpace(stripANSI(body))
-			if trimmed == "" {
+			if trimmed == "" || rolledUp[tc.Name] {
 				if leaf {
 					cumul += tc.Time
 				}
@@ -462,14 +552,24 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 	return rep, nil
 }
 
-// Render writes the markdown report (Stage 1).
+// Render writes the markdown report.
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
 
-	totalFail := 0
+	totalFail, totalTests := 0, 0
 	for _, s := range r.Suites {
 		totalFail += s.FailCount
+		totalTests += s.Total
+	}
+
+	// A cell that dies before the suites run emits junit with no testcases at
+	// all. Reporting that as clean turns "nothing ran" into "nothing wrong", so
+	// name it and point above the suites.
+	if totalTests == 0 {
+		b.WriteString("⚠️ No tests ran — no testcases in any junit file. " +
+			"The failure is upstream of the suites (provisioning, bootstrap, or reimage); check the job log.\n")
+		return b.String()
 	}
 	if totalFail == 0 {
 		b.WriteString("✅ No failures across any suite.\n")
@@ -484,12 +584,22 @@ func Render(r Report) string {
 		fmt.Fprintf(&b, "### Suite `%s`: %d failed, 1 root cause likely\n\n", s.Label, s.FailCount)
 
 		if s.Root != nil {
-			b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			// XML order is completion order, so "earliest" only means earliest
+			// in time for a suite whose tests ran one after another.
+			if s.Root.SuiteSpan > 0 {
+				b.WriteString("**First non-cascade failure** (the suite ran in parallel, so this is report order, not time order)\n\n")
+			} else {
+				b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			}
 			fmt.Fprintf(&b, "- Test: `%s`\n", s.Root.Name)
-			if !s.Root.StartAt.IsZero() {
+			switch {
+			case s.Root.SuiteSpan > 0:
+				fmt.Fprintf(&b, "- Start: unknown, the suite ran in parallel (duration %.1fs)\n",
+					s.Root.Duration)
+			case !s.Root.StartAt.IsZero():
 				fmt.Fprintf(&b, "- Start: %s (duration %.1fs)\n",
 					s.Root.StartAt.Format("15:04:05"), s.Root.Duration)
-			} else {
+			default:
 				fmt.Fprintf(&b, "- Duration: %.1fs\n", s.Root.Duration)
 			}
 			if s.Root.Error != "" {

.github/actions/e2e-analyze/analyze_test.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 )
 
 // updateGolden lets `go test ./.github/actions/e2e-analyze/ -update` rewrite
@@ -124,6 +125,22 @@ func TestExtractErrorLine_FallsBackToLastTestLine(t *testing.T) {
 	}
 }
 
+// A test's cleanup logs after it has failed, so its lines are the last in the
+// body — and the cleanup line here is one passing tests emit too.
+func TestExtractErrorLine_SkipsCleanupLoggedAfterTheFailure(t *testing.T) {
+	body := `    failure_test.go:42: DB instance rds-e2e-failure-1 reached status available
+    failure_test.go:60: EventuallyErr: condition not met within 4m0s: rds-e2e-failure-1 status=available want=failed
+    rds.go:364: db diagnostics rds-e2e-failure-1: no VM to capture a console from (<nil>)
+    main_test.go:325: DB instance rds-e2e-failure-1 is gone`
+	got := extractErrorLine(body)
+	if !strings.HasPrefix(got, "EventuallyErr:") {
+		t.Errorf("extractErrorLine = %q, want the assertion at failure_test.go:60", got)
+	}
+	if hint := extractFileHint(body); hint != "failure_test.go:60" {
+		t.Errorf("extractFileHint = %q, want failure_test.go:60", hint)
+	}
+}
+
 func TestExtractFileHint_PicksLastMatch(t *testing.T) {
 	body := "ec2helpers.go:50: setup\n    vpc_test.go:227: Eventually: condition not met"
 	got := extractFileHint(body)
@@ -157,6 +174,116 @@ func TestParseFile_SkipsEmptyParentFailure(t *testing.T) {
 	}
 }
 
+func TestParseFile_SkipsParentRolledUpBySubtests(t *testing.T) {
+	// A parent that logs during cleanup gets a non-empty body, and a t.Log like
+	// "DB instance … is gone" — which passing tests emit too — reads exactly
+	// like an assertion site. Its subtest holds the real failure.
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="2">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    main_test.go:325: DB instance rds-e2e-backup-1 is gone
+]]></failure></testcase>
+  <testcase name="TestX/RealFailure" time="0.5"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 {
+		t.Fatalf("FailCount = %d, want 1 (the parent rollup should be skipped)", sr.FailCount)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX/RealFailure" {
+		t.Fatalf("expected the subtest as root, got %+v", sr.Root)
+	}
+}
+
+// A parent that fails on its own after its subtests passed is a real failure,
+// and dropping it would lose the only record of it.
+func TestParseFile_KeepsParentWithNoFailingSubtest(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    foo_test.go:42:
+        Messages: the parent asserted after its subtests
+]]></failure></testcase>
+  <testcase name="TestX/Passing" time="0.5"></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX" {
+		t.Fatalf("expected the parent as root, got %+v", sr.Root)
+	}
+}
+
+// suiteFixture writes a junit file plus its .start sidecar into a temp log dir
+// and returns the parsed, start-file-corrected report for one suite.
+func suiteFixture(t *testing.T, start string, xml []byte) SuiteReport {
+	t.Helper()
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "test-x.start"), []byte(start), 0o644); err != nil {
+		t.Fatalf("write start file: %v", err)
+	}
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	rep := Report{Suites: []SuiteReport{sr}}
+	ApplySuiteStartFiles(&rep, dir)
+	return rep.Suites[0]
+}
+
+// Parallel tests overlap, so summing the ones before a failure puts its start
+// minutes past where it really was — and the journal slice cut from that window
+// misses the failure entirely. The tell is the top-level tests taking longer in
+// total than the suite's own wall clock.
+func TestApplySuiteStartFiles_DropsTheOffsetWhenTheSuiteRanInParallel(t *testing.T) {
+	// timestamp is when go-junit-report wrote the XML, i.e. the suite's end.
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestSlow" time="100.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 2*time.Minute {
+		t.Errorf("SuiteSpan = %s, want 2m0s to mark the suite parallel", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 0, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start %s rather than a summed offset", s.Root.StartAt, want)
+	}
+}
+
+// The sequential case must keep its offsets: they are the only thing that makes
+// a per-failure journal slice narrow enough to read.
+func TestApplySuiteStartFiles_KeepsTheOffsetWhenTheSuiteRanSequentially(t *testing.T) {
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestFirst" time="30.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 0 {
+		t.Errorf("SuiteSpan = %s, want 0 for a sequential suite", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 30, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start plus 30s of preceding tests %s", s.Root.StartAt, want)
+	}
+}
+
 func TestParseFile_CascadeFallbackWhenAllCascades(t *testing.T) {
 	// If every failure is a cascade marker, the earliest one becomes root
 	// so the report isn't blank.
@@ -189,3 +316,25 @@ func TestRender_NoFailuresIsClean(t *testing.T) {
 		t.Errorf("expected zero-failure banner, got:\n%s", out)
 	}
 }
+
+// A cell that fails in provisioning writes junit with no testcases. Rendering
+// the clean banner there reports "nothing ran" as "nothing wrong".
+func TestRender_NoTestsRanIsNotClean(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		suites []SuiteReport
+	}{
+		{"empty junit file", []SuiteReport{{File: "junit-baremetal.xml", Label: "baremetal", Total: 0, FailCount: 0}}},
+		{"no junit files at all", nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			out := Render(Report{Title: "E2E failure analysis", Suites: tc.suites})
+			if strings.Contains(out, "No failures across any suite") {
+				t.Errorf("clean banner rendered when no test ran, got:\n%s", out)
+			}
+			if !strings.Contains(out, "No tests ran") {
+				t.Errorf("expected a no-tests-ran warning, got:\n%s", out)
+			}
+		})
+	}
+}

.github/actions/e2e-analyze/bundle.go

@@ -17,9 +17,9 @@ import (
 )
 
 // bundlePad is the per-failure window padding either side of the
-// [StartAt, StartAt+Duration] testcase span. Five seconds matches the
-// plan doc: long enough to catch the daemon's pre-test setup and
-// post-test cleanup chatter, short enough to keep the slice focused.
+// [StartAt, StartAt+Duration] testcase span. Five seconds catches the
+// daemon's pre-test setup and post-test cleanup chatter while keeping
+// the slice focused.
 const bundlePad = 5 * time.Second
 
 // nameSanitiseRe replaces characters unsafe in directory names with `_`.
@@ -58,21 +58,57 @@ func ApplySuiteStartFiles(rep *Report, logDir string) {
 			fmt.Fprintf(os.Stderr, "::warning::e2e-analyze: parse %s: %v\n", startPath, err)
 			continue
 		}
-		if s.Root != nil {
-			s.Root.StartAt = ts.Add(s.Root.OffsetFromSuiteStart)
-		}
-		for ci := range s.Cascades {
-			s.Cascades[ci].StartAt = ts.Add(s.Cascades[ci].OffsetFromSuiteStart)
-		}
-		for bi := range s.Buckets {
-			for fi := range s.Buckets[bi] {
-				f := &s.Buckets[bi][fi]
-				f.StartAt = ts.Add(f.OffsetFromSuiteStart)
+		// The start file is the first wall clock the analyzer has, so it is also
+		// the first chance to tell a parallel suite from a sequential one — and
+		// a parallel one's per-test offsets are fiction, since overlapping tests
+		// never started at the sum of the ones before them.
+		span := suiteSpanIfParallel(s, ts)
+		for _, f := range collectFailures(s) {
+			f.SuiteSpan = span
+			if span > 0 {
+				f.StartAt = ts
+				continue
 			}
+			f.StartAt = ts.Add(f.OffsetFromSuiteStart)
 		}
 	}
 }
 
+// suiteSpanIfParallel returns the suite's wall-clock span when its top-level
+// tests overlapped, and zero when they ran one after another. Consumers read a
+// non-zero span as "no per-test offset is derivable from this suite".
+func suiteSpanIfParallel(s *SuiteReport, start time.Time) time.Duration {
+	if s.EndedAt.IsZero() || s.TopLevelTotal == 0 {
+		return 0
+	}
+	wall := s.EndedAt.Sub(start)
+	// The 5% margin keeps per-testcase rounding from calling a sequential
+	// suite parallel; real overlap runs to several times the wall clock.
+	if wall <= 0 || s.TopLevelTotal <= time.Duration(float64(wall)*1.05) {
+		return 0
+	}
+	return wall
+}
+
+// collectFailures returns every failure in the report by pointer. Root, cascades
+// and buckets hold independent copies of the same failures, so a field that has
+// to reach the bundle writer must be set on all three.
+func collectFailures(s *SuiteReport) []*Failure {
+	var all []*Failure
+	if s.Root != nil {
+		all = append(all, s.Root)
+	}
+	for ci := range s.Cascades {
+		all = append(all, &s.Cascades[ci])
+	}
+	for bi := range s.Buckets {
+		for fi := range s.Buckets[bi] {
+			all = append(all, &s.Buckets[bi][fi])
+		}
+	}
+	return all
+}
+
 // WriteBundles materialises per-failure bundles under <logDir>/analysis/.
 // Mutates rep in-place so each Failure.BundlePath holds the relative
 // path the renderer will link from the summary.
@@ -143,7 +179,11 @@ func writeOneBundle(logDir, base, suiteLabel string, f *Failure) {
 
 	writeFailureSummary(dir, suiteLabel, f)
 
-	if !f.StartAt.IsZero() {
+	// A parallel suite offers no derivable per-test window, and the narrowing is
+	// the whole point of the slice: cutting one from the wrong minute leaves out
+	// the lines that explain the failure, and widening it to the suite just
+	// copies the artifact's own journal into every bundle.
+	if !f.StartAt.IsZero() && f.SuiteSpan == 0 {
 		start := f.StartAt.Add(-bundlePad)
 		end := f.StartAt.Add(time.Duration(f.Duration * float64(time.Second))).Add(bundlePad)
 		journalSrc := filepath.Join(logDir, "spinifex-journal.log")
@@ -188,7 +228,11 @@ func writeFailureSummary(dir, suiteLabel string, f *Failure) {
 	var b strings.Builder
 	fmt.Fprintf(&b, "Suite:    %s\n", suiteLabel)
 	fmt.Fprintf(&b, "Test:     %s\n", f.Name)
-	if !f.StartAt.IsZero() {
+	switch {
+	case f.SuiteSpan > 0:
+		b.WriteString("Start:    unknown, the suite ran in parallel; " +
+			"no journal window is derivable — read ../../spinifex-journal.log\n")
+	case !f.StartAt.IsZero():
 		fmt.Fprintf(&b, "Start:    %s\n", f.StartAt.Format(time.RFC3339))
 	}
 	fmt.Fprintf(&b, "Duration: %.3fs\n", f.Duration)

.github/actions/e2e-analyze/main.go

@@ -45,7 +45,7 @@ func main() {
 		report.Suites = append(report.Suites, sr)
 	}
 
-	// Stage 2: prefer authoritative per-suite start times written by the
+	// Prefer authoritative per-suite start times written by the
 	// workflow over junit-report's conversion-time `timestamp` attribute,
 	// then materialise per-failure bundles so Render can link them.
 	if logDir != "" {

.github/actions/e2e-analyze/slice.go

@@ -1,6 +1,6 @@
 package main
 
-// Stage 2: per-failure log slicers used by bundle.go to materialise time-window /
+// Per-failure log slicers used by bundle.go to materialise time-window /
 // testcase-window log excerpts next to each named failure.
 
 import (

.github/dependabot.yml

@@ -5,7 +5,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -19,7 +19,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -38,7 +38,7 @@ updates:
     directory: /spinifex/services/spinifexui/frontend
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3

.github/scripts/suite-sets.sh

@@ -20,7 +20,11 @@
 # 8192 halves reverted a generation) and passes against the fixed one (0 of
 # 8192). A suite that has never been shown to go red for the reason it claims
 # does not belong here.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock"
+# rds is single-only for the opposite reason to lb: v1 has no multi-AZ
+# behaviour, so a second node adds no assertion. Its DB VMs are capped at four
+# concurrent by the suite's own semaphore, which is what keeps it inside the
+# job's 50-minute budget alongside the suites above.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
 
 # Suites runnable against a multi-node environment.
 E2E_SUITES_MULTI="multinode lb cert"

.github/workflows/apt-packages.yml

@@ -0,0 +1,48 @@
+name: Apt Packages
+
+# Path-filtered like image-scripts.yml: this resolves setup.sh's runtime package
+# list against each supported suite's real indices, so a renamed, dropped, or
+# ambiguously virtual package fails here rather than on a user's machine.
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  apt_packages:
+    name: Resolve apt list (${{ matrix.image }})
+    runs-on: ubuntu-26.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # The suites the nightly E2E matrix actually installs on. Ubuntu 22.04 is
+        # accepted by detect_os but has never been exercised and does not resolve.
+        image:
+          - ubuntu:24.04
+          - ubuntu:26.04
+          - debian:13
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Resolve runtime package list
+        run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}

.github/workflows/e2e-ddil.yml

@@ -1,8 +1,7 @@
 name: E2E DDIL
 
-# Phase 1 trigger: workflow_dispatch only. Nightly cron + on-main-merge
-# land in mulga-siv-5.8 once ≥4 scenarios are asserting. See
-# docs/development/improvements/ddil-e2e-test-harness.md §4 / §Dependencies.
+# This workflow is manually triggered while the DDIL scenarios are being
+# validated.
 on:
   workflow_dispatch:
     inputs:

.github/workflows/e2e-nightly.yml

@@ -20,7 +20,7 @@ permissions:
 # trigger must stay in sync on main for cron to fire.
 env:
   GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
-  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
+  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
   # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
   # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
@@ -36,17 +36,17 @@ jobs:
       matrix:
         include:
           - { cell: 1,  install: binary, topo: single,     net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
+          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
+          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
           - { cell: 8,  install: binary, topo: real-multi, net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
+          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
+          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
           - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
           - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
           - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }
 
     runs-on: [ self-hosted, "${{ matrix.runner }}" ]
-    timeout-minutes: 35
+    timeout-minutes: ${{ matrix.extra == 'tofu-examples' && 45 || 35 }}
     env:
       TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
@@ -128,10 +128,13 @@ jobs:
         run: |
           export TF_VAR_git_branch="${GIT_BRANCH}"
 
+          # Match the family, not a pinned version: an exact match silently stops
+          # overriding the image when the matrix bumps release, dropping the cell
+          # back onto the debian gold.
           TOFU_VAR_ARGS=""
-          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
-            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
-          fi
+          case "$MATRIX_HOST" in
+            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
+          esac
 
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
@@ -248,6 +251,18 @@ jobs:
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
              done; exit \$FAIL"
 
+      - name: Import RDS PostgreSQL AMI
+        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "spx admin images import --name spinifex-rds-postgres"
+
       - name: Run Tofu Workbooks
         if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
         working-directory: mulga/scripts/tofu-cluster
@@ -370,10 +385,12 @@ jobs:
         env:
           MATRIX_HOST: ${{ matrix.host }}
         run: |
+          # Must mirror the Provision step's override, or destroy runs against a
+          # different image path than apply did.
           TOFU_VAR_ARGS=""
-          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
-            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
-          fi
+          case "$MATRIX_HOST" in
+            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
+          esac
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1

.github/workflows/e2e-suites.yml

@@ -1,7 +1,5 @@
-# E2E (per-suite blocks) — EXPERIMENTAL, workflow_dispatch only.
-#
-# Restructures the looped e2e.yml into env-grouped job blocks so each suite
-# surfaces as its own node in the Actions graph (mulga-siv-283):
+# E2E per-suite blocks. Each suite appears as its own node in the Actions
+# graph:
 #
 #   single_provision ─┬─ single_suite (matrix: the single-node-eligible suites)
 #                     └─ single_teardown (if: always)

.github/workflows/e2e.yml

@@ -15,7 +15,7 @@ on:
   workflow_dispatch:
     inputs:
       suites:
-        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam')"
+        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam e2e-rds')"
         required: false
         default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
       test_run:
@@ -50,9 +50,9 @@ jobs:
     # provision a tofu VM and pay the Spinifex install only to run nothing and
     # tear down. Push triggers keep running via the null-check on
     # inputs.suites.
-    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock')
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 50
+    timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -139,7 +139,8 @@ jobs:
           ./build-install-artifacts.sh "${GIT_BRANCH}" \
             "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
             "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar"
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
 
       - name: Provision VM
         working-directory: mulga/scripts/tofu-cluster
@@ -249,6 +250,59 @@ jobs:
           EOF
           ok "kubectl installed on VM"
 
+      # CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
+      # must be in the VM's image store before the suite runs — same split as the
+      # eks-node pair above: build on the runner (needs source + network), import
+      # on the VM (needs predastore/NATS). The suite's psql and mariadb clients
+      # need no step here: they run inside a client VM that installs them from
+      # its own user-data, because a private-only endpoint is unreachable from
+      # the runner and from the cluster host alike.
+      - name: Build RDS AMIs (libguestfs, runner)
+        if: contains(env.SUITES, 'e2e-rds')
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
+          export LIBGUESTFS_BACKEND=direct
+          sudo chmod 0644 /boot/vmlinuz-* || true
+          for IMAGE in rds-postgres rds-mariadb; do
+            banner "build ${IMAGE} AMI (libguestfs — no host devices)"
+            group_open "make build-${IMAGE}-image"
+            make "build-${IMAGE}-image"
+            group_close
+            ok "raw: $(du -h "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
+          done
+
+      - name: Import RDS AMIs on VM
+        if: contains(env.SUITES, 'e2e-rds')
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          for IMAGE in rds-postgres rds-mariadb; do
+            eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+              "spinifex/scripts/images/${IMAGE}/manifest.conf")"
+            # SYSTEM_TAG carries four tags, not one: managed-by marks it a system
+            # image, engine/engine-version are what an EngineVersion request
+            # resolves against, and the contract tag gates the data-volume format
+            # grant. --tag is repeatable, so expand rather than pass the whole
+            # string as a single value.
+            TAG_ARGS=""
+            for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
+            banner "import ${IMAGE} AMI to VM image store"
+            scp "${SSH_OPTS[@]}" \
+              "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
+              "$VM":"/tmp/${IMAGE}-alpine.raw"
+            ssh "${SSH_OPTS[@]}" "$VM" \
+              "spx admin images import --file /tmp/${IMAGE}-alpine.raw \
+                 --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
+            ok "${IMAGE} AMI imported"
+          done
+
       - name: Build E2E test binaries on runner
         working-directory: spinifex/tests/e2e
         run: |
@@ -334,9 +388,10 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               case \$SUITE in rds) TMO=45m ;; *) TMO=30m ;; esac; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \
@@ -568,7 +623,8 @@ jobs:
           ./build-install-artifacts.sh "${GIT_BRANCH}" \
             "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
             "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar"
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
 
       - name: Provision VMs
         working-directory: mulga/scripts/tofu-cluster

.github/workflows/gpu-e2e.yml

@@ -1,6 +1,6 @@
 # GPU E2E — drives wattle's persistent, shared-host Spinifex stack over SSH to
 # exercise GPU passthrough (raw EC2, ECS task GPU exposure, EKS pod GPU
-# exposure). See docs/development/feature/wattle-deploy-and-ci.md (Part B).
+# exposure).
 #
 # ---------------------------------------------------------------------------
 # ONE-TIME HOST-SIDE SETUP — NOT part of this repo, a human must do this:
@@ -32,7 +32,6 @@
 #      to skip cleanly (job succeeds, all work steps skipped) instead of
 #      queuing behind manual work indefinitely.
 #
-#   Runbook: docs/development/feature/wattle-deploy-and-ci.md (Part B).
 # ---------------------------------------------------------------------------
 
 name: GPU E2E
@@ -236,9 +235,8 @@ jobs:
       # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
       # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
       # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
-      # the advertised gateway (mulga-7suds), so the override is required
-      # here (proper fix tracked separately: tests/e2e/harness/env.go should
-      # prefer spinifex.toml's advertised bridge).
+      # the advertised gateway, so the override is required here. The harness
+      # should prefer spinifex.toml's advertised bridge.
       - name: Run GPU e2e suite on wattle
         id: e2e
         if: steps.gate.outputs.paused != 'true'

.github/workflows/image-scripts.yml

@@ -0,0 +1,39 @@
+name: Image Scripts
+
+# Separate from spinifex.yml on purpose: scripts/images/ and images/ are shell
+# and mkosi assets baked into system images, not Go, so they get their own
+# path-filtered gate instead of running on every Go contributor's push (see
+# ui.yml for the same pattern applied to the frontend).
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  image_scripts:
+    name: Image Scripts
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Run image script tests + shellcheck
+        run: make test-images

.github/workflows/publish-system-ami.yml

@@ -1,17 +1,21 @@
-name: Publish EKS + ECS node AMIs
+name: Publish System AMIs
 
 # Builds the system AMIs (libguestfs, on the runner) and publishes each
 # compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Four images ship:
+# iso.mulgadc.com/system-ami/. Six images ship:
 #   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
 #   spinifex-ecs-node       Alpine ECS container instance
 #   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
 #   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
+#   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
+#   spinifex-rds-mariadb    Alpine RDS MariaDB instance
 # Operators then install them on any cluster with:
 #   spx admin images import --name spinifex-eks-node
 #   spx admin images import --name spinifex-ecs-node
 #   spx admin images import --name spinifex-eks-node-gpu
 #   spx admin images import --name spinifex-ecs-node-gpu
+#   spx admin images import --name spinifex-rds-postgres
+#   spx admin images import --name spinifex-rds-mariadb
 #
 # Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
 # action, not something every push should trigger. The `variant` input scopes
@@ -29,6 +33,7 @@ on:
           - all
           - cpu
           - gpu
+          - rds
       dry_run:
         description: "Build + convert only; skip the R2 upload"
         type: boolean
@@ -56,10 +61,12 @@ jobs:
           set -euo pipefail
           cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
           gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
+          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
           case "${VARIANT}" in
             cpu) items="${cpu}" ;;
             gpu) items="${gpu}" ;;
-            *)   items="${cpu},${gpu}" ;;
+            rds) items="${rds}" ;;
+            *)   items="${cpu},${gpu},${rds}" ;;
           esac
           echo "images={\"include\":[${items}]}" >> "$GITHUB_OUTPUT"
 

.github/workflows/segscan-oracle.yml

@@ -0,0 +1,81 @@
+# Segscan storage oracle — push-triggered (main/dev) + manual dispatch only,
+# deliberately NOT wired into spinifex.yml's pull_request path.
+#
+# tests/integration/segscan_oracle_test.go needs the mulga umbrella repo's
+# scripts/segscan checked out alongside spinifex (it is a separate Go module,
+# `package main`, and cannot be imported — see spinifex/testutil/segscanoracle).
+# spinifex.yml's jobs check out only spinifex/viperblock/predastore and run on
+# every PR; adding a second-repo checkout there would make every spinifex PR
+# depend on mulga main being healthy and pay that checkout cost on every run,
+# for one storage-accounting oracle test. Mirrors e2e.yml's push-only trigger
+# This never gates a PR before merge, matching how e2e.yml's own
+# "front gate" job works.
+name: Segscan Storage Oracle
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  segscan_oracle:
+    name: Segscan Oracle
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - name: Checkout Mulga (scripts/segscan only)
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/segscan
+          path: mulga
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore
+
+      - name: Run Segscan Oracle Test
+        env:
+          SEGSCAN_DIR: ${{ github.workspace }}/mulga/scripts/segscan
+        run: make -C spinifex test-segscan-oracle

.github/workflows/ui.yml

@@ -57,3 +57,6 @@ jobs:
 
       - name: Build Frontend
         run: pnpm build
+
+      - name: Check for generated route tree drift
+        run: git diff --exit-code -- src/routeTree.gen.ts

.gitignore

@@ -37,3 +37,6 @@ Thumbs.db
 
 # Test reports
 *-report.txt
+
+# Per-worktree golangci-lint cache
+/.cache/

Makefile

@@ -1,6 +1,11 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
+# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
+# path, not worktree path, so parallel worktrees of this module collide and
+# surface each other's issues. Scope it per worktree.
+export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
+
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
 ifeq ($(ARCH),x86_64)
@@ -44,7 +49,7 @@ else
   _SECQ  = 2>&1 | tee
 endif
 
-build: go_build build-installer build-lb-agent
+build: go_build build-installer build-lb-agent generate-aws-model-coverage
 
 # Build spinifex-ui frontend (requires pnpm)
 build-ui:
@@ -100,6 +105,18 @@ build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd +
 import-ecs-node-image: ## Build + register the ecs-node AMI (requires a running cluster)
 	$(MAKE) build-system-image IMAGE=ecs-agent IMPORT=1
 
+build-rds-postgres-image: ## Build the spinifex-rds-postgres AMI (Alpine + PostgreSQL 18 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-postgres
+
+import-rds-postgres-image: ## Build + register the rds-postgres AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-postgres IMPORT=1
+
+build-rds-mariadb-image: ## Build the spinifex-rds-mariadb AMI (Alpine + MariaDB 11.8 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-mariadb
+
+import-rds-mariadb-image: ## Build + register the rds-mariadb AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-mariadb IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -120,15 +137,10 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
 
-# Preflight — the pre-commit gate: manifest checks, lint, vuln, and the unit,
-# race and e2e-harness tiers.
-#
-# The integration tier is deliberately NOT here. CI runs it as its own parallel
-# job, so it is still gated before merge, and keeping it out leaves preflight
-# the fast common-path check — run `make test-integration` directly when a
-# change touches the gateway router or the NATS subjects it drives.
+# the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
+# integration and race tests skipped to keep quick, they run in CI.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-race test-harness
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-harness
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
@@ -139,25 +151,40 @@ test-harness:
 	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)
 
 # In-process integration tier: the real gateway router against embedded NATS
-# JetStream, with only the daemon-side NATS subjects stubbed. Build-tagged
-# `integration` so it's skipped by the default `go test ./spinifex/...` and by
-# `test-cover`/`test-race`. Nothing provisioned — no tofu, no docker, no
-# Spinifex daemons, so the whole package runs in well under a minute. It gets
-# its own PR-blocking CI job rather than the self-hosted, push-triggered live
-# e2e tiers, so a regression here is caught before it can be merged, not just
-# after.
+# JetStream, with only the daemon-side NATS subjects stubbed.
+AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
+AWS_MODEL_CONFORMANCE_MODE ?= fail
+AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
-	$(_Q)LOG_IGNORE=1 go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
+	@$(MAKE) --no-print-directory generate-aws-model-coverage
+	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
+
+generate-aws-model-coverage:
+	@mkdir -p $(dir $(AWS_MODEL_OPERATION_COVERAGE_REPORT))
+	@go run ./cmd/aws-model-coverage > $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+
+aws-model-coverage: generate-aws-model-coverage
+	@cat $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+
+# Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
+# checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
+# is not the default local or CI layout, so this is a separate target from
+# test-integration rather than folded into it. Skips itself when segscan's
+# source isn't found.
+test-segscan-oracle:
+	@echo -e "\n....Running segscan storage oracle test...."
+	$(_Q)LOG_IGNORE=1 go test -tags=integration,segscanoracle -timeout 120s ./tests/integration/... -run TestSegscanOracle $(_RACEQ)
 
 # Validate docs/service-interfaces.yaml. Schema check + cross-reference
-# of services/suites/fixtures + on-disk path existence. Subject content
-# vs source is enforced separately in Bead 5 drift lint.
+# of services/suites/fixtures + on-disk path existence.
 manifest-check:
 	@echo -e "\n....Checking service-interfaces.yaml...."
 	@go run ./tests/e2e/manifest-check/cmd/manifest-check -repo-root . -manifest docs/service-interfaces.yaml
 
-# Drift guards (Bead 5): direct-create fixture lint + NATS subject lint,
+# Drift guards: direct-create fixture lint + NATS subject lint,
 # ratcheted against tests/e2e/manifest-lint/baseline.txt. Fails only on NEW
 # drift beyond the baseline.
 manifest-lint:
@@ -171,19 +198,19 @@ manifest-lint-update:
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	LOG_IGNORE=1 go test -timeout 120s ./spinifex/...
+	GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -timeout 120s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... $(_COVQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... $(_RACEQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -191,6 +218,39 @@ test-actions:
 	@echo -e "\n....Running action tests...."
 	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
 
+# Shell suites + shellcheck for scripts/images/ and images/mkosi.profiles/
+# helpers baked into system images. Kept out of `preflight` (a dedicated CI
+# job gates it on scripts/images/**+images/mkosi.profiles/** changes instead)
+# so image-asset churn doesn't run on every Go contributor's commit.
+#
+# images/mkosi.profiles/ ships scripts with no .sh suffix (mkosi's own
+# mkosi.*.chroot lifecycle hooks, and wrapper binaries like vllm-serve under
+# mkosi.extra/) so a plain `-name '*.sh'` glob silently skips exactly the
+# files most worth linting. Sources there are found by shebang instead of
+# name; this also means a profile that ships no scripts at all yields an
+# empty match rather than an error.
+test-images:
+	@echo -e "\n....Running scripts/images/**/*_test.sh...."
+	@for t in $$(find scripts/images -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running images/mkosi.profiles/**/*_test.sh...."
+	@for t in $$(find images/mkosi.profiles -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over scripts/images/**/*.sh...."
+	shellcheck -S warning $$(find scripts/images -name '*.sh' | sort)
+	@echo -e "\n....Running shellcheck over images/mkosi.profiles/** shell sources...."
+	@srcs="$$(grep -rlIE '^#!/bin/(bash|sh)' images/mkosi.profiles 2>/dev/null | sort)"; \
+	if [ -n "$$srcs" ]; then \
+		shellcheck -S warning $$srcs; \
+	else \
+		echo "  (no shell sources found under images/mkosi.profiles)"; \
+	fi
+	@echo "  test-images ok"
+
 # Check that new/changed code meets coverage threshold (runs tests first)
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
@@ -206,6 +266,20 @@ bench:
 # `make install-microvm` explicitly the first time.
 deploy: build
 	sudo install -m 755 bin/spx /usr/local/bin/spx
+	@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
+		if ! sudo /usr/local/bin/spx admin preflight; then \
+			echo ""; \
+			echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
+			echo "  grants don't match the binary just installed. Remediate with:"; \
+			echo "    scripts/update-nodes.sh   (from the mulga root)"; \
+			echo "  or:"; \
+			echo "    make reinstall"; \
+			echo "  Set SKIP_ASSET_PREFLIGHT=1 to bypass for a deliberate binary-only push."; \
+			exit 1; \
+		fi; \
+	else \
+		echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
+	fi
 	@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
 		$(MAKE) install-microvm; \
 	else \
@@ -228,7 +302,7 @@ install-system:
 	sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
 		-o Dpkg::Options::="--force-confdef" \
 		-o Dpkg::Options::="--force-confold" \
-		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils qemu-kvm \
+		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
 		ovmf qemu-efi-aarch64 \
 		libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
 		iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
@@ -260,7 +334,7 @@ quickinstall: install-system install-go install-aws
 
 lint:
 	@echo "Running golangci-lint..."
-	$(_Q)golangci-lint run ./...
+	$(_Q)scripts/run-gate.sh golangci-lint golangci-lint run ./...
 	@echo "  golangci-lint ok"
 
 fix:
@@ -268,14 +342,13 @@ fix:
 
 govulncheck:
 	@echo "Running govulncheck..."
-	$(_Q)go tool govulncheck ./...
+	$(_Q)scripts/run-gate.sh govulncheck go tool govulncheck ./...
 	@echo "  govulncheck ok"
 
-# NilAway — advisory nil-panic analysis. Not in preflight: it has a known
-# false-positive rate, so findings are triaged by hand rather than gating commits.
+# NilAway — advisory nil-panic analysis. Not in preflight due to false positives
 nilaway:
 	@echo "Running nilaway..."
-	$(_Q)go tool nilaway -include-pkgs=github.com/mulgadc/spinifex -exclude-test-files ./...
+	$(_Q)scripts/run-gate.sh nilaway go tool nilaway -include-pkgs=github.com/mulgadc/spinifex -exclude-test-files ./...
 	@echo "  nilaway ok"
 
 # Build release tarballs — use distro-ARCH for single arch, distro for both
@@ -321,7 +394,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness test-integration manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -5,7 +5,7 @@
 <p align="center">
   <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
-  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-docs.mulgadc.com-DC2626?style=flat-square" alt="Docs"></a>
+  <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
 </p>
 
 <p align="center">
@@ -23,22 +23,13 @@
 
 # Spinifex: the open, AWS-compatible cloud you run yourself
 
-Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your
-software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control.
-Move workloads off the hyperscalers, run AI at a fraction of the cost, or take
-the cloud to the edge, without rewriting a line. The exact build you ship to AWS,
-Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
+Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line. The exact build you ship to AWS, Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
 
 ## What is Spinifex?
 
-Spinifex is an open-source infrastructure platform that recreates the AWS service
-surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises
-environments. Run cloud-native software without a hyperscaler.
+Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
 
-Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering
-underneath: distributed object storage with erasure coding, block storage built
-to survive failure, and compute on bare metal. That depth is the reason your
-software runs unchanged, instead of rewritten.
+Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering underneath: distributed object storage with erasure coding, block storage built to survive failure, and compute on bare metal. That depth is the reason your software runs unchanged, instead of rewritten.
 
 It's built for teams that need:
 
@@ -59,34 +50,23 @@ From commodity hardware up to unmodified AWS tooling, every layer is replaceable
 
 ## Three ways to deploy
 
-One AWS-compatible surface, three deployment shapes. Pick the one that matches
-your reality; the platform on top is the same.
+One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
 
 ### Neocloud
 
-Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and
-onto our Neocloud partner ecosystem without rewriting them. GPU capacity
-(H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the
-software.
+Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and onto our Neocloud partner ecosystem without rewriting them. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
 
 ### On-premise
 
-Bring your own hardware and host Spinifex in your own data centre. A real
-multi-node HA cluster integrated with your storage and networking, with full
-control of stack, data, and jurisdiction. A predictable bill, no egress
-surprises, an auditable open-source core.
+Bring your own hardware and host Spinifex in your own data centre. A real multi-node HA cluster integrated with your storage and networking, with full control of stack, data, and jurisdiction. A predictable bill, no egress surprises, an auditable open-source core.
 
 ### Edge
 
-Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels,
-factories, and clinics. Compute next to the data, running through disconnection,
-on hardware you own. The same AWS APIs your software already uses.
+Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
 
 ## AWS compatibility
 
-Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform:
-everything you deploy on AWS deploys on Spinifex unchanged. At the edge,
-on-premise, or on a partner Neocloud.
+Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
@@ -99,11 +79,10 @@ on-premise, or on a partner Neocloud.
 | **EKS** | Kubernetes | Available |
 | **ECR** | Container registry | Available |
 | **ECS** | Container service | Available |
-| **RDS** | Databases | Q3 2026 |
+| **RDS** | Databases | Available |
+| **Bedrock** | AI Deployment | Q3 2026 |
 
-Roadmap items (Q3 2026) ship under the same AWS API surface. Code written for AWS
-today keeps working the moment they land.
-[Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
+Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 
 ## Core Components
 
@@ -139,11 +118,7 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
   <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
 </p>
 
-Every AWS API call is authenticated at the gateway, published to a NATS subject,
-and answered by whichever daemon claims it. Daemons are stateless: scale
-horizontally by starting more. No etcd, no Kubernetes, no external control plane.
-Just systemd units, a NATS cluster, and your hardware. Deep dive:
-**[`docs/DESIGN.md`](docs/DESIGN.md)**.
+Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
 ## Key Features
 

build/logrotate/spinifex

@@ -1,4 +1,8 @@
 /var/log/spinifex/*.log {
+    # The directory is 0775 root:spinifex so each service user can write its own
+    # log. logrotate refuses to rotate inside a group-writable directory unless
+    # it is told which user to rotate as.
+    su root spinifex
     daily
     rotate 14
     compress

build/scripts/nats-js-watchdog.sh

@@ -1,21 +1,32 @@
 #!/bin/sh
 # JetStream liveness watchdog for spinifex-nats.
 #
-# NATS can disable JetStream at runtime (e.g. a filesystem permission error while
-# flushing stream state) and keep the process alive serving KV as read-only. That
-# is not a process failure, so Restart=on-failure never fires and KV-backed
-# services (awsgw IAM bucket init, etc.) stall until NATS is restarted by hand.
+# NATS can stop accepting JetStream writes at runtime while the process stays
+# alive and keeps serving reads. That is not a process failure, so
+# Restart=on-failure never fires and KV-backed services (awsgw IAM bucket init,
+# etc.) stall until NATS is restarted by hand.
 #
-# This probes the NATS monitoring endpoint's JS health and restarts the service
-# only when JetStream is sustainedly unhealthy while the service is running. The
-# multi-sample window avoids restarting during transient startup/quorum 503s.
+# Two distinct latches produce that symptom, and they need different probes:
+#
+#   1. Server-wide. handleOutOfSpace calls ShutdownJetStream and sets an
+#      internal disabled flag that only a restart clears. healthz reports it.
+#   2. Per message block. filestore records a write error on the block and
+#      returns it on every later write; nothing clears it short of a restart.
+#      healthz does NOT report it, because the server-wide state is untouched —
+#      only an actual write observes it.
+#
+# So a health check alone misses case 2 entirely. Both probes run here, and a
+# sustained failure of either restarts the service. The multi-sample window
+# avoids restarting during transient startup/quorum 503s.
 
 set -eu
 
 HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
 SERVICE="spinifex-nats.service"
+SPX="${SPX_BIN:-/usr/local/bin/spx}"
 SAMPLES=3        # consecutive failing samples required before acting
 SAMPLE_GAP=2     # seconds between samples
+PROBE_TIMEOUT=5  # seconds for one canary round-trip
 
 # Only act on a running service: a stopped/activating unit is systemd's to manage,
 # and try-restart on it would either no-op or fight an in-flight (re)start.
@@ -30,15 +41,29 @@ if ! command -v curl >/dev/null 2>&1; then
     exit 0
 fi
 
-# A single 200 means JetStream is healthy — exit fast on the common case.
 js_healthy() {
     curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
 }
 
+# Round-trips a canary key through the cluster-state bucket. Absent spx this
+# degrades to the health check alone rather than restarting on a missing tool.
+js_writable() {
+    if [ ! -x "$SPX" ]; then
+        return 0
+    fi
+    "$SPX" admin node js-probe --timeout "${PROBE_TIMEOUT}s" >/dev/null 2>&1
+}
+
+failure=""
 i=1
 while [ "$i" -le "$SAMPLES" ]; do
     if js_healthy; then
-        exit 0
+        if js_writable; then
+            exit 0
+        fi
+        failure="JetStream accepts no writes (canary round-trip failed) while $HEALTH_URL reports healthy"
+    else
+        failure="JetStream unhealthy at $HEALTH_URL"
     fi
     if [ "$i" -lt "$SAMPLES" ]; then
         sleep "$SAMPLE_GAP"
@@ -46,5 +71,5 @@ while [ "$i" -le "$SAMPLES" ]; do
     i=$((i + 1))
 done
 
-echo "nats-js-watchdog: JetStream unhealthy on $SAMPLES consecutive probes ($HEALTH_URL); restarting $SERVICE" >&2
+echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2
 systemctl try-restart "$SERVICE"

build/scripts/predastore-start.sh

@@ -1,30 +0,0 @@
-#!/bin/sh
-# Helper script for spinifex-predastore.service
-# Detects host, port, backend, and node-id from spinifex.toml
-# then execs spx. Falls back to single-node defaults if no config found.
-
-CONF=/etc/spinifex/spinifex.toml
-BIND="0.0.0.0:8443"
-# -1 = co-located mode (all [[db]] peers in one process). Single-node spinifex
-# templates list 3 DB peers on 127.0.0.1; only -1 can serve that topology.
-# Multi-node spinifex.toml emits node_id = N (N >= 1) which overrides below.
-# predastore rejects NODE_ID=0 since d3a02f0 (strconv.Atoi-of-garbage guard).
-NODE_ID="-1"
-
-if [ -f "$CONF" ]; then
-    H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
-    if [ -n "$H" ]; then
-        BIND="$H"
-    fi
-
-    N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
-    if [ -n "$N" ]; then
-        NODE_ID="$N"
-    fi
-fi
-
-export SPINIFEX_PREDASTORE_HOST="${BIND%%:*}"
-export SPINIFEX_PREDASTORE_PORT="${BIND##*:}"
-export SPINIFEX_PREDASTORE_NODE_ID="$NODE_ID"
-
-exec /usr/local/bin/spx service predastore start

build/systemd/regenerate-ssh-host-keys.service

@@ -7,6 +7,9 @@ Before=ssh.service
 Type=oneshot
 ExecStart=/usr/bin/ssh-keygen -A
 RemainAfterExit=yes
+# No ExecStop: the keygen process has already exited by the time this unit
+# is stopped, so there is nothing to signal.
+TimeoutStopSec=5
 
 [Install]
 WantedBy=multi-user.target

build/systemd/spinifex-awsgw.service

@@ -14,8 +14,15 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service awsgw start
 Restart=on-failure
 RestartSec=5
+# No SIGTERM handler in this process: it exits immediately on signal, so this
+# only bounds an unexpectedly wedged process before SIGKILL.
+TimeoutStopSec=10
 
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/
 Environment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem

build/systemd/spinifex-daemon.service

@@ -7,6 +7,8 @@ Wants=openvswitch-ipsec.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
+# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see
+EnvironmentFile=-/etc/spinifex/host.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-daemon
@@ -19,13 +21,17 @@ KillMode=process
 TimeoutStopSec=15
 # RG-4: infra tier — reaped only after every guest
 OOMScoreAdjust=-500
+# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead
+MemoryHigh=75%
+MemoryMax=85%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
 # Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)
 Environment=XDG_RUNTIME_DIR=/run/spinifex
 
-# RG-9: daemon tier — privileged by necessity (GPU vfio). NoNewPrivileges omitted (RG-10: sudo ip/ovs-vsctl/dhcpcd)
+# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,
+# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.
 AmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE
 ProtectSystem=full
 PrivateDevices=no

build/systemd/spinifex-nats-watchdog.service

@@ -5,10 +5,16 @@ After=spinifex-nats.service
 
 [Service]
 Type=oneshot
-# Runs as root: the probe is a local curl to the NATS monitoring port, and the
-# remedy is `systemctl try-restart spinifex-nats` when JetStream is disabled while
-# the process stays alive (which Restart=on-failure cannot catch).
+# Runs as root: the probes are a local curl to the NATS monitoring port and a
+# canary KV round-trip via spx, and the remedy is `systemctl try-restart
+# spinifex-nats` when JetStream stops accepting writes while the process stays
+# alive (which Restart=on-failure cannot catch).
 ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
+# Sized for the probe loop's own worst case (3 samples x (5s curl + 5s canary)
+# + 2 gaps x 2s = 34s): a stop mid-cycle lets the script finish rather than
+# cutting it off. Only the failing path runs that long; a healthy node exits on
+# the first sample.
+TimeoutStopSec=45
 
 # Hardening (kept compatible with talking to PID1 over /run/systemd).
 NoNewPrivileges=yes

build/systemd/spinifex-nats.service

@@ -12,7 +12,15 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service nats start
 Restart=on-failure
 RestartSec=5
+# nats-server installs its own SIGTERM handler (JetStream store shutdown, raft
+# stepdown, client drain) with no internal deadline; every dependent service
+# stops before nats does, so size this for nats's own graceful-stop needs.
+TimeoutStopSec=20
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf
 
 # Filesystem access

build/systemd/spinifex-northstar.service

@@ -13,7 +13,14 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service northstar start
 Restart=on-failure
 RestartSec=5
+# server.Shutdown() is called with an internal 5s context; margin covers the
+# NATS unsubscribe/close and process exit that follow it.
+TimeoutStopSec=10
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/northstar/
 

build/systemd/spinifex-predastore.service

@@ -14,10 +14,20 @@ Slice=spinifex-system.slice
 User=spinifex-storage
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
-ExecStart=/var/lib/spinifex/predastore-start.sh
+ExecStart=/usr/local/bin/spx service predastore start
 Restart=on-failure
 RestartSec=5
+# Matches viperblock's drain budget: viperblock's WAL flush writes into
+# predastore, and After= already stops viperblock first, but this is the
+# backstop if that ordering is ever bypassed (direct stop, future edit).
+TimeoutStopSec=30
 OOMScoreAdjust=-500
+# Legitimately the most memory-hungry infra service (S3 buffering/caching);
+# High/Max stay well above idle so normal load isn't throttled, while still
+# bounding a leak short of the slice-wide cap.
+MemoryHigh=40%
+MemoryMax=55%
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml
 Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
 Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem

build/systemd/spinifex-qmp-collector.service

@@ -16,6 +16,13 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service qmp-collector start
 Restart=on-failure
 RestartSec=5
+# Poll loop selects on ctx.Done() directly and closes NATS locally; shutdown
+# is near-instant, so this only bounds an unexpectedly wedged process.
+TimeoutStopSec=10
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 # Same runtime dir as the daemon: telemetry sockets + metadata live there

build/systemd/spinifex-shutdown.service

@@ -9,9 +9,13 @@ After=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.serv
 Type=oneshot
 RemainAfterExit=yes
 ExecStart=/bin/true
-# ExecStop only runs when the target/host goes down, not on a single-service
-# restart — so a daemon restart still leaves guests running (DDIL).
-ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s
+# ExecStop fires on ANY target stop, including the stop half of `systemctl
+# restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless
+# of whether the host is actually going down. --only-if-host-stopping drains
+# only when systemd itself is unwinding into shutdown.target; a plain restart
+# or `systemctl stop spinifex.target` falls through so QEMU/nbdkit survive
+# (KillMode=process) and the daemon reconnects to them on the way back up.
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --only-if-host-stopping
 TimeoutStopSec=180
 
 [Install]

build/systemd/spinifex-ui.service

@@ -12,16 +12,24 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service spinifex-ui start
 Restart=on-failure
 RestartSec=5
+# server.Shutdown() uses an internal 30s context (WriteTimeout=120s covers
+# long streaming proxy responses, e.g. console passthrough); add margin above it.
+TimeoutStopSec=40
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
+Environment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/
 
 # Filesystem access
 NoNewPrivileges=yes
 ProtectSystem=strict
 PrivateDevices=yes
 ReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key
-ReadWritePaths=/var/log/spinifex
+ReadWritePaths=/var/lib/spinifex/spinifex-ui /var/log/spinifex
 
 # Hardening
 ProtectHome=yes

build/systemd/spinifex-viperblock.service

@@ -15,6 +15,16 @@ ExecStart=/usr/local/bin/spx service viperblock start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=15%
+MemoryMax=25%
+# KillMode=process: nbdkit still serving a guest survives a restart here, the
+# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts
+# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.
+KillMode=process
+# Bounds this process's own WAL drain, which the plugin caps at 20s.
+TimeoutStopSec=30
 EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/

build/systemd/spinifex-vpcd.service

@@ -14,13 +14,23 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service vpcd start
 Restart=on-failure
 RestartSec=5
+# Waits for an in-flight OVN drift-reconcile pass to return before exiting;
+# no sub-timeout bounds that call today, so this covers a normal pass.
+TimeoutStopSec=20
 OOMScoreAdjust=-500
+# Per-service ceiling: bounds a leak here instead of the slice-wide cap.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/
 
 # RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no
 # longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem
-# sandbox is restored. NoNewPrivileges stays off (RG-10: sudo ip/ovs-vsctl/dhcpcd).
+# sandbox is restored.
+#
+# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and
+# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR
+# update that retires the last escalation path.
 AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
 CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
 NoNewPrivileges=no

cmd/aws-model-coverage/main.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/mulgadc/spinifex/internal/awsmodel"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+)
+
+func main() {
+	dispatch := gateway.AWSOperationInventory()
+	coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
+	for _, service := range awsmodel.Services() {
+		inventory, ok := dispatch[string(service)]
+		var modelInventory awsmodel.DispatchInventory
+		switch {
+		case ok:
+			modelInventory = awsmodel.DispatchInventory{
+				Registered:  inventory.Registered,
+				Stubbed:     inventory.Stubbed,
+				Unsupported: inventory.Unsupported,
+			}
+		case service == awsmodel.S3:
+			modelInventory = awsmodel.DispatchInventory{
+				Opaque: true,
+				Note:   "Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.",
+			}
+		default:
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: no gateway inventory for %s\n", service)
+			os.Exit(1)
+		}
+
+		coverage, err := awsmodel.CompareOperations(service, modelInventory)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
+			os.Exit(1)
+		}
+		coverages = append(coverages, coverage)
+	}
+
+	fmt.Print(awsmodel.RenderCoverageMarkdown(coverages))
+}

cmd/dhcptest/main.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/insomniacslk/dhcp/dhcpv4"
 	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
 	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
 )
@@ -36,6 +37,7 @@ func main() {
 	timeout := flag.Duration("timeout", 10*time.Second, "Per-attempt DHCP timeout")
 	retries := flag.Int("retries", 3, "Number of DORA attempts before giving up")
 	verbose := flag.Bool("verbose", false, "Print full DHCP packet contents")
+	broadcast := flag.Bool("broadcast", false, "Set the BOOTP broadcast flag, as vpcd's Acquire path does, so replies come back to ff:ff:ff:ff:ff:ff rather than unicast to the bridge MAC")
 	flag.Parse()
 
 	// Resolve the hardware address to use.
@@ -114,6 +116,9 @@ func main() {
 	// Identity modifiers — client-id / hostname / vendor-class for upstream lease
 	// tagging. Built by the vpcd code path so the probe cannot drift from it.
 	mods := dhcp.IdentityModifiers(cid, *hostname, *vendorClass, hwAddr)
+	if *broadcast {
+		mods = append(mods, dhcpv4.WithBroadcast(true))
+	}
 
 	fmt.Println("Sending DHCP Discover...")
 	start := time.Now()

cmd/ecr-credential-provider/main.go

@@ -12,17 +12,16 @@
 package main
 
 import (
-	"bufio"
 	"encoding/json"
 	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/guestenv"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 
 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
@@ -144,13 +143,7 @@ func cacheDurationFrom(expiresAt time.Time) string {
 // loadConfig reads the cloud-init env file then lets real env vars override. The
 // IMDS base is fixed (not in the env file) but overridable for tests.
 func loadConfig(envFile string) config {
-	env := parseEnvFile(envFile)
-	get := func(key string) string {
-		if v := os.Getenv(key); v != "" {
-			return v
-		}
-		return env[key]
-	}
+	get := guestenv.Load(envFile).Get
 
 	cfg := config{
 		GatewayURL: get("EKS_GATEWAY_URL"),
@@ -167,29 +160,6 @@ func loadConfig(envFile string) config {
 	return cfg
 }
 
-// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
-func parseEnvFile(path string) map[string]string {
-	out := map[string]string{}
-	f, err := os.Open(path)
-	if err != nil {
-		return out
-	}
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		key, val, ok := strings.Cut(line, "=")
-		if !ok {
-			continue
-		}
-		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
-	}
-	return out
-}
-
 // emitEmpty writes a valid CredentialProviderResponse with no auth entries.
 func emitEmpty(out io.Writer) {
 	resp := CredentialProviderResponse{

cmd/ecr-credential-provider/main_test.go

@@ -18,12 +18,14 @@ func fakeBackend(t *testing.T) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
 
-	// IMDSv2 token.
+	// IMDSv2 token. Real IMDS echoes the requested TTL back as a response
+	// header; the SDK client parses it from there, not the body.
 	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPut {
 			w.WriteHeader(http.StatusMethodNotAllowed)
 			return
 		}
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("imds-v2-token"))
 	})
 	// IMDS role name.

cmd/ecs-agent/config.go

@@ -1,11 +1,10 @@
 package main
 
 import (
-	"bufio"
-	"os"
 	"strconv"
-	"strings"
 	"time"
+
+	"github.com/mulgadc/spinifex/internal/guestenv"
 )
 
 const (
@@ -40,13 +39,7 @@ type config struct {
 
 // loadConfig reads the cloud-init env file then lets real env vars override.
 func loadConfig(envFile string) config {
-	env := parseEnvFile(envFile)
-	get := func(key string) string {
-		if v := os.Getenv(key); v != "" {
-			return v
-		}
-		return env[key]
-	}
+	get := guestenv.Load(envFile).Get
 
 	cfg := config{
 		GatewayURL:       get("ECS_GATEWAY_URL"),
@@ -94,26 +87,3 @@ func loadConfig(envFile string) config {
 	}
 	return cfg
 }
-
-// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
-func parseEnvFile(path string) map[string]string {
-	out := map[string]string{}
-	f, err := os.Open(path)
-	if err != nil {
-		return out
-	}
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		key, val, ok := strings.Cut(line, "=")
-		if !ok {
-			continue
-		}
-		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
-	}
-	return out
-}

cmd/ecs-agent/config_test.go

@@ -71,18 +71,3 @@ func TestLoadConfig_HeartbeatOverride(t *testing.T) {
 		t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
 	}
 }
-
-func TestParseEnvFile_SkipsBlankAndComments(t *testing.T) {
-	dir := t.TempDir()
-	p := filepath.Join(dir, "f.env")
-	if err := os.WriteFile(p, []byte("\n# c\nA=1\nnokeyval\nB = 2 \n"), 0o600); err != nil {
-		t.Fatal(err)
-	}
-	m := parseEnvFile(p)
-	if m["A"] != "1" || m["B"] != "2" {
-		t.Errorf("parse mismatch: %#v", m)
-	}
-	if _, ok := m["nokeyval"]; ok {
-		t.Errorf("line without = should be skipped")
-	}
-}

cmd/ecs-agent/credentials/imds.go

@@ -3,57 +3,39 @@ package credentials
 import (
 	"context"
 	"net/http"
-	"sync"
-	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 )
 
-// refreshMargin is how long before expiry the cache is considered stale.
-const refreshMargin = 5 * time.Minute
-
-// IMDSProvider fetches instance-role credentials from IMDS and caches them until
-// they near expiry. Safe for concurrent use.
+// IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
+// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache
+// (which refetches once the real expiry passes; ec2rolecreds also caps
+// Expires to at most an hour out). Safe for concurrent use.
 type IMDSProvider struct {
-	client *http.Client
-	base   string
-
-	mu     sync.Mutex
-	cached Credentials
+	cache *aws.CredentialsCache
 }
 
 var _ CredentialsProvider = (*IMDSProvider)(nil)
 
 // NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
-// "http://169.254.169.254/latest"). A nil client uses http.DefaultClient.
+// "http://169.254.169.254/latest"). A nil client uses the SDK IMDS client's
+// default HTTP client.
 func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
-	if client == nil {
-		client = http.DefaultClient
-	}
-	return &IMDSProvider{client: client, base: base}
+	provider := imdscreds.NewProvider(client, base)
+	return &IMDSProvider{cache: aws.NewCredentialsCache(provider)}
 }
 
 // Retrieve returns cached credentials when still valid, else fetches fresh ones.
 func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	if p.cached.Valid(refreshMargin) {
-		return p.cached, nil
-	}
-	if err := ctx.Err(); err != nil {
-		return Credentials{}, err
-	}
-
-	raw, err := imdscreds.Fetch(p.client, p.base)
+	creds, err := p.cache.Retrieve(ctx)
 	if err != nil {
 		return Credentials{}, err
 	}
-	p.cached = Credentials{
-		AccessKeyID:     raw.AccessKeyID,
-		SecretAccessKey: raw.SecretAccessKey,
-		SessionToken:    raw.SessionToken,
-		Expiration:      raw.Expiration,
-	}
-	return p.cached, nil
+	return Credentials{
+		AccessKeyID:     creds.AccessKeyID,
+		SecretAccessKey: creds.SecretAccessKey,
+		SessionToken:    creds.SessionToken,
+		Expiration:      creds.Expires,
+	}, nil
 }

cmd/ecs-agent/credentials/imds_test.go

@@ -10,14 +10,19 @@ import (
 	"time"
 )
 
+// imdsStub serves the IMDSv2 token + role + credentials endpoints under the
+// real /latest prefix the SDK IMDS client always requests.
 func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		// Real IMDS echoes the requested TTL back as a response header; the SDK
+		// client parses it from there, not the body.
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("v2-token"))
 	})
-	mux.HandleFunc("/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/meta-data/iam/security-credentials/" {
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
 			_, _ = w.Write([]byte("node-role"))
 			return
 		}
@@ -33,15 +38,17 @@ func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Serv
 
 func TestRetrieve_FetchesAndCaches(t *testing.T) {
 	var hits int32
-	exp := time.Now().Add(6 * time.Hour).UTC().Round(time.Second)
+	// Kept under an hour: ec2rolecreds caps Expires to now+1h.
+	exp := time.Now().Add(30 * time.Minute).UTC().Round(time.Second)
 	srv := imdsStub(t, &hits, map[string]string{
+		"Code":            "Success",
 		"AccessKeyId":     "AKIA",
 		"SecretAccessKey": "secret",
 		"Token":           "session",
 		"Expiration":      exp.Format(time.RFC3339),
 	})
 
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	got, err := p.Retrieve(context.Background())
 	if err != nil {
 		t.Fatalf("Retrieve: %v", err)
@@ -50,7 +57,7 @@ func TestRetrieve_FetchesAndCaches(t *testing.T) {
 		t.Errorf("creds mismatch: %+v", got)
 	}
 
-	// Second call within validity window must not re-hit IMDS.
+	// Second call well within validity must not re-hit IMDS.
 	if _, err := p.Retrieve(context.Background()); err != nil {
 		t.Fatalf("Retrieve 2: %v", err)
 	}
@@ -59,15 +66,17 @@ func TestRetrieve_FetchesAndCaches(t *testing.T) {
 	}
 }
 
-func TestRetrieve_RefetchesWhenStale(t *testing.T) {
+func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
 	var hits int32
-	// Expiry inside the refresh margin → never cached, always refetched.
+	// Already expired: aws.CredentialsCache refetches once the stored
+	// credentials' real expiry has passed, regardless of ExpiryWindow.
 	srv := imdsStub(t, &hits, map[string]string{
+		"Code":            "Success",
 		"AccessKeyId":     "AKIA",
 		"SecretAccessKey": "secret",
-		"Expiration":      time.Now().Add(time.Minute).UTC().Format(time.RFC3339),
+		"Expiration":      time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
 	})
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	if _, err := p.Retrieve(context.Background()); err != nil {
 		t.Fatal(err)
 	}
@@ -75,13 +84,13 @@ func TestRetrieve_RefetchesWhenStale(t *testing.T) {
 		t.Fatal(err)
 	}
 	if hits != 2 {
-		t.Errorf("hits = %d, want 2 (stale refetch)", hits)
+		t.Errorf("hits = %d, want 2 (expired refetch)", hits)
 	}
 }
 
 func TestRetrieve_CancelledContext(t *testing.T) {
-	srv := imdsStub(t, nil, map[string]string{"AccessKeyId": "A", "SecretAccessKey": "B"})
-	p := NewIMDSProvider(srv.Client(), srv.URL)
+	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 	if _, err := p.Retrieve(ctx); err == nil {
@@ -89,16 +98,10 @@ func TestRetrieve_CancelledContext(t *testing.T) {
 	}
 }
 
-func TestCredentials_Valid(t *testing.T) {
-	if (Credentials{}).Valid(0) {
-		t.Error("empty creds should be invalid")
-	}
-	noExp := Credentials{AccessKeyID: "a", SecretAccessKey: "b"}
-	if !noExp.Valid(time.Hour) {
-		t.Error("zero expiry should be treated valid")
-	}
-	soon := Credentials{AccessKeyID: "a", SecretAccessKey: "b", Expiration: time.Now().Add(time.Minute)}
-	if soon.Valid(5 * time.Minute) {
-		t.Error("expiry inside margin should be invalid")
+func TestRetrieve_CodeNotSuccessErrors(t *testing.T) {
+	srv := imdsStub(t, nil, map[string]string{"Code": "Failure", "AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
+	if _, err := p.Retrieve(context.Background()); err == nil {
+		t.Fatal("expected error when Code != Success")
 	}
 }

cmd/ecs-agent/credentials/provider.go

@@ -17,18 +17,6 @@ type Credentials struct {
 	Expiration      time.Time
 }
 
-// Valid reports whether the credentials are usable now, leaving a margin so a
-// caller does not start a request with a token about to expire mid-flight.
-func (c Credentials) Valid(margin time.Duration) bool {
-	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
-		return false
-	}
-	if c.Expiration.IsZero() {
-		return true
-	}
-	return time.Now().Add(margin).Before(c.Expiration)
-}
-
 // CredentialsProvider yields current credentials, refreshing as needed. Retrieve
 // must be safe for concurrent use.
 type CredentialsProvider interface {

cmd/ecs-agent/metadata.go

@@ -1,11 +1,14 @@
 package main
 
 import (
-	"encoding/json"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
 )
 
 // instanceMetadata is the IMDS-derived identity of the host the agent runs on.
@@ -15,83 +18,44 @@ type instanceMetadata struct {
 	AZ         string
 }
 
-// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2.
-// base is the IMDS root, e.g. "http://169.254.169.254/latest".
+// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2
+// via the SDK IMDS client. base is the IMDS root, e.g.
+// "http://169.254.169.254/latest".
 func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
-	token, err := imdsToken(client, base)
-	if err != nil {
-		return instanceMetadata{}, err
-	}
+	imdsClient := imdscreds.NewClient(client, base)
+	ctx := context.Background()
 
-	instanceID, err := imdsGet(client, base+"/meta-data/instance-id", token)
+	instanceID, err := getMetadata(ctx, imdsClient, "instance-id")
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
 	}
-	az, err := imdsGet(client, base+"/meta-data/placement/availability-zone", token)
+	az, err := getMetadata(ctx, imdsClient, "placement/availability-zone")
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
 	}
 
-	doc, err := imdsGet(client, base+"/dynamic/instance-identity/document", token)
+	doc, err := imdsClient.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
 	if err != nil {
 		return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
 	}
-	var ident struct {
-		AccountID string `json:"accountId"`
-	}
-	if err := json.Unmarshal([]byte(doc), &ident); err != nil {
-		return instanceMetadata{}, fmt.Errorf("parse identity document: %w", err)
-	}
 
 	return instanceMetadata{
-		AccountID:  ident.AccountID,
+		AccountID:  doc.AccountID,
 		InstanceID: strings.TrimSpace(instanceID),
 		AZ:         strings.TrimSpace(az),
 	}, nil
 }
 
-// imdsToken fetches an IMDSv2 session token.
-func imdsToken(client *http.Client, base string) (string, error) {
-	req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
+// getMetadata reads a single meta-data path and returns its body as a string.
+func getMetadata(ctx context.Context, imdsClient *imds.Client, path string) (string, error) {
+	out, err := imdsClient.GetMetadata(ctx, &imds.GetMetadataInput{Path: path})
 	if err != nil {
 		return "", err
 	}
-	req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("imds token: %w", err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
+	defer out.Content.Close()
+	body, err := io.ReadAll(out.Content)
 	if err != nil {
 		return "", err
 	}
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("imds token: status %d", resp.StatusCode)
-	}
-	return string(body), nil
-}
-
-// imdsGet performs a token-authenticated IMDS GET.
-func imdsGet(client *http.Client, url, token string) (string, error) {
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return "", err
-	}
-	if token != "" {
-		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
-	}
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("status %d", resp.StatusCode)
-	}
 	return string(body), nil
 }

cmd/ecs-agent/metadata_test.go

@@ -9,11 +9,14 @@ import (
 func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPut {
 			w.WriteHeader(http.StatusMethodNotAllowed)
 			return
 		}
+		// Real IMDS echoes the requested TTL back as a response header; the SDK
+		// client parses it from there, not the body.
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
 		_, _ = w.Write([]byte("v2-token"))
 	})
 	guard := func(w http.ResponseWriter, r *http.Request) bool {
@@ -23,17 +26,17 @@ func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 		}
 		return true
 	}
-	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte("i-0abc123"))
 		}
 	})
-	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte("us-east-1a"))
 		}
 	})
-	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
 		if guard(w, r) {
 			_, _ = w.Write([]byte(`{"accountId":"123456789012","region":"us-east-1"}`))
 		}
@@ -45,7 +48,7 @@ func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
 
 func TestFetchInstanceMetadata_OK(t *testing.T) {
 	srv := metadataStub(t, true)
-	got, err := fetchInstanceMetadata(srv.Client(), srv.URL)
+	got, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest")
 	if err != nil {
 		t.Fatalf("fetch: %v", err)
 	}
@@ -57,20 +60,23 @@ func TestFetchInstanceMetadata_OK(t *testing.T) {
 func TestFetchInstanceMetadata_TokenRequired(t *testing.T) {
 	// Server returns 405 for non-PUT token; a GET-only client would 401 downstream.
 	srv := metadataStub(t, true)
-	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err != nil {
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err != nil {
 		t.Fatalf("v2 path should succeed: %v", err)
 	}
 }
 
 func TestFetchInstanceMetadata_BadDocument(t *testing.T) {
 	mux := http.NewServeMux()
-	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("t")) })
-	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
-	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
-	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("t"))
+	})
+	mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
+	mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
+	mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
 	srv := httptest.NewServer(mux)
 	t.Cleanup(srv.Close)
-	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err == nil {
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err == nil {
 		t.Fatal("expected parse error on bad identity document")
 	}
 }

cmd/ecs-agent/netns.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"hash/crc32"
+	"os"
 	"os/exec"
 	"strings"
 	"time"
@@ -21,6 +22,12 @@ const (
 	credEndpointCIDR = defaultCredEndpointIP + "/32"
 )
 
+// udhcpcScript is the config script busybox udhcpc execs to apply a lease. It
+// is the reason presence of the udhcpc binary is not enough to use it: Debian
+// and Ubuntu's busybox ship the applet without this script, and udhcpc then
+// takes the lease and exits nonzero having configured nothing.
+const udhcpcScript = "/usr/share/udhcpc/default.script"
+
 // netCmdRunner executes a host networking command. Abstracted so unit tests can
 // assert the ip/netns sequence without touching the kernel.
 type netCmdRunner interface {
@@ -43,10 +50,47 @@ type taskNetns struct {
 	run     netCmdRunner
 	nicWait time.Duration
 	poll    time.Duration
+	// Injected so the DHCP-client choice below is unit-testable on a host that
+	// has neither client installed.
+	lookPath   func(string) (string, error)
+	fileIsExec func(string) bool
 }
 
 func newTaskNetns(run netCmdRunner) *taskNetns {
-	return &taskNetns{run: run, nicWait: 30 * time.Second, poll: time.Second}
+	return &taskNetns{
+		run:        run,
+		nicWait:    30 * time.Second,
+		poll:       time.Second,
+		lookPath:   exec.LookPath,
+		fileIsExec: func(p string) bool { fi, err := os.Stat(p); return err == nil && fi.Mode()&0o111 != 0 },
+	}
+}
+
+// dhcpStep returns the in-netns DHCP command that leases the task ENI's
+// OVN-assigned address.
+//
+// This image ships on two distributions and they do not agree on a DHCP client.
+// Alpine has busybox udhcpc; Ubuntu has no udhcpc at all, and `ip netns exec`
+// exits 1 — not 127 — when it cannot exec the binary, so a hardcoded udhcpc
+// fails as an opaque "network setup failed" with the task stuck in STOPPED.
+// Resolve the client at setup time instead, preferring udhcpc so Alpine keeps
+// its existing behaviour exactly, then falling back to dhcpcd. Same order and
+// same reasoning as dhcp_oneshot() in mulga-mgmt-net.sh.
+//
+// Both invocations are one-shot: udhcpc -q exits once it has the lease and -n
+// gives up rather than backgrounding, and dhcpcd -1 exits after configuring the
+// interface. Neither leaves a daemon behind in the netns.
+func (n *taskNetns) dhcpStep(name, iface string) ([]string, error) {
+	if _, err := n.lookPath("udhcpc"); err == nil && n.fileIsExec(udhcpcScript) {
+		return []string{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"}, nil
+	}
+	// Lease state is keyed by interface name, which is unique per task here:
+	// each task ENI is a separate hot-plug, so it lands on its own PCI slot and
+	// gets its own predictable name before being moved into the netns.
+	if _, err := n.lookPath("dhcpcd"); err == nil {
+		return []string{"ip", "netns", "exec", name, "dhcpcd", "-q", "-1", "-t", "20", iface}, nil
+	}
+	return nil, fmt.Errorf("no DHCP client for %s: need udhcpc (with %s) or dhcpcd", iface, udhcpcScript)
 }
 
 func netnsName(taskID string) string { return "ecs-" + taskID }
@@ -68,11 +112,16 @@ func (n *taskNetns) Setup(taskID, mac string) (string, error) {
 	}
 	hostVeth := credVethName(taskID)
 	hostIP, taskIP := credSubnet(taskID)
+	dhcp, err := n.dhcpStep(name, iface)
+	if err != nil {
+		_ = n.Teardown(taskID)
+		return "", err
+	}
 	steps := [][]string{
 		{"ip", "link", "set", iface, "netns", name},
 		{"ip", "-n", name, "link", "set", "lo", "up"},
 		{"ip", "-n", name, "link", "set", iface, "up"},
-		{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"},
+		dhcp,
 		// Credentials path: a veth into the host netns plus a host-routed /32 to the
 		// credential endpoint, so the container reaches 169.254.170.2 without it
 		// leaking onto the task ENI's VPC route.

cmd/ecs-agent/netns_test.go

@@ -55,10 +55,27 @@ const linkWithENI = `1: lo: <LOOPBACK,UP> mtu 65536 ... link/loopback 00:00:00:0
 2: eth0: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:aa:bb:cc ...
 3: eth1: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:de:ad:01 ...`
 
+// withDHCPClients pins which DHCP clients the netns sees, so a test asserts the
+// branch it names rather than whatever happens to be installed on the machine
+// running it. Without this the DHCP step is host-dependent: an Alpine CI box and
+// an Ubuntu dev box would exercise different code from the same test.
+func withDHCPClients(n *taskNetns, present map[string]bool, scriptOK bool) {
+	n.lookPath = func(f string) (string, error) {
+		if present[f] {
+			return "/sbin/" + f, nil
+		}
+		return "", errors.New("not found")
+	}
+	n.fileIsExec = func(string) bool { return scriptOK }
+}
+
+// newTestNetns defaults to the Alpine arrangement — udhcpc plus its config
+// script — so the sequence assertions below keep describing that platform.
 func newTestNetns(f *fakeNetRunner) *taskNetns {
 	n := newTaskNetns(f)
 	n.nicWait = 50 * time.Millisecond
 	n.poll = 5 * time.Millisecond
+	withDHCPClients(n, map[string]bool{"udhcpc": true}, true)
 	return n
 }
 
@@ -142,6 +159,79 @@ func TestNetns_SetupTearsDownOnStepFailure(t *testing.T) {
 	}
 }
 
+// The image runs on two distributions with different DHCP clients, and picking
+// the wrong one fails as an opaque "network setup failed" with the task stuck
+// in STOPPED — `ip netns exec` exits 1, not 127, when it cannot exec the
+// binary. Each branch is asserted so that misresolution is a test failure.
+func TestNetns_DHCPStepPerPlatform(t *testing.T) {
+	tests := []struct {
+		name     string
+		present  map[string]bool
+		scriptOK bool
+		want     string
+		wantErr  bool
+	}{
+		{
+			name:     "alpine: udhcpc with its config script",
+			present:  map[string]bool{"udhcpc": true, "dhcpcd": true},
+			scriptOK: true,
+			want:     "ip netns exec ecs-t-001 udhcpc -i eth1 -q -n",
+		},
+		{
+			// Ubuntu ships no udhcpc at all — the case that broke awsvpc tasks.
+			name:    "ubuntu: no udhcpc, falls back to dhcpcd",
+			present: map[string]bool{"dhcpcd": true},
+			want:    "ip netns exec ecs-t-001 dhcpcd -q -1 -t 20 eth1",
+		},
+		{
+			// Debian/Ubuntu busybox ships the applet without the script, so
+			// presence of the binary alone must not select it.
+			name:     "udhcpc present but config script missing: still dhcpcd",
+			present:  map[string]bool{"udhcpc": true, "dhcpcd": true},
+			scriptOK: false,
+			want:     "ip netns exec ecs-t-001 dhcpcd -q -1 -t 20 eth1",
+		},
+		{
+			name:    "neither client: explicit error, not a silent bad command",
+			present: map[string]bool{},
+			wantErr: true,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			n := newTestNetns(&fakeNetRunner{linkOut: linkWithENI})
+			withDHCPClients(n, tc.present, tc.scriptOK)
+			got, err := n.dhcpStep("ecs-t-001", "eth1")
+			if tc.wantErr {
+				if err == nil {
+					t.Fatalf("want error with no DHCP client, got %v", got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("dhcpStep: %v", err)
+			}
+			if joined := strings.Join(got, " "); joined != tc.want {
+				t.Errorf("want %q, got %q", tc.want, joined)
+			}
+		})
+	}
+}
+
+// A missing DHCP client must not leave the half-built netns behind for the next
+// attempt to trip over, same as any other failed setup step.
+func TestNetns_SetupTearsDownWhenNoDHCPClient(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	withDHCPClients(n, map[string]bool{}, false)
+	if _, err := n.Setup("t-001", "52:54:00:de:ad:01"); err == nil {
+		t.Fatal("want error when no DHCP client is available")
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected teardown, got %v", f.joined())
+	}
+}
+
 func TestNetns_Teardown(t *testing.T) {
 	f := &fakeNetRunner{}
 	n := newTestNetns(f)

cmd/ecs-agent/reconcile.go

@@ -8,12 +8,18 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
-// reconcile re-adopts containers that survived an agent restart. It lists the
-// runtime's containers and, for each task still running in this cluster,
-// re-registers its IAM credentials, re-attaches the exit-wait, and refreshes its
-// RUNNING state on the gateway. The returned set seeds pollAssignments so the
+// reconcile re-syncs the gateway with what this instance is actually running
+// after an agent restart. It lists the runtime's containers and, for each task
+// in this cluster, either re-adopts it (still running: re-register IAM
+// credentials, re-attach the exit-wait, refresh RUNNING) or reports it STOPPED
+// and clears its leftovers. The returned set seeds pollAssignments so the
 // gateway's re-delivered assignments for these tasks are acked but not re-run.
 //
+// Reporting the dead ones is the load-bearing half. The exit-wait is cancelled
+// when the agent shuts down, so a container that dies while the agent is down
+// has nobody to report its exit; the gateway holds the task RUNNING forever,
+// its service never replaces it, and the target never recovers.
+//
 // It is best-effort: a nil runner or a List error returns an empty set and the
 // agent falls back to its pre-reconcile behaviour (poll + run).
 func (a *Agent) reconcile(ctx context.Context) map[string]bool {
@@ -27,15 +33,17 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 		return adopted
 	}
 
-	// Group this cluster's running containers back into their tasks. The Assign is
-	// reconstructed from labels — enough to re-register creds and tear down netns.
+	// Group this cluster's containers back into their tasks, running or not. The
+	// Assign is reconstructed from labels — enough to re-register creds and tear
+	// down netns.
 	type task struct {
-		as         *bus.Assign
-		containers []ctrruntime.Container
+		as      *bus.Assign
+		running []ctrruntime.Container
+		dead    []ctrruntime.Container
 	}
 	tasks := map[string]*task{}
 	for _, c := range containers {
-		if !c.Running || c.Labels[labelClusterName] != a.id.ClusterName {
+		if c.Labels[labelClusterName] != a.id.ClusterName {
 			continue
 		}
 		taskID := c.Labels[labelTaskID]
@@ -53,20 +61,50 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 			}}
 			tasks[taskID] = t
 		}
-		t.containers = append(t.containers, c)
+		if c.Running {
+			t.running = append(t.running, c)
+		} else {
+			t.dead = append(t.dead, c)
+		}
 	}
 
 	for taskID, t := range tasks {
+		// A task with nothing running is over, whatever the gateway still believes.
+		// Mixed running/dead is left to the running path: the exit-wait re-attached
+		// below reports the individual container when it is reaped.
+		if len(t.running) == 0 {
+			a.reportStoppedTask(ctx, t.as, t.dead)
+			adopted[taskID] = true
+			continue
+		}
+
 		a.registerTaskCreds(t.as)
-		statuses := make([]bus.ContainerStatus, 0, len(t.containers))
-		for _, c := range t.containers {
+		statuses := make([]bus.ContainerStatus, 0, len(t.running))
+		for _, c := range t.running {
 			name := c.Labels[labelContainerName]
 			go a.waitContainer(ctx, t.as, name, c.ID)
 			statuses = append(statuses, bus.ContainerStatus{Name: name, Status: bus.TaskStatusRunning, ContainerID: c.ID})
 		}
 		a.reportTaskState(t.as, bus.TaskStatusRunning, "", statuses)
 		adopted[taskID] = true
-		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.containers))
+		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.running))
 	}
 	return adopted
 }
+
+// reportStoppedTask clears a task's leftover containers and tells the gateway it
+// is STOPPED, so the scheduler stops counting a task this instance is not
+// running and its service can place a replacement.
+func (a *Agent) reportStoppedTask(ctx context.Context, as *bus.Assign, dead []ctrruntime.Container) {
+	statuses := make([]bus.ContainerStatus, 0, len(dead))
+	for _, c := range dead {
+		if err := a.runner.Remove(ctx, c.ID); err != nil {
+			slog.Warn("ecs-agent: reconcile remove failed", "task", as.TaskID, "container", c.ID, "err", err)
+		}
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
+		})
+	}
+	a.reportTaskState(as, bus.TaskStatusStopped, "Container exited while the agent was not running", statuses)
+	slog.Info("ecs-agent: reported stopped task", "task", as.TaskID, "containers", len(dead))
+}

cmd/ecs-agent/reconcile_test.go

@@ -72,6 +72,63 @@ func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
 	waitedFor(t, rt, containerID("t-001", "web"))
 }
 
+// A task whose container died while the agent was down must be reported STOPPED,
+// not silently skipped. The exit-wait is cancelled on agent shutdown, so nothing
+// else ever reports that exit: skipping it leaves the gateway holding the task
+// RUNNING forever, its service never places a replacement, and the load
+// balancer target stays unhealthy with no path back.
+func TestReconcile_ReportsTaskWhoseContainerDied(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-dead", "web", "default", "t-dead", testTaskRole, "52:54:00:de:ad:02", false),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusStopped); n != 1 {
+		t.Errorf("STOPPED reports for t-dead = %d, want 1", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusRunning); n != 0 {
+		t.Errorf("t-dead must not be reported RUNNING, got %d such reports", n)
+	}
+	// Seeded so a re-delivered assignment for the dead task is acked, not re-run.
+	if !adopted["t-dead"] {
+		t.Errorf("t-dead not seeded into the adopted set; got %v", adopted)
+	}
+	if want := containerID("t-dead", "web"); !slices.Contains(rt.Removed, want) {
+		t.Errorf("leftover container %s not removed; removed=%v", want, rt.Removed)
+	}
+}
+
+// A task with one live and one dead container is still running: the live half is
+// re-adopted and the re-attached exit-wait reports the dead one when it is
+// reaped, so reconcile must not declare the whole task STOPPED.
+func TestReconcile_PartiallyDeadTaskStaysRunning(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-mixed", "web", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", true),
+			adoptedContainer("t-mixed", "sidecar", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", false),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	a.reconcile(context.Background())
+
+	if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("RUNNING reports for t-mixed = %d, want 1", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusStopped); n != 0 {
+		t.Errorf("t-mixed must not be reported STOPPED while a container still runs, got %d", n)
+	}
+}
+
 // pollAssignments seeded with an adopted task acks its re-delivered assignment
 // but does not re-run it, while a genuinely new task still dispatches.
 func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
@@ -123,23 +180,35 @@ func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
 	}
 }
 
-// Containers for another cluster or with a stopped task are not adopted.
-func TestReconcile_FiltersOtherClusterAndStopped(t *testing.T) {
+// Containers belonging to another cluster are ignored entirely: this agent
+// speaks for its own cluster only, and must neither adopt nor report them.
+//
+// A stopped container of this cluster IS handled — see
+// TestReconcile_ReportsTaskWhoseContainerDied. It lands in the adopted set so
+// its re-delivered assignment is acked rather than re-run, after reconcile has
+// reported the task STOPPED.
+func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
+	cp := &fakeCP{}
 	rt := &ctrruntime.FakePuller{
 		WaitErr: errors.New("blocked"),
 		Containers: []ctrruntime.Container{
 			adoptedContainer("t-other", "web", "other", "", "", "", true),
-			adoptedContainer("t-stop", "web", "default", "", "", "", false),
+			adoptedContainer("t-other-dead", "web", "other", "", "", "", false),
 			adoptedContainer("t-ok", "web", "default", "", "", "", true),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
 
 	adopted := a.reconcile(context.Background())
 
 	if len(adopted) != 1 || !adopted["t-ok"] {
 		t.Fatalf("adopted = %v, want only t-ok", adopted)
 	}
+	for _, id := range []string{"t-other", "t-other-dead"} {
+		if n := countStatus(cp.taskStates(), id, bus.TaskStatusStopped); n != 0 {
+			t.Errorf("reported %d STOPPED states for another cluster's task %s, want 0", n, id)
+		}
+	}
 }
 
 // A nil runner or a List error degrades to an empty set without panicking.

cmd/ecs-agent/taskrun_test.go

@@ -70,8 +70,8 @@ func TestRunTask_PullRunReportsRunning(t *testing.T) {
 	}
 }
 
-// TestRunTask_GPUCarriedToRunSpec covers mulga-11opz (Epic C task C1): the
-// AssignContainer's GPU count reaches the runtime RunSpec unchanged; a
+// TestRunTask_GPUCarriedToRunSpec verifies that the AssignContainer's GPU count
+// reaches the runtime RunSpec unchanged; a
 // container with no GPU request keeps RunSpec.GPU at zero (regression).
 func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
 	cp := &fakeCP{}
@@ -534,7 +534,7 @@ func (sentinelResolver) Authorize(context.Context, string) (string, string, stri
 	return "", "", "", nil
 }
 
-// TestPullResolver covers siv-459: an assign with no execution role (or no
+// TestPullResolver verifies that an assign with no execution role (or no
 // credential endpoint) uses the instance resolver; with both, a distinct
 // exec-role resolver is built.
 func TestPullResolver(t *testing.T) {

cmd/installer/autoinstall/autoinstall.go

@@ -4,12 +4,14 @@
 package autoinstall
 
 import (
+	"cmp"
 	"fmt"
 	"log/slog"
 	"net"
 	"os"
 	"os/exec"
-	"sort"
+	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"syscall"
@@ -18,6 +20,10 @@ import (
 	"github.com/mulgadc/spinifex/cmd/installer/install"
 )
 
+// listDisks is the block-device scan, indirected so tests can drive selection
+// without real hardware.
+var listDisks = install.ListDisks
+
 // Load returns an install.Config built from SPINIFEX_* environment variables,
 // or nil if SPINIFEX_AUTO is not set to "1" (interactive mode).
 func Load() (*install.Config, error) {
@@ -65,9 +71,9 @@ func buildConfig() (*install.Config, error) {
 		hostname = "spinifex-node"
 	}
 
-	disk, err := resolveDisk(os.Getenv("SPINIFEX_DISK"))
+	storage, err := resolveStorage()
 	if err != nil {
-		return nil, fmt.Errorf("disk: %w", err)
+		return nil, fmt.Errorf("storage: %w", err)
 	}
 
 	wanIface, err := resolveNIC(os.Getenv("SPINIFEX_WAN_IFACE"), "")
@@ -76,73 +82,217 @@ func buildConfig() (*install.Config, error) {
 	}
 
 	cfg := &install.Config{
-		Disk:         disk,
+		Storage:      storage,
 		Hostname:     hostname,
 		RootPassword: password,
 		// Optional on the headless path — empty means no --email passed to
 		// spx admin init, which omits the operator identity from telemetry.
-		Email:        strings.TrimSpace(os.Getenv("SPINIFEX_EMAIL")),
-		WANInterface: wanIface,
-		WANDHCPMode:  strings.ToLower(os.Getenv("SPINIFEX_WAN_MODE")) != "static",
+		Email: strings.TrimSpace(os.Getenv("SPINIFEX_EMAIL")),
 	}
 
-	if !cfg.WANDHCPMode {
-		ip := os.Getenv("SPINIFEX_WAN_IP")
-		mask := os.Getenv("SPINIFEX_WAN_MASK")
-		gw := os.Getenv("SPINIFEX_WAN_GW")
-		if ip == "" || mask == "" || gw == "" {
-			return nil, fmt.Errorf("SPINIFEX_WAN_IP, SPINIFEX_WAN_MASK, SPINIFEX_WAN_GW required for static mode")
-		}
-		cfg.WANAddress = ip
-		cfg.WANMask = mask
-		cfg.WANGateway = gw
-		if dns := os.Getenv("SPINIFEX_WAN_DNS"); dns != "" {
-			cfg.WANDNS = strings.Split(dns, ",")
-		}
+	cfg.WAN, err = parseRole("WAN", wanIface)
+	if err != nil {
+		return nil, err
+	}
+	if cfg.WAN.DHCPMode {
+		cfg.WAN.Gateway = ""
+	} else if cfg.WAN.Gateway == "" {
+		return nil, fmt.Errorf("SPINIFEX_WAN_IP, SPINIFEX_WAN_MASK, SPINIFEX_WAN_GW required for static mode")
 	}
 
-	if lanIface := os.Getenv("SPINIFEX_LAN_IFACE"); lanIface != "" {
-		lan, err := resolveNIC(lanIface, wanIface)
+	// lan and vpc are optional: an unset _IFACE leaves the role folded, which
+	// is how a single- or two-NIC node collapses onto the planes above it.
+	for _, r := range []struct {
+		plane string
+		dst   *install.NetworkRole
+	}{{"LAN", &cfg.LAN}, {"VPC", &cfg.VPC}} {
+		name := os.Getenv("SPINIFEX_" + r.plane + "_IFACE")
+		if name == "" {
+			continue
+		}
+		iface, err := resolveNIC(name, wanIface)
 		if err != nil {
-			return nil, fmt.Errorf("LAN NIC: %w", err)
+			return nil, fmt.Errorf("%s NIC: %w", r.plane, err)
 		}
-		cfg.LANInterface = lan
-		cfg.LANDHCPMode = strings.ToLower(os.Getenv("SPINIFEX_LAN_MODE")) != "static"
-		if !cfg.LANDHCPMode {
-			cfg.LANAddress = os.Getenv("SPINIFEX_LAN_IP")
-			cfg.LANMask = os.Getenv("SPINIFEX_LAN_MASK")
-			if dns := os.Getenv("SPINIFEX_LAN_DNS"); dns != "" {
-				cfg.LANDNS = strings.Split(dns, ",")
-			}
+		if *r.dst, err = parseRole(r.plane, iface); err != nil {
+			return nil, err
 		}
+		// Only the wan plane installs a default route.
+		r.dst.Gateway = ""
+	}
+
+	if err := cfg.Validate(); err != nil {
+		return nil, fmt.Errorf("network roles: %w", err)
 	}
 
 	if os.Getenv("SPINIFEX_GPU_PASSTHROUGH") == "1" {
 		cfg.GPUPassthrough = true
 	}
 
-	role := strings.ToLower(os.Getenv("SPINIFEX_ROLE"))
-	if role == "" {
-		role = "init"
-	}
-	cfg.ClusterRole = role
-	skipFormation := os.Getenv("SPINIFEX_SKIP_FORMATION") == "1"
-	if skipFormation {
+	if os.Getenv("SPINIFEX_SKIP_FORMATION") == "1" {
 		cfg.SkipFormation = true
 	}
 
-	if role == "join" && !skipFormation {
-		joinAddr := os.Getenv("SPINIFEX_JOIN_ADDR")
-		if joinAddr == "" {
-			return nil, fmt.Errorf("SPINIFEX_JOIN_ADDR required when SPINIFEX_ROLE=join")
-		}
-		cfg.JoinAddr = joinAddr
-	}
-
 	return cfg, nil
 }
 
-// resolveDisk maps the spx_disk value from grub.cfg to a block device path.
+// resolveStorage builds the disk configuration from the kernel cmdline.
+//
+//	SPINIFEX_FS               ext4 (default) or one of the zfs-* topologies
+//	SPINIFEX_DISKS            explicit ordered member list, required for multi-disk pools
+//	SPINIFEX_DISK             the OS disk, as before
+//	SPINIFEX_DISK_SPINIFEX    ext4 only: dedicated disk for /var/lib/spinifex
+//	SPINIFEX_DISK_PREDASTORE  ext4 only: dedicated disk for /var/lib/spinifex/predastore
+func resolveStorage() (install.DiskConfig, error) {
+	cfg := install.DiskConfig{FS: install.FSExt4}
+	if v := strings.TrimSpace(os.Getenv("SPINIFEX_FS")); v != "" {
+		fs, err := install.ParseFSType(v)
+		if err != nil {
+			return cfg, err
+		}
+		cfg.FS = fs
+	}
+
+	var err error
+	switch list := strings.TrimSpace(os.Getenv("SPINIFEX_DISKS")); {
+	case list != "":
+		if cfg.Disks, err = disksByName(strings.Split(list, ",")); err != nil {
+			return cfg, err
+		}
+	case cfg.FS.MinDisks() > 1:
+		// No auto-selection for a multi-disk pool. Choosing which disks to
+		// erase is not a decision an unattended install gets to make, and the
+		// member order determines the RAID10 pairing.
+		return cfg, fmt.Errorf("%s needs at least %d disks — set SPINIFEX_DISKS to an explicit comma-separated list",
+			cfg.FS.Label(), cfg.FS.MinDisks())
+	default:
+		d, derr := resolveDisk(os.Getenv("SPINIFEX_DISK"))
+		if derr != nil {
+			return cfg, derr
+		}
+		cfg.Disks = []install.Disk{d}
+	}
+
+	if err := applyDiskRoles(&cfg); err != nil {
+		return cfg, err
+	}
+	if cfg.ZFS, err = parseZFSOpts(); err != nil {
+		return cfg, err
+	}
+	if err := cfg.Validate(); err != nil {
+		return cfg, err
+	}
+	slog.Info("autoinstall: storage selected", "fs", cfg.FS, "disks", cfg.Paths())
+	return cfg, nil
+}
+
+// roleVars maps each optional data role to the variable that assigns it. The OS
+// role is not here: it always comes from SPINIFEX_DISK.
+var roleVars = []struct {
+	role install.DiskRole
+	env  string
+}{
+	{install.RoleSpinifex, "SPINIFEX_DISK_SPINIFEX"},
+	{install.RolePredastore, "SPINIFEX_DISK_PREDASTORE"},
+}
+
+// applyDiskRoles attaches dedicated data drives to an ext4 install. Roles are
+// always set on ext4, even with no extra drives, so the OS assignment is
+// explicit rather than implied by position.
+func applyDiskRoles(cfg *install.DiskConfig) error {
+	if cfg.FS.IsZFS() {
+		for _, rv := range roleVars {
+			if strings.TrimSpace(os.Getenv(rv.env)) != "" {
+				return fmt.Errorf("%s applies to ext4 only — %s spans every member already", rv.env, cfg.FS.Label())
+			}
+		}
+		return nil
+	}
+	if len(cfg.Disks) != 1 {
+		return fmt.Errorf("ext4 takes one OS disk, %d given — set SPINIFEX_DISK, and SPINIFEX_DISK_SPINIFEX or SPINIFEX_DISK_PREDASTORE for dedicated drives",
+			len(cfg.Disks))
+	}
+
+	roles := []install.RoleMount{{Role: install.RoleOS, Disk: cfg.Disks[0]}}
+	for _, rv := range roleVars {
+		spec := strings.TrimSpace(os.Getenv(rv.env))
+		if spec == "" {
+			continue
+		}
+		// Resolved with the same fail-closed selector as the OS disk, so an
+		// ambiguous value lists the candidates rather than picking one.
+		d, err := resolveDisk(spec)
+		if err != nil {
+			return fmt.Errorf("%s: %w", rv.env, err)
+		}
+		roles = append(roles, install.RoleMount{Role: rv.role, Disk: d})
+	}
+	*cfg = cfg.WithRoles(roles)
+	return nil
+}
+
+// parseZFSOpts reads the advanced pool tunables. Every one is optional; unset
+// values are computed from the hardware at install time.
+func parseZFSOpts() (install.ZFSOpts, error) {
+	var o install.ZFSOpts
+	ints := []struct {
+		env string
+		dst *int
+	}{
+		{"SPINIFEX_ZFS_ASHIFT", &o.Ashift},
+		{"SPINIFEX_ZFS_COPIES", &o.Copies},
+		{"SPINIFEX_ZFS_ARC_MAX_MIB", &o.ARCMaxMiB},
+	}
+	for _, f := range ints {
+		v := strings.TrimSpace(os.Getenv(f.env))
+		if v == "" {
+			continue
+		}
+		n, err := strconv.Atoi(v)
+		if err != nil {
+			return o, fmt.Errorf("%s: %w", f.env, err)
+		}
+		*f.dst = n
+	}
+	o.Compress = strings.TrimSpace(os.Getenv("SPINIFEX_ZFS_COMPRESS"))
+	o.Checksum = strings.TrimSpace(os.Getenv("SPINIFEX_ZFS_CHECKSUM"))
+	return o, nil
+}
+
+// disksByName resolves an explicit member list. Names may be kernel names
+// ("sdb"), device paths or by-id paths; the order given is preserved because
+// RAID10 pairs members in it.
+func disksByName(names []string) ([]install.Disk, error) {
+	available, err := listDisks()
+	if err != nil {
+		return nil, err
+	}
+	out := make([]install.Disk, 0, len(names))
+	for _, raw := range names {
+		name := strings.TrimSpace(raw)
+		if name == "" {
+			continue
+		}
+		// A by-id path or a symlink resolves to the kernel device the scan
+		// reported, so both spellings name the same disk.
+		resolved := name
+		if !strings.HasPrefix(resolved, "/dev/") {
+			resolved = "/dev/" + resolved
+		}
+		if target, err := filepath.EvalSymlinks(resolved); err == nil {
+			resolved = target
+		}
+
+		idx := slices.IndexFunc(available, func(d install.Disk) bool { return d.Path == resolved })
+		if idx < 0 {
+			return nil, fmt.Errorf("%q is not an available disk — found:\n%s", name, diskList(available))
+		}
+		out = append(out, available[idx])
+	}
+	return out, nil
+}
+
+// resolveDisk maps the SPINIFEX_DISK value from grub.cfg to a disk.
 //
 // Supported values:
 //
@@ -151,141 +301,78 @@ func buildConfig() (*install.Config, error) {
 //	smallest        — smallest non-removable disk (typical OS-on-SSD pattern)
 //	nvme            — the only NVMe disk; fail if multiple found
 //	/dev/sda (etc.) — exact device path; fail if not present
-func resolveDisk(target string) (string, error) {
-	switch strings.ToLower(target) {
+func resolveDisk(target string) (install.Disk, error) {
+	disks, err := candidateDisks()
+	if err != nil {
+		return install.Disk{}, err
+	}
+	switch strings.ToLower(strings.TrimSpace(target)) {
 	case "", "auto":
-		return singleDisk()
-	case "largest":
-		return diskBySize(true)
-	case "smallest":
-		return diskBySize(false)
-	case "nvme":
-		return nvmeDisk()
-	default:
-		if _, err := os.Stat(target); err != nil {
-			return "", fmt.Errorf("%q not found", target)
+		if len(disks) != 1 {
+			return install.Disk{}, fmt.Errorf(
+				"expected exactly one disk, found %d — set SPINIFEX_DISK to one of:\n%s\n"+
+					"  largest   (largest disk)\n"+
+					"  smallest  (smallest disk)\n"+
+					"  nvme      (NVMe only)\n"+
+					"  /dev/sdX  (exact path)",
+				len(disks), diskList(disks))
 		}
-		return target, nil
+		return disks[0], nil
+	case "largest":
+		return pickBySize(disks, true)
+	case "smallest":
+		return pickBySize(disks, false)
+	case "nvme":
+		nvmes := slices.DeleteFunc(slices.Clone(disks), func(d install.Disk) bool {
+			return !strings.HasPrefix(filepath.Base(d.Path), "nvme")
+		})
+		if len(nvmes) != 1 {
+			return install.Disk{}, fmt.Errorf("expected exactly one NVMe disk, found %d:\n%s",
+				len(nvmes), diskList(nvmes))
+		}
+		return nvmes[0], nil
+	default:
+		found, err := disksByName([]string{target})
+		if err != nil {
+			return install.Disk{}, err
+		}
+		return found[0], nil
 	}
 }
 
-type diskCandidate struct {
-	dev   string
-	bytes int64
-}
-
-// nonRemovableDisks returns all non-removable, non-virtual block devices.
-func nonRemovableDisks() ([]diskCandidate, error) {
-	entries, err := os.ReadDir("/sys/block")
+// candidateDisks lists the disks an unattended install may erase: fixed, and
+// not the installer's own media.
+func candidateDisks() ([]install.Disk, error) {
+	all, err := listDisks()
 	if err != nil {
 		return nil, err
 	}
-	var out []diskCandidate
-	for _, e := range entries {
-		dev := e.Name()
-		switch {
-		case strings.HasPrefix(dev, "loop"),
-			strings.HasPrefix(dev, "ram"),
-			strings.HasPrefix(dev, "dm-"),
-			strings.HasPrefix(dev, "sr"):
-			continue
-		}
-		removable, _ := os.ReadFile("/sys/block/" + dev + "/removable")
-		if strings.TrimSpace(string(removable)) == "1" {
-			continue
-		}
-		sizeRaw, _ := os.ReadFile("/sys/block/" + dev + "/size")
-		sectors, parseErr := strconv.ParseInt(strings.TrimSpace(string(sizeRaw)), 10, 64)
-		if parseErr != nil || sectors == 0 {
-			continue
-		}
-		out = append(out, diskCandidate{dev: dev, bytes: sectors * 512})
-	}
-	return out, nil
+	return slices.DeleteFunc(all, func(d install.Disk) bool {
+		return d.Removable || d.LiveMedia
+	}), nil
 }
 
-// diskList returns a human-readable list of candidates for error messages.
-func diskList(disks []diskCandidate) string {
+func pickBySize(disks []install.Disk, largest bool) (install.Disk, error) {
+	if len(disks) == 0 {
+		return install.Disk{}, fmt.Errorf("no non-removable disks found")
+	}
+	sorted := slices.Clone(disks)
+	slices.SortFunc(sorted, func(a, b install.Disk) int { return cmp.Compare(a.Bytes, b.Bytes) })
+	if largest {
+		return sorted[len(sorted)-1], nil
+	}
+	return sorted[0], nil
+}
+
+// diskList renders candidates for an error message.
+func diskList(disks []install.Disk) string {
 	lines := make([]string, len(disks))
 	for i, d := range disks {
-		lines[i] = fmt.Sprintf("  /dev/%s (%dG)", d.dev, d.bytes>>30)
+		lines[i] = fmt.Sprintf("  %s (%s, %s)", d.Path, d.SizeHuman(), d.Content)
 	}
 	return strings.Join(lines, "\n")
 }
 
-// singleDisk selects the only non-removable disk, or fails listing all found.
-func singleDisk() (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	switch len(disks) {
-	case 0:
-		return "", fmt.Errorf("no non-removable disks found")
-	case 1:
-		slog.Info("autoinstall: single disk selected", "disk", disks[0].dev, "size_gb", disks[0].bytes>>30)
-		return "/dev/" + disks[0].dev, nil
-	default:
-		return "", fmt.Errorf(
-			"multiple disks found — set SPINIFEX_DISK to one of:\n%s\n"+
-				"  largest   (largest disk)\n"+
-				"  smallest  (smallest disk)\n"+
-				"  nvme      (NVMe only)\n"+
-				"  /dev/sdX  (exact path)",
-			diskList(disks),
-		)
-	}
-}
-
-// diskBySize selects the largest (largest=true) or smallest non-removable disk.
-func diskBySize(largest bool) (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	if len(disks) == 0 {
-		return "", fmt.Errorf("no non-removable disks found")
-	}
-	sort.Slice(disks, func(i, j int) bool {
-		if largest {
-			return disks[i].bytes > disks[j].bytes
-		}
-		return disks[i].bytes < disks[j].bytes
-	})
-	label := "smallest"
-	if largest {
-		label = "largest"
-	}
-	slog.Info("autoinstall: disk selected by size", "mode", label, "disk", disks[0].dev, "size_gb", disks[0].bytes>>30)
-	return "/dev/" + disks[0].dev, nil
-}
-
-// nvmeDisk selects the only NVMe disk, or fails listing all NVMe disks found.
-func nvmeDisk() (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	var nvmes []diskCandidate
-	for _, d := range disks {
-		if strings.HasPrefix(d.dev, "nvme") {
-			nvmes = append(nvmes, d)
-		}
-	}
-	switch len(nvmes) {
-	case 0:
-		return "", fmt.Errorf("no NVMe disks found")
-	case 1:
-		slog.Info("autoinstall: NVMe disk selected", "disk", nvmes[0].dev, "size_gb", nvmes[0].bytes>>30)
-		return "/dev/" + nvmes[0].dev, nil
-	default:
-		return "", fmt.Errorf(
-			"multiple NVMe disks found — set SPINIFEX_DISK to one of:\n%s",
-			diskList(nvmes),
-		)
-	}
-}
-
 // virtualNICPrefixes lists interface name prefixes that identify non-physical
 // interfaces (bridges, tunnels, container/OVS veth pairs, etc.). These are
 // skipped when auto-selecting a NIC so we don't configure docker0 or
@@ -297,6 +384,42 @@ var virtualNICPrefixes = []string{
 
 // resolveNIC returns the interface name to use. "auto" or empty picks the
 // first physical (non-loopback, non-virtual) interface that is not exclude.
+// parseRole reads the SPINIFEX_<PLANE>_* kernel-cmdline vars for one plane.
+// iface is the already-resolved interface name; _VLAN and _MTU are optional
+// and default to untagged at the link MTU.
+func parseRole(plane, iface string) (install.NetworkRole, error) {
+	role := install.NetworkRole{
+		Interface: iface,
+		DHCPMode:  strings.ToLower(os.Getenv("SPINIFEX_"+plane+"_MODE")) != "static",
+	}
+	if v := os.Getenv("SPINIFEX_" + plane + "_VLAN"); v != "" {
+		id, err := strconv.Atoi(v)
+		if err != nil {
+			return role, fmt.Errorf("SPINIFEX_%s_VLAN: %w", plane, err)
+		}
+		role.VLAN = id
+	}
+	if v := os.Getenv("SPINIFEX_" + plane + "_MTU"); v != "" {
+		mtu, err := strconv.Atoi(v)
+		if err != nil {
+			return role, fmt.Errorf("SPINIFEX_%s_MTU: %w", plane, err)
+		}
+		role.MTU = mtu
+	}
+	if !role.DHCPMode {
+		role.Address = os.Getenv("SPINIFEX_" + plane + "_IP")
+		role.Mask = os.Getenv("SPINIFEX_" + plane + "_MASK")
+		role.Gateway = os.Getenv("SPINIFEX_" + plane + "_GW")
+		if role.Address == "" || role.Mask == "" {
+			return role, fmt.Errorf("SPINIFEX_%s_IP and SPINIFEX_%s_MASK required for static mode", plane, plane)
+		}
+	}
+	if dns := os.Getenv("SPINIFEX_" + plane + "_DNS"); dns != "" {
+		role.DNS = strings.Split(dns, ",")
+	}
+	return role, nil
+}
+
 func resolveNIC(name, exclude string) (string, error) {
 	if name != "" && name != "auto" {
 		return name, nil

cmd/installer/autoinstall/autoinstall_test.go

@@ -1,12 +1,57 @@
 package autoinstall
 
 import (
+	"errors"
+	"fmt"
+	"maps"
+	"net"
 	"os"
+	"slices"
+	"strings"
 	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
 )
 
+const gib = 1 << 30
+
+// setEnv clears every SPINIFEX_* variable inherited from the test runner before
+// applying want, so one case cannot leak a kernel-cmdline value into the next.
+func setEnv(t *testing.T, want map[string]string) {
+	t.Helper()
+	for _, kv := range os.Environ() {
+		if k, _, ok := strings.Cut(kv, "="); ok && strings.HasPrefix(k, "SPINIFEX_") {
+			t.Setenv(k, "")
+		}
+	}
+	for k, v := range want {
+		t.Setenv(k, v)
+	}
+}
+
+// withDisks replaces the block-device scan. The clone matters: candidateDisks
+// filters with slices.DeleteFunc, which would otherwise mutate the fixture.
+func withDisks(t *testing.T, disks []install.Disk, err error) {
+	t.Helper()
+	prev := listDisks
+	listDisks = func() ([]install.Disk, error) { return slices.Clone(disks), err }
+	t.Cleanup(func() { listDisks = prev })
+}
+
+func mkDisk(path string, bytes int64) install.Disk {
+	return install.Disk{
+		Path:              path,
+		ByID:              "/dev/disk/by-id/test-" + strings.TrimPrefix(path, "/dev/"),
+		Bytes:             bytes,
+		Model:             "TESTDISK",
+		LogicalBlockSize:  512,
+		PhysicalBlockSize: 512,
+		Content:           "empty",
+	}
+}
+
 func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
-	os.Unsetenv("SPINIFEX_AUTO")
+	setEnv(t, nil)
 	cfg, err := Load()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -15,3 +60,629 @@ func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
 		t.Fatal("expected nil config when SPINIFEX_AUTO is not set")
 	}
 }
+
+func TestLoadBuildsConfigFromEnv(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_AUTO":            "1",
+		"SPINIFEX_PASSWORD":        "s3cret",
+		"SPINIFEX_HOSTNAME":        "node-7",
+		"SPINIFEX_EMAIL":           "  ops@example.com  ",
+		"SPINIFEX_WAN_IFACE":       "eno1",
+		"SPINIFEX_WAN_DNS":         "1.1.1.1,8.8.8.8",
+		"SPINIFEX_LAN_IFACE":       "eno2",
+		"SPINIFEX_LAN_VLAN":        "100",
+		"SPINIFEX_LAN_MTU":         "9000",
+		"SPINIFEX_GPU_PASSTHROUGH": "1",
+	})
+
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.Hostname != "node-7" || cfg.RootPassword != "s3cret" {
+		t.Errorf("identity = %q/%q", cfg.Hostname, cfg.RootPassword)
+	}
+	if cfg.Email != "ops@example.com" {
+		t.Errorf("Email = %q, want trimmed", cfg.Email)
+	}
+	if cfg.Storage.FS != install.FSExt4 || len(cfg.Storage.Disks) != 1 {
+		t.Errorf("Storage = %v / %d disks", cfg.Storage.FS, len(cfg.Storage.Disks))
+	}
+	if cfg.WAN.Interface != "eno1" || !cfg.WAN.DHCPMode {
+		t.Errorf("WAN = %+v, want eno1 on DHCP", cfg.WAN)
+	}
+	if !slices.Equal(cfg.WAN.DNS, []string{"1.1.1.1", "8.8.8.8"}) {
+		t.Errorf("WAN.DNS = %v", cfg.WAN.DNS)
+	}
+	if cfg.LAN.Interface != "eno2" || cfg.LAN.VLAN != 100 || cfg.LAN.MTU != 9000 {
+		t.Errorf("LAN = %+v", cfg.LAN)
+	}
+	// Only the wan plane installs a default route.
+	if cfg.LAN.Gateway != "" {
+		t.Errorf("LAN.Gateway = %q, want cleared", cfg.LAN.Gateway)
+	}
+	if !cfg.VPC.Folded() {
+		t.Errorf("VPC = %+v, want folded when no _IFACE is set", cfg.VPC)
+	}
+	if !cfg.GPUPassthrough {
+		t.Error("GPUPassthrough not enabled")
+	}
+}
+
+func TestLoadDefaultsHostname(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_AUTO":      "1",
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+	})
+
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.Hostname != "spinifex-node" {
+		t.Errorf("Hostname = %q, want spinifex-node", cfg.Hostname)
+	}
+}
+
+func TestLoadRequiresPassword(t *testing.T) {
+	setEnv(t, map[string]string{"SPINIFEX_AUTO": "1"})
+
+	cfg, err := Load()
+	if err == nil {
+		t.Fatalf("expected error, got config %+v", cfg)
+	}
+	if !strings.Contains(err.Error(), "SPINIFEX_PASSWORD") {
+		t.Errorf("err = %v, want it to name SPINIFEX_PASSWORD", err)
+	}
+	if !strings.HasPrefix(err.Error(), "autoinstall config:") {
+		t.Errorf("err = %v, want the autoinstall wrapper", err)
+	}
+}
+
+func TestBuildConfigStaticWAN(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     map[string]string
+		wantErr string
+	}{
+		{
+			name: "full static",
+			env: map[string]string{
+				"SPINIFEX_WAN_MODE": "static",
+				"SPINIFEX_WAN_IP":   "10.0.0.5",
+				"SPINIFEX_WAN_MASK": "255.255.255.0",
+				"SPINIFEX_WAN_GW":   "10.0.0.1",
+			},
+		},
+		{
+			name: "missing gateway",
+			env: map[string]string{
+				"SPINIFEX_WAN_MODE": "static",
+				"SPINIFEX_WAN_IP":   "10.0.0.5",
+				"SPINIFEX_WAN_MASK": "255.255.255.0",
+			},
+			wantErr: "SPINIFEX_WAN_GW required",
+		},
+		{
+			name:    "missing address",
+			env:     map[string]string{"SPINIFEX_WAN_MODE": "static"},
+			wantErr: "SPINIFEX_WAN_IP and SPINIFEX_WAN_MASK required",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+			env := map[string]string{
+				"SPINIFEX_PASSWORD":  "s3cret",
+				"SPINIFEX_WAN_IFACE": "eno1",
+			}
+			maps.Copy(env, tt.env)
+			setEnv(t, env)
+
+			cfg, err := buildConfig()
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("buildConfig: %v", err)
+			}
+			if cfg.WAN.DHCPMode {
+				t.Error("WAN.DHCPMode = true, want static")
+			}
+			if cfg.WAN.Address != "10.0.0.5" || cfg.WAN.Gateway != "10.0.0.1" {
+				t.Errorf("WAN = %+v", cfg.WAN)
+			}
+		})
+	}
+}
+
+// SPINIFEX_ROLE and SPINIFEX_JOIN_ADDR no longer exist: the installer cannot
+// form a multi-node cluster, so a stale cmdline from an older ISO has to be
+// inert rather than producing a half-configured node.
+func TestBuildConfigSkipFormation(t *testing.T) {
+	tests := []struct {
+		name         string
+		env          map[string]string
+		wantSkipForm bool
+	}{
+		{
+			name:         "default initializes a single-node cluster",
+			wantSkipForm: false,
+		},
+		{
+			name:         "formation deferred to the provisioner",
+			env:          map[string]string{"SPINIFEX_SKIP_FORMATION": "1"},
+			wantSkipForm: true,
+		},
+		{
+			name:         "stale join cmdline is ignored",
+			env:          map[string]string{"SPINIFEX_ROLE": "join", "SPINIFEX_JOIN_ADDR": "10.0.0.1:9999"},
+			wantSkipForm: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+			env := map[string]string{
+				"SPINIFEX_PASSWORD":  "s3cret",
+				"SPINIFEX_WAN_IFACE": "eno1",
+			}
+			maps.Copy(env, tt.env)
+			setEnv(t, env)
+
+			cfg, err := buildConfig()
+			if err != nil {
+				t.Fatalf("buildConfig: %v", err)
+			}
+			if cfg.SkipFormation != tt.wantSkipForm {
+				t.Errorf("SkipFormation = %v, want %v", cfg.SkipFormation, tt.wantSkipForm)
+			}
+		})
+	}
+}
+
+func TestBuildConfigRejectsDuplicatePlaneBinding(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+		"SPINIFEX_LAN_IFACE": "eno1",
+	})
+
+	if _, err := buildConfig(); err == nil || !strings.Contains(err.Error(), "network roles") {
+		t.Fatalf("err = %v, want a network roles validation failure", err)
+	}
+}
+
+func TestBuildConfigPropagatesStorageError(t *testing.T) {
+	setEnv(t, map[string]string{
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+		"SPINIFEX_FS":        "btrfs",
+	})
+
+	_, err := buildConfig()
+	if err == nil || !strings.HasPrefix(err.Error(), "storage:") {
+		t.Fatalf("err = %v, want a storage: prefix", err)
+	}
+}
+
+func TestParseRole(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     map[string]string
+		want    install.NetworkRole
+		wantErr string
+	}{
+		{
+			name: "dhcp by default",
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true},
+		},
+		{
+			name: "vlan and mtu",
+			env:  map[string]string{"SPINIFEX_LAN_VLAN": "42", "SPINIFEX_LAN_MTU": "9000"},
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true, VLAN: 42, MTU: 9000},
+		},
+		{
+			name: "dns list is comma separated",
+			env:  map[string]string{"SPINIFEX_LAN_DNS": "1.1.1.1,9.9.9.9"},
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true, DNS: []string{"1.1.1.1", "9.9.9.9"}},
+		},
+		{
+			name: "static",
+			env: map[string]string{
+				"SPINIFEX_LAN_MODE": "STATIC",
+				"SPINIFEX_LAN_IP":   "192.168.1.2",
+				"SPINIFEX_LAN_MASK": "255.255.255.0",
+				"SPINIFEX_LAN_GW":   "192.168.1.1",
+			},
+			want: install.NetworkRole{
+				Interface: "eno1",
+				Address:   "192.168.1.2",
+				Mask:      "255.255.255.0",
+				Gateway:   "192.168.1.1",
+			},
+		},
+		{
+			name:    "static without mask",
+			env:     map[string]string{"SPINIFEX_LAN_MODE": "static", "SPINIFEX_LAN_IP": "192.168.1.2"},
+			wantErr: "SPINIFEX_LAN_IP and SPINIFEX_LAN_MASK required",
+		},
+		{
+			name:    "unparsable vlan",
+			env:     map[string]string{"SPINIFEX_LAN_VLAN": "trunk"},
+			wantErr: "SPINIFEX_LAN_VLAN",
+		},
+		{
+			name:    "unparsable mtu",
+			env:     map[string]string{"SPINIFEX_LAN_MTU": "jumbo"},
+			wantErr: "SPINIFEX_LAN_MTU",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			setEnv(t, tt.env)
+
+			got, err := parseRole("LAN", "eno1")
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("parseRole: %v", err)
+			}
+			if fmt.Sprintf("%+v", got) != fmt.Sprintf("%+v", tt.want) {
+				t.Errorf("role = %+v, want %+v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseZFSOpts(t *testing.T) {
+	t.Run("unset leaves the computed defaults", func(t *testing.T) {
+		setEnv(t, nil)
+		got, err := parseZFSOpts()
+		if err != nil {
+			t.Fatalf("parseZFSOpts: %v", err)
+		}
+		if got != (install.ZFSOpts{}) {
+			t.Errorf("opts = %+v, want zero", got)
+		}
+	})
+
+	t.Run("all tunables", func(t *testing.T) {
+		setEnv(t, map[string]string{
+			"SPINIFEX_ZFS_ASHIFT":      " 12 ",
+			"SPINIFEX_ZFS_COPIES":      "2",
+			"SPINIFEX_ZFS_ARC_MAX_MIB": "4096",
+			"SPINIFEX_ZFS_COMPRESS":    " lz4 ",
+			"SPINIFEX_ZFS_CHECKSUM":    "blake3",
+		})
+		got, err := parseZFSOpts()
+		if err != nil {
+			t.Fatalf("parseZFSOpts: %v", err)
+		}
+		want := install.ZFSOpts{Ashift: 12, Copies: 2, ARCMaxMiB: 4096, Compress: "lz4", Checksum: "blake3"}
+		if got != want {
+			t.Errorf("opts = %+v, want %+v", got, want)
+		}
+	})
+
+	for _, env := range []string{"SPINIFEX_ZFS_ASHIFT", "SPINIFEX_ZFS_COPIES", "SPINIFEX_ZFS_ARC_MAX_MIB"} {
+		t.Run("unparsable "+env, func(t *testing.T) {
+			setEnv(t, map[string]string{env: "auto"})
+			if _, err := parseZFSOpts(); err == nil || !strings.Contains(err.Error(), env) {
+				t.Fatalf("err = %v, want it to name %s", err, env)
+			}
+		})
+	}
+}
+
+func TestResolveStorage(t *testing.T) {
+	four := []install.Disk{
+		mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 40*gib),
+		mkDisk("/dev/vdd", 40*gib), mkDisk("/dev/vde", 40*gib),
+	}
+
+	tests := []struct {
+		name      string
+		disks     []install.Disk
+		env       map[string]string
+		wantFS    install.FSType
+		wantPaths []string
+		wantErr   string
+	}{
+		{
+			name:      "ext4 single disk by default",
+			disks:     four[:1],
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb"},
+		},
+		{
+			name:      "explicit member list preserves order",
+			disks:     four,
+			env:       map[string]string{"SPINIFEX_FS": "zfs-raid10", "SPINIFEX_DISKS": "vde,/dev/vdd,vdc, vdb "},
+			wantFS:    install.FSZFSRAID10,
+			wantPaths: []string{"/dev/vde", "/dev/vdd", "/dev/vdc", "/dev/vdb"},
+		},
+		{
+			name:    "unknown filesystem",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_FS": "btrfs"},
+			wantErr: "unknown filesystem",
+		},
+		{
+			// Choosing which disks to erase is not an unattended decision.
+			name:    "multi-disk pool without an explicit list",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_FS": "zfs-raid1"},
+			wantErr: "set SPINIFEX_DISKS to an explicit comma-separated list",
+		},
+		{
+			name:    "member that is not present",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_DISKS": "vdz"},
+			wantErr: `"vdz" is not an available disk`,
+		},
+		{
+			name:    "member list is for pools, not ext4 roles",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_DISKS": "vdb,vdc"},
+			wantErr: "ext4 takes one OS disk",
+		},
+		{
+			name:  "dedicated role drives, os first",
+			disks: four,
+			env: map[string]string{
+				"SPINIFEX_DISK":            "vdb",
+				"SPINIFEX_DISK_SPINIFEX":   "vdc",
+				"SPINIFEX_DISK_PREDASTORE": "vdd",
+			},
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb", "/dev/vdc", "/dev/vdd"},
+		},
+		{
+			// The two-drive layout: predastore gets its own disk while
+			// /var/lib/spinifex stays on the root filesystem.
+			name:      "predastore alone",
+			disks:     four,
+			env:       map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_PREDASTORE": "vdc"},
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb", "/dev/vdc"},
+		},
+		{
+			name:  "role drive on a pool",
+			disks: four,
+			env: map[string]string{
+				"SPINIFEX_FS": "zfs-raid1", "SPINIFEX_DISKS": "vdb,vdc",
+				"SPINIFEX_DISK_SPINIFEX": "vdd",
+			},
+			wantErr: "SPINIFEX_DISK_SPINIFEX applies to ext4 only",
+		},
+		{
+			name:    "role drive that is not present",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_PREDASTORE": "vdz"},
+			wantErr: "SPINIFEX_DISK_PREDASTORE",
+		},
+		{
+			// Two roles cannot share a drive; the disk is erased once and the
+			// second mount would land on top of the first.
+			name:    "role drive repeats the os disk",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_SPINIFEX": "vdb"},
+			wantErr: "selected more than once",
+		},
+		{
+			name:    "bad zfs tunable",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_ZFS_ASHIFT": "big"},
+			wantErr: "SPINIFEX_ZFS_ASHIFT",
+		},
+		{
+			name:    "scan failure",
+			disks:   nil,
+			env:     map[string]string{"SPINIFEX_DISKS": "vdb"},
+			wantErr: "sysfs unavailable",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var scanErr error
+			if tt.disks == nil {
+				scanErr = errors.New("sysfs unavailable")
+			}
+			withDisks(t, tt.disks, scanErr)
+			setEnv(t, tt.env)
+
+			got, err := resolveStorage()
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveStorage: %v", err)
+			}
+			if got.FS != tt.wantFS {
+				t.Errorf("FS = %v, want %v", got.FS, tt.wantFS)
+			}
+			if !slices.Equal(got.Paths(), tt.wantPaths) {
+				t.Errorf("Paths = %v, want %v", got.Paths(), tt.wantPaths)
+			}
+		})
+	}
+}
+
+func TestResolveDisk(t *testing.T) {
+	small := mkDisk("/dev/vdb", 40*gib)
+	large := mkDisk("/dev/vdc", 400*gib)
+	nvme := mkDisk("/dev/nvme0n1", 200*gib)
+
+	tests := []struct {
+		name     string
+		disks    []install.Disk
+		target   string
+		wantPath string
+		wantErr  string
+	}{
+		{name: "auto with one candidate", disks: []install.Disk{small}, target: "auto", wantPath: "/dev/vdb"},
+		{name: "empty target is auto", disks: []install.Disk{small}, wantPath: "/dev/vdb"},
+		{
+			name:    "auto with several candidates",
+			disks:   []install.Disk{small, large},
+			target:  "auto",
+			wantErr: "expected exactly one disk, found 2",
+		},
+		{name: "largest", disks: []install.Disk{small, large, nvme}, target: "LARGEST", wantPath: "/dev/vdc"},
+		{name: "smallest", disks: []install.Disk{large, small, nvme}, target: " smallest ", wantPath: "/dev/vdb"},
+		{name: "nvme", disks: []install.Disk{small, large, nvme}, target: "nvme", wantPath: "/dev/nvme0n1"},
+		{
+			name:    "nvme with none present",
+			disks:   []install.Disk{small, large},
+			target:  "nvme",
+			wantErr: "expected exactly one NVMe disk, found 0",
+		},
+		{name: "exact path", disks: []install.Disk{small, large}, target: "/dev/vdc", wantPath: "/dev/vdc"},
+		{
+			name:    "path that is not present",
+			disks:   []install.Disk{small},
+			target:  "/dev/vdz",
+			wantErr: "is not an available disk",
+		},
+		{name: "no candidates at all", disks: []install.Disk{}, target: "largest", wantErr: "no non-removable disks found"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, tt.disks, nil)
+
+			got, err := resolveDisk(tt.target)
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveDisk(%q): %v", tt.target, err)
+			}
+			if got.Path != tt.wantPath {
+				t.Errorf("path = %q, want %q", got.Path, tt.wantPath)
+			}
+		})
+	}
+}
+
+func TestCandidateDisksExcludesRemovableAndLiveMedia(t *testing.T) {
+	fixed := mkDisk("/dev/vdb", 40*gib)
+	usb := mkDisk("/dev/sdz", 16*gib)
+	usb.Removable = true
+	boot := mkDisk("/dev/sdy", 16*gib)
+	boot.LiveMedia = true
+	withDisks(t, []install.Disk{usb, fixed, boot}, nil)
+
+	got, err := candidateDisks()
+	if err != nil {
+		t.Fatalf("candidateDisks: %v", err)
+	}
+	if len(got) != 1 || got[0].Path != "/dev/vdb" {
+		t.Fatalf("candidates = %v, want only /dev/vdb", got)
+	}
+}
+
+func TestCandidateDisksPropagatesScanError(t *testing.T) {
+	withDisks(t, nil, errors.New("read /sys/block: denied"))
+	if _, err := candidateDisks(); err == nil {
+		t.Fatal("expected the scan error to propagate")
+	}
+}
+
+func TestPickBySizeRejectsEmptySet(t *testing.T) {
+	if _, err := pickBySize(nil, true); err == nil {
+		t.Fatal("expected an error for an empty disk set")
+	}
+}
+
+func TestPickBySizeLeavesInputOrderIntact(t *testing.T) {
+	disks := []install.Disk{mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 400*gib)}
+	if _, err := pickBySize(disks, false); err != nil {
+		t.Fatalf("pickBySize: %v", err)
+	}
+	if disks[0].Path != "/dev/vdb" {
+		t.Errorf("input reordered: %v", disks)
+	}
+}
+
+func TestDiskList(t *testing.T) {
+	got := diskList([]install.Disk{mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 2048*gib)})
+	want := "  /dev/vdb (40.0G, empty)\n  /dev/vdc (2.0T, empty)"
+	if got != want {
+		t.Errorf("diskList =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestResolveNICExplicitName(t *testing.T) {
+	got, err := resolveNIC("eno1", "")
+	if err != nil {
+		t.Fatalf("resolveNIC: %v", err)
+	}
+	if got != "eno1" {
+		t.Errorf("nic = %q, want eno1", got)
+	}
+}
+
+// The auto path reads the host's real interfaces, so assert the selection rules
+// rather than a specific name.
+func TestResolveNICAutoSkipsVirtualInterfaces(t *testing.T) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		t.Skipf("no interface list on this host: %v", err)
+	}
+
+	got, err := resolveNIC("auto", "")
+	if err != nil {
+		if len(ifaces) > 0 {
+			t.Logf("no physical NIC among %d interfaces: %v", len(ifaces), err)
+		}
+		return
+	}
+
+	idx := slices.IndexFunc(ifaces, func(i net.Interface) bool { return i.Name == got })
+	if idx < 0 {
+		t.Fatalf("resolveNIC returned %q, which is not a host interface", got)
+	}
+	sel := ifaces[idx]
+	if sel.Flags&net.FlagLoopback != 0 || sel.Flags&net.FlagBroadcast == 0 || len(sel.HardwareAddr) == 0 {
+		t.Errorf("selected %q with flags %v and MAC %q", got, sel.Flags, sel.HardwareAddr)
+	}
+	for _, pfx := range virtualNICPrefixes {
+		if strings.HasPrefix(got, pfx) {
+			t.Errorf("selected virtual interface %q (prefix %q)", got, pfx)
+		}
+	}
+}
+
+func TestResolveNICAutoHonoursExclude(t *testing.T) {
+	first, err := resolveNIC("auto", "")
+	if err != nil {
+		t.Skipf("no physical NIC on this host: %v", err)
+	}
+	got, err := resolveNIC("", first)
+	if err == nil && got == first {
+		t.Errorf("resolveNIC returned the excluded interface %q", first)
+	}
+}

cmd/installer/firstboot/firstboot.go

@@ -14,14 +14,23 @@ import (
 // Config holds the values the firstboot service needs to configure the node.
 type Config struct {
 	Hostname string
-	// EncapIP is the Geneve tunnel IP for OVN. Set to the LAN bridge IP when a
-	// dedicated LAN NIC is present, otherwise the WAN bridge IP. Empty when DHCP
-	// is used — setup-ovn.sh auto-detects the IP from the default route in that case.
+	// EncapIP is the Geneve tunnel IP for OVN, taken from the vpc plane after
+	// collapsing (vpc <- lan <- wan). Empty when that plane uses DHCP —
+	// setup-ovn.sh auto-detects the IP from the default route in that case.
 	EncapIP string
-	// ClusterRole is "init" or "join".
-	ClusterRole string
-	// JoinAddr is host:port of the primary node, only used when ClusterRole is "join".
-	JoinAddr string
+	// LANIP is the internal cluster address, taken from the lan plane after
+	// collapsing. Passed as --bind and --cluster-bind so predastore replication,
+	// the NATS mesh and OVN control traffic stay off the public interface.
+	// Empty leaves the 0.0.0.0 wildcard default, which is correct for a
+	// single-NIC node where lan folds onto wan.
+	LANIP string
+	// WANIP is the public address, taken from the wan plane. Passed as
+	// --advertise so northstar's :53 listener, the awsgw registry host and the
+	// dial target recorded for off-host clients stay on the public interface.
+	// Without it a concrete --bind is echoed back as the advertise address,
+	// which would move all three onto the internal plane. Empty when wan uses
+	// DHCP, leaving spx to auto-detect from the default route.
+	WANIP string
 	// Email is the operator email collected by the TUI or SPINIFEX_EMAIL on
 	// the headless path. Passed to `spx admin init --email=<value>` when set;
 	// omitted entirely when empty.
@@ -147,7 +156,8 @@ resolvconf -u
 # Write the banner (first time, spinifex-banner service will do this on reboot)
 /usr/local/bin/spx admin banner --boot-check
 
-# Cluster formation — capture credentials to file for display on console.
+# Cluster formation. Output goes to the journal via the systemd unit; spx
+# admin init writes credentials to disk and never prints them.
 %s 2>&1
 
 # Fix ownership of files spx admin init wrote. spx runs as root under
@@ -197,6 +207,9 @@ fi
 
 # Enable services to start, on reboot
 systemctl enable spinifex.target spinifex-banner.service
+# WantedBy=timers.target only self-activates once enabled — without this the
+# JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart.
+systemctl enable --now spinifex-nats-watchdog.timer
 
 # Mark complete only after every step above has succeeded. Until this point,
 # any failure (set -e) leaves the marker absent and the next reboot retries
@@ -244,14 +257,67 @@ func buildClusterCmd(cfg Config) string {
 	if cfg.GPUPassthrough {
 		gpuFlag = " --gpu-passthrough"
 	}
-	switch cfg.ClusterRole {
-	case "join":
-		return fmt.Sprintf("spx admin join --node %s --host %s%s", cfg.Hostname, cfg.JoinAddr, emailFlag)
-	default:
-		return fmt.Sprintf("spx admin init --node %s --nodes 1%s%s", cfg.Hostname, emailFlag, gpuFlag)
+	// Without these the node takes the 0.0.0.0 wildcard default and every
+	// internal service resolves onto the auto-detected WAN address, which is
+	// exactly what the three-plane model exists to prevent.
+	bindFlags := ""
+	if cfg.LANIP != "" {
+		bindFlags = fmt.Sprintf(" --bind %s --cluster-bind %s", cfg.LANIP, cfg.LANIP)
 	}
+	// --advertise must be explicit whenever --bind is: spx echoes a concrete
+	// bind address straight back as the advertise address and never reaches its
+	// WAN auto-detection, which would silently publish the internal plane as
+	// this node's public dial target.
+	preamble := ""
+	switch {
+	case bindFlags == "":
+		// Nothing pinned, so spx auto-detects both and a guessed advertise
+		// address would only get in the way.
+	case cfg.WANIP != "":
+		bindFlags += " --advertise " + cfg.WANIP
+	default:
+		// A DHCP wan has no address at install time, but it does by the time
+		// firstboot runs, so read it off the bridge instead of shipping the
+		// bind address as the public one.
+		preamble = wanAdvertisePreamble
+		bindFlags += ` $SPX_ADVERTISE`
+	}
+
+	// Always a single-node cluster. The installer cannot form a multi-node one:
+	// cluster membership decides the OVN database topology and the join token
+	// only exists once the primary has booted, so neither is knowable while the
+	// nodes are still being installed. Multi-node is a post-install conversion,
+	// documented in the multi-node install guide.
+	cmd := fmt.Sprintf("spx admin init --node %s --nodes 1%s%s%s", cfg.Hostname, bindFlags, emailFlag, gpuFlag)
+	return preamble + cmd
 }
 
+// wanAdvertisePreamble resolves the wan plane's DHCP lease into $SPX_ADVERTISE
+// just before formation. br-wan is always the wan bridge — that plane is the one
+// role that cannot fold — so its address is the node's public identity.
+//
+// Falling through with SPX_ADVERTISE empty is deliberate: no lease means there
+// is no public address to advertise yet, and spx echoing the bind address is
+// still better than formation failing outright.
+const wanAdvertisePreamble = `# The wan plane leases its address, so it is only knowable at boot. Without this
+# spx would echo --bind back as the advertise address and publish the internal
+# plane as this node's public dial target.
+SPX_ADVERTISE=""
+echo "[firstboot] waiting for the wan plane to acquire an address..."
+for _i in $(seq 1 60); do
+    _wan_ip=$(ip -4 -o addr show br-wan scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1)
+    if [ -n "$_wan_ip" ]; then
+        SPX_ADVERTISE=" --advertise $_wan_ip"
+        echo "[firstboot] wan address $_wan_ip (${_i}s)"
+        break
+    fi
+    sleep 1
+done
+if [ -z "$SPX_ADVERTISE" ]; then
+    echo "[firstboot] warning: br-wan has no address after 60s — this node will advertise its bind address"
+fi
+`
+
 // shellEscapeSingle wraps s in single quotes with any embedded single
 // quotes escaped. Minimal — we only need this because the email value is
 // interpolated into a shell script written by Write().

cmd/installer/firstboot/firstboot_test.go

@@ -2,6 +2,7 @@ package firstboot
 
 import (
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -24,7 +25,7 @@ func TestWriteScriptNoCallbackWhenEmpty(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	cfg := Config{Hostname: "test-node"}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -45,7 +46,6 @@ func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
 	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	cfg := Config{
 		Hostname:        "test-node",
-		ClusterRole:     "init",
 		InstallCallback: callbackURL,
 	}
 	if err := Write(root, cfg); err != nil {
@@ -70,7 +70,7 @@ func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	cfg := Config{Hostname: "test-node"}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -88,7 +88,7 @@ func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
+	cfg := Config{Hostname: "test-node", SkipFormation: true}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -114,6 +114,25 @@ func readScript(t *testing.T, root string) string {
 	return string(b)
 }
 
+// TestWriteScriptEnablesNatsWatchdogTimer pins that firstboot activates the
+// JetStream ENOSPC-latch watchdog timer. The unit file's WantedBy=timers.target
+// only takes effect once enabled — without this line the watchdog is dropped
+// onto disk but never runs, and a full disk needs a manual nats restart forever.
+func TestWriteScriptEnablesNatsWatchdogTimer(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node"}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if !strings.Contains(content, "systemctl enable --now spinifex-nats-watchdog.timer") {
+		t.Error("firstboot script must enable --now spinifex-nats-watchdog.timer")
+	}
+}
+
 func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
@@ -121,7 +140,6 @@ func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	cfg := Config{
 		Hostname:        "node1",
-		ClusterRole:     "init",
 		InstallCallback: callbackURL,
 	}
 	if err := Write(root, cfg); err != nil {
@@ -146,3 +164,121 @@ func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 		t.Error("curl must appear after done marker write")
 	}
 }
+
+// A multi-NIC node must keep cluster traffic on the internal plane while still
+// publishing the public one. --advertise has to be explicit to do that: spx
+// returns a concrete --bind verbatim as the advertise address and never reaches
+// its WAN auto-detection, so omitting it moves northstar's :53 listener and the
+// off-host dial target onto the internal plane.
+func TestBuildClusterCmdSeparatesBindFromAdvertise(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "hydrogen",
+		LANIP:    "10.0.0.3",
+		WANIP:    "216.218.163.99",
+	})
+	for _, want := range []string{
+		"--bind 10.0.0.3",
+		"--cluster-bind 10.0.0.3",
+		"--advertise 216.218.163.99",
+	} {
+		if !strings.Contains(cmd, want) {
+			t.Errorf("missing %q in:\n%s", want, cmd)
+		}
+	}
+}
+
+// Without a bind address there is nothing to correct: spx auto-detects both,
+// and a stray --advertise would pin the node to an address the installer only
+// guessed at. A single-NIC node also collapses wan and lan onto one address,
+// where the split is meaningless.
+func TestBuildClusterCmdOmitsAdvertiseWithoutBind(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "node1",
+		WANIP:    "216.218.163.99",
+	})
+	if strings.Contains(cmd, "--advertise") {
+		t.Errorf("advertise must not be passed without a bind address:\n%s", cmd)
+	}
+}
+
+// A DHCP wan has no address at install time, so the advertise value has to be
+// resolved at boot. Shipping --bind alone here would republish the internal
+// plane as the node's public dial target, which is the case the static path
+// already guards against.
+func TestBuildClusterCmdResolvesDHCPWanAtBoot(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "hydrogen",
+		LANIP:    "10.0.0.3",
+		// WANIP empty — the wan plane is on DHCP.
+	})
+	if !strings.Contains(cmd, "ip -4 -o addr show br-wan") {
+		t.Errorf("must read the wan address off the bridge:\n%s", cmd)
+	}
+	if !strings.Contains(cmd, "$SPX_ADVERTISE") {
+		t.Errorf("resolved address must reach the command:\n%s", cmd)
+	}
+	// The preamble has to run before the command that consumes it.
+	if strings.Index(cmd, "SPX_ADVERTISE=\"\"") > strings.Index(cmd, "spx admin init") {
+		t.Errorf("preamble must precede the spx invocation:\n%s", cmd)
+	}
+	if !strings.Contains(cmd, "--bind 10.0.0.3") {
+		t.Errorf("bind still comes from the lan plane:\n%s", cmd)
+	}
+}
+
+// With no bind address there is nothing to correct, so neither the literal flag
+// nor the boot-time lookup should appear — spx auto-detects both ends.
+func TestBuildClusterCmdSkipsWanLookupWithoutBind(t *testing.T) {
+	cmd := buildClusterCmd(Config{Hostname: "node1"})
+	if strings.Contains(cmd, "SPX_ADVERTISE") || strings.Contains(cmd, "--advertise") {
+		t.Errorf("no bind means no advertise handling at all:\n%s", cmd)
+	}
+}
+
+// The advertise handling injects a shell preamble into the generated script, so
+// the permutations that change its shape are syntax-checked rather than only
+// string-matched — a broken quote here bricks first boot with no installer error.
+func TestGeneratedScriptIsValidShell(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		cfg  Config
+	}{
+		{"dhcp wan resolves at boot", Config{Hostname: "hydrogen", LANIP: "10.0.0.3"}},
+		{"static wan is inlined", Config{Hostname: "hydrogen", LANIP: "10.0.0.3", WANIP: "216.218.163.99"}},
+		{"single nic, nothing pinned", Config{Hostname: "node1"}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			root := t.TempDir()
+			makeRootDirs(t, root)
+			if err := Write(root, tc.cfg); err != nil {
+				t.Fatalf("Write: %v", err)
+			}
+			path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
+			if out, err := exec.Command("bash", "-n", path).CombinedOutput(); err != nil {
+				script, _ := os.ReadFile(path)
+				t.Fatalf("generated script is not valid shell: %v\n%s\n---\n%s", err, out, script)
+			}
+		})
+	}
+}
+
+// The installer cannot form a multi-node cluster: membership decides the OVN
+// database topology, and the join token only exists once the primary has
+// booted. Neither is knowable at install time, so firstboot must only ever
+// initialize a single-node cluster and leave multi-node to a post-install
+// conversion.
+func TestBuildClusterCmdNeverJoins(t *testing.T) {
+	for _, cfg := range []Config{
+		{Hostname: "node1"},
+		{Hostname: "node1", LANIP: "10.0.0.3", WANIP: "216.218.163.99"},
+		{Hostname: "node1", LANIP: "10.0.0.3"},
+	} {
+		cmd := buildClusterCmd(cfg)
+		if strings.Contains(cmd, "spx admin join") {
+			t.Errorf("firstboot must never join a cluster:\n%s", cmd)
+		}
+		if !strings.Contains(cmd, "--nodes 1") {
+			t.Errorf("want a single-node init, got:\n%s", cmd)
+		}
+	}
+}

cmd/installer/install/assets/grub-install-wrapper

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Diversion wrapper for grub-install on a spinifex-boot-tool system.
+#
+# On ZFS root the bootloader lives on per-disk ESPs, not in /boot. A plain
+# grub-install writes a core image that tries to read the pool, which works
+# until the next OpenZFS feature flag lands and then silently does not. Worse,
+# the grub-pc and shim packages run grub-install from their own postinst, so an
+# ordinary `apt upgrade` would undo the layout with no visible error.
+#
+# Mirrors the wrapper Proxmox ships for the same reason.
+
+set -e
+
+REAL="/usr/sbin/grub-install.real"
+TOOL="/usr/sbin/spinifex-boot-tool"
+
+# Not a managed system: behave exactly as the real grub-install.
+if ! "$TOOL" status --quiet 2>/dev/null; then
+	exec "$REAL" "$@"
+fi
+
+# dpkg sets this when a maintainer script is running. The bootloader packages
+# reinstalling themselves is expected and is redirected, not refused.
+case "${DPKG_MAINTSCRIPT_PACKAGE:-}" in
+grub-* | shim-*)
+	echo "This system is managed by spinifex-boot-tool; redirecting grub-install to 'spinifex-boot-tool reinit'." >&2
+	exec "$TOOL" reinit
+	;;
+esac
+
+cat >&2 <<EOF
+E: This system's bootloader is managed by spinifex-boot-tool.
+
+Running grub-install directly would install GRUB somewhere the firmware does
+not read, and would leave the per-disk ESPs holding a stale kernel.
+
+Use instead:
+  spinifex-boot-tool refresh     re-sync kernels and grub.cfg to every ESP
+  spinifex-boot-tool reinit      reinstall GRUB on every ESP
+  spinifex-boot-tool status      list the ESPs under management
+
+To run the real grub-install anyway (you probably do not want this):
+  $REAL $*
+EOF
+exit 1

cmd/installer/install/assets/spinifex-boot-tool

@@ -0,0 +1,412 @@
+#!/bin/bash
+# spinifex-boot-tool — keep a bootloader on every disk of a ZFS root pool.
+#
+# GRUB cannot be trusted to read a ZFS pool: its ZFS reader lags OpenZFS by
+# years, and enabling a new feature flag on the pool can make the machine
+# unbootable with no warning. So /boot on the pool is never read by firmware.
+# Instead each pool member carries a small FAT32 ESP holding its own copy of
+# GRUB, the kernels and the initramfs images, and this tool keeps them in step.
+#
+# Modelled directly on Proxmox's proxmox-boot-tool, which has run this design
+# across their entire installed base for years.
+
+set -u
+set -o pipefail
+
+ESP_LIST="/etc/kernel/spinifex-boot-uuids"
+ESPMOUNTS="/var/tmp/espmounts"
+MARKER="spinifex-boot-uuids"
+
+# Kernels are only ever copied out of the real /boot, which lives on the pool.
+# Once an ESP is bind-mounted over /boot this variable is how we still find it.
+BOOTROOT="/boot"
+
+warn() { echo "W: $*" >&2; }
+die() { echo "E: $*" >&2; exit 1; }
+
+usage() {
+	cat <<EOF
+Usage: $(basename "$0") <command>
+
+  format <partition> [--force]  make <partition> a Spinifex ESP
+  init <partition>              register <partition> and install a bootloader on it
+  clean [--dry-run]             drop ESPs that are no longer present
+  refresh [--hook <name>]       re-sync kernels and grub.cfg to every ESP
+  reinit                        re-run refresh, reinstalling GRUB on every ESP
+  kernel <add|remove> <ver>     pin or unpin a kernel version
+  status [--quiet]              list registered ESPs
+EOF
+}
+
+# reexec_in_mountns re-runs this script inside a private mount namespace. Every
+# code path here bind-mounts an ESP over /boot; if the script is killed at the
+# wrong moment outside a namespace, the running system is left with its real
+# /boot masked and the next kernel upgrade writes into a FAT partition.
+reexec_in_mountns() {
+	if [ "${SPINIFEX_BOOT_TOOL_NS:-0}" = "1" ]; then
+		return 0
+	fi
+	export SPINIFEX_BOOT_TOOL_NS=1
+	# --propagation private stops the bind mounts leaking back to the host.
+	exec unshare --mount --propagation private -- "$0" "$@"
+}
+
+# --- ESP registry -----------------------------------------------------------
+
+# esp_list prints the registered ESP filesystem UUIDs.
+esp_list() {
+	[ -f "$ESP_LIST" ] || return 0
+	grep -v '^[[:space:]]*\(#\|$\)' "$ESP_LIST" || true
+}
+
+# uuid_of prints the FAT filesystem UUID of a partition.
+uuid_of() {
+	blkid "$1" -o value -s UUID 2>/dev/null
+}
+
+# path_for_uuid resolves a registered UUID back to a device, or nothing if the
+# disk has been pulled.
+path_for_uuid() {
+	local dev
+	dev="/dev/disk/by-uuid/$1"
+	[ -e "$dev" ] || return 1
+	readlink -f "$dev"
+}
+
+# mount_esp mounts the ESP for a UUID and echoes its mountpoint.
+mount_esp() {
+	local uuid="$1" dev mp
+	dev=$(path_for_uuid "$uuid") || return 1
+	mp="$ESPMOUNTS/$uuid"
+	mkdir -p "$mp"
+	mountpoint -q "$mp" || mount "$dev" "$mp" || return 1
+	echo "$mp"
+}
+
+umount_esp() {
+	local mp="$ESPMOUNTS/$1"
+	mountpoint -q "$mp" && umount "$mp"
+	rmdir "$mp" 2>/dev/null || true
+}
+
+# --- format / init ----------------------------------------------------------
+
+do_format() {
+	local part="$1" force="${2:-}"
+	[ -b "$part" ] || die "$part is not a block device"
+
+	if [ "$force" != "--force" ]; then
+		local existing
+		existing=$(blkid -o value -s TYPE "$part" 2>/dev/null || true)
+		[ -z "$existing" ] || die "$part already contains a $existing filesystem; pass --force to overwrite"
+	fi
+	mkfs.vfat -F 32 "$part" || die "mkfs.vfat failed on $part"
+}
+
+do_init() {
+	local part="$1"
+	[ -b "$part" ] || die "$part is not a block device"
+
+	local uuid
+	uuid=$(uuid_of "$part") || true
+	[ -n "$uuid" ] || die "$part has no filesystem UUID — run '$(basename "$0") format $part' first"
+
+	mkdir -p "$(dirname "$ESP_LIST")"
+	touch "$ESP_LIST"
+	if ! grep -qx "$uuid" "$ESP_LIST"; then
+		echo "$uuid" >>"$ESP_LIST"
+	fi
+	echo "Registered $part ($uuid)"
+
+	install_hooks
+	# Only the ESP just registered: the others are already current, and
+	# refreshing all of them here makes registering N disks an N-squared
+	# sequence of grub-installs.
+	refresh_one "$uuid" || die "could not make $part bootable"
+}
+
+do_clean() {
+	local dry="${1:-}" uuid keep=()
+	for uuid in $(esp_list); do
+		if path_for_uuid "$uuid" >/dev/null 2>&1; then
+			keep+=("$uuid")
+		else
+			echo "Removing stale ESP $uuid"
+		fi
+	done
+	if [ "$dry" = "--dry-run" ]; then
+		return 0
+	fi
+	printf '%s\n' "${keep[@]+"${keep[@]}"}" >"$ESP_LIST"
+}
+
+# --- refresh ----------------------------------------------------------------
+
+# kernel_versions lists the kernels present in the real /boot, newest first, so
+# the default menu entry is the newest kernel.
+kernel_versions() {
+	local img
+	for img in "$BOOTROOT"/vmlinuz-*; do
+		[ -e "$img" ] || continue
+		basename "$img" | sed 's/^vmlinuz-//'
+	done | sort -Vr
+}
+
+# sync_esp copies the current kernels and initramfs images onto one ESP and
+# removes anything left from a kernel that is no longer installed — an ESP is
+# small and a stale 100MB initramfs is what makes the next upgrade fail.
+sync_esp() {
+	local mp="$1" kver f base
+
+	local -A wanted=()
+	for kver in $(kernel_versions); do
+		wanted["vmlinuz-$kver"]=1
+		wanted["initrd.img-$kver"]=1
+	done
+
+	for f in "$mp"/vmlinuz-* "$mp"/initrd.img-*; do
+		[ -e "$f" ] || continue
+		base=$(basename "$f")
+		[ -n "${wanted[$base]:-}" ] || rm -f "$f"
+	done
+
+	for kver in $(kernel_versions); do
+		for base in "vmlinuz-$kver" "initrd.img-$kver"; do
+			[ -e "$BOOTROOT/$base" ] || continue
+			# Copy to a temp name and rename: a half-written kernel on the ESP
+			# is a machine that does not come back from a reboot.
+			cp -f "$BOOTROOT/$base" "$mp/$base.tmp" || return 1
+			mv -f "$mp/$base.tmp" "$mp/$base" || return 1
+		done
+	done
+
+	# GRUB reads its theme from $prefix, which is the ESP's own grub directory,
+	# so the branding has to be copied out of /boot alongside the kernels.
+	mkdir -p "$mp/grub/fonts"
+	for f in grub/theme.txt grub/splash.png grub/fonts/unicode.pf2; do
+		[ -e "$BOOTROOT/$f" ] && cp -f "$BOOTROOT/$f" "$mp/$f"
+	done
+
+	# Marks the partition as ours, so a later `format` on the wrong disk is
+	# obviously a mistake rather than a silent overwrite.
+	echo "$MARKER" >"$mp/$MARKER"
+	return 0
+}
+
+# grub_install_bin echoes the real grub-install. On a managed system the normal
+# name is a wrapper that refuses to run, and it must not intercept this tool's
+# own calls — that would make the refusal a self-deadlock.
+grub_install_bin() {
+	if [ -x /usr/sbin/grub-install.real ]; then
+		echo /usr/sbin/grub-install.real
+	else
+		echo grub-install
+	fi
+}
+
+# install_grub puts GRUB itself on one ESP, for both firmware types.
+#
+# --boot-directory points at the ESP, not at /boot, so the GRUB core image
+# embeds a FAT reader and a search for the ESP's own UUID. It never needs to
+# read the pool, which is the entire point of this tool.
+install_grub() {
+	local mp="$1" dev="$2" rc=0 grub
+	grub="$(grub_install_bin)"
+
+	if [ ! -d /sys/firmware/efi ]; then
+		"$grub" --target=i386-pc --boot-directory="$mp" --recheck "$dev" || rc=1
+		return $rc
+	fi
+
+	# NVRAM is written at install time only. A refresh runs on every kernel
+	# upgrade, and rewriting boot variables that often has bricked firmware on
+	# more than one vendor's boards.
+	local nvram="--no-nvram"
+	[ "${SPINIFEX_BOOT_TOOL_NVRAM:-0}" = "1" ] && nvram=""
+
+	# Twice: once as a named entry, and once at the removable path, which is all
+	# some firmware will ever look at and is what keeps the disk bootable after
+	# its NVRAM entry is lost or the drive is moved to another machine.
+	# shellcheck disable=SC2086 # $nvram is a single optional flag, not a path
+	"$grub" --target=x86_64-efi \
+		--efi-directory="$mp" --boot-directory="$mp" \
+		--bootloader-id=spinifex $nvram --recheck "$dev" || rc=1
+	"$grub" --target=x86_64-efi \
+		--efi-directory="$mp" --boot-directory="$mp" \
+		--bootloader-id=spinifex --removable --no-nvram --recheck "$dev" || rc=1
+	return $rc
+}
+
+# update_grub_on regenerates grub.cfg for one ESP.
+#
+# The bind mount is the trick that makes this work: grub-mkconfig probes the
+# device behind /boot to decide how to find the kernel at boot. With the ESP
+# mounted there it emits a FAT UUID search and paths relative to the ESP root,
+# so the generated config matches what the embedded core image can actually do.
+update_grub_on() {
+	local mp="$1" rc=0
+	mount --bind "$mp" "$BOOTROOT" || return 1
+	grub-mkconfig -o "$BOOTROOT/grub/grub.cfg" || rc=1
+	umount "$BOOTROOT" || warn "could not unmount the bind mount over $BOOTROOT"
+	return $rc
+}
+
+# refresh_one makes a single registered ESP independently bootable: kernels,
+# bootloader, and a grub.cfg generated against that ESP's own UUID.
+refresh_one() {
+	local uuid="$1" mp dev disk rc=0
+
+	if ! dev=$(path_for_uuid "$uuid"); then
+		warn "ESP $uuid is not present, skipping"
+		return 1
+	fi
+	if ! mp=$(mount_esp "$uuid"); then
+		warn "could not mount ESP $uuid"
+		return 1
+	fi
+
+	# The whole disk, not the partition: BIOS GRUB embeds into the boot
+	# sector and the BIOS boot partition, both of which live on the disk.
+	disk=$(lsblk -no pkname "$dev")
+	disk="/dev/$disk"
+
+	if sync_esp "$mp" && install_grub "$mp" "$disk" && update_grub_on "$mp"; then
+		echo "Synced $dev ($uuid)"
+	else
+		warn "failed to update ESP $uuid on $dev"
+		rc=1
+	fi
+	umount_esp "$uuid"
+	return $rc
+}
+
+do_refresh() {
+	local uuid failed=0 ok=0 total=0
+
+	local uuids
+	uuids=$(esp_list)
+	if [ -z "$uuids" ]; then
+		echo "No ESPs registered, nothing to do."
+		return 0
+	fi
+
+	for uuid in $uuids; do
+		total=$((total + 1))
+		if refresh_one "$uuid"; then
+			ok=$((ok + 1))
+		else
+			failed=$((failed + 1))
+		fi
+	done
+
+	if [ "$ok" -eq 0 ]; then
+		die "no ESP could be updated ($failed of $total failed) — this system may not boot"
+	fi
+	if [ "$failed" -gt 0 ]; then
+		# Strict mode is used by the installer, where a partly-updated set of
+		# ESPs means the install is broken and should fail loudly now rather
+		# than at the first reboot.
+		if [ "${SPINIFEX_BOOT_TOOL_STRICT:-0}" = "1" ]; then
+			die "$failed of $total ESPs failed to update"
+		fi
+		warn "$failed of $total ESPs failed to update; $ok succeeded, so the system will still boot"
+	fi
+	return 0
+}
+
+do_status() {
+	local quiet="${1:-}" uuid dev
+	if [ ! -s "$ESP_LIST" ]; then
+		[ "$quiet" = "--quiet" ] || echo "No ESPs registered — this system does not use spinifex-boot-tool."
+		return 1
+	fi
+	[ "$quiet" = "--quiet" ] && return 0
+	for uuid in $(esp_list); do
+		if dev=$(path_for_uuid "$uuid"); then
+			echo "$uuid  $dev"
+		else
+			echo "$uuid  MISSING"
+		fi
+	done
+}
+
+do_kernel() {
+	local action="$1" kver="${2:-}"
+	local pinfile="/etc/kernel/spinifex-boot-pin"
+	case "$action" in
+	add)
+		[ -n "$kver" ] || die "kernel add needs a version"
+		touch "$pinfile"
+		grep -qx "$kver" "$pinfile" || echo "$kver" >>"$pinfile"
+		;;
+	remove)
+		[ -n "$kver" ] || die "kernel remove needs a version"
+		[ -f "$pinfile" ] && sed -i "\|^$kver\$|d" "$pinfile"
+		;;
+	*) die "unknown kernel action: $action" ;;
+	esac
+	do_refresh
+}
+
+# --- hooks ------------------------------------------------------------------
+
+# install_hooks wires this tool into every package operation that would
+# otherwise write a bootloader to the wrong place.
+install_hooks() {
+	mkdir -p /etc/kernel/postinst.d /etc/kernel/postrm.d /etc/initramfs/post-update.d
+
+	# Debian's own hook runs update-grub against the real /boot, which is on the
+	# pool. Firmware never reads that, so it would produce a config nothing uses
+	# while the ESPs silently kept an old kernel.
+	if [ -e /etc/kernel/postinst.d/zz-update-grub ]; then
+		dpkg-divert --divert /etc/kernel/postinst.d/zz-update-grub.distrib \
+			--rename --add /etc/kernel/postinst.d/zz-update-grub >/dev/null 2>&1 || true
+	fi
+
+	for hook in /etc/kernel/postinst.d/zz-spinifex-boot-tool \
+		/etc/kernel/postrm.d/zz-spinifex-boot-tool \
+		/etc/initramfs/post-update.d/spinifex-boot-tool; do
+		cat >"$hook" <<'HOOK'
+#!/bin/sh
+# Re-sync the ESPs after a kernel or initramfs change.
+set -e
+exec /usr/sbin/spinifex-boot-tool refresh --hook "$(basename "$0")"
+HOOK
+		chmod 0755 "$hook"
+	done
+}
+
+# --- dispatch ---------------------------------------------------------------
+
+cmd="${1:-}"
+[ $# -gt 0 ] && shift
+
+case "$cmd" in
+format) do_format "$@" ;;
+init)
+	reexec_in_mountns init "$@"
+	do_init "$@"
+	;;
+refresh)
+	# --hook <name> is passed by the kernel hooks purely so the reason for a
+	# refresh is visible in the log.
+	if [ "${1:-}" = "--hook" ]; then shift 2 || shift; fi
+	reexec_in_mountns refresh
+	do_refresh
+	;;
+reinit)
+	reexec_in_mountns reinit
+	do_refresh
+	;;
+clean) do_clean "$@" ;;
+kernel)
+	reexec_in_mountns kernel "$@"
+	do_kernel "$@"
+	;;
+status) do_status "$@" ;;
+help | --help | -h | "") usage ;;
+*)
+	usage
+	exit 1
+	;;
+esac

cmd/installer/install/blockdev.go

@@ -0,0 +1,267 @@
+package install
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// Disk is one candidate block device with every attribute the installer needs
+// to validate a selection: sizes for the same-size rule, block sizes for ashift
+// and the 4Kn/BIOS check, rotational for autotrim, and ByID for pool members.
+type Disk struct {
+	Path  string // /dev/sda
+	ByID  string // /dev/disk/by-id/ata-… — stable across controller reordering
+	Bytes int64
+	Model string
+
+	LogicalBlockSize  int
+	PhysicalBlockSize int
+
+	Rotational bool
+	Removable  bool
+
+	// LiveMedia marks the device the installer itself booted from. Selecting it
+	// would erase the running system mid-install.
+	LiveMedia bool
+
+	// Content is a human-readable summary of what is on the disk today
+	// ("ext4 filesystem", "empty"), shown before the operator confirms erasure.
+	Content string
+}
+
+// skipPrefixes are kernel block devices that are never install targets:
+// virtual, mapped, or removable media. Mirrors Proxmox's hd_list filter.
+var skipPrefixes = []string{"ram", "loop", "md", "dm-", "fd", "sr", "zram", "nbd"}
+
+// The kernel-owned trees the scan reads, indirected so tests can point it at a
+// fixture instead of the running machine's hardware.
+var (
+	sysBlockDir      = "/sys/block"
+	sysClassBlockDir = "/sys/class/block"
+	devByIDDir       = "/dev/disk/by-id"
+	procMountsPath   = "/proc/mounts"
+)
+
+// ListDisks enumerates install-candidate disks from /sys/block. It reads sysfs
+// and blkid only — pool detection is separate (ImportablePools) because it is
+// far slower and not every caller needs it.
+func ListDisks() ([]Disk, error) {
+	entries, err := os.ReadDir(sysBlockDir)
+	if err != nil {
+		return nil, fmt.Errorf("read /sys/block: %w", err)
+	}
+
+	byID := buildByIDIndex()
+	live := liveMediaDevices()
+
+	var disks []Disk
+	for _, e := range entries {
+		name := e.Name()
+		if hasAnyPrefix(name, skipPrefixes) {
+			continue
+		}
+		// A zero-sized device is an empty card reader slot, not a disk.
+		sectors := sysfsInt(name, "size")
+		if sectors == 0 {
+			continue
+		}
+		d := Disk{
+			Path:              "/dev/" + name,
+			ByID:              byID[name],
+			Bytes:             sectors * 512,
+			Model:             sysfsString(name, "device/model"),
+			LogicalBlockSize:  sysfsIntDefault(name, "queue/logical_block_size", 512),
+			PhysicalBlockSize: sysfsIntDefault(name, "queue/physical_block_size", 512),
+			Rotational:        sysfsInt(name, "queue/rotational") == 1,
+			Removable:         sysfsInt(name, "removable") == 1,
+			LiveMedia:         live[name],
+		}
+		d.Content = describeContent(d.Path)
+		disks = append(disks, d)
+	}
+	slices.SortFunc(disks, func(a, b Disk) int { return strings.Compare(a.Path, b.Path) })
+	return disks, nil
+}
+
+// SizeHuman renders the disk size for display, e.g. "1.7T".
+func (d Disk) SizeHuman() string {
+	switch {
+	case d.Bytes >= 1<<40:
+		return fmt.Sprintf("%.1fT", float64(d.Bytes)/(1<<40))
+	case d.Bytes >= 1<<30:
+		return fmt.Sprintf("%.1fG", float64(d.Bytes)/(1<<30))
+	case d.Bytes >= 1<<20:
+		return fmt.Sprintf("%.1fM", float64(d.Bytes)/(1<<20))
+	default:
+		return fmt.Sprintf("%dB", d.Bytes)
+	}
+}
+
+// Stable returns the path to use when building a pool: the by-id path when one
+// exists, else the kernel name. Pools referencing /dev/sdX break when a
+// controller enumerates disks in a different order after a reboot.
+func (d Disk) Stable() string {
+	if d.ByID != "" {
+		return d.ByID
+	}
+	return d.Path
+}
+
+// PartitionPath returns the device path for partition n of this disk. NVMe and
+// other devices whose name ends in a digit take a 'p' separator.
+func (d Disk) PartitionPath(n int) string { return partitionPath(d.Path, n) }
+
+// StablePartitionPath is PartitionPath against the by-id path, which udev
+// publishes with a "-partN" suffix rather than the kernel's 'p' convention.
+func (d Disk) StablePartitionPath(n int) string {
+	if d.ByID == "" {
+		return d.PartitionPath(n)
+	}
+	return fmt.Sprintf("%s-part%d", d.ByID, n)
+}
+
+func partitionPath(disk string, n int) string {
+	if len(disk) > 0 && disk[len(disk)-1] >= '0' && disk[len(disk)-1] <= '9' {
+		return fmt.Sprintf("%sp%d", disk, n)
+	}
+	return fmt.Sprintf("%s%d", disk, n)
+}
+
+// byIDPreference orders the /dev/disk/by-id/ namespaces we prefer to name a
+// pool member by. wwn- ids are last: they are stable but opaque, so a degraded
+// pool reports a number nobody can map to a drive bay.
+var byIDPreference = []string{"nvme-", "ata-", "scsi-", "virtio-", "usb-", "wwn-"}
+
+// buildByIDIndex maps kernel device name to its preferred by-id path.
+func buildByIDIndex() map[string]string {
+	entries, err := os.ReadDir(devByIDDir)
+	if err != nil {
+		return map[string]string{}
+	}
+	// Track the chosen preference rank per device so a later, less-preferred
+	// id cannot displace a better one already found.
+	rank := map[string]int{}
+	out := map[string]string{}
+	for _, e := range entries {
+		id := e.Name()
+		// Partition links describe a slice of the disk, not the disk.
+		if strings.Contains(id, "-part") {
+			continue
+		}
+		target, err := filepath.EvalSymlinks(filepath.Join(devByIDDir, id))
+		if err != nil {
+			continue
+		}
+		dev := filepath.Base(target)
+		r := prefixRank(id)
+		if existing, ok := rank[dev]; ok && existing <= r {
+			continue
+		}
+		rank[dev] = r
+		out[dev] = filepath.Join(devByIDDir, id)
+	}
+	return out
+}
+
+func prefixRank(id string) int {
+	for i, p := range byIDPreference {
+		if strings.HasPrefix(id, p) {
+			return i
+		}
+	}
+	return len(byIDPreference)
+}
+
+// liveMediaDevices returns the set of parent disk names backing the live
+// installer's own media, so they can never be selected as install targets.
+func liveMediaDevices() map[string]bool {
+	out := map[string]bool{}
+	data, err := os.ReadFile(procMountsPath)
+	if err != nil {
+		return out
+	}
+	liveMounts := []string{"/cdrom", "/run/live/medium", "/lib/live/mount/medium"}
+	for line := range strings.SplitSeq(string(data), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) < 2 || !strings.HasPrefix(fields[0], "/dev/") {
+			continue
+		}
+		if !slices.Contains(liveMounts, fields[1]) {
+			continue
+		}
+		if parent := parentDisk(filepath.Base(fields[0])); parent != "" {
+			out[parent] = true
+		}
+	}
+	return out
+}
+
+// parentDisk resolves a partition's kernel name to its whole-disk name by
+// walking /sys/class/block/<part>/.. — the sysfs parent of a partition is its
+// disk. Returns the input unchanged when it is already a whole disk.
+func parentDisk(dev string) string {
+	link, err := filepath.EvalSymlinks(filepath.Join(sysClassBlockDir, dev))
+	if err != nil {
+		return dev
+	}
+	// A whole disk lives directly under a bus path; a partition sits one level
+	// deeper, inside its disk's directory.
+	if _, err := os.Stat(filepath.Join(link, "partition")); err != nil {
+		return dev
+	}
+	return filepath.Base(filepath.Dir(link))
+}
+
+// describeContent summarises what is currently on a disk for the confirmation
+// screen. Best-effort: an unreadable disk is reported as unknown, not empty,
+// because "empty" invites an operator to erase something that is not.
+func describeContent(dev string) string {
+	out, err := exec.Command("blkid", "-p", "-o", "value", "-s", "TYPE", dev).Output()
+	if err == nil {
+		if t := strings.TrimSpace(string(out)); t != "" {
+			return t + " filesystem"
+		}
+	}
+	// No filesystem on the whole device: report the partition table instead,
+	// since that is what tells the operator the disk is in use.
+	out, err = exec.Command("blkid", "-p", "-o", "value", "-s", "PTTYPE", dev).Output()
+	if err != nil {
+		return "unknown"
+	}
+	if t := strings.TrimSpace(string(out)); t != "" {
+		return t + " partition table"
+	}
+	return "empty"
+}
+
+func sysfsString(dev, file string) string {
+	data, err := os.ReadFile(filepath.Join(sysBlockDir, dev, file))
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(data))
+}
+
+func sysfsInt(dev, file string) int64 {
+	n, err := strconv.ParseInt(sysfsString(dev, file), 10, 64)
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+func sysfsIntDefault(dev, file string, def int) int {
+	if n := sysfsInt(dev, file); n > 0 {
+		return int(n)
+	}
+	return def
+}
+
+func hasAnyPrefix(s string, prefixes []string) bool {
+	return slices.ContainsFunc(prefixes, func(p string) bool { return strings.HasPrefix(s, p) })
+}

cmd/installer/install/blockdev_test.go

@@ -0,0 +1,354 @@
+package install
+
+import (
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"testing"
+)
+
+// fakeDisk describes one device to lay down in the fixture sysfs tree.
+type fakeDisk struct {
+	name       string
+	sectors    int64
+	model      string
+	logical    int
+	physical   int
+	rotational bool
+	removable  bool
+	partitions []string
+}
+
+// fakeSysfs builds a /sys/block, /sys/class/block, /dev/disk/by-id and
+// /proc/mounts fixture and points the scan at it for the duration of the test.
+type fakeSysfs struct {
+	root string
+	t    *testing.T
+}
+
+func newFakeSysfs(t *testing.T, disks ...fakeDisk) *fakeSysfs {
+	t.Helper()
+	f := &fakeSysfs{root: t.TempDir(), t: t}
+
+	swap := func(dst *string, v string) {
+		prev := *dst
+		*dst = v
+		t.Cleanup(func() { *dst = prev })
+	}
+	swap(&sysBlockDir, filepath.Join(f.root, "sys/block"))
+	swap(&sysClassBlockDir, filepath.Join(f.root, "sys/class/block"))
+	swap(&devByIDDir, filepath.Join(f.root, "dev/disk/by-id"))
+	swap(&procMountsPath, filepath.Join(f.root, "proc/mounts"))
+
+	f.mkdirAll(sysBlockDir)
+	f.mkdirAll(sysClassBlockDir)
+	for _, d := range disks {
+		f.addDisk(d)
+	}
+	return f
+}
+
+func (f *fakeSysfs) mkdirAll(dir string) {
+	f.t.Helper()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		f.t.Fatalf("mkdir %s: %v", dir, err)
+	}
+}
+
+func (f *fakeSysfs) write(path, content string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Dir(path))
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		f.t.Fatalf("write %s: %v", path, err)
+	}
+}
+
+// addDisk lays out one device the way the kernel does: the attribute files
+// under /sys/block/<dev>, and a /sys/class/block symlink per disk and
+// partition, since parentDisk walks the latter.
+func (f *fakeSysfs) addDisk(d fakeDisk) {
+	f.t.Helper()
+	dir := filepath.Join(sysBlockDir, d.name)
+	f.write(filepath.Join(dir, "size"), strconv.FormatInt(d.sectors, 10)+"\n")
+	if d.model != "" {
+		f.write(filepath.Join(dir, "device/model"), d.model+"\n")
+	}
+	if d.logical > 0 {
+		f.write(filepath.Join(dir, "queue/logical_block_size"), strconv.Itoa(d.logical)+"\n")
+	}
+	if d.physical > 0 {
+		f.write(filepath.Join(dir, "queue/physical_block_size"), strconv.Itoa(d.physical)+"\n")
+	}
+	f.write(filepath.Join(dir, "queue/rotational"), boolAttr(d.rotational))
+	f.write(filepath.Join(dir, "removable"), boolAttr(d.removable))
+
+	f.symlink(dir, filepath.Join(sysClassBlockDir, d.name))
+	for _, part := range d.partitions {
+		partDir := filepath.Join(dir, part)
+		f.write(filepath.Join(partDir, "partition"), "1\n")
+		f.symlink(partDir, filepath.Join(sysClassBlockDir, part))
+	}
+}
+
+func (f *fakeSysfs) symlink(target, link string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Dir(link))
+	if err := os.Symlink(target, link); err != nil {
+		f.t.Fatalf("symlink %s -> %s: %v", link, target, err)
+	}
+}
+
+// addByID creates a by-id alias pointing at a kernel device name. The link
+// target must exist for EvalSymlinks, so a placeholder /dev tree is used.
+func (f *fakeSysfs) addByID(id, dev string) {
+	f.t.Helper()
+	devPath := filepath.Join(f.root, "dev", dev)
+	f.write(devPath, "")
+	f.symlink(devPath, filepath.Join(devByIDDir, id))
+}
+
+func (f *fakeSysfs) setMounts(content string) {
+	f.t.Helper()
+	f.write(procMountsPath, content)
+}
+
+func boolAttr(b bool) string {
+	if b {
+		return "1\n"
+	}
+	return "0\n"
+}
+
+func TestListDisksReadsSysfsAttributes(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{
+		name: "sdb", sectors: 2 << 20, model: "SAMSUNG MZ7", logical: 512, physical: 4096, rotational: true,
+	})
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	if len(disks) != 1 {
+		t.Fatalf("got %d disks, want 1: %+v", len(disks), disks)
+	}
+	d := disks[0]
+	if d.Path != "/dev/sdb" || d.Bytes != (2<<20)*512 {
+		t.Errorf("path/size = %s/%d", d.Path, d.Bytes)
+	}
+	if d.Model != "SAMSUNG MZ7" {
+		t.Errorf("Model = %q", d.Model)
+	}
+	if d.LogicalBlockSize != 512 || d.PhysicalBlockSize != 4096 {
+		t.Errorf("block sizes = %d/%d", d.LogicalBlockSize, d.PhysicalBlockSize)
+	}
+	if !d.Rotational || d.Removable || d.LiveMedia {
+		t.Errorf("flags = rot %v, rem %v, live %v", d.Rotational, d.Removable, d.LiveMedia)
+	}
+	if d.Content == "" {
+		t.Error("Content is empty; an unreadable device must report something")
+	}
+}
+
+func TestListDisksSkipsVirtualAndEmptyDevices(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "loop0", sectors: 1 << 20},
+		fakeDisk{name: "dm-0", sectors: 1 << 20},
+		fakeDisk{name: "zram0", sectors: 1 << 20},
+		fakeDisk{name: "sr0", sectors: 1 << 20},
+		fakeDisk{name: "nbd0", sectors: 1 << 20},
+		// An empty card-reader slot reports zero sectors.
+		fakeDisk{name: "mmcblk0", sectors: 0},
+	)
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	var paths []string
+	for _, d := range disks {
+		paths = append(paths, d.Path)
+	}
+	if !slices.Equal(paths, []string{"/dev/sda"}) {
+		t.Errorf("paths = %v, want only /dev/sda", paths)
+	}
+}
+
+func TestListDisksSortsByPath(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sdc", sectors: 1 << 20},
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "sdb", sectors: 1 << 20},
+	)
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	var paths []string
+	for _, d := range disks {
+		paths = append(paths, d.Path)
+	}
+	if !slices.Equal(paths, []string{"/dev/sda", "/dev/sdb", "/dev/sdc"}) {
+		t.Errorf("paths = %v, want sorted", paths)
+	}
+}
+
+func TestListDisksDefaultsBlockSizesWhenSysfsIsSilent(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "vda", sectors: 1 << 20})
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	if disks[0].LogicalBlockSize != 512 || disks[0].PhysicalBlockSize != 512 {
+		t.Errorf("block sizes = %d/%d, want the 512 default",
+			disks[0].LogicalBlockSize, disks[0].PhysicalBlockSize)
+	}
+}
+
+func TestListDisksErrorsWhenSysfsIsUnreadable(t *testing.T) {
+	newFakeSysfs(t)
+	sysBlockDir = filepath.Join(t.TempDir(), "does-not-exist")
+
+	if _, err := ListDisks(); err == nil {
+		t.Fatal("expected an error when /sys/block cannot be read")
+	}
+}
+
+func TestListDisksMarksTheLiveMedium(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "sdz", sectors: 1 << 20, removable: true, partitions: []string{"sdz1"}},
+	)
+	// The installer boots from a partition; the whole disk must be excluded.
+	f.setMounts("/dev/sdz1 /run/live/medium iso9660 ro 0 0\n" +
+		"proc /proc proc rw 0 0\n" +
+		"/dev/sda1 / ext4 rw 0 0\n")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	byPath := map[string]Disk{}
+	for _, d := range disks {
+		byPath[d.Path] = d
+	}
+	if !byPath["/dev/sdz"].LiveMedia {
+		t.Error("/dev/sdz backs the live medium and was not marked")
+	}
+	if byPath["/dev/sda"].LiveMedia {
+		t.Error("/dev/sda is not the live medium but was marked")
+	}
+}
+
+func TestLiveMediaDevicesIsEmptyWithoutProcMounts(t *testing.T) {
+	newFakeSysfs(t)
+	if got := liveMediaDevices(); len(got) != 0 {
+		t.Errorf("liveMediaDevices = %v, want empty when /proc/mounts is missing", got)
+	}
+}
+
+func TestParentDiskLeavesWholeDisksAlone(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20, partitions: []string{"sda1"}})
+
+	if got := parentDisk("sda"); got != "sda" {
+		t.Errorf("parentDisk(sda) = %q, want sda", got)
+	}
+	if got := parentDisk("sda1"); got != "sda" {
+		t.Errorf("parentDisk(sda1) = %q, want sda", got)
+	}
+	// An unknown name cannot be resolved and is returned unchanged.
+	if got := parentDisk("sdq9"); got != "sdq9" {
+		t.Errorf("parentDisk(sdq9) = %q, want it unchanged", got)
+	}
+}
+
+func TestBuildByIDIndexPrefersTheMostLegibleAlias(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20})
+	// wwn- is stable but opaque, so ata- must win regardless of scan order.
+	f.addByID("wwn-0x5000c500", "sda")
+	f.addByID("ata-SAMSUNG_MZ7_S1", "sda")
+	f.addByID("ata-SAMSUNG_MZ7_S1-part1", "sda")
+
+	idx := buildByIDIndex()
+	want := filepath.Join(devByIDDir, "ata-SAMSUNG_MZ7_S1")
+	if idx["sda"] != want {
+		t.Errorf("by-id[sda] = %q, want %q", idx["sda"], want)
+	}
+}
+
+func TestBuildByIDIndexIsEmptyWhenTheDirIsAbsent(t *testing.T) {
+	newFakeSysfs(t)
+	if got := buildByIDIndex(); len(got) != 0 {
+		t.Errorf("buildByIDIndex = %v, want empty", got)
+	}
+}
+
+func TestListDisksAttachesByIDPath(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20})
+	f.addByID("ata-TEST_SERIAL0", "sda")
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	want := filepath.Join(devByIDDir, "ata-TEST_SERIAL0")
+	if disks[0].ByID != want {
+		t.Errorf("ByID = %q, want %q", disks[0].ByID, want)
+	}
+	if disks[0].Stable() != want {
+		t.Errorf("Stable() = %q, want the by-id path", disks[0].Stable())
+	}
+}
+
+func TestPrefixRank(t *testing.T) {
+	if prefixRank("nvme-Foo") >= prefixRank("wwn-0x1") {
+		t.Error("nvme- must outrank wwn-")
+	}
+	if prefixRank("md-uuid-x") != len(byIDPreference) {
+		t.Error("an unknown namespace must rank last")
+	}
+}
+
+func TestSizeHuman(t *testing.T) {
+	tests := []struct {
+		bytes int64
+		want  string
+	}{
+		{2 << 40, "2.0T"},
+		{500 << 30, "500.0G"},
+		{64 << 20, "64.0M"},
+		{512, "512B"},
+	}
+	for _, tt := range tests {
+		if got := (Disk{Bytes: tt.bytes}).SizeHuman(); got != tt.want {
+			t.Errorf("SizeHuman(%d) = %q, want %q", tt.bytes, got, tt.want)
+		}
+	}
+}
+
+func TestStableFallsBackToKernelName(t *testing.T) {
+	d := Disk{Path: "/dev/sda"}
+	if d.Stable() != "/dev/sda" {
+		t.Errorf("Stable() = %q", d.Stable())
+	}
+	if d.StablePartitionPath(2) != "/dev/sda2" {
+		t.Errorf("StablePartitionPath(2) = %q", d.StablePartitionPath(2))
+	}
+}
+
+func TestHasAnyPrefix(t *testing.T) {
+	if !hasAnyPrefix("loop3", skipPrefixes) {
+		t.Error("loop3 should match")
+	}
+	if hasAnyPrefix("sda", skipPrefixes) {
+		t.Error("sda should not match")
+	}
+}

cmd/installer/install/boottool.go

@@ -0,0 +1,202 @@
+package install
+
+import (
+	_ "embed"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+//go:embed assets/spinifex-boot-tool
+var bootToolScript []byte
+
+//go:embed assets/grub-install-wrapper
+var grubInstallWrapper []byte
+
+const (
+	bootToolPath     = "/usr/sbin/spinifex-boot-tool"
+	grubInstallPath  = "/usr/sbin/grub-install"
+	grubInstallReal  = "/usr/sbin/grub-install.real"
+	bootToolUUIDList = "/etc/kernel/spinifex-boot-uuids"
+)
+
+// installBootToolZFS sets up per-disk bootloaders on a ZFS root system.
+//
+// Nothing in the boot path is allowed to read the pool: GRUB's ZFS support
+// trails OpenZFS, so a pool feature flag enabled years from now could make the
+// machine unbootable. Each disk instead carries a self-contained ESP with its
+// own GRUB, kernels and initramfs.
+func installBootToolZFS(cfg DiskConfig) error {
+	if err := writeTargetFile(bootToolPath, bootToolScript, 0o755); err != nil {
+		return err
+	}
+	if err := writeGrubDefaults(cfg); err != nil {
+		return err
+	}
+
+	if err := bindChrootMounts(); err != nil {
+		return err
+	}
+	defer unbindChrootMounts()
+
+	// Diverted before the first init: init runs a refresh, and refresh calls
+	// grub-install. With the wrapper already in place that call has to reach
+	// the real binary, which is what the diversion arranges.
+	if err := divertGrubInstall(); err != nil {
+		return err
+	}
+
+	// A partially-updated set of ESPs is an install that boots today and fails
+	// after the next kernel upgrade, so every disk must succeed here.
+	env := []string{"SPINIFEX_BOOT_TOOL_STRICT=1"}
+	for _, d := range cfg.Disks {
+		esp := d.PartitionPath(espPartNum)
+		if err := runEnv(env, "chroot", mountRoot, bootToolPath, "init", esp); err != nil {
+			return fmt.Errorf("spinifex-boot-tool init %s: %w", esp, err)
+		}
+	}
+
+	// The one moment firmware boot variables are written. Refreshes run on
+	// every kernel upgrade and deliberately leave NVRAM alone: repeatedly
+	// rewriting boot entries has bricked more than one vendor's boards. Losing
+	// the entry is survivable anyway, because each ESP also carries the
+	// removable-path loader that firmware falls back to.
+	if err := runEnv(append(env, "SPINIFEX_BOOT_TOOL_NVRAM=1"),
+		"chroot", mountRoot, bootToolPath, "refresh"); err != nil {
+		return fmt.Errorf("spinifex-boot-tool refresh: %w", err)
+	}
+	return verifyBootConfig(cfg)
+}
+
+// divertGrubInstall moves the real grub-install aside and puts the wrapper in
+// its place. dpkg-divert makes the move survive package upgrades, which is the
+// point: grub-pc's postinst runs grub-install and would otherwise quietly
+// replace a working boot layout with one that reads the pool.
+func divertGrubInstall() error {
+	if err := run("chroot", mountRoot, "dpkg-divert", "--divert", grubInstallReal,
+		"--rename", "--add", grubInstallPath); err != nil {
+		return fmt.Errorf("divert grub-install: %w", err)
+	}
+	return writeTargetFile(grubInstallPath, grubInstallWrapper, 0o755)
+}
+
+// verifyBootConfig asserts that the config the machine will actually boot from
+// names the ZFS root. grub-mkconfig degrades quietly when grub-probe cannot
+// read the pool, emitting a config that gets as far as the GRUB menu and then
+// panics with "unable to mount root fs" — which looks like a hardware fault.
+func verifyBootConfig(cfg DiskConfig) error {
+	uuids, err := os.ReadFile(filepath.Join(mountRoot, strings.TrimPrefix(bootToolUUIDList, "/")))
+	if err != nil {
+		return fmt.Errorf("read ESP list: %w", err)
+	}
+	if len(strings.Fields(string(uuids))) != len(cfg.Disks) {
+		return fmt.Errorf("expected %d registered ESPs, found %d — some disks have no bootloader",
+			len(cfg.Disks), len(strings.Fields(string(uuids))))
+	}
+
+	// One ESP is enough to prove grub-mkconfig produced the right thing; they
+	// are all generated by the same code path in the same run.
+	mp := filepath.Join(mountRoot, "var/tmp/verify-esp")
+	if err := os.MkdirAll(mp, 0o755); err != nil {
+		return err
+	}
+	defer func() { _ = os.Remove(mp) }()
+
+	esp := cfg.Disks[0].PartitionPath(espPartNum)
+	if err := run("mount", esp, mp); err != nil {
+		return fmt.Errorf("mount %s for verification: %w", esp, err)
+	}
+	defer func() { _ = run("umount", mp) }()
+
+	grubCfg, err := os.ReadFile(filepath.Join(mp, "grub/grub.cfg"))
+	if err != nil {
+		return fmt.Errorf("read grub.cfg from %s: %w", esp, err)
+	}
+	want := "root=ZFS=" + ZFSRootDataset
+	if !strings.Contains(string(grubCfg), want) {
+		return fmt.Errorf("grub.cfg on %s does not contain %q — the installed system would not find its root filesystem", esp, want)
+	}
+	if !strings.Contains(string(grubCfg), "vmlinuz-") {
+		return fmt.Errorf("grub.cfg on %s references no kernel", esp)
+	}
+	return nil
+}
+
+// writeTargetFile writes a file into the install target.
+func writeTargetFile(path string, content []byte, mode os.FileMode) error {
+	full := filepath.Join(mountRoot, strings.TrimPrefix(path, "/"))
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		return err
+	}
+	if err := os.WriteFile(full, content, mode); err != nil {
+		return fmt.Errorf("write %s: %w", path, err)
+	}
+	// WriteFile does not apply the mode to a file that already exists, and this
+	// path is re-run on a retried install.
+	return os.Chmod(full, mode)
+}
+
+// writeGrubDefaults writes /etc/default/grub and the branding snippet.
+//
+// Both consoles are listed: tty0 stays primary for a local install while ttyS0
+// mirrors kernel output for headless racks and CI. The last console= on the
+// cmdline becomes the system console, so serial-only hardware still works.
+func writeGrubDefaults(cfg DiskConfig) error {
+	cmdline := "console=tty0 console=ttyS0,115200 systemd.show_status=1"
+	if cfg.FS.IsZFS() {
+		// Stated explicitly rather than relying on grub-mkconfig's probe. It
+		// appears after the root= that 10_linux computes, so it wins if the
+		// probe came back with something wrong.
+		cmdline = fmt.Sprintf("root=ZFS=%s boot=zfs %s", ZFSRootDataset, cmdline)
+	}
+	grubDefault := fmt.Sprintf(`GRUB_DEFAULT=0
+GRUB_TIMEOUT=5
+GRUB_DISTRIBUTOR=Spinifex
+GRUB_CMDLINE_LINUX_DEFAULT=""
+GRUB_CMDLINE_LINUX="%s"
+`, cmdline)
+	if err := os.WriteFile(filepath.Join(mountRoot, "etc/default/grub"), []byte(grubDefault), 0o644); err != nil {
+		return fmt.Errorf("write /etc/default/grub: %w", err)
+	}
+
+	// gfxterm must be activated before serial is appended — background_image
+	// silently does nothing in text mode. $prefix is used rather than a literal
+	// /boot/grub so the same snippet is correct whether GRUB's root is the ext4
+	// filesystem or a per-disk ESP.
+	grubTheme := `#!/bin/sh
+exec tail -n +3 $0
+insmod all_video
+insmod font
+if loadfont $prefix/fonts/unicode.pf2; then
+  set gfxmode=auto
+  insmod gfxterm
+  terminal_output gfxterm
+fi
+insmod serial
+if serial --unit=0 --speed=115200 --timeout=1; then
+  terminal_input  --append serial
+  terminal_output --append serial
+fi
+
+# --- Branding ---
+insmod png
+set theme=$prefix/theme.txt
+export theme
+`
+	if err := os.WriteFile(filepath.Join(mountRoot, "etc/grub.d/06_spinifex"), []byte(grubTheme), 0o755); err != nil {
+		return fmt.Errorf("write /etc/grub.d/06_spinifex: %w", err)
+	}
+	return nil
+}
+
+// refreshBootToolNote logs the command an operator needs after replacing a
+// disk. Nothing in the installer can do this for them later.
+func refreshBootToolNote(cfg DiskConfig) {
+	if cfg.FS.IsZFS() {
+		slog.Info("bootloader installed on every pool member",
+			"disks", len(cfg.Disks),
+			"after_disk_replacement", "spinifex-boot-tool format <esp> && spinifex-boot-tool init <esp>")
+	}
+}

cmd/installer/install/config.go

@@ -2,42 +2,107 @@
 // collected configuration.
 package install
 
+import (
+	"fmt"
+	"net"
+	"strings"
+)
+
+// Plane names the three traffic planes a node presents. Each plane is a role
+// bound to a target, not a fixed NIC: the same three roles collapse onto one,
+// two or three interfaces depending on the hardware.
+type Plane string
+
+const (
+	// PlaneWAN carries public north-south traffic: ingress/egress, EIPs and
+	// SNAT/DNAT. It is the OVN external bridge-mapping.
+	PlaneWAN Plane = "wan"
+
+	// PlaneLAN carries internal cluster traffic: predastore replication, the
+	// NATS cluster mesh, OVN NB/SB control, Raft and S3DB.
+	PlaneLAN Plane = "lan"
+
+	// PlaneVPC carries the tenant overlay: OVN Geneve tunnels between nodes.
+	// It is the ovn-encap-ip endpoint.
+	PlaneVPC Plane = "vpc"
+)
+
+// Bridge returns the Linux bridge that carries this plane.
+func (p Plane) Bridge() string { return "br-" + string(p) }
+
+// NetworkRole binds one plane to a network target. An empty Interface means
+// the role is folded onto the next role up the chain (vpc <- lan <- wan) and
+// no bridge of its own is created.
+//
+// VLAN is what allows two roles to share one physical NIC while staying
+// separate planes: each gets its own NIC.<vlan> subinterface and bridge. Two
+// roles on the same Interface with no VLAN are the same plane, not two.
+type NetworkRole struct {
+	// Interface is the physical NIC this role binds to; empty means folded.
+	Interface string
+
+	// VLAN is the 802.1Q tag, or 0 for untagged.
+	VLAN int
+
+	// DHCPMode leaves addressing to the DHCP client; Address/Mask are then unset.
+	DHCPMode bool
+
+	Address string
+	Mask    string
+
+	// Gateway is only meaningful on the wan role — the other planes are
+	// link-local to the rack and must not install a default route.
+	Gateway string
+
+	DNS []string
+
+	// MTU is written as MTUBytes= on the link; 0 leaves the kernel default.
+	// East-west planes typically run 9000, wan stays at 1500.
+	MTU int
+
+	// WiFiSSID and WiFiPass are set when the selected NIC is wireless.
+	WiFiSSID string
+	WiFiPass string
+}
+
+// Folded reports whether this role has no interface of its own and inherits
+// the plane of the role above it.
+func (r NetworkRole) Folded() bool { return r.Interface == "" }
+
+// Link returns the name of the interface carrying this role's address: the
+// physical NIC, or its VLAN subinterface when tagged. Empty when folded.
+func (r NetworkRole) Link() string {
+	if r.Folded() {
+		return ""
+	}
+	if r.VLAN > 0 {
+		return fmt.Sprintf("%s.%d", r.Interface, r.VLAN)
+	}
+	return r.Interface
+}
+
 // Config holds all values collected by the installer UI.
 type Config struct {
-	// Disk is the block device path to install onto (e.g. /dev/sda).
-	Disk string
+	// Storage is the filesystem choice and the ordered set of disks it is
+	// built from. In ext4 mode it holds exactly one disk.
+	Storage DiskConfig
 
-	// WAN interface — always present. The management IP lives on br-wan (a Linux
-	// bridge over the NIC), not on the physical NIC itself. This allows OVN/OVS
-	// to safely attach to the bridge without disrupting host connectivity.
-	WANInterface string
-	WANDHCPMode  bool
-	WANAddress   string   // empty if WANDHCPMode
-	WANMask      string   // empty if WANDHCPMode
-	WANGateway   string   // empty if WANDHCPMode
-	WANDNS       []string // empty if WANDHCPMode; e.g. ["1.1.1.1", "8.8.8.8"]
-	WANWiFiSSID  string   // non-empty if a WiFi NIC was selected
-	WANWiFiPass  string   // non-empty if a WiFi NIC was selected
+	// WAN is always present and can never be folded — a node needs an uplink.
+	// The address lives on br-wan (a Linux bridge over the NIC), not on the
+	// physical NIC itself, so OVN/OVS can attach to the bridge without
+	// disrupting host connectivity.
+	WAN NetworkRole
 
-	// LAN interface — only set when 2+ NICs are present. The Geneve encap IP
-	// for OVN tunnels lives on br-lan. When empty, the WAN address is used.
-	LANInterface string
-	LANDHCPMode  bool
-	LANAddress   string
-	LANMask      string
-	LANDNS       []string
-	LANWiFiSSID  string
-	LANWiFiPass  string
+	// LAN carries internal cluster traffic. Folded onto WAN on single-NIC nodes.
+	LAN NetworkRole
+
+	// VPC carries Geneve overlay traffic and supplies ovn-encap-ip. Folded onto
+	// LAN unless a node dedicates an interface or VLAN to it.
+	VPC NetworkRole
 
 	// Node identity
 	Hostname string
 
-	// ClusterRole is "init" or "join".
-	ClusterRole string
-
-	// JoinAddr is the primary node address (host:port) when ClusterRole is "join".
-	JoinAddr string
-
 	// SkipFormation skips spx admin init/join in firstboot; the provisioning
 	// controller (e.g. bm-bootstrap.sh) owns cluster formation instead.
 	SkipFormation bool
@@ -60,3 +125,111 @@ type Config struct {
 	// Passes --gpu-passthrough to `spx admin init` during firstboot.
 	GPUPassthrough bool
 }
+
+// Resolve returns the role that actually carries the given plane after
+// collapsing folded roles up the chain vpc <- lan <- wan, along with the plane
+// it landed on. A folded vpc on a single-NIC node resolves to the wan role.
+func (c *Config) Resolve(p Plane) (NetworkRole, Plane) {
+	switch p {
+	case PlaneVPC:
+		if !c.VPC.Folded() {
+			return c.VPC, PlaneVPC
+		}
+		return c.Resolve(PlaneLAN)
+	case PlaneLAN:
+		if !c.LAN.Folded() {
+			return c.LAN, PlaneLAN
+		}
+		return c.WAN, PlaneWAN
+	default:
+		return c.WAN, PlaneWAN
+	}
+}
+
+// PlaneAddress returns the configured address for a plane after collapsing.
+// Empty when the resolved role uses DHCP, since no static address is known
+// until the lease is taken.
+func (c *Config) PlaneAddress(p Plane) string {
+	role, _ := c.Resolve(p)
+	if role.DHCPMode {
+		return ""
+	}
+	return role.Address
+}
+
+// PlaneBridge returns the bridge carrying a plane after collapsing, so callers
+// asking for the vpc bridge on a single-NIC node correctly get br-wan.
+func (c *Config) PlaneBridge(p Plane) string {
+	_, landed := c.Resolve(p)
+	return landed.Bridge()
+}
+
+// RoleBinding pairs a plane with the role that carries it.
+type RoleBinding struct {
+	Plane Plane
+	Role  NetworkRole
+}
+
+// Roles returns the bound roles in bridge-creation order, skipping folded ones.
+// wan comes first because br-lan and br-vpc are activated manually after it.
+func (c *Config) Roles() []RoleBinding {
+	out := []RoleBinding{{PlaneWAN, c.WAN}}
+	if !c.LAN.Folded() {
+		out = append(out, RoleBinding{PlaneLAN, c.LAN})
+	}
+	if !c.VPC.Folded() {
+		out = append(out, RoleBinding{PlaneVPC, c.VPC})
+	}
+	return out
+}
+
+// Validate enforces the sharing rules the UI and the headless path must both
+// obey: wan always needs a target, and two roles may only share an interface
+// when distinct VLAN ids keep them on separate broadcast domains.
+func (c *Config) Validate() error {
+	if c.WAN.Folded() {
+		return fmt.Errorf("wan role must be bound to an interface")
+	}
+
+	seen := map[string]Plane{}
+	for _, r := range c.Roles() {
+		link := r.Role.Link()
+		if prev, dup := seen[link]; dup {
+			return fmt.Errorf("%s and %s both bind %s: give each a distinct VLAN id or fold one onto the other",
+				prev, r.Plane, link)
+		}
+		seen[link] = r.Plane
+		if r.Role.VLAN < 0 || r.Role.VLAN > 4094 {
+			return fmt.Errorf("%s: VLAN id %d out of range (1-4094)", r.Plane, r.Role.VLAN)
+		}
+		if r.Role.MTU != 0 && (r.Role.MTU < 68 || r.Role.MTU > 9216) {
+			return fmt.Errorf("%s: MTU %d out of range (68-9216)", r.Plane, r.Role.MTU)
+		}
+		// Resolvers belong to the plane that holds the default route. Accepting
+		// one here and dropping it during generation would leave an unattended
+		// install believing it had set a resolver that was never written.
+		if r.Plane != PlaneWAN && len(r.Role.DNS) > 0 {
+			return fmt.Errorf("%s: DNS servers belong to the wan plane, which holds the default route", r.Plane)
+		}
+		// Netgen converts the mask with net.ParseIP, so a prefix length reaches
+		// it as garbage and fails the install after the disk is partitioned.
+		// Catch it here, where the operator can still fix it.
+		if m := strings.TrimSpace(r.Role.Mask); m != "" && !validDottedMask(m) {
+			return fmt.Errorf("%s: subnet mask %q must be dotted-decimal, e.g. 255.255.255.0", r.Plane, m)
+		}
+	}
+	return nil
+}
+
+// validDottedMask reports whether s is a contiguous dotted-decimal IPv4 subnet
+// mask. Prefix-length form ("24", "/24") is deliberately not accepted: netgen
+// parses the mask with net.ParseIP, so anything else fails the install itself.
+func validDottedMask(s string) bool {
+	ip := net.ParseIP(strings.TrimSpace(s))
+	if ip == nil || ip.To4() == nil {
+		return false
+	}
+	// Size reports zero bits for a non-contiguous mask, which netgen rejects too.
+	_, bits := net.IPMask(ip.To4()).Size()
+	return bits != 0
+}

cmd/installer/install/config_test.go

@@ -0,0 +1,463 @@
+package install
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// Node layouts drawn from the us-west-1 racks, which between them exercise
+// every collapse permutation the installer has to express.
+func threeNIC() *Config { // hydrogen / radon
+	return &Config{
+		WAN: NetworkRole{Interface: "eno6", Address: "216.218.163.99", Mask: "255.255.255.224", Gateway: "216.218.163.97"},
+		LAN: NetworkRole{Interface: "ens1f0np0", Address: "10.0.0.3", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "ens1f1np1", Address: "10.1.0.3", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func vlanSplit() *Config { // helium — one 100gbe carrying lan and vpc
+	return &Config{
+		WAN: NetworkRole{Interface: "enx6c1ff7c1475a", DHCPMode: true},
+		LAN: NetworkRole{Interface: "enp129s0np0", VLAN: 10, Address: "10.0.0.2", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "enp129s0np0", VLAN: 20, Address: "10.1.0.2", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func dualPortTagged() *Config { // us-west-1-az2 — wan and lan share port 1
+	return &Config{
+		WAN: NetworkRole{Interface: "enp1s0f0", VLAN: 30, Address: "216.218.163.10", Mask: "255.255.255.224", Gateway: "216.218.163.97"},
+		LAN: NetworkRole{Interface: "enp1s0f0", VLAN: 10, Address: "10.0.0.10", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "enp1s0f1", VLAN: 20, Address: "10.1.0.10", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func TestResolveCollapsesUpTheChain(t *testing.T) {
+	singleNIC := &Config{WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"}}
+	twoNIC := &Config{
+		WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"},
+		LAN: NetworkRole{Interface: "eth1", Address: "10.0.0.5", Mask: "255.255.0.0"},
+	}
+
+	tests := []struct {
+		name      string
+		cfg       *Config
+		plane     Plane
+		wantAddr  string
+		wantPlane Plane
+	}{
+		{"1 NIC: lan folds onto wan", singleNIC, PlaneLAN, "192.168.1.10", PlaneWAN},
+		{"1 NIC: vpc folds all the way to wan", singleNIC, PlaneVPC, "192.168.1.10", PlaneWAN},
+		{"2 NIC: vpc folds onto lan", twoNIC, PlaneVPC, "10.0.0.5", PlaneLAN},
+		{"2 NIC: lan stays put", twoNIC, PlaneLAN, "10.0.0.5", PlaneLAN},
+		{"3 NIC: vpc is its own plane", threeNIC(), PlaneVPC, "10.1.0.3", PlaneVPC},
+		{"VLAN split: vpc is its own plane", vlanSplit(), PlaneVPC, "10.1.0.2", PlaneVPC},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.cfg.PlaneAddress(tt.plane); got != tt.wantAddr {
+				t.Errorf("PlaneAddress(%s) = %q, want %q", tt.plane, got, tt.wantAddr)
+			}
+			if _, landed := tt.cfg.Resolve(tt.plane); landed != tt.wantPlane {
+				t.Errorf("Resolve(%s) landed on %s, want %s", tt.plane, landed, tt.wantPlane)
+			}
+		})
+	}
+}
+
+// A DHCP plane has no address known at install time; setup-ovn.sh auto-detects
+// it from the default route at boot instead.
+func TestPlaneAddressEmptyUnderDHCP(t *testing.T) {
+	if got := vlanSplit().PlaneAddress(PlaneWAN); got != "" {
+		t.Errorf("PlaneAddress(wan) = %q, want empty for a DHCP role", got)
+	}
+}
+
+func TestValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     *Config
+		wantErr string
+	}{
+		{"three physical NICs", threeNIC(), ""},
+		{"VLAN split on one NIC", vlanSplit(), ""},
+		{"dual port, all tagged", dualPortTagged(), ""},
+		{
+			"wan cannot be folded",
+			&Config{LAN: NetworkRole{Interface: "eth1"}},
+			"wan role must be bound",
+		},
+		{
+			"two roles share a NIC untagged",
+			&Config{
+				WAN: NetworkRole{Interface: "eth0"},
+				LAN: NetworkRole{Interface: "eth0"},
+			},
+			"both bind eth0",
+		},
+		{
+			"duplicate VLAN id on one NIC",
+			&Config{
+				WAN: NetworkRole{Interface: "eth0", VLAN: 10},
+				LAN: NetworkRole{Interface: "eth0", VLAN: 10},
+			},
+			"both bind eth0.10",
+		},
+		{
+			"VLAN id out of range",
+			&Config{WAN: NetworkRole{Interface: "eth0", VLAN: 5000}},
+			"out of range",
+		},
+		{
+			"MTU out of range",
+			&Config{WAN: NetworkRole{Interface: "eth0", MTU: 70000}},
+			"MTU 70000 out of range",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("Validate() = %v, want nil", err)
+				}
+				return
+			}
+			if err == nil {
+				t.Fatalf("Validate() = nil, want error containing %q", tt.wantErr)
+			}
+			if !strings.Contains(err.Error(), tt.wantErr) {
+				t.Errorf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Two roles sharing one NIC must produce a single parent .network carrying both
+// VLAN declarations — not two files racing to configure the same interface.
+func TestWriteParentNICTaggedSharesOneFile(t *testing.T) {
+	dir := t.TempDir()
+	p := &parentNIC{vlans: []string{"enp129s0np0.10", "enp129s0np0.20"}, mtu: 9000}
+	if err := writeParentNIC(dir, "enp129s0np0", p); err != nil {
+		t.Fatal(err)
+	}
+
+	got := readFile(t, dir, "05-spinifex-nic-enp129s0np0.network")
+	for _, want := range []string{
+		"Name=enp129s0np0",
+		"MTUBytes=9000",
+		"VLAN=enp129s0np0.10",
+		"VLAN=enp129s0np0.20",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("parent .network missing %q:\n%s", want, got)
+		}
+	}
+	// A tagged parent is never itself enslaved — the VLAN devices are.
+	if strings.Contains(got, "Bridge=") {
+		t.Errorf("tagged parent must not be bridged directly:\n%s", got)
+	}
+}
+
+func TestWriteNetworkdBridgeVLAN(t *testing.T) {
+	dir := t.TempDir()
+	cfg := vlanSplit()
+	if err := writeNetworkdBridge(dir, PlaneVPC, cfg.VPC, true); err != nil {
+		t.Fatal(err)
+	}
+
+	netdev := readFile(t, dir, "10-spinifex-vpc-vlan.netdev")
+	if !strings.Contains(netdev, "Kind=vlan") || !strings.Contains(netdev, "Id=20") {
+		t.Errorf("VLAN netdev wrong:\n%s", netdev)
+	}
+	if !strings.Contains(netdev, "Name=enp129s0np0.20") {
+		t.Errorf("VLAN netdev name wrong:\n%s", netdev)
+	}
+
+	vlanNet := readFile(t, dir, "11-spinifex-vpc-vlan.network")
+	if !strings.Contains(vlanNet, "Bridge=br-vpc") {
+		t.Errorf("VLAN device must be enslaved to br-vpc:\n%s", vlanNet)
+	}
+
+	brNet := readFile(t, dir, "20-spinifex-vpc.network")
+	if !strings.Contains(brNet, "Address=10.1.0.2/16") {
+		t.Errorf("bridge address wrong:\n%s", brNet)
+	}
+	// Only wan installs a default route; an east-west plane must not.
+	if strings.Contains(brNet, "Gateway=") {
+		t.Errorf("vpc plane must not install a default route:\n%s", brNet)
+	}
+	if !strings.Contains(brNet, "ActivationPolicy=manual") {
+		t.Errorf("east-west bridge must not auto-activate:\n%s", brNet)
+	}
+}
+
+// The wan plane is the one that carries a default route and auto-activates.
+func TestWriteNetworkdBridgeWANUntagged(t *testing.T) {
+	dir := t.TempDir()
+	cfg := threeNIC()
+	if err := writeNetworkdBridge(dir, PlaneWAN, cfg.WAN, false); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := os.Stat(filepath.Join(dir, "10-spinifex-wan-vlan.netdev")); !os.IsNotExist(err) {
+		t.Error("untagged role must not produce a VLAN netdev")
+	}
+	brNet := readFile(t, dir, "20-spinifex-wan.network")
+	if !strings.Contains(brNet, "Gateway=216.218.163.97") {
+		t.Errorf("wan must install its default route:\n%s", brNet)
+	}
+	if strings.Contains(brNet, "ActivationPolicy=manual") {
+		t.Errorf("wan bridge must auto-activate:\n%s", brNet)
+	}
+}
+
+func TestToFirstbootConfigSourcesPlanes(t *testing.T) {
+	cfg := threeNIC()
+	cfg.Hostname = "hydrogen"
+	fb := cfg.toFirstbootConfig()
+
+	if fb.EncapIP != "10.1.0.3" {
+		t.Errorf("EncapIP = %q, want the vpc plane address", fb.EncapIP)
+	}
+	if fb.LANIP != "10.0.0.3" {
+		t.Errorf("LANIP = %q, want the lan plane address", fb.LANIP)
+	}
+	// The public address has to travel separately: spx echoes a concrete --bind
+	// back as the advertise address, so leaving this empty would publish the
+	// internal plane as the node's public dial target.
+	if fb.WANIP != "216.218.163.99" {
+		t.Errorf("WANIP = %q, want the wan plane address", fb.WANIP)
+	}
+
+	// On a single-NIC node all three collapse onto wan.
+	single := &Config{WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"}}
+	fb = single.toFirstbootConfig()
+	if fb.EncapIP != "192.168.1.10" || fb.LANIP != "192.168.1.10" || fb.WANIP != "192.168.1.10" {
+		t.Errorf("single-NIC collapse wrong: EncapIP=%q LANIP=%q WANIP=%q", fb.EncapIP, fb.LANIP, fb.WANIP)
+	}
+}
+
+func readFile(t *testing.T, dir, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(dir, name))
+	if err != nil {
+		t.Fatalf("reading %s: %v", name, err)
+	}
+	return string(b)
+}
+
+// Resolvers belong to the plane holding the default route. The east-west
+// bridges are link-local and activate manually, so a resolver pinned to one is
+// unreachable while resolved still tries it — a timeout in front of every
+// lookup. The TUI cannot express this; the kernel cmdline can, so it is caught.
+func TestValidateRejectsDNSOffWAN(t *testing.T) {
+	cfg := threeNIC()
+	cfg.LAN.DNS = []string{"10.0.0.1"}
+	if err := cfg.Validate(); err == nil {
+		t.Fatal("lan must not carry DNS servers")
+	}
+
+	cfg = threeNIC()
+	cfg.VPC.DNS = []string{"10.1.0.1"}
+	if err := cfg.Validate(); err == nil {
+		t.Fatal("vpc must not carry DNS servers")
+	}
+
+	cfg = threeNIC()
+	cfg.WAN.DNS = []string{"8.8.8.8", "1.1.1.1"}
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("wan carries the resolvers: %v", err)
+	}
+}
+
+// Even if a role somehow carries DNS, generation must not write it onto an
+// east-west bridge.
+func TestWriteNetworkdBridgeKeepsDNSOnWAN(t *testing.T) {
+	dir := t.TempDir()
+	lan := NetworkRole{Interface: "ens1f0np0", Address: "10.0.0.3", Mask: "255.255.0.0", DNS: []string{"10.0.0.1"}}
+	if err := writeNetworkdBridge(dir, PlaneLAN, lan, true); err != nil {
+		t.Fatalf("writeNetworkdBridge: %v", err)
+	}
+	if got := readFile(t, dir, "20-spinifex-lan.network"); strings.Contains(got, "DNS=") {
+		t.Errorf("lan bridge must carry no resolver:\n%s", got)
+	}
+
+	wan := NetworkRole{Interface: "eno6", Address: "216.218.163.99", Mask: "255.255.255.224", Gateway: "216.218.163.97", DNS: []string{"8.8.8.8"}}
+	if err := writeNetworkdBridge(dir, PlaneWAN, wan, false); err != nil {
+		t.Fatalf("writeNetworkdBridge: %v", err)
+	}
+	if got := readFile(t, dir, "20-spinifex-wan.network"); !strings.Contains(got, "DNS=8.8.8.8") {
+		t.Errorf("wan bridge must carry the resolver:\n%s", got)
+	}
+}
+
+func TestSwapSize(t *testing.T) {
+	dir := t.TempDir()
+
+	// Without an override, RAM drives the size and the disk caps it.
+	ram, err := totalRAM()
+	if err != nil {
+		t.Fatalf("totalRAM: %v", err)
+	}
+	got, err := swapSize(dir)
+	if err != nil {
+		t.Fatalf("swapSize: %v", err)
+	}
+	want := ram / 2
+	if ram >= swapRAMThreshold {
+		want = swapLargeBytes
+	}
+	if got > want {
+		t.Errorf("swapSize = %d, must not exceed what RAM implies (%d)", got, want)
+	}
+	if got != 0 && got < swapMinBytes {
+		t.Errorf("swapSize = %d, must be 0 rather than below the %d floor", got, int64(swapMinBytes))
+	}
+
+	// An explicit size wins, so a large node can be told to take the full 32G
+	// and a constrained one can opt out entirely.
+	for _, tt := range []struct {
+		env  string
+		want int64
+	}{
+		{"32G", 32 << 30},
+		{"512M", 512 << 20},
+		{"0", 0},
+		{"1073741824", 1 << 30},
+	} {
+		t.Setenv("SPINIFEX_SWAP_SIZE", tt.env)
+		got, err := swapSize(dir)
+		if err != nil {
+			t.Fatalf("swapSize(%s): %v", tt.env, err)
+		}
+		if got != tt.want {
+			t.Errorf("swapSize(%s) = %d, want %d", tt.env, got, tt.want)
+		}
+	}
+
+	t.Setenv("SPINIFEX_SWAP_SIZE", "banana")
+	if _, err := swapSize(dir); err == nil {
+		t.Error("an unparseable size must be reported, not silently ignored")
+	}
+}
+
+// The swap line must only appear when the file backing it exists — systemd
+// fails the boot's swap unit on a missing file rather than skipping it.
+func TestFstabSwapLineTracksTheFile(t *testing.T) {
+	if _, err := parseSize("8G"); err != nil {
+		t.Fatalf("parseSize: %v", err)
+	}
+	if n, _ := parseSize("2G"); n != 2<<30 {
+		t.Errorf("parseSize(2G) = %d, want %d", n, int64(2)<<30)
+	}
+	if _, err := parseSize("-5G"); err == nil {
+		t.Error("negative sizes must be rejected")
+	}
+}
+
+// Bridge ports and VLAN trunks never reach the "degraded" state
+// systemd-networkd-wait-online requires — the address sits on the bridge above
+// them. Leaving them required makes that unit fail on every boot and stall
+// until it times out, which is what a three-NIC node was doing.
+func TestSlaveLinksAreNotRequiredForOnline(t *testing.T) {
+	dir := t.TempDir()
+
+	// Untagged bridge port.
+	if err := writeParentNIC(dir, "ens4", &parentNIC{bridge: "br-lan", mtu: 9000}); err != nil {
+		t.Fatal(err)
+	}
+	got := readFile(t, dir, "05-spinifex-nic-ens4.network")
+	if !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("bridge port must not gate network-online:\n%s", got)
+	}
+	if !strings.Contains(got, "MTUBytes=9000") {
+		t.Errorf("MTU must survive alongside it:\n%s", got)
+	}
+
+	// VLAN trunk carrying tagged planes.
+	if err := writeParentNIC(dir, "enp129s0np0", &parentNIC{vlans: []string{"enp129s0np0.10"}}); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "05-spinifex-nic-enp129s0np0.network"); !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("VLAN trunk must not gate network-online:\n%s", got)
+	}
+
+	// The VLAN device itself is enslaved too.
+	cfg := vlanSplit()
+	if err := writeNetworkdBridge(dir, PlaneVPC, cfg.VPC, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "11-spinifex-vpc-vlan.network"); !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("VLAN device must not gate network-online:\n%s", got)
+	}
+
+	// The wan bridge is what should gate it, and still does.
+	three := threeNIC()
+	if err := writeNetworkdBridge(dir, PlaneWAN, three.WAN, false); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "20-spinifex-wan.network"); strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("wan bridge must remain required for network-online:\n%s", got)
+	}
+}
+
+// A prefix length reaches netgen's net.ParseIP as garbage and fails the install
+// after the disk has been partitioned, so it has to be rejected up front. The
+// TUI advertised "255.255.255.0 or 24" while only the first ever worked.
+func TestValidateRejectsPrefixLengthMask(t *testing.T) {
+	for _, bad := range []string{"24", "/24", "16", "255.255.0", "255.0.255.0", "banana"} {
+		cfg := threeNIC()
+		cfg.LAN.Mask = bad
+		if err := cfg.Validate(); err == nil {
+			t.Errorf("mask %q must be rejected", bad)
+		} else if !strings.Contains(err.Error(), "dotted-decimal") {
+			t.Errorf("mask %q: error should name the expected form, got: %v", bad, err)
+		}
+	}
+
+	for _, good := range []string{"255.255.255.0", "255.255.0.0", "255.255.255.252", "255.255.255.255"} {
+		cfg := threeNIC()
+		cfg.LAN.Mask = good
+		if err := cfg.Validate(); err != nil {
+			t.Errorf("mask %q must be accepted: %v", good, err)
+		}
+	}
+
+	// A DHCP role carries no mask, and that must stay valid.
+	cfg := threeNIC()
+	cfg.LAN = NetworkRole{Interface: "ens1f0np0", DHCPMode: true}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("a DHCP role has no mask to check: %v", err)
+	}
+}
+
+// A bridged uplink must present the enslaved NIC's address. Left to its default
+// policy, systemd-networkd generates one for the bridge, so the node appears on
+// the wire as an address belonging to no NIC — DHCP requests carry a source that
+// disagrees with their own chaddr and the unicast reply is never delivered.
+func TestBridgeInheritsPortMAC(t *testing.T) {
+	dir := t.TempDir()
+	cfg := threeNIC()
+
+	for _, tc := range []struct {
+		plane  Plane
+		role   NetworkRole
+		manual bool
+	}{
+		{PlaneWAN, cfg.WAN, false},
+		{PlaneLAN, cfg.LAN, true},
+		{PlaneVPC, cfg.VPC, true},
+	} {
+		if err := writeNetworkdBridge(dir, tc.plane, tc.role, tc.manual); err != nil {
+			t.Fatal(err)
+		}
+		got := readFile(t, dir, "20-spinifex-"+string(tc.plane)+".netdev")
+		if !strings.Contains(got, "MACAddressPolicy=none") {
+			t.Errorf("%s bridge must inherit its port's MAC:\n%s", tc.plane, got)
+		}
+	}
+}

cmd/installer/install/disk.go

@@ -0,0 +1,196 @@
+package install
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"time"
+)
+
+// Partition numbers, identical on every disk the installer touches so that a
+// replacement drive can be cloned with `sgdisk -R` and nothing has to be
+// recomputed at 3am.
+const (
+	biosPartNum = 1
+	espPartNum  = 2
+	rootPartNum = 3
+
+	// dataPartNum is the sole partition on a role drive. Data drives carry no
+	// bootloader, so they start at 1 rather than mirroring the boot layout.
+	dataPartNum = 1
+)
+
+// GPT partition type GUIDs, by their sgdisk short code.
+const (
+	typeBIOSBoot = "EF02"
+	typeESP      = "EF00"
+	typeLinuxFS  = "8300"
+	typeZFS      = "BF00"
+)
+
+// Partition geometry. The ESP is sized as Proxmox does: 512MiB is enough for
+// two kernels, but a 1GiB ESP on a real disk costs nothing and keeps room for
+// the extra kernels an operator pins during an upgrade.
+const (
+	espSmallMiB     = 512
+	espLargeMiB     = 1024
+	espLargeDiskGiB = 100
+
+	// biosBootMiB is the BIOS Boot Partition GRUB's i386-pc core.img is embedded
+	// into. GPT leaves no post-MBR gap, so without it there is nowhere to put it.
+	biosBootMiB = 1
+
+	// tailReserveMiB is left unallocated at the end of every disk. A replacement
+	// drive of the "same" size is routinely a few thousand sectors smaller, and
+	// without slack `sgdisk -R` refuses to clone the table onto it — which is
+	// discovered while standing in front of a degraded pool.
+	tailReserveMiB = 1024
+)
+
+// espSizeMiB picks the ESP size for a disk.
+func espSizeMiB(d Disk) int {
+	if d.Bytes >= espLargeDiskGiB<<30 {
+		return espLargeMiB
+	}
+	return espSmallMiB
+}
+
+// partitionDisks lays down an identical GPT on every selected disk. Each disk
+// gets a BIOS boot partition and an ESP whether or not this machine booted in
+// EFI mode: the cost is 1GiB, and it means a board swap or a firmware change
+// from BIOS to UEFI does not leave an unbootable pool.
+func partitionDisks(cfg DiskConfig) error {
+	ptype := typeLinuxFS
+	if cfg.FS.IsZFS() {
+		ptype = typeZFS
+	}
+	for _, d := range cfg.bootDisks() {
+		if err := partitionOne(d, ptype); err != nil {
+			return fmt.Errorf("partition %s: %w", d.Path, err)
+		}
+	}
+	for _, rm := range cfg.DataMounts() {
+		if err := partitionDataDisk(rm); err != nil {
+			return fmt.Errorf("partition %s (%s): %w", rm.Disk.Path, rm.Role, err)
+		}
+	}
+	return waitForPartitions(cfg)
+}
+
+// partitionDataDisk writes the single-partition GPT for a role drive. It gets
+// no bios_boot and no ESP: nothing installs a bootloader there, and a formatted
+// ESP no firmware entry points at only misleads whoever reads lsblk in an
+// incident.
+func partitionDataDisk(rm RoleMount) error {
+	if err := clearDisk(rm.Disk); err != nil {
+		return fmt.Errorf("clear: %w", err)
+	}
+	// Same tail reserve as a boot drive, for the same reason: a replacement of
+	// the "same" size is routinely a few thousand sectors smaller.
+	if err := run("sgdisk",
+		"-n", fmt.Sprintf("%d:1M:-%dM", dataPartNum, tailReserveMiB),
+		"-t", fmt.Sprintf("%d:%s", dataPartNum, typeLinuxFS),
+		"-c", fmt.Sprintf("%d:%s", dataPartNum, rm.Role.Label()),
+		rm.Disk.Path,
+	); err != nil {
+		return err
+	}
+	if err := run("sgdisk", "-G", rm.Disk.Path); err != nil {
+		slog.Warn("could not randomise GPT GUIDs", "disk", rm.Disk.Path, "err", err)
+	}
+	return nil
+}
+
+// partitionOne writes the GPT for a single disk.
+func partitionOne(d Disk, rootType string) error {
+	// Stale ZFS labels and filesystem signatures survive a new partition table
+	// and make a retried install fail with errors that read like hardware faults.
+	if err := clearDisk(d); err != nil {
+		return fmt.Errorf("clear: %w", err)
+	}
+
+	esp := espSizeMiB(d)
+	if err := run("sgdisk",
+		"-n", fmt.Sprintf("%d:1M:+%dM", biosPartNum, biosBootMiB),
+		"-t", fmt.Sprintf("%d:%s", biosPartNum, typeBIOSBoot),
+		"-c", fmt.Sprintf("%d:bios_boot", biosPartNum),
+		"-n", fmt.Sprintf("%d:0:+%dM", espPartNum, esp),
+		"-t", fmt.Sprintf("%d:%s", espPartNum, typeESP),
+		"-c", fmt.Sprintf("%d:ESP", espPartNum),
+		"-n", fmt.Sprintf("%d:0:-%dM", rootPartNum, tailReserveMiB),
+		"-t", fmt.Sprintf("%d:%s", rootPartNum, rootType),
+		"-c", fmt.Sprintf("%d:root", rootPartNum),
+		d.Path,
+	); err != nil {
+		return err
+	}
+	// A fresh disk-level GUID: cloning a table with `sgdisk -R` copies it, and
+	// two disks sharing one leaves the pool ambiguous to udev.
+	if err := run("sgdisk", "-G", d.Path); err != nil {
+		slog.Warn("could not randomise GPT GUIDs", "disk", d.Path, "err", err)
+	}
+	return nil
+}
+
+// waitForPartitions ensures every partition device node the installer is about
+// to use exists. Trixie's udev is slow enough after a table rewrite that mkfs
+// races it and fails with ENOENT on a device the kernel has already accepted.
+func waitForPartitions(cfg DiskConfig) error {
+	for _, d := range cfg.Disks {
+		// Best-effort: partprobe failing is not fatal, since udev may still act
+		// on the BLKRRPART ioctl sgdisk issued itself.
+		if err := run("partprobe", d.Path); err != nil {
+			slog.Warn("partprobe failed, continuing", "disk", d.Path, "err", err)
+		}
+	}
+	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
+		slog.Warn("udevadm settle failed, continuing", "err", err)
+	}
+
+	deadline := time.Now().Add(15 * time.Second)
+	for _, d := range cfg.bootDisks() {
+		wanted := []string{d.PartitionPath(espPartNum), d.PartitionPath(rootPartNum)}
+		// The pool is built from by-id paths, so their symlinks have to exist
+		// too — udev publishes them later than the kernel device node.
+		if d.ByID != "" {
+			wanted = append(wanted, d.StablePartitionPath(rootPartNum))
+		}
+		for _, part := range wanted {
+			if err := waitForPath(part, deadline); err != nil {
+				return err
+			}
+		}
+	}
+	for _, rm := range cfg.DataMounts() {
+		if err := waitForPath(rm.Disk.PartitionPath(dataPartNum), deadline); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func waitForPath(path string, deadline time.Time) error {
+	for {
+		if _, err := os.Stat(path); err == nil {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("device %s did not appear within timeout — the kernel or udev did not pick up the new partition table", path)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// formatESPs makes a FAT32 filesystem on every disk's ESP. Each one is
+// independent — there is no RAID here, they are kept in step by content.
+func formatESPs(cfg DiskConfig) error {
+	for _, d := range cfg.bootDisks() {
+		esp := d.PartitionPath(espPartNum)
+		// A distinct label per member so `lsblk -f` on a degraded machine tells
+		// the operator which disk each ESP belongs to.
+		if err := run("mkfs.fat", "-F", "32", "-n", "EFI", esp); err != nil {
+			return fmt.Errorf("format ESP %s: %w", esp, err)
+		}
+	}
+	return nil
+}

cmd/installer/install/diskconfig.go

@@ -0,0 +1,483 @@
+package install
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"strings"
+)
+
+// Pool and dataset names. rpool matches Proxmox, which is what every ZFS
+// recovery guide and forum answer assumes — the name an operator will be
+// reading off a serial console at 3am.
+const (
+	ZFSPoolName    = "rpool"
+	ZFSRootDataset = ZFSPoolName + "/ROOT/spinifex"
+)
+
+// FSType selects the root filesystem and, for ZFS, the vdev topology.
+type FSType string
+
+const (
+	FSExt4      FSType = "ext4"
+	FSZFSRAID0  FSType = "zfs-raid0"
+	FSZFSRAID1  FSType = "zfs-raid1"
+	FSZFSRAID10 FSType = "zfs-raid10"
+	FSZFSRAIDZ1 FSType = "zfs-raidz1"
+	FSZFSRAIDZ2 FSType = "zfs-raidz2"
+	FSZFSRAIDZ3 FSType = "zfs-raidz3"
+)
+
+// AllFSTypes is the selector order presented by the UI. ext4 is first and is
+// the default, so ZFS — which holds the kernel at its shipped version and
+// reserves RAM for the ARC — is always an explicit choice.
+var AllFSTypes = []FSType{
+	FSExt4, FSZFSRAID0, FSZFSRAID1, FSZFSRAID10, FSZFSRAIDZ1, FSZFSRAIDZ2, FSZFSRAIDZ3,
+}
+
+// fsMeta carries the per-topology facts that drive display and validation.
+var fsMeta = map[FSType]struct {
+	label     string
+	minDisks  int
+	redundant bool   // subject to the same-size rule
+	vdev      string // zpool create keyword; empty means a plain stripe
+}{
+	FSExt4:      {"ext4", 1, false, ""},
+	FSZFSRAID0:  {"zfs (RAID0)", 1, false, ""},
+	FSZFSRAID1:  {"zfs (RAID1)", 2, true, "mirror"},
+	FSZFSRAID10: {"zfs (RAID10)", 4, true, "mirror"},
+	FSZFSRAIDZ1: {"zfs (RAIDZ-1)", 3, true, "raidz1"},
+	FSZFSRAIDZ2: {"zfs (RAIDZ-2)", 4, true, "raidz2"},
+	FSZFSRAIDZ3: {"zfs (RAIDZ-3)", 5, true, "raidz3"},
+}
+
+// IsZFS reports whether this mode creates a pool rather than a plain filesystem.
+func (f FSType) IsZFS() bool { return strings.HasPrefix(string(f), "zfs-") }
+
+// Label is the display name, matching proxinstall's wording so the screen is
+// recognisable to anyone who has installed Proxmox.
+func (f FSType) Label() string { return fsMeta[f].label }
+
+// MinDisks is the smallest member count the topology can be built from.
+func (f FSType) MinDisks() int { return fsMeta[f].minDisks }
+
+// Redundant reports whether the topology stores parity or mirrors, and so
+// requires members of matching size. A RAID0 stripe over unequal disks is
+// legitimate and is deliberately not covered.
+func (f FSType) Redundant() bool { return fsMeta[f].redundant }
+
+// vdevKeyword is the zpool create keyword introducing each vdev.
+func (f FSType) vdevKeyword() string { return fsMeta[f].vdev }
+
+// ParseFSType maps a SPINIFEX_FS value to an FSType.
+func ParseFSType(s string) (FSType, error) {
+	f := FSType(strings.ToLower(strings.TrimSpace(s)))
+	if _, ok := fsMeta[f]; !ok {
+		names := make([]string, 0, len(AllFSTypes))
+		for _, t := range AllFSTypes {
+			names = append(names, string(t))
+		}
+		return "", fmt.Errorf("unknown filesystem %q — must be one of: %s", s, strings.Join(names, ", "))
+	}
+	return f, nil
+}
+
+// DiskRole is the purpose a selected drive serves on an ext4 install. A ZFS
+// pool carries every workload on one set of members, so roles do not apply
+// there — the topology already spans all the disks.
+type DiskRole string
+
+const (
+	RoleOS         DiskRole = "os"
+	RoleSpinifex   DiskRole = "spinifex"
+	RolePredastore DiskRole = "predastore"
+)
+
+// AllDiskRoles is canonical order, outermost mountpoint first. Mounting, fstab
+// emission and rsync protect filters all walk it forwards; teardown walks it
+// back, because predastore nests inside spinifex.
+var AllDiskRoles = []DiskRole{RoleOS, RoleSpinifex, RolePredastore}
+
+// roleMountpoints is where each role lands on the installed system. A role left
+// unassigned simply stays on the filesystem above it.
+var roleMountpoints = map[DiskRole]string{
+	RoleOS:         "/",
+	RoleSpinifex:   "/var/lib/spinifex",
+	RolePredastore: "/var/lib/spinifex/predastore",
+}
+
+// Label is the role name as shown in the UI and written to partition labels.
+func (r DiskRole) Label() string { return string(r) }
+
+// Mountpoint is the role's path on the installed system.
+func (r DiskRole) Mountpoint() string { return roleMountpoints[r] }
+
+// RoleMount binds one drive to one role.
+type RoleMount struct {
+	Role DiskRole
+	Disk Disk
+}
+
+// Mountpoint is where this drive's filesystem is mounted on the target.
+func (r RoleMount) Mountpoint() string { return r.Role.Mountpoint() }
+
+// ZFSOpts are the advanced tunables. A zero value means "use the computed
+// default", which is why Ashift and ARCMaxMiB are plain ints — 0 is not legal
+// for either, so it is unambiguous as a sentinel.
+type ZFSOpts struct {
+	Ashift    int
+	Compress  string
+	Checksum  string
+	Copies    int
+	ARCMaxMiB int
+}
+
+// DiskConfig is the storage half of the installer's configuration. Disks is
+// ordered and the order is significant: RAID10 pairs members two at a time, and
+// on ext4 the first entry is the OS drive.
+type DiskConfig struct {
+	FS    FSType
+	Disks []Disk
+	ZFS   ZFSOpts
+
+	// Roles assigns ext4 drives to mountpoints, in AllDiskRoles order. Empty
+	// means a single-disk install with everything on the root filesystem.
+	Roles []RoleMount
+}
+
+// WithRoles returns a copy with ext4 drive roles assigned and Disks rebuilt
+// from them in canonical order, so the list of drives to erase and the list of
+// roles cannot drift apart — there is one place a drive can be named, not two.
+func (d DiskConfig) WithRoles(roles []RoleMount) DiskConfig {
+	d.Roles, d.Disks = nil, nil
+	add := func(rm RoleMount) {
+		d.Roles = append(d.Roles, rm)
+		d.Disks = append(d.Disks, rm.Disk)
+	}
+	for _, want := range AllDiskRoles {
+		for _, rm := range roles {
+			if rm.Role == want {
+				add(rm)
+			}
+		}
+	}
+	// Anything with an unknown or duplicate role is kept so Validate can name
+	// it, rather than being dropped into a silently smaller selection.
+	for _, rm := range roles {
+		if _, known := roleMountpoints[rm.Role]; !known {
+			add(rm)
+		}
+	}
+	return d
+}
+
+// RolesForDisks assigns roles positionally, which is the default the UI offers:
+// first disk picked is the OS, second is spinifex, third is predastore.
+func RolesForDisks(disks []Disk) []RoleMount {
+	out := make([]RoleMount, 0, len(disks))
+	for i, disk := range disks {
+		var role DiskRole
+		if i < len(AllDiskRoles) {
+			role = AllDiskRoles[i]
+		}
+		out = append(out, RoleMount{Role: role, Disk: disk})
+	}
+	return out
+}
+
+// DataMounts is every role except the OS drive, shallowest mountpoint first.
+// This is the traversal every stage of the install shares.
+func (d DiskConfig) DataMounts() []RoleMount {
+	var out []RoleMount
+	for _, rm := range d.Roles {
+		if rm.Role != RoleOS {
+			out = append(out, rm)
+		}
+	}
+	return out
+}
+
+// bootDisks are the drives a bootloader is installed to. Every ZFS member gets
+// its own ESP; on ext4 only the OS drive does, because a data drive holding a
+// bootloader that nothing points at is a trap during recovery.
+func (d DiskConfig) bootDisks() []Disk {
+	if d.FS.IsZFS() || len(d.Disks) == 0 {
+		return d.Disks
+	}
+	return d.Disks[:1]
+}
+
+// Primary is the first selected disk — the OS drive in ext4 mode, and the
+// disk shown in single-disk summaries.
+func (d DiskConfig) Primary() Disk {
+	if len(d.Disks) == 0 {
+		return Disk{}
+	}
+	return d.Disks[0]
+}
+
+// Paths lists the selected device paths, for logs and confirmation screens.
+func (d DiskConfig) Paths() []string {
+	out := make([]string, len(d.Disks))
+	for i, disk := range d.Disks {
+		out[i] = disk.Path
+	}
+	return out
+}
+
+// Buildable reports whether the selection has enough members, of the right
+// count, to form the topology at all. Capacity and fault tolerance are
+// meaningless until it does — a 3-disk RAIDZ-3 has no geometry to describe.
+func (d DiskConfig) Buildable() bool {
+	if _, ok := fsMeta[d.FS]; !ok {
+		return false
+	}
+	if len(d.Disks) < d.FS.MinDisks() {
+		return false
+	}
+	return d.FS != FSZFSRAID10 || len(d.Disks)%2 == 0
+}
+
+// Requirement describes what the topology still needs, for the preview line
+// shown in place of a capacity that cannot be computed yet.
+func (d DiskConfig) Requirement() string {
+	req := fmt.Sprintf("needs %d disks", d.FS.MinDisks())
+	if d.FS == FSZFSRAID10 {
+		req = "needs an even number of disks, at least 4"
+	}
+	if d.FS.Redundant() {
+		req += " of the same size"
+	}
+	return req
+}
+
+// UsableBytes estimates post-parity capacity, for the geometry preview. It is
+// an operator-facing approximation and deliberately ignores metadata overhead.
+// Returns 0 for a selection that cannot form the topology, since the parity
+// arithmetic below goes to zero and then negative as members run out.
+func (d DiskConfig) UsableBytes() int64 {
+	n := int64(len(d.Disks))
+	if n == 0 || !d.Buildable() {
+		return 0
+	}
+	smallest := d.Disks[0].Bytes
+	for _, disk := range d.Disks {
+		smallest = min(smallest, disk.Bytes)
+	}
+	switch d.FS {
+	case FSExt4, FSZFSRAID0:
+		var total int64
+		for _, disk := range d.Disks {
+			total += disk.Bytes
+		}
+		return total
+	case FSZFSRAID1:
+		return smallest
+	case FSZFSRAID10:
+		return smallest * (n / 2)
+	case FSZFSRAIDZ1:
+		return smallest * (n - 1)
+	case FSZFSRAIDZ2:
+		return smallest * (n - 2)
+	case FSZFSRAIDZ3:
+		return smallest * (n - 3)
+	}
+	return 0
+}
+
+// Tolerated returns the number of simultaneous disk failures the topology
+// survives, for the geometry preview.
+func (d DiskConfig) Tolerated() int {
+	if !d.Buildable() {
+		return 0
+	}
+	switch d.FS {
+	case FSZFSRAID1:
+		return max(len(d.Disks)-1, 0)
+	case FSZFSRAID10:
+		return 1 // one per mirror pair; the honest worst case is one
+	case FSZFSRAIDZ1:
+		return 1
+	case FSZFSRAIDZ2:
+		return 2
+	case FSZFSRAIDZ3:
+		return 3
+	}
+	return 0
+}
+
+// sizeTolerance is the fraction by which pool members may differ in size.
+// Matches Proxmox's zfs_mirror_size_check: anything beyond 10% and the pool
+// silently truncates every member to the smallest.
+const sizeTolerance = 0.10
+
+// bootedEFI reports whether firmware booted this machine in EFI mode. GRUB's
+// i386-pc target cannot read a 4Kn drive, so legacy BIOS constrains selection.
+func bootedEFI() bool {
+	_, err := os.Stat("/sys/firmware/efi")
+	return err == nil
+}
+
+// Validate checks a disk selection against every rule that must hold before any
+// destructive command runs. The UI calls it on each keystroke so failures block
+// Continue, rather than aborting an install that has already repartitioned.
+func (d DiskConfig) Validate() error {
+	if _, ok := fsMeta[d.FS]; !ok {
+		return fmt.Errorf("no filesystem selected")
+	}
+	if len(d.Disks) == 0 {
+		return fmt.Errorf("%s: select at least one disk", d.FS.Label())
+	}
+	if err := d.validateRoles(); err != nil {
+		return err
+	}
+	if n := len(d.Disks); n < d.FS.MinDisks() {
+		return fmt.Errorf("%s: needs at least %d disks, %d selected", d.FS.Label(), d.FS.MinDisks(), n)
+	}
+	if d.FS == FSZFSRAID10 && len(d.Disks)%2 != 0 {
+		return fmt.Errorf("%s: needs an even number of disks, %d selected", d.FS.Label(), len(d.Disks))
+	}
+
+	seen := map[string]bool{}
+	for _, disk := range d.Disks {
+		if seen[disk.Path] {
+			return fmt.Errorf("%s is selected more than once", disk.Path)
+		}
+		seen[disk.Path] = true
+
+		if disk.LiveMedia {
+			return fmt.Errorf("%s is the installer's own boot media and cannot be erased", disk.Path)
+		}
+		if disk.Bytes < minRootBytes {
+			return fmt.Errorf("%s is too small (%s, need at least %dGiB)",
+				disk.Path, disk.SizeHuman(), minRootBytes>>30)
+		}
+	}
+
+	// GRUB's i386-pc target cannot read a 4Kn drive, so on legacy BIOS a drive
+	// holding a bootloader would leave the installed system unreachable. Only
+	// boot drives are constrained: a data role is never booted from.
+	for _, disk := range d.bootDisks() {
+		if !bootedEFI() && disk.LogicalBlockSize == 4096 {
+			return fmt.Errorf("%s is a 4Kn drive (4096-byte sectors) and this machine booted in legacy BIOS mode — booting from it is not supported; enable UEFI in firmware",
+				disk.Path)
+		}
+	}
+
+	return d.validateSizes()
+}
+
+// validateRoles checks the ext4 role assignment. Roles are the only way to use
+// more than one disk without ZFS, so a multi-disk selection without them is
+// rejected rather than quietly installed onto the first drive.
+func (d DiskConfig) validateRoles() error {
+	if d.FS.IsZFS() {
+		if len(d.Roles) > 0 {
+			return fmt.Errorf("%s: drive roles apply to ext4 only — a pool spans every member already", d.FS.Label())
+		}
+		return nil
+	}
+	if len(d.Roles) == 0 {
+		if len(d.Disks) > 1 {
+			return fmt.Errorf("%s: %d disks selected but no roles assigned — assign os, spinifex and predastore, or choose a zfs mode",
+				d.FS.Label(), len(d.Disks))
+		}
+		return nil
+	}
+	if len(d.Roles) > len(AllDiskRoles) {
+		return fmt.Errorf("%s: at most %d disks, %d selected — the roles are os, spinifex and predastore",
+			d.FS.Label(), len(AllDiskRoles), len(d.Roles))
+	}
+
+	seen := map[DiskRole]bool{}
+	for _, rm := range d.Roles {
+		if _, ok := roleMountpoints[rm.Role]; !ok {
+			return fmt.Errorf("%s has no role assigned — every selected disk needs one", rm.Disk.Path)
+		}
+		if seen[rm.Role] {
+			return fmt.Errorf("two disks are assigned the %s role", rm.Role)
+		}
+		seen[rm.Role] = true
+	}
+	if !seen[RoleOS] {
+		return fmt.Errorf("%s: no disk assigned the os role — one disk must hold the operating system", d.FS.Label())
+	}
+	if len(d.Roles) != len(d.Disks) {
+		return fmt.Errorf("internal: %d roles for %d disks", len(d.Roles), len(d.Disks))
+	}
+	return nil
+}
+
+// minRootBytes matches Proxmox's hard floor for a root disk.
+const minRootBytes = 2 << 30
+
+// validateSizes enforces the same-size rule where the topology needs it:
+// across all members for mirrors and raidz, and per-pair for RAID10 — a
+// striped mirror only requires each pair to match, not every disk in the pool.
+func (d DiskConfig) validateSizes() error {
+	if !d.FS.Redundant() {
+		return nil
+	}
+	if d.FS == FSZFSRAID10 {
+		for i := 0; i+1 < len(d.Disks); i += 2 {
+			if err := sizesMatch(d.Disks[i], d.Disks[i+1]); err != nil {
+				return fmt.Errorf("%s mirror pair %d: %w", d.FS.Label(), i/2+1, err)
+			}
+		}
+		return nil
+	}
+	for _, disk := range d.Disks[1:] {
+		if err := sizesMatch(d.Disks[0], disk); err != nil {
+			return fmt.Errorf("%s: %w", d.FS.Label(), err)
+		}
+	}
+	return nil
+}
+
+// SizesWithinTolerance reports whether two disks could be members of the same
+// redundant pool. The UI uses it to decide whether suggesting ZFS is honest.
+func SizesWithinTolerance(a, b Disk) bool { return sizesMatch(a, b) == nil }
+
+// sizesMatch reports whether two members are within the tolerance, naming both
+// disks and their sizes — that message is the operator's only diagnostic.
+func sizesMatch(a, b Disk) error {
+	if math.Abs(float64(a.Bytes-b.Bytes)) > float64(a.Bytes)*sizeTolerance {
+		return fmt.Errorf("disks must be the same size within %.0f%%: %s is %s, %s is %s",
+			sizeTolerance*100, a.Path, a.SizeHuman(), b.Path, b.SizeHuman())
+	}
+	return nil
+}
+
+// Warnings returns non-blocking advice for a valid selection. A RAID0 stripe
+// over mismatched disks is legal but almost always a mistake, so it is
+// surfaced without preventing the install — which is what Proxmox does.
+func (d DiskConfig) Warnings() []string {
+	var out []string
+	if d.FS == FSZFSRAID0 && len(d.Disks) > 1 {
+		for _, disk := range d.Disks[1:] {
+			if sizesMatch(d.Disks[0], disk) != nil {
+				out = append(out, "striping disks of different sizes — capacity is usable but a failure of any disk loses the pool")
+				break
+			}
+		}
+	}
+	if d.FS.IsZFS() && d.FS != FSZFSRAID0 && d.Buildable() && d.Tolerated() == 0 {
+		out = append(out, "this topology stores no redundancy")
+	}
+	// A three-drive role layout looks like an array and is not one: each drive
+	// is an independent point of failure, so the node loses whatever sat on it.
+	if len(d.DataMounts()) > 0 {
+		out = append(out, "role-assigned drives store no redundancy — losing any one drive loses what it held")
+	}
+	for _, rm := range d.DataMounts() {
+		if rm.Role == RolePredastore && rm.Disk.Bytes < smallPredastoreBytes {
+			out = append(out, fmt.Sprintf("%s is small for object storage (%s) — predastore will fill quickly",
+				rm.Disk.Path, rm.Disk.SizeHuman()))
+		}
+	}
+	return out
+}
+
+// smallPredastoreBytes is where a dedicated object-store drive stops being
+// worth the drive. Advisory only: the install is legal below it.
+const smallPredastoreBytes = 64 << 30

cmd/installer/install/diskconfig_test.go

@@ -0,0 +1,267 @@
+package install
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDiskConfigValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     DiskConfig
+		wantErr string
+	}{
+		{"ext4 on one disk", DiskConfig{FS: FSExt4, Disks: disks(1, 40)}, ""},
+		{"raidz1 on three matching disks", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)}, ""},
+		{"raid10 on four matching disks", DiskConfig{FS: FSZFSRAID10, Disks: disks(4, 40)}, ""},
+		// The single-drive ZFS case: no redundancy, but checksums, compression
+		// and snapshots still apply.
+		{"raid0 on one disk", DiskConfig{FS: FSZFSRAID0, Disks: disks(1, 40)}, ""},
+
+		{"no filesystem chosen", DiskConfig{Disks: disks(1, 40)}, "no filesystem selected"},
+		{"no disks", DiskConfig{FS: FSZFSRAIDZ1}, "select at least one disk"},
+		{"ext4 spans disks only with roles", DiskConfig{FS: FSExt4, Disks: disks(2, 40)}, "no roles assigned"},
+		{"raidz1 needs three", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(2, 40)}, "at least 3 disks"},
+		{"raidz2 needs four", DiskConfig{FS: FSZFSRAIDZ2, Disks: disks(3, 40)}, "at least 4 disks"},
+		{"raidz3 needs five", DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(4, 40)}, "at least 5 disks"},
+		{"raid1 needs two", DiskConfig{FS: FSZFSRAID1, Disks: disks(1, 40)}, "at least 2 disks"},
+		{"raid10 rejects an odd count", DiskConfig{FS: FSZFSRAID10, Disks: disks(5, 40)}, "even number"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			switch {
+			case tt.wantErr == "" && err != nil:
+				t.Fatalf("Validate() = %v, want nil", err)
+			case tt.wantErr != "" && err == nil:
+				t.Fatalf("Validate() = nil, want error containing %q", tt.wantErr)
+			case tt.wantErr != "" && !strings.Contains(err.Error(), tt.wantErr):
+				t.Fatalf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestValidateRejectsDuplicateDisk(t *testing.T) {
+	d := disk("a", 40)
+	err := DiskConfig{FS: FSZFSRAID1, Disks: []Disk{d, d}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "more than once") {
+		t.Fatalf("Validate() = %v, want a duplicate-selection error", err)
+	}
+}
+
+func TestValidateRejectsLiveMedia(t *testing.T) {
+	d := disk("a", 40)
+	d.LiveMedia = true
+	err := DiskConfig{FS: FSExt4, Disks: []Disk{d}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "boot media") {
+		t.Fatalf("Validate() = %v, want the installer's own media to be refused", err)
+	}
+}
+
+func TestValidateRejectsTooSmallDisk(t *testing.T) {
+	err := DiskConfig{FS: FSExt4, Disks: []Disk{disk("a", 1)}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "too small") {
+		t.Fatalf("Validate() = %v, want a minimum-size error", err)
+	}
+}
+
+// A pool truncates every member to the smallest, so mismatched disks in a
+// redundant topology silently throw capacity away.
+func TestValidateSizeTolerance(t *testing.T) {
+	tests := []struct {
+		name    string
+		fs      FSType
+		sizes   []int64
+		wantErr bool
+	}{
+		{"identical disks", FSZFSRAIDZ1, []int64{40, 40, 40}, false},
+		{"within 10%", FSZFSRAIDZ1, []int64{100, 95, 92}, false},
+		{"beyond 10%", FSZFSRAIDZ1, []int64{100, 100, 80}, true},
+		{"mirror with a mismatched member", FSZFSRAID1, []int64{100, 50}, true},
+		// A stripe has no parity to waste, so mismatched sizes are a warning
+		// rather than a refusal — matching Proxmox.
+		{"raid0 permits mismatched disks", FSZFSRAID0, []int64{100, 40}, false},
+		{"raid10 checks each pair, not the whole pool", FSZFSRAID10, []int64{40, 40, 100, 100}, false},
+		{"raid10 rejects a mismatched pair", FSZFSRAID10, []int64{40, 100, 100, 100}, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			members := make([]Disk, len(tt.sizes))
+			for i, gib := range tt.sizes {
+				members[i] = disk(string(rune('a'+i)), gib)
+			}
+			err := DiskConfig{FS: tt.fs, Disks: members}.Validate()
+			if tt.wantErr && err == nil {
+				t.Fatalf("Validate() = nil, want a size-mismatch error")
+			}
+			if !tt.wantErr && err != nil {
+				t.Fatalf("Validate() = %v, want nil", err)
+			}
+		})
+	}
+}
+
+func TestWarningsFlagMismatchedStripe(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAID0, Disks: []Disk{disk("a", 100), disk("b", 40)}}
+	warnings := cfg.Warnings()
+	if len(warnings) == 0 {
+		t.Fatal("a stripe over mismatched disks should warn")
+	}
+}
+
+func TestUsableBytes(t *testing.T) {
+	tests := []struct {
+		name    string
+		fs      FSType
+		n       int
+		wantGiB int64
+	}{
+		{"stripe adds every disk", FSZFSRAID0, 3, 120},
+		{"mirror gives one disk", FSZFSRAID1, 2, 40},
+		{"raid10 gives half", FSZFSRAID10, 4, 80},
+		{"raidz1 loses one to parity", FSZFSRAIDZ1, 3, 80},
+		{"raidz2 loses two", FSZFSRAIDZ2, 4, 80},
+		{"raidz3 loses three", FSZFSRAIDZ3, 5, 80},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if got := cfg.UsableBytes() >> 30; got != tt.wantGiB {
+				t.Errorf("UsableBytes() = %dG, want %dG", got, tt.wantGiB)
+			}
+		})
+	}
+}
+
+// The parity arithmetic goes to zero and then negative as members run out, and
+// a preview reading "-40GiB usable" is worse than no figure at all.
+func TestUsableBytesIsZeroWhenTopologyCannotBeBuilt(t *testing.T) {
+	tests := []struct {
+		name string
+		fs   FSType
+		n    int
+	}{
+		{"raidz3 with exactly its parity count", FSZFSRAIDZ3, 3},
+		{"raidz3 with fewer disks than parity", FSZFSRAIDZ3, 2},
+		{"raidz2 with two disks", FSZFSRAIDZ2, 2},
+		{"raidz1 with one disk", FSZFSRAIDZ1, 1},
+		{"mirror with one disk", FSZFSRAID1, 1},
+		{"raid10 with an odd count", FSZFSRAID10, 3},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if cfg.Buildable() {
+				t.Fatalf("Buildable() = true for %s across %d disks", tt.fs, tt.n)
+			}
+			if got := cfg.UsableBytes(); got != 0 {
+				t.Errorf("UsableBytes() = %d, want 0", got)
+			}
+			if got := cfg.Tolerated(); got != 0 {
+				t.Errorf("Tolerated() = %d, want 0", got)
+			}
+		})
+	}
+}
+
+// A selection that cannot be built must not also be described as unredundant:
+// the operator's problem is the disk count, not the topology.
+func TestWarningsStaySilentUntilBuildable(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(3, 40)}
+	if got := cfg.Warnings(); len(got) != 0 {
+		t.Errorf("Warnings() = %v, want none", got)
+	}
+}
+
+func TestRequirementNamesTheSameSizeRule(t *testing.T) {
+	if got := (DiskConfig{FS: FSZFSRAIDZ2}).Requirement(); !strings.Contains(got, "4 disks") ||
+		!strings.Contains(got, "same size") {
+		t.Errorf("Requirement() = %q, want the disk count and the same-size rule", got)
+	}
+	if got := (DiskConfig{FS: FSZFSRAID10}).Requirement(); !strings.Contains(got, "even") {
+		t.Errorf("Requirement() = %q, want the even-count rule", got)
+	}
+	// A stripe has no matching requirement, so claiming one would be wrong.
+	if got := (DiskConfig{FS: FSZFSRAID0}).Requirement(); strings.Contains(got, "same size") {
+		t.Errorf("Requirement() = %q, want no same-size rule for a stripe", got)
+	}
+}
+
+func TestTolerated(t *testing.T) {
+	tests := []struct {
+		fs   FSType
+		n    int
+		want int
+	}{
+		{FSExt4, 1, 0},
+		{FSZFSRAID0, 3, 0},
+		{FSZFSRAID1, 2, 1},
+		{FSZFSRAID1, 3, 2},
+		{FSZFSRAID10, 4, 1},
+		{FSZFSRAIDZ1, 3, 1},
+		{FSZFSRAIDZ2, 4, 2},
+		{FSZFSRAIDZ3, 5, 3},
+	}
+	for _, tt := range tests {
+		t.Run(string(tt.fs), func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if got := cfg.Tolerated(); got != tt.want {
+				t.Errorf("Tolerated() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseFSType(t *testing.T) {
+	for _, fs := range AllFSTypes {
+		got, err := ParseFSType(strings.ToUpper(string(fs)))
+		if err != nil {
+			t.Errorf("ParseFSType(%q): %v", fs, err)
+			continue
+		}
+		if got != fs {
+			t.Errorf("ParseFSType(%q) = %q", fs, got)
+		}
+	}
+	// The error has to list the valid values: it is read off a serial console
+	// by someone whose unattended install just stopped.
+	_, err := ParseFSType("btrfs")
+	if err == nil || !strings.Contains(err.Error(), "zfs-raidz1") {
+		t.Fatalf("ParseFSType(btrfs) = %v, want an error listing the accepted values", err)
+	}
+}
+
+func TestPartitionPathSeparator(t *testing.T) {
+	tests := []struct{ dev, want string }{
+		{"/dev/sda", "/dev/sda3"},
+		// A trailing digit needs a 'p' or the name is ambiguous.
+		{"/dev/nvme0n1", "/dev/nvme0n1p3"},
+		{"/dev/vda", "/dev/vda3"},
+	}
+	for _, tt := range tests {
+		if got := (Disk{Path: tt.dev}).PartitionPath(3); got != tt.want {
+			t.Errorf("PartitionPath(%s) = %s, want %s", tt.dev, got, tt.want)
+		}
+	}
+}
+
+// udev names partition links with a -partN suffix regardless of the kernel's
+// separator, so the by-id form must not reuse the 'p' rule.
+func TestStablePartitionPathUsesPartSuffix(t *testing.T) {
+	d := Disk{Path: "/dev/nvme0n1", ByID: "/dev/disk/by-id/nvme-SAMSUNG_1234"}
+	if got := d.StablePartitionPath(3); got != "/dev/disk/by-id/nvme-SAMSUNG_1234-part3" {
+		t.Errorf("StablePartitionPath() = %s", got)
+	}
+}
+
+func TestESPSizeMiB(t *testing.T) {
+	if got := espSizeMiB(disk("a", 40)); got != espSmallMiB {
+		t.Errorf("espSizeMiB(40G) = %d, want %d", got, espSmallMiB)
+	}
+	if got := espSizeMiB(disk("a", 500)); got != espLargeMiB {
+		t.Errorf("espSizeMiB(500G) = %d, want %d", got, espLargeMiB)
+	}
+}

cmd/installer/install/install_test.go

@@ -9,7 +9,7 @@ func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
 	const url = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	t.Setenv("SPINIFEX_INSTALL_CALLBACK", url)
 
-	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	cfg := &Config{Hostname: "node1"}
 	fb := cfg.toFirstbootConfig()
 	if fb.InstallCallback != url {
 		t.Errorf("InstallCallback = %q, want %q", fb.InstallCallback, url)
@@ -19,7 +19,7 @@ func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
 func TestToFirstbootConfigEmptyCallbackWhenEnvUnset(t *testing.T) {
 	os.Unsetenv("SPINIFEX_INSTALL_CALLBACK")
 
-	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	cfg := &Config{Hostname: "node1"}
 	fb := cfg.toFirstbootConfig()
 	if fb.InstallCallback != "" {
 		t.Errorf("InstallCallback = %q, want empty", fb.InstallCallback)

cmd/installer/install/roles_test.go

@@ -0,0 +1,391 @@
+package install
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// roleCfg builds an ext4 configuration from role/disk-name pairs, in whatever
+// order they are given — WithRoles is responsible for normalising them.
+func roleCfg(pairs ...any) DiskConfig {
+	var roles []RoleMount
+	for i := 0; i+1 < len(pairs); i += 2 {
+		roles = append(roles, RoleMount{
+			Role: pairs[i].(DiskRole),
+			Disk: disk(pairs[i+1].(string), 100),
+		})
+	}
+	return DiskConfig{FS: FSExt4}.WithRoles(roles)
+}
+
+// fakeUUIDs makes partUUID return a value derived from the device path, so a
+// generated fstab can be checked without touching a real disk.
+func fakeUUIDs(t *testing.T) {
+	t.Helper()
+	prev := partUUID
+	partUUID = func(dev string) (string, error) { return "uuid" + strings.ReplaceAll(dev, "/", "-"), nil }
+	t.Cleanup(func() { partUUID = prev })
+}
+
+func TestWithRolesNormalisesOrderAndDerivesDisks(t *testing.T) {
+	// Given deepest-first, which is the order that breaks mounting.
+	cfg := roleCfg(RolePredastore, "c", RoleSpinifex, "b", RoleOS, "a")
+
+	wantRoles := []DiskRole{RoleOS, RoleSpinifex, RolePredastore}
+	for i, rm := range cfg.Roles {
+		if rm.Role != wantRoles[i] {
+			t.Fatalf("role %d = %s, want %s", i, rm.Role, wantRoles[i])
+		}
+	}
+	// Disks is derived, so the OS drive is first and every role drive appears
+	// exactly once in the list the installer erases.
+	if got := cfg.Paths(); !slices.Equal(got, []string{"/dev/a", "/dev/b", "/dev/c"}) {
+		t.Fatalf("Paths() = %v, want the roles in canonical order", got)
+	}
+	if cfg.Primary().Path != "/dev/a" {
+		t.Errorf("Primary() = %s, want the os drive", cfg.Primary().Path)
+	}
+}
+
+func TestDataMountsExcludeTheOSDriveAndNestShallowestFirst(t *testing.T) {
+	got := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c").DataMounts()
+	want := []string{"/var/lib/spinifex", "/var/lib/spinifex/predastore"}
+	for i, rm := range got {
+		if rm.Mountpoint() != want[i] {
+			t.Fatalf("mount %d = %s, want %s", i, rm.Mountpoint(), want[i])
+		}
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d data mounts, want %d", len(got), len(want))
+	}
+}
+
+// A predastore drive without a spinifex drive is the two-drive layout, and it
+// nests a mount inside the root filesystem rather than inside another mount.
+func TestPredastoreRoleWithoutSpinifexIsValid(t *testing.T) {
+	cfg := roleCfg(RoleOS, "a", RolePredastore, "b")
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("Validate() = %v, want nil", err)
+	}
+	if got := cfg.DataMounts(); len(got) != 1 || got[0].Mountpoint() != "/var/lib/spinifex/predastore" {
+		t.Fatalf("DataMounts() = %v, want predastore only", got)
+	}
+}
+
+func TestValidateRoles(t *testing.T) {
+	twoRoles := func(a, b DiskRole) DiskConfig { return roleCfg(a, "a", b, "b") }
+
+	tests := []struct {
+		name    string
+		cfg     DiskConfig
+		wantErr string
+	}{
+		{"single disk needs no roles", DiskConfig{FS: FSExt4, Disks: disks(1, 40)}, ""},
+		{"os alone", roleCfg(RoleOS, "a"), ""},
+		{"all three", roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"), ""},
+		{"no os role", twoRoles(RoleSpinifex, RolePredastore), "no disk assigned the os role"},
+		{"duplicate role", twoRoles(RoleOS, RoleOS), "two disks are assigned the os role"},
+		{"unknown role", roleCfg(RoleOS, "a", DiskRole("cache"), "b"), "no role assigned"},
+		{"more disks than roles", roleCfg(
+			RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c", DiskRole("cache"), "d"), "at most 3 disks"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			switch {
+			case tt.wantErr == "" && err != nil:
+				t.Fatalf("Validate() = %v, want nil", err)
+			case tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)):
+				t.Fatalf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Roles describe where a workload lives on a plain filesystem. A pool already
+// spans every member, so accepting them there would silently do nothing.
+func TestValidateRejectsRolesOnZFS(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	cfg.Roles = []RoleMount{{Role: RoleOS, Disk: cfg.Disks[0]}}
+	if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "ext4 only") {
+		t.Fatalf("Validate() = %v, want a roles-are-ext4-only error", err)
+	}
+}
+
+func TestBootDisksAreOnlyTheOSDriveOnExt4(t *testing.T) {
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if got := cfg.bootDisks(); len(got) != 1 || got[0].Path != "/dev/a" {
+		t.Fatalf("bootDisks() = %v, want the os drive only", got)
+	}
+	// Every ZFS member carries its own ESP, so all of them stay boot disks.
+	zfs := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	if got := zfs.bootDisks(); len(got) != 2 {
+		t.Fatalf("bootDisks() = %v, want every pool member", got)
+	}
+}
+
+func TestWriteFstabEmitsRoleMountsShallowestFirst(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if err := writeFstab(cfg); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	got := readFile(t, root, "etc/fstab")
+
+	// Order matters: `mount -a` in a rescue shell walks the file top-down, and
+	// predastore cannot be mounted before the filesystem it sits inside.
+	wantOrder := []string{" / ", "/boot/efi", " /var/lib/spinifex ", " /var/lib/spinifex/predastore "}
+	at := -1
+	for _, want := range wantOrder {
+		i := strings.Index(got, want)
+		if i < 0 {
+			t.Fatalf("fstab is missing %q:\n%s", want, got)
+		}
+		if i < at {
+			t.Fatalf("%q is out of order:\n%s", want, got)
+		}
+		at = i
+	}
+
+	for _, dev := range []string{"/dev/b1", "/dev/c1"} {
+		if want := "UUID=uuid-dev-" + strings.TrimPrefix(dev, "/dev/"); !strings.Contains(got, want) {
+			t.Errorf("fstab does not mount %s by UUID:\n%s", dev, got)
+		}
+	}
+	// Data mounts are fsck'd after root, never alongside it.
+	for line := range strings.SplitSeq(got, "\n") {
+		if strings.Contains(line, "/var/lib/spinifex") && !strings.HasSuffix(line, "0 2") {
+			t.Errorf("data mount should be fsck pass 2: %q", line)
+		}
+	}
+}
+
+// Fail-closed by design: a missing data drive must stop the boot for an admin,
+// not let predastore write objects onto the OS drive while everything reports
+// healthy. nofail would silently produce exactly that.
+func TestWriteFstabNeverMarksRoleMountsNofail(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	if err := writeFstab(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	if got := readFile(t, root, "etc/fstab"); strings.Contains(got, "nofail") {
+		t.Errorf("role mounts must not be nofail:\n%s", got)
+	}
+}
+
+func TestWriteFstabSingleDiskIsUnchanged(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	if err := writeFstab(roleCfg(RoleOS, "a")); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	if got := readFile(t, root, "etc/fstab"); strings.Contains(got, "/var/lib/spinifex") {
+		t.Errorf("a single-disk install should mount nothing extra:\n%s", got)
+	}
+}
+
+// rsync runs with --delete, so every mountpoint that exists before the copy
+// needs protecting or the copy can delete across a mount boundary on a retry.
+func TestMountProtectFiltersCoverEveryRoleMount(t *testing.T) {
+	got := mountProtectFilters(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"))
+	want := []string{"--filter=P /var/lib/spinifex", "--filter=P /var/lib/spinifex/predastore"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("mountProtectFilters() = %v, want %v", got, want)
+	}
+	if got := mountProtectFilters(roleCfg(RoleOS, "a")); len(got) != 0 {
+		t.Errorf("a single-disk install has no mounts to protect, got %v", got)
+	}
+	// ZFS keeps its dataset filters — the union is by mode, not additive.
+	if got := mountProtectFilters(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}); len(got) == 0 {
+		t.Error("zfs mode should still protect its dataset mountpoints")
+	}
+}
+
+func TestMountPartitionsMountsRolesAfterRootAndBeforeTheCopy(t *testing.T) {
+	f := fakeCommands(t)
+	root := withMountRoot(t)
+
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if err := mountPartitions(cfg); err != nil {
+		t.Fatalf("mountPartitions: %v", err)
+	}
+
+	// The nested mount cannot precede the one it sits inside, or it is masked.
+	want := []string{
+		"mount /dev/a3 " + root,
+		"mount /dev/a2 " + root + "/boot/efi",
+		"mount /dev/b1 " + root + "/var/lib/spinifex",
+		"mount /dev/c1 " + root + "/var/lib/spinifex/predastore",
+	}
+	var mounts []string
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "mount ") {
+			mounts = append(mounts, c)
+		}
+	}
+	if !slices.Equal(mounts, want) {
+		t.Fatalf("mount order:\n got %v\nwant %v", mounts, want)
+	}
+}
+
+func TestCleanupTargetUnmountsDeepestFirst(t *testing.T) {
+	f := fakeCommands(t)
+	root := withMountRoot(t)
+
+	cleanupTarget(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"))
+
+	// Unmounting the outer filesystem first fails with EBUSY and leaves a
+	// half-mounted target behind for the next attempt.
+	want := []string{
+		"umount " + root + "/var/lib/spinifex/predastore",
+		"umount " + root + "/var/lib/spinifex",
+		"umount " + root + "/boot/efi",
+		"umount " + root,
+	}
+	// The chroot binds are released first and are not part of the target order.
+	var got []string
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "umount ") && slices.Contains(want, c) {
+			got = append(got, c)
+		}
+	}
+	if !slices.Equal(got, want) {
+		t.Fatalf("unmount order:\n got %v\nwant %v", got, want)
+	}
+}
+
+func TestPartitionDisksGivesDataDrivesOnePartitionAndNoESP(t *testing.T) {
+	f := fakeCommands(t)
+	nodes := tempDisks(t, 2, 200)
+	cfg := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: nodes[0]}, {Role: RoleSpinifex, Disk: nodes[1]}})
+	osDisk, data := nodes[0].Path, nodes[1].Path
+
+	if err := partitionDisks(cfg); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+
+	f.mustRun(t, "sgdisk", osDisk, "-n 3:0:-1024M", "-t 3:8300")
+	f.mustRun(t, "sgdisk", osDisk, "-t 2:EF00")
+	// A data drive is never booted from, so a bios_boot or an ESP there would
+	// only mislead whoever reads lsblk during an incident.
+	for _, c := range f.calls {
+		if strings.Contains(c, data) && (strings.Contains(c, "EF02") || strings.Contains(c, "EF00")) {
+			t.Errorf("data drive must get no bios_boot or ESP: %q", c)
+		}
+	}
+	f.mustRun(t, "sgdisk", data, "-n 1:1M:-1024M", "-t 1:8300", "-c 1:spinifex")
+}
+
+func TestFormatPartitionsLabelsEachRoleFilesystem(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+
+	if err := formatPartitions(cfg); err != nil {
+		t.Fatalf("formatPartitions: %v", err)
+	}
+	// One ESP only: a data drive holding a bootloader nothing points at would
+	// mislead whoever reads lsblk during an incident.
+	esps := 0
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "mkfs.fat") {
+			esps++
+		}
+	}
+	if esps != 1 {
+		t.Errorf("want one ESP, got %d:\n%v", esps, f.calls)
+	}
+	f.mustRun(t, "mkfs.ext4", "-L", "spinifex", "/dev/b1")
+	f.mustRun(t, "mkfs.ext4", "-L", "predastore", "/dev/c1")
+}
+
+func TestWarningsFlagTheAbsenceOfRedundancy(t *testing.T) {
+	got := strings.Join(roleCfg(RoleOS, "a", RoleSpinifex, "b").Warnings(), "\n")
+	if !strings.Contains(got, "no redundancy") {
+		t.Errorf("a multi-drive role layout must warn it is not an array, got %q", got)
+	}
+	if got := roleCfg(RoleOS, "a").Warnings(); len(got) != 0 {
+		t.Errorf("a single-disk install has nothing to warn about, got %v", got)
+	}
+}
+
+func TestWarningsFlagASmallPredastoreDrive(t *testing.T) {
+	cfg := DiskConfig{FS: FSExt4}.WithRoles([]RoleMount{
+		{Role: RoleOS, Disk: disk("a", 500)},
+		{Role: RolePredastore, Disk: disk("b", 8)},
+	})
+	got := strings.Join(cfg.Warnings(), "\n")
+	if !strings.Contains(got, "small for object storage") {
+		t.Errorf("an 8GiB predastore drive should be flagged, got %q", got)
+	}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("the warning must not block the install: %v", err)
+	}
+}
+
+func TestRolesForDisksAssignsPositionally(t *testing.T) {
+	got := RolesForDisks(disks(3, 40))
+	for i, rm := range got {
+		if want := AllDiskRoles[i]; rm.Role != want {
+			t.Errorf("disk %d got role %s, want %s", i, rm.Role, want)
+		}
+	}
+	// A fourth disk has no role, which Validate is left to reject by name.
+	if extra := RolesForDisks(disks(4, 40)); extra[3].Role != "" {
+		t.Errorf("disk 4 got role %q, want none", extra[3].Role)
+	}
+}
+
+func TestFourKnDataDriveIsAllowedWhereABootDriveIsNot(t *testing.T) {
+	if bootedEFI() {
+		t.Skip("machine booted in EFI mode; the 4Kn rule does not apply")
+	}
+	fourKn := func(name string) Disk {
+		d := disk(name, 100)
+		d.LogicalBlockSize = 4096
+		return d
+	}
+	// GRUB's i386-pc target cannot read a 4Kn drive, but nothing boots from a
+	// data drive, so only the OS role is constrained.
+	ok := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: disk("a", 100)}, {Role: RoleSpinifex, Disk: fourKn("b")}})
+	if err := ok.Validate(); err != nil {
+		t.Errorf("a 4Kn data drive should be accepted: %v", err)
+	}
+
+	bad := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: fourKn("a")}, {Role: RoleSpinifex, Disk: disk("b", 100)}})
+	if err := bad.Validate(); err == nil || !strings.Contains(err.Error(), "4Kn") {
+		t.Errorf("Validate() = %v, want a 4Kn rejection for the os drive", err)
+	}
+}
+
+func TestVerifyRoleLayoutCatchesAMountThatNeverHappened(t *testing.T) {
+	root := withMountRoot(t)
+	// Plain directories on one filesystem: exactly what a mount performed after
+	// the rootfs copy, or not at all, leaves behind.
+	if err := os.MkdirAll(filepath.Join(root, "var/lib/spinifex/predastore"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	err := verifyRoleLayout(roleCfg(RoleOS, "a", RoleSpinifex, "b"))
+	if err == nil || !strings.Contains(err.Error(), "not a separate filesystem") {
+		t.Fatalf("verifyRoleLayout() = %v, want a missing-mount error", err)
+	}
+	if !strings.Contains(fmt.Sprint(err), "/dev/b") {
+		t.Errorf("the error must name the drive that was not mounted: %v", err)
+	}
+}

cmd/installer/install/steps_test.go

@@ -0,0 +1,821 @@
+package install
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// fakeExec records every command the install steps shell out to and lets a
+// test fail a chosen one. Without it these steps can only be exercised by
+// running a real install against real hardware.
+type fakeExec struct {
+	calls []string
+	fail  func(name string, args []string) error
+}
+
+func fakeCommands(t *testing.T) *fakeExec {
+	t.Helper()
+	f := &fakeExec{}
+	prevRun, prevQuiet, prevEnv := run, runQuiet, runEnv
+
+	record := func(name string, args ...string) error {
+		f.calls = append(f.calls, strings.TrimSpace(name+" "+strings.Join(args, " ")))
+		if f.fail != nil {
+			return f.fail(name, args)
+		}
+		return nil
+	}
+	run = record
+	runQuiet = record
+	runEnv = func(env []string, name string, args ...string) error {
+		return record(name, append(env, args...)...)
+	}
+
+	t.Cleanup(func() { run, runQuiet, runEnv = prevRun, prevQuiet, prevEnv })
+	return f
+}
+
+// ran reports whether any recorded command contains every given fragment.
+func (f *fakeExec) ran(fragments ...string) bool {
+	for _, c := range f.calls {
+		all := true
+		for _, frag := range fragments {
+			if !strings.Contains(c, frag) {
+				all = false
+				break
+			}
+		}
+		if all {
+			return true
+		}
+	}
+	return false
+}
+
+func (f *fakeExec) mustRun(t *testing.T, fragments ...string) {
+	t.Helper()
+	if !f.ran(fragments...) {
+		t.Errorf("no command matched %v; recorded:\n  %s", fragments, strings.Join(f.calls, "\n  "))
+	}
+}
+
+func (f *fakeExec) mustNotRun(t *testing.T, fragments ...string) {
+	t.Helper()
+	if f.ran(fragments...) {
+		t.Errorf("unexpected command matching %v; recorded:\n  %s", fragments, strings.Join(f.calls, "\n  "))
+	}
+}
+
+// indexOf returns the position of the first command matching frag, or -1.
+func (f *fakeExec) indexOf(frag string) int {
+	for i, c := range f.calls {
+		if strings.Contains(c, frag) {
+			return i
+		}
+	}
+	return -1
+}
+
+// indexOfExact is indexOf for a whole command, for cases where one command
+// line is a prefix of another.
+func (f *fakeExec) indexOfExact(cmd string) int {
+	for i, c := range f.calls {
+		if c == cmd {
+			return i
+		}
+	}
+	return -1
+}
+
+// withMountRoot points the install target at a temp tree.
+func withMountRoot(t *testing.T) string {
+	t.Helper()
+	prev := mountRoot
+	mountRoot = t.TempDir()
+	t.Cleanup(func() { mountRoot = prev })
+	return mountRoot
+}
+
+// targetSkeleton creates the directories rsync would already have copied in by
+// the time these steps run.
+func targetSkeleton(t *testing.T, root string) {
+	t.Helper()
+	for _, d := range []string{"etc/default", "etc/grub.d", "etc/systemd/system", "etc"} {
+		if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// tempDisks builds disks whose device nodes are real files, so the steps that
+// wait for partition nodes to appear can complete.
+func tempDisks(t *testing.T, n int, gib int64) []Disk {
+	t.Helper()
+	dir := t.TempDir()
+	out := make([]Disk, n)
+	for i := range out {
+		d := Disk{
+			Path:              filepath.Join(dir, fmt.Sprintf("sd%c", 'a'+i)),
+			Bytes:             gib << 30,
+			LogicalBlockSize:  512,
+			PhysicalBlockSize: 512,
+		}
+		for _, part := range []int{biosPartNum, espPartNum, rootPartNum} {
+			if err := os.WriteFile(d.PartitionPath(part), nil, 0o644); err != nil {
+				t.Fatalf("create partition node: %v", err)
+			}
+		}
+		out[i] = d
+	}
+	return out
+}
+
+func TestPartitionDisksWritesTheSameLayoutOnEveryDisk(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 200)}
+
+	if err := partitionDisks(cfg); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+
+	for _, d := range cfg.Disks {
+		// Stale signatures survive a new table, so the wipe comes first.
+		f.mustRun(t, "wipefs -a "+d.Path)
+		f.mustRun(t, "sgdisk -Z "+d.Path)
+		f.mustRun(t,
+			"sgdisk", d.Path,
+			"-n 1:1M:+1M", "-t 1:EF02", "-c 1:bios_boot",
+			// A 200GiB disk is a real host, so it gets the large ESP.
+			"-n 2:0:+1024M", "-t 2:EF00", "-c 2:ESP",
+			"-n 3:0:-1024M", "-t 3:BF00", "-c 3:root",
+		)
+		f.mustRun(t, "sgdisk -G "+d.Path)
+		f.mustRun(t, "partprobe "+d.Path)
+
+		if wipe, part := f.indexOf("wipefs -a "+d.Path), f.indexOf("-c 3:root "+d.Path); wipe > part {
+			t.Errorf("wipefs ran after the table was written (%d > %d)", wipe, part)
+		}
+	}
+	f.mustRun(t, "udevadm settle --timeout=10")
+}
+
+func TestPartitionDisksTypesTheRootPartitionForTheFilesystem(t *testing.T) {
+	f := fakeCommands(t)
+	if err := partitionDisks(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+	f.mustRun(t, "-t 3:8300")
+	f.mustNotRun(t, "-t 3:BF00")
+	// Under 100GiB the ESP stays at the smaller size.
+	f.mustRun(t, "-n 2:0:+512M")
+}
+
+func TestPartitionDisksNamesTheDiskThatFailed(t *testing.T) {
+	f := fakeCommands(t)
+	disks := tempDisks(t, 2, 40)
+	f.fail = func(name string, args []string) error {
+		if name == "sgdisk" && strings.Contains(strings.Join(args, " "), "bios_boot") {
+			return errors.New("device busy")
+		}
+		return nil
+	}
+
+	err := partitionDisks(DiskConfig{FS: FSExt4, Disks: disks})
+	if err == nil || !strings.Contains(err.Error(), disks[0].Path) {
+		t.Fatalf("err = %v, want it to name %s", err, disks[0].Path)
+	}
+}
+
+func TestPartitionDisksSurvivesBestEffortFailures(t *testing.T) {
+	f := fakeCommands(t)
+	// partprobe, udevadm and the GUID randomisation are all advisory: sgdisk
+	// has already issued BLKRRPART and the nodes exist.
+	f.fail = func(name string, args []string) error {
+		switch {
+		case name == "partprobe", name == "udevadm":
+			return errors.New("not available")
+		case name == "sgdisk" && len(args) > 0 && args[0] == "-G":
+			return errors.New("no")
+		}
+		return nil
+	}
+	if err := partitionDisks(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+}
+
+func TestWaitForPathFailsWhenTheNodeNeverAppears(t *testing.T) {
+	missing := filepath.Join(t.TempDir(), "sda3")
+
+	err := waitForPath(missing, time.Now().Add(-time.Second))
+	if err == nil || !strings.Contains(err.Error(), missing) {
+		t.Fatalf("err = %v, want a timeout naming the missing node", err)
+	}
+}
+
+func TestWaitForPathReturnsOnceTheNodeExists(t *testing.T) {
+	f, err := os.CreateTemp(t.TempDir(), "node")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := waitForPath(f.Name(), time.Now().Add(-time.Second)); err != nil {
+		t.Errorf("waitForPath on an existing node: %v", err)
+	}
+}
+
+func TestEspSizeMiB(t *testing.T) {
+	if got := espSizeMiB(Disk{Bytes: 99 << 30}); got != espSmallMiB {
+		t.Errorf("99GiB -> %d, want %d", got, espSmallMiB)
+	}
+	if got := espSizeMiB(Disk{Bytes: 100 << 30}); got != espLargeMiB {
+		t.Errorf("100GiB -> %d, want %d", got, espLargeMiB)
+	}
+}
+
+func TestClearDiskOnlyLabelclearsAnExistingPartition(t *testing.T) {
+	f := fakeCommands(t)
+	disks := tempDisks(t, 1, 40)
+	if err := clearDisk(disks[0]); err != nil {
+		t.Fatalf("clearDisk: %v", err)
+	}
+	f.mustRun(t, "zpool labelclear -f "+disks[0].PartitionPath(rootPartNum))
+
+	f2 := fakeCommands(t)
+	blank := Disk{Path: filepath.Join(t.TempDir(), "sdb")}
+	if err := clearDisk(blank); err != nil {
+		t.Fatalf("clearDisk: %v", err)
+	}
+	f2.mustNotRun(t, "labelclear")
+	f2.mustRun(t, "wipefs -a "+blank.Path)
+}
+
+func TestFormatESPsLabelsEveryMember(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 40)}
+	if err := formatESPs(cfg); err != nil {
+		t.Fatalf("formatESPs: %v", err)
+	}
+	for _, d := range cfg.Disks {
+		f.mustRun(t, "mkfs.fat -F 32 -n EFI "+d.PartitionPath(espPartNum))
+	}
+}
+
+func TestFormatESPsNamesTheFailingESP(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 40)}
+	f.fail = func(name string, _ []string) error {
+		if name == "mkfs.fat" {
+			return errors.New("no such device")
+		}
+		return nil
+	}
+	err := formatESPs(cfg)
+	if err == nil || !strings.Contains(err.Error(), cfg.Disks[0].PartitionPath(espPartNum)) {
+		t.Fatalf("err = %v, want it to name the ESP", err)
+	}
+}
+
+func TestFormatPartitionsMakesTheExt4Root(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}
+	if err := formatPartitions(cfg); err != nil {
+		t.Fatalf("formatPartitions: %v", err)
+	}
+	f.mustRun(t, "mkfs.ext4 -F "+cfg.Disks[0].PartitionPath(rootPartNum))
+}
+
+func TestMountPartitionsMountsRootThenESP(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}
+
+	if err := mountPartitions(cfg); err != nil {
+		t.Fatalf("mountPartitions: %v", err)
+	}
+	d := cfg.Disks[0]
+	f.mustRun(t, "mount "+d.PartitionPath(rootPartNum)+" "+root)
+	f.mustRun(t, "mount "+d.PartitionPath(espPartNum)+" "+efiPart())
+	if _, err := os.Stat(efiPart()); err != nil {
+		t.Errorf("ESP mountpoint was not created: %v", err)
+	}
+	if f.indexOf("mount "+d.PartitionPath(rootPartNum)) > f.indexOf("mount "+d.PartitionPath(espPartNum)) {
+		t.Error("the ESP was mounted before the root it lives under")
+	}
+}
+
+func TestMountPartitionsStopsWhenTheRootWillNotMount(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "mount" {
+			return errors.New("wrong fs type")
+		}
+		return nil
+	}
+	if err := mountPartitions(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err == nil {
+		t.Fatal("expected an error when the root mount fails")
+	}
+	f.mustNotRun(t, "boot/efi")
+}
+
+func TestCleanupTargetExportsThePoolOnZFS(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	cleanupTarget(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
+
+	// Recursive: one dataset left mounted is enough to block the export.
+	f.mustRun(t, "umount -R "+root)
+	f.mustRun(t, "zpool export "+ZFSPoolName)
+	if f.indexOf("umount -R") > f.indexOf("zpool export") {
+		t.Error("the pool was exported before its datasets were unmounted")
+	}
+}
+
+func TestCleanupTargetUnmountsESPBeforeRootOnExt4(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	cleanupTarget(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
+
+	f.mustNotRun(t, "zpool export")
+	esp, rootIdx := f.indexOfExact("umount "+efiPart()), f.indexOfExact("umount "+root)
+	if esp < 0 || rootIdx < 0 {
+		t.Fatalf("missing umounts; recorded:\n  %s", strings.Join(f.calls, "\n  "))
+	}
+	if esp > rootIdx {
+		t.Error("the root was unmounted before the ESP nested inside it")
+	}
+}
+
+func TestCreatePoolBuildsTheZpoolCommand(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	opts := ZFSOpts{Ashift: 12, Compress: "lz4", Checksum: "blake3", Copies: 1, ARCMaxMiB: 4096}
+
+	if err := createPool(cfg, opts); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f.mustRun(t, "zpool create -f",
+		"cachefile=none", "ashift=12", "compression=lz4", "checksum=blake3", "copies=1",
+		// altroot keeps the datasets under the target rather than over the live system.
+		"-R "+root, ZFSPoolName, "mirror",
+		cfg.Disks[0].StablePartitionPath(rootPartNum),
+		cfg.Disks[1].StablePartitionPath(rootPartNum),
+	)
+}
+
+func TestCreatePoolEnablesAutotrimOnFlashOnly(t *testing.T) {
+	withMountRoot(t)
+	flash := disks(2, 40)
+	f := fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: flash}, ZFSOpts{Ashift: 12, Copies: 1}); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f.mustRun(t, "zpool set autotrim=on "+ZFSPoolName)
+
+	spinning := disks(2, 40)
+	spinning[1].Rotational = true
+	f2 := fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: spinning}, ZFSOpts{Ashift: 12, Copies: 1}); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f2.mustNotRun(t, "autotrim")
+}
+
+func TestCreatePoolWrapsTheZpoolFailure(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, args []string) error {
+		if name == "zpool" && len(args) > 0 && args[0] == "create" {
+			return errors.New("device is in use")
+		}
+		return nil
+	}
+	err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}, ZFSOpts{Ashift: 12, Copies: 1})
+	if err == nil || !strings.Contains(err.Error(), "zpool create") {
+		t.Fatalf("err = %v, want a zpool create failure", err)
+	}
+}
+
+func TestCreatePoolRejectsAnUnbuildableTopology(t *testing.T) {
+	withMountRoot(t)
+	fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: disks(1, 40)}, ZFSOpts{Ashift: 12}); err == nil {
+		t.Fatal("expected a vdev error for a one-disk mirror")
+	}
+}
+
+func TestCreateDatasetsCreatesEveryDataset(t *testing.T) {
+	f := fakeCommands(t)
+	if err := createDatasets(); err != nil {
+		t.Fatalf("createDatasets: %v", err)
+	}
+	for _, ds := range datasets {
+		f.mustRun(t, "zfs create", "mountpoint="+ds.mountpoint, ds.name)
+	}
+}
+
+func TestCreateDatasetsNamesTheDatasetThatFailed(t *testing.T) {
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "zfs" {
+			return errors.New("out of space")
+		}
+		return nil
+	}
+	err := createDatasets()
+	if err == nil || !strings.Contains(err.Error(), datasets[0].name) {
+		t.Fatalf("err = %v, want it to name %s", err, datasets[0].name)
+	}
+}
+
+func TestExportPoolToleratesABusyPool(t *testing.T) {
+	f := fakeCommands(t)
+	f.fail = func(string, []string) error { return errors.New("pool is busy") }
+	exportPool()
+	f.mustRun(t, "zpool export "+ZFSPoolName)
+}
+
+func TestWriteZFSSystemConfigDeclaresTheARCCeilingToBothConsumers(t *testing.T) {
+	root := withMountRoot(t)
+
+	if err := writeZFSSystemConfig(ZFSOpts{ARCMaxMiB: 4096}); err != nil {
+		t.Fatalf("writeZFSSystemConfig: %v", err)
+	}
+
+	modprobe := readFile(t, root, "etc/modprobe.d/zfs.conf")
+	if !strings.Contains(modprobe, fmt.Sprintf("zfs_arc_max=%d", int64(4096)<<20)) {
+		t.Errorf("zfs.conf does not cap the ARC in bytes:\n%s", modprobe)
+	}
+
+	// The daemon cannot see the ARC, so the reserve has to account for it or
+	// guests get admitted into memory the ARC already owns.
+	env := readFile(t, root, "etc/spinifex/host.env")
+	want := fmt.Sprintf("SPINIFEX_RESERVED_MEM_GB=%.1f", defaultHostReserveGB+4.0)
+	if !strings.Contains(env, want) {
+		t.Errorf("host.env = %q, want it to contain %q", env, want)
+	}
+}
+
+func TestWriteTargetFileAppliesTheModeOnRewrite(t *testing.T) {
+	root := withMountRoot(t)
+
+	if err := writeTargetFile("/usr/sbin/tool", []byte("#!/bin/sh\n"), 0o600); err != nil {
+		t.Fatalf("writeTargetFile: %v", err)
+	}
+	// A retried install rewrites an existing file, and WriteFile would leave
+	// the original mode in place.
+	if err := writeTargetFile("/usr/sbin/tool", []byte("#!/bin/sh\ntrue\n"), 0o755); err != nil {
+		t.Fatalf("writeTargetFile rewrite: %v", err)
+	}
+
+	st, err := os.Stat(filepath.Join(root, "usr/sbin/tool"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st.Mode().Perm() != 0o755 {
+		t.Errorf("mode = %o, want 755", st.Mode().Perm())
+	}
+	if got := readFile(t, root, "usr/sbin/tool"); !strings.Contains(got, "true") {
+		t.Errorf("content not rewritten: %q", got)
+	}
+}
+
+func TestWriteGrubDefaultsPinsTheZFSRoot(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeGrubDefaults(DiskConfig{FS: FSZFSRAID1}); err != nil {
+		t.Fatalf("writeGrubDefaults: %v", err)
+	}
+	got := readFile(t, root, "etc/default/grub")
+	if !strings.Contains(got, "root=ZFS="+ZFSRootDataset) || !strings.Contains(got, "boot=zfs") {
+		t.Errorf("grub defaults do not name the ZFS root:\n%s", got)
+	}
+	// Serial is listed last so it wins as the system console on headless racks.
+	if !strings.Contains(got, "console=tty0 console=ttyS0,115200") {
+		t.Errorf("grub defaults do not set both consoles:\n%s", got)
+	}
+	if theme := readFile(t, root, "etc/grub.d/06_spinifex"); !strings.Contains(theme, "terminal_output gfxterm") {
+		t.Errorf("branding snippet missing gfxterm:\n%s", theme)
+	}
+}
+
+func TestWriteGrubDefaultsLeavesExt4RootToGrubProbe(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	if err := writeGrubDefaults(DiskConfig{FS: FSExt4}); err != nil {
+		t.Fatalf("writeGrubDefaults: %v", err)
+	}
+	if got := readFile(t, root, "etc/default/grub"); strings.Contains(got, "root=ZFS=") {
+		t.Errorf("ext4 install should not pin a ZFS root:\n%s", got)
+	}
+}
+
+func TestDivertGrubInstallReplacesTheBinary(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	if err := divertGrubInstall(); err != nil {
+		t.Fatalf("divertGrubInstall: %v", err)
+	}
+	// The diversion is what makes the replacement survive a grub-pc upgrade.
+	f.mustRun(t, "dpkg-divert --divert "+grubInstallReal, "--rename --add "+grubInstallPath)
+	if _, err := os.Stat(filepath.Join(root, strings.TrimPrefix(grubInstallPath, "/"))); err != nil {
+		t.Errorf("wrapper not written: %v", err)
+	}
+}
+
+func TestDivertGrubInstallFailsLoudly(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(string, []string) error { return errors.New("dpkg is locked") }
+
+	if err := divertGrubInstall(); err == nil || !strings.Contains(err.Error(), "divert grub-install") {
+		t.Fatalf("err = %v, want a diversion failure", err)
+	}
+}
+
+// espFixture writes the state a successful boot-tool run leaves behind: the
+// registered ESP list, and a grub.cfg on the ESP that verifyBootConfig mounts.
+func espFixture(t *testing.T, root string, uuids string, grubCfg string) {
+	t.Helper()
+	list := filepath.Join(root, strings.TrimPrefix(bootToolUUIDList, "/"))
+	if err := os.MkdirAll(filepath.Dir(list), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(list, []byte(uuids), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if grubCfg == "" {
+		return
+	}
+	dir := filepath.Join(root, "var/tmp/verify-esp/grub")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "grub.cfg"), []byte(grubCfg), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestVerifyBootConfig(t *testing.T) {
+	goodCfg := "linux /vmlinuz-6.12.0 root=ZFS=" + ZFSRootDataset + " boot=zfs\n"
+
+	tests := []struct {
+		name    string
+		uuids   string
+		grubCfg string
+		wantErr string
+	}{
+		{name: "every ESP registered and pointing at the pool", uuids: "AAAA-1111\nBBBB-2222\n", grubCfg: goodCfg},
+		{
+			name:    "a disk got no bootloader",
+			uuids:   "AAAA-1111\n",
+			grubCfg: goodCfg,
+			wantErr: "expected 2 registered ESPs, found 1",
+		},
+		{
+			// grub-mkconfig degrades quietly when grub-probe cannot read the
+			// pool, and the failure only shows up as a panic at boot.
+			name:    "grub.cfg names no root",
+			uuids:   "AAAA-1111\nBBBB-2222\n",
+			grubCfg: "linux /vmlinuz-6.12.0 root=/dev/sda3\n",
+			wantErr: "would not find its root filesystem",
+		},
+		{
+			name:    "grub.cfg names no kernel",
+			uuids:   "AAAA-1111\nBBBB-2222\n",
+			grubCfg: "set root=ZFS=" + ZFSRootDataset + "\n",
+			wantErr: "references no kernel",
+		},
+		{name: "no ESP list at all", wantErr: "read ESP list"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			root := withMountRoot(t)
+			fakeCommands(t)
+			cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+			if tt.uuids != "" {
+				espFixture(t, root, tt.uuids, tt.grubCfg)
+			}
+
+			err := verifyBootConfig(cfg)
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("verifyBootConfig: %v", err)
+				}
+				return
+			}
+			if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+				t.Fatalf("err = %v, want %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestInstallBootToolZFSSetsUpEveryESP(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	targetSkeleton(t, root)
+	espFixture(t, root, "AAAA-1111\nBBBB-2222\n", "linux /vmlinuz-6.12.0 root=ZFS="+ZFSRootDataset+"\n")
+
+	if err := installBootToolZFS(cfg); err != nil {
+		t.Fatalf("installBootToolZFS: %v", err)
+	}
+
+	if _, err := os.Stat(filepath.Join(root, strings.TrimPrefix(bootToolPath, "/"))); err != nil {
+		t.Errorf("boot tool not installed: %v", err)
+	}
+	// A partially-updated set of ESPs boots today and fails on the next kernel
+	// upgrade, so every disk is initialised under strict mode.
+	for _, d := range cfg.Disks {
+		f.mustRun(t, "SPINIFEX_BOOT_TOOL_STRICT=1", "init", d.PartitionPath(espPartNum))
+	}
+	// NVRAM is written exactly once, by the initial refresh.
+	f.mustRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1", "refresh")
+	f.mustRun(t, "dpkg-divert")
+	for _, m := range chrootMountPaths {
+		f.mustRun(t, "mount --bind /"+m)
+	}
+}
+
+func TestInstallBootToolZFSStopsWhenADiskFails(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	targetSkeleton(t, root)
+	espFixture(t, root, "AAAA-1111\nBBBB-2222\n", "linux /vmlinuz root=ZFS="+ZFSRootDataset+"\n")
+	f.fail = func(name string, args []string) error {
+		if strings.Contains(strings.Join(args, " "), "init") {
+			return errors.New("no ESP")
+		}
+		return nil
+	}
+
+	err := installBootToolZFS(cfg)
+	if err == nil || !strings.Contains(err.Error(), "spinifex-boot-tool init") {
+		t.Fatalf("err = %v, want the init failure", err)
+	}
+	f.mustNotRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1")
+}
+
+func TestRefreshBootToolNoteOnlyAppliesToZFS(t *testing.T) {
+	// Log-only, but it is the operator's only record of the disk-replacement
+	// command, so both branches are walked.
+	refreshBootToolNote(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
+	refreshBootToolNote(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
+}
+
+func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
+	root := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(root, "etc/systemd/system"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	for range 2 {
+		if err := maskSystemdUnit(root, "getty@tty1.service"); err != nil {
+			t.Fatalf("maskSystemdUnit: %v", err)
+		}
+	}
+	target, err := os.Readlink(filepath.Join(root, "etc/systemd/system/getty@tty1.service"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if target != "/dev/null" {
+		t.Errorf("mask target = %q, want /dev/null", target)
+	}
+}
+
+func TestCopyAssetsAreBestEffort(t *testing.T) {
+	root := t.TempDir()
+	// Neither asset exists outside the live ISO; a missing one must not fail
+	// the install, only the branding.
+	copyGrubFont(root)
+	copySplashImage(root)
+}
+
+func TestWriteFstabOnZFSHoldsNothingButSwap(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeFstab(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	got := readFile(t, root, "etc/fstab")
+	// Root and every dataset are mounted from pool properties, and the ESPs on
+	// demand by the boot tool.
+	if strings.Contains(got, " / ") || strings.Contains(got, "/boot/efi") {
+		t.Errorf("ZFS fstab should carry no root or ESP line:\n%s", got)
+	}
+}
+
+func TestWriteNetworkConfigWritesTheWholeNetworkdTree(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeNetworkConfig(threeNIC()); err != nil {
+		t.Fatalf("writeNetworkConfig: %v", err)
+	}
+
+	netd := filepath.Join(root, "etc/systemd/network")
+	for _, plane := range []Plane{PlaneWAN, PlaneLAN, PlaneVPC} {
+		if got := readFile(t, netd, fmt.Sprintf("20-spinifex-%s.netdev", plane)); !strings.Contains(got, "Kind=bridge") {
+			t.Errorf("%s netdev is not a bridge:\n%s", plane, got)
+		}
+	}
+	// Only br-wan may block network-online.target; the east-west bridges are
+	// brought up by their own unit afterwards.
+	if got := readFile(t, netd, "20-spinifex-wan.network"); strings.Contains(got, "ActivationPolicy=manual") {
+		t.Errorf("br-wan must auto-activate:\n%s", got)
+	}
+	if got := readFile(t, netd, "20-spinifex-lan.network"); !strings.Contains(got, "ActivationPolicy=manual") {
+		t.Errorf("br-lan must not auto-activate:\n%s", got)
+	}
+
+	// The live ISO masks wait-online; the mask is copied in by rsync and has to
+	// be lifted, then scoped to br-wan so br-lan never blocks the boot.
+	dropin := readFile(t, root, "etc/systemd/system/systemd-networkd-wait-online.service.d/spinifex-wan-only.conf")
+	if !strings.Contains(dropin, "--interface=br-wan") || !strings.Contains(dropin, "--timeout=60") {
+		t.Errorf("wait-online drop-in is not scoped to br-wan:\n%s", dropin)
+	}
+
+	sysctl := readFile(t, root, "etc/sysctl.d/99-spinifex-network.conf")
+	for _, br := range []string{"br-wan", "br-lan", "br-vpc"} {
+		if !strings.Contains(sysctl, "net.ipv6.conf."+br+".disable_ipv6=1") {
+			t.Errorf("IPv6 not disabled on %s:\n%s", br, sysctl)
+		}
+	}
+}
+
+func TestWriteNetworkConfigRejectsAnInvalidRoleSet(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	cfg := threeNIC()
+	cfg.LAN.Interface = cfg.WAN.Interface
+
+	if err := writeNetworkConfig(cfg); err == nil || !strings.Contains(err.Error(), "network roles") {
+		t.Fatalf("err = %v, want the role validation to reject two planes on one link", err)
+	}
+}
+
+func TestWriteNetworkConfigLiftsTheWaitOnlineMask(t *testing.T) {
+	root := withMountRoot(t)
+	mask := filepath.Join(root, "etc/systemd/system/systemd-networkd-wait-online.service")
+	if err := os.MkdirAll(filepath.Dir(mask), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink("/dev/null", mask); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := writeNetworkConfig(threeNIC()); err != nil {
+		t.Fatalf("writeNetworkConfig: %v", err)
+	}
+	if _, err := os.Lstat(mask); err == nil {
+		t.Error("the wait-online mask survived; firstboot would run before br-wan has a lease")
+	}
+}
+
+func TestBindChrootMountsCreatesEveryMountpoint(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	if err := bindChrootMounts(); err != nil {
+		t.Fatalf("bindChrootMounts: %v", err)
+	}
+	for _, m := range chrootMountPaths {
+		if _, err := os.Stat(filepath.Join(root, m)); err != nil {
+			t.Errorf("mountpoint /%s not created: %v", m, err)
+		}
+		f.mustRun(t, "mount --bind /"+m+" "+filepath.Join(root, m))
+	}
+
+	unbindChrootMounts()
+	// Reverse order: /sys and /proc come out before /dev they were nested under.
+	if f.indexOf("umount "+filepath.Join(root, "sys")) > f.indexOf("umount "+filepath.Join(root, "dev")) {
+		t.Error("chroot mounts were unbound in bind order, not reverse")
+	}
+}
+
+func TestBindChrootMountsReportsAFailedBind(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "mount" {
+			return errors.New("permission denied")
+		}
+		return nil
+	}
+	if err := bindChrootMounts(); err == nil || !strings.Contains(err.Error(), "bind-mount") {
+		t.Fatalf("err = %v, want a bind-mount failure", err)
+	}
+}

cmd/installer/install/zfs.go

@@ -0,0 +1,362 @@
+package install
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math"
+	"math/bits"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// ARC bounds. Spinifex RAM is guest RAM: Proxmox defaults non-PVE products to
+// half of system memory, which on a 256GB compute node would remove 128GB of
+// schedulable instance capacity. 10% capped at 16GiB is their PVE behaviour and
+// the right trade for a hypervisor.
+const (
+	arcMinMiB     = 64
+	arcMaxCapMiB  = 16 * 1024
+	arcFraction   = 0.10
+	arcHeadroomMi = 1024 // always leave the OS a GiB above the ARC ceiling
+)
+
+// zfsDataset describes one dataset created at install time. Order matters:
+// parents must precede children so mountpoints nest correctly.
+type zfsDataset struct {
+	name       string
+	mountpoint string
+	props      []string
+}
+
+// datasets is the layout described in the plan. Recordsizes are matched to each
+// service's access pattern; 16K on the predastore index keeps RAIDZ-1 parity
+// overhead at 33% rather than the ~50% a 4-8K record would cost.
+var datasets = []zfsDataset{
+	{ZFSPoolName + "/ROOT", "none", []string{"canmount=off"}},
+	{ZFSRootDataset, "/", []string{"acltype=posix"}},
+	{ZFSPoolName + "/data", "/var/lib/spinifex", []string{"canmount=off"}},
+	{ZFSPoolName + "/data/viperblock", "/var/lib/spinifex/viperblock",
+		[]string{"recordsize=128K", "logbias=throughput"}},
+	{ZFSPoolName + "/data/predastore", "/var/lib/spinifex/predastore",
+		[]string{"recordsize=1M"}},
+	// Broken out so the later move of object data onto dedicated drives is a
+	// zfs send | zfs recv rather than an rsync with downtime.
+	{ZFSPoolName + "/data/predastore-nodes", "/var/lib/spinifex/predastore/distributed/nodes", nil},
+	// BadgerDB index plus the Raft BoltDB: small random, fsync-heavy.
+	{ZFSPoolName + "/data/predastore-db", "/var/lib/spinifex/predastore/distributed/db",
+		[]string{"recordsize=16K", "logbias=latency"}},
+	{ZFSPoolName + "/data/nats", "/var/lib/spinifex/nats", []string{"recordsize=16K"}},
+	{ZFSPoolName + "/log", "/var/log/spinifex", []string{"compression=zstd"}},
+}
+
+// buildVdev renders the vdev specification for zpool create. Pure: it is the
+// function that decides pool shape, so it is exhaustively unit-tested rather
+// than discovered to be wrong on hardware.
+func buildVdev(cfg DiskConfig) (string, error) {
+	if err := cfg.Validate(); err != nil {
+		return "", err
+	}
+	if !cfg.FS.IsZFS() {
+		return "", fmt.Errorf("buildVdev: %s is not a zfs topology", cfg.FS)
+	}
+
+	// Members are named by their partition-3 by-id path: a pool built on
+	// /dev/sdX breaks the first time a controller enumerates in a new order.
+	members := make([]string, len(cfg.Disks))
+	for i, d := range cfg.Disks {
+		members[i] = d.StablePartitionPath(rootPartNum)
+	}
+
+	keyword := cfg.FS.vdevKeyword()
+	if keyword == "" {
+		return strings.Join(members, " "), nil // plain stripe
+	}
+	// RAID10 is a stripe of two-disk mirrors, paired in selection order — which
+	// is why the UI numbers selected disks instead of just ticking them.
+	if cfg.FS == FSZFSRAID10 {
+		var parts []string
+		for i := 0; i+1 < len(members); i += 2 {
+			parts = append(parts, fmt.Sprintf("mirror %s %s", members[i], members[i+1]))
+		}
+		return strings.Join(parts, " "), nil
+	}
+	return keyword + " " + strings.Join(members, " "), nil
+}
+
+// detectAshift derives the pool sector size from the widest sector any member
+// reports. ashift cannot be changed after creation, and one that is too small
+// costs read-modify-write on every block for the life of the pool, so the floor
+// is 12 (4K) even when every disk claims 512.
+func detectAshift(disks []Disk) int {
+	widest := 4096
+	for _, d := range disks {
+		widest = max(widest, d.PhysicalBlockSize, d.LogicalBlockSize)
+	}
+	// ashift is the base-2 exponent of the sector size.
+	return bits.Len(uint(widest)) - 1
+}
+
+// defaultARCMaxMiB computes the ARC ceiling from total system memory.
+func defaultARCMaxMiB(totalMiB int) int {
+	want := int(math.Round(float64(totalMiB) * arcFraction))
+	want = min(want, arcMaxCapMiB)
+	// Never let the ARC crowd the OS out of its last GiB on a small node.
+	if ceiling := totalMiB - arcHeadroomMi; ceiling > 0 {
+		want = min(want, ceiling)
+	}
+	return max(want, arcMinMiB)
+}
+
+// totalMemoryMiB reads MemTotal from /proc/meminfo.
+func totalMemoryMiB() int {
+	data, err := os.ReadFile("/proc/meminfo")
+	if err != nil {
+		return 0
+	}
+	for line := range strings.SplitSeq(string(data), "\n") {
+		if !strings.HasPrefix(line, "MemTotal:") {
+			continue
+		}
+		var kb int
+		if _, err := fmt.Sscanf(line, "MemTotal: %d kB", &kb); err == nil {
+			return kb / 1024
+		}
+	}
+	return 0
+}
+
+// resolveZFSOpts fills unset advanced options with their computed defaults, so
+// every value written to the pool is explicit and auditable afterwards.
+func resolveZFSOpts(cfg DiskConfig) ZFSOpts {
+	o := cfg.ZFS
+	if o.Ashift == 0 {
+		o.Ashift = detectAshift(cfg.Disks)
+	}
+	if o.Compress == "" {
+		// Named explicitly rather than "on" so a future OpenZFS default change
+		// cannot silently alter what this pool does.
+		o.Compress = "lz4"
+	}
+	if o.Checksum == "" {
+		o.Checksum = "on"
+	}
+	if o.Copies == 0 {
+		o.Copies = 1
+	}
+	if o.ARCMaxMiB == 0 {
+		o.ARCMaxMiB = defaultARCMaxMiB(totalMemoryMiB())
+	}
+	return o
+}
+
+// loadZFSModule brings up the ZFS kernel module in the live environment and
+// waits for /dev/zfs. The module is prebuilt against the shipped kernel at ISO
+// build time, so failure here means the ISO is broken, not the hardware.
+func loadZFSModule() error {
+	if err := run("modprobe", "zfs"); err != nil {
+		slog.Warn("modprobe zfs failed, checking for /dev/zfs anyway", "err", err)
+	}
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		if _, err := os.Stat("/dev/zfs"); err == nil {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("/dev/zfs did not appear after modprobe zfs — the ZFS module is missing or does not match the running kernel")
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// ImportablePools returns the names of pools visible to `zpool import`. There
+// is still no machine-readable output in OpenZFS 2.3, so the human format is
+// parsed — the same approach Proxmox::Sys::ZFS takes.
+func ImportablePools() ([]string, error) {
+	out, err := exec.Command("zpool", "import").Output()
+	if err != nil {
+		// zpool exits non-zero when there is nothing to import, which is the
+		// normal case. Anything else — no binary, no /dev/zfs — must surface, or
+		// a machine with an existing pool looks empty to the caller.
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("zpool import: %w", err)
+	}
+	return parseImportablePools(string(out)), nil
+}
+
+// parseImportablePools extracts pool names from `zpool import` output. Split
+// out from the command call so it can be tested against captured fixtures.
+func parseImportablePools(out string) []string {
+	var pools []string
+	scanner := bufio.NewScanner(strings.NewReader(out))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		// Only the "pool:" key introduces a pool; "id:", "state:" and the
+		// config tree that follows are attributes of the one before it.
+		if name, ok := strings.CutPrefix(line, "pool:"); ok {
+			if n := strings.TrimSpace(name); n != "" {
+				pools = append(pools, n)
+			}
+		}
+	}
+	return pools
+}
+
+// createPool builds the root pool across the selected disks. altroot keeps
+// every dataset mounted under the install target instead of over the live
+// filesystem, and cachefile=none stops the live environment from claiming it.
+func createPool(cfg DiskConfig, opts ZFSOpts) error {
+	vdev, err := buildVdev(cfg)
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"create", "-f",
+		"-o", "cachefile=none",
+		"-o", fmt.Sprintf("ashift=%d", opts.Ashift),
+		"-O", "atime=on", "-O", "relatime=on",
+		"-O", "xattr=sa",
+		"-O", fmt.Sprintf("compression=%s", opts.Compress),
+		"-O", fmt.Sprintf("checksum=%s", opts.Checksum),
+		"-O", fmt.Sprintf("copies=%d", opts.Copies),
+		"-O", "mountpoint=none",
+		"-R", mountRoot,
+		ZFSPoolName,
+	}
+	args = append(args, strings.Fields(vdev)...)
+	if err := run("zpool", args...); err != nil {
+		return fmt.Errorf("zpool create: %w", err)
+	}
+
+	// TRIM matters on a pool that churns EBS volumes: without it the drives'
+	// garbage collection degrades write latency over months, and diagnosing
+	// that after the fact is miserable. Detected, never prompted.
+	if allFlash(cfg.Disks) {
+		if err := run("zpool", "set", "autotrim=on", ZFSPoolName); err != nil {
+			slog.Warn("could not enable autotrim", "err", err)
+		}
+	}
+	return nil
+}
+
+// allFlash reports whether every member is non-rotational.
+func allFlash(disks []Disk) bool {
+	for _, d := range disks {
+		if d.Rotational {
+			return false
+		}
+	}
+	return len(disks) > 0
+}
+
+// createDatasets lays out the service datasets. Mountpoints are set relative to
+// the pool's altroot, so they land under the install target now and at their
+// real paths once the installed system imports the pool.
+func createDatasets() error {
+	for _, ds := range datasets {
+		args := []string{"create", "-o", "mountpoint=" + ds.mountpoint}
+		for _, p := range ds.props {
+			args = append(args, "-o", p)
+		}
+		args = append(args, ds.name)
+		if err := run("zfs", args...); err != nil {
+			return fmt.Errorf("create dataset %s: %w", ds.name, err)
+		}
+	}
+	return nil
+}
+
+// datasetProtectFilters keeps rsync --delete away from the dataset mountpoints.
+//
+// The datasets are mounted before the copy so files land on the right one, but
+// the live source has nothing at those paths, so --delete tries to rmdir a busy
+// mountpoint and fails the whole transfer. Protect rules exempt them.
+func datasetProtectFilters(cfg DiskConfig) []string {
+	if !cfg.FS.IsZFS() {
+		return nil
+	}
+	var out []string
+	for _, ds := range datasets {
+		if ds.mountpoint == "/" || ds.mountpoint == "none" {
+			continue
+		}
+		out = append(out, "--filter=P "+ds.mountpoint)
+	}
+	return out
+}
+
+// writeZFSSystemConfig writes the target's ZFS tunables: the ARC ceiling, and
+// the matching host reserve so the daemon's admission gate stops handing out
+// memory the ARC is holding.
+func writeZFSSystemConfig(opts ZFSOpts) error {
+	modprobeDir := filepath.Join(mountRoot, "etc/modprobe.d")
+	if err := os.MkdirAll(modprobeDir, 0o755); err != nil {
+		return err
+	}
+	arcBytes := int64(opts.ARCMaxMiB) << 20
+	conf := fmt.Sprintf("# Generated by the Spinifex installer.\n"+
+		"# Guest RAM is the scarce resource on a hypervisor, so the ARC is\n"+
+		"# capped well below the ZFS default of half of system memory.\n"+
+		"options zfs zfs_arc_max=%d\n", arcBytes)
+	if err := os.WriteFile(filepath.Join(modprobeDir, "zfs.conf"), []byte(conf), 0o644); err != nil {
+		return fmt.Errorf("write zfs.conf: %w", err)
+	}
+
+	// The daemon computes schedulable memory as host total minus this reserve
+	// (defaultHostReserve in daemon/resource.go). ARC is invisible to it, so
+	// the ceiling has to be declared or instances get admitted into memory the
+	// ARC already owns.
+	reserve := defaultHostReserveGB + float64(opts.ARCMaxMiB)/1024.0
+	env := fmt.Sprintf("# Generated by the Spinifex installer.\n"+
+		"# %.1fGB base reserve + %.1fGB ZFS ARC ceiling.\n"+
+		"SPINIFEX_RESERVED_MEM_GB=%.1f\n",
+		defaultHostReserveGB, float64(opts.ARCMaxMiB)/1024.0, reserve)
+	confDir := filepath.Join(mountRoot, "etc/spinifex")
+	if err := os.MkdirAll(confDir, 0o755); err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(confDir, "host.env"), []byte(env), 0o644)
+}
+
+// defaultHostReserveGB mirrors defaultHostReserve.memGB in
+// spinifex/daemon/resource.go. Changing one without the other silently
+// misreports how much memory a ZFS node can schedule.
+const defaultHostReserveGB = 2.0
+
+// exportPool releases the pool so a retry after a failed install is not blocked
+// by "pool is busy". The live environment runs spinifex-init as PID 1, so
+// nothing else will ever clean this up.
+func exportPool() {
+	if err := run("zpool", "export", ZFSPoolName); err != nil {
+		slog.Warn("zpool export failed", "pool", ZFSPoolName, "err", err)
+	}
+}
+
+// clearDisk removes any prior filesystem, partition table and ZFS label from a
+// disk so a retry after a partial install is not rejected for stale metadata —
+// a failure that otherwise reads as mysterious rather than as leftover debris.
+//
+// The probes are quiet because a blank disk makes all of them fail, and their
+// stderr on the install console reads like the install is going wrong.
+func clearDisk(d Disk) error {
+	// ZFS labels live at both ends of the device and wipefs does not know about
+	// them, so an old pool member has to be cleared before its partition goes.
+	if old := d.PartitionPath(rootPartNum); exists(old) {
+		_ = runQuiet("zpool", "labelclear", "-f", old)
+	}
+	_ = runQuiet("wipefs", "-a", d.Path)
+	return runQuiet("sgdisk", "-Z", d.Path)
+}
+
+func exists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}

cmd/installer/install/zfs_test.go

@@ -0,0 +1,311 @@
+package install
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+// disk builds a test disk with a by-id path, since that is what pool members
+// are actually named by.
+func disk(name string, gib int64) Disk {
+	return Disk{
+		Path:              "/dev/" + name,
+		ByID:              "/dev/disk/by-id/virtio-" + name,
+		Bytes:             gib << 30,
+		LogicalBlockSize:  512,
+		PhysicalBlockSize: 512,
+	}
+}
+
+func disks(n int, gib int64) []Disk {
+	out := make([]Disk, n)
+	for i := range out {
+		out[i] = disk(string(rune('a'+i)), gib)
+	}
+	return out
+}
+
+func TestBuildVdev(t *testing.T) {
+	tests := []struct {
+		name  string
+		fs    FSType
+		disks []Disk
+		want  string
+	}{
+		{
+			"raid0 is a bare stripe with no keyword",
+			FSZFSRAID0, disks(2, 40),
+			"/dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3",
+		},
+		{
+			"single-disk raid0 is the one-drive case",
+			FSZFSRAID0, disks(1, 40),
+			"/dev/disk/by-id/virtio-a-part3",
+		},
+		{
+			"raid1 is one mirror vdev",
+			FSZFSRAID1, disks(2, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3",
+		},
+		{
+			"raid1 across three disks is a three-way mirror, not a stripe",
+			FSZFSRAID1, disks(3, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 /dev/disk/by-id/virtio-c-part3",
+		},
+		{
+			"raid10 pairs members in selection order",
+			FSZFSRAID10, disks(4, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 " +
+				"mirror /dev/disk/by-id/virtio-c-part3 /dev/disk/by-id/virtio-d-part3",
+		},
+		{
+			"raidz1 across three disks",
+			FSZFSRAIDZ1, disks(3, 40),
+			"raidz1 /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 /dev/disk/by-id/virtio-c-part3",
+		},
+		{
+			"raidz2 needs four",
+			FSZFSRAIDZ2, disks(4, 40),
+			"raidz2 /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 " +
+				"/dev/disk/by-id/virtio-c-part3 /dev/disk/by-id/virtio-d-part3",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := buildVdev(DiskConfig{FS: tt.fs, Disks: tt.disks})
+			if err != nil {
+				t.Fatalf("buildVdev: %v", err)
+			}
+			if got != tt.want {
+				t.Errorf("buildVdev()\n got: %s\nwant: %s", got, tt.want)
+			}
+		})
+	}
+}
+
+// A pool built on /dev/sdX silently breaks when the controller enumerates in a
+// different order, so the fallback only applies when there is no by-id path.
+func TestBuildVdevFallsBackToKernelNameWithoutByID(t *testing.T) {
+	d := disk("a", 40)
+	d.ByID = ""
+	got, err := buildVdev(DiskConfig{FS: FSZFSRAID0, Disks: []Disk{d}})
+	if err != nil {
+		t.Fatalf("buildVdev: %v", err)
+	}
+	if got != "/dev/a3" {
+		t.Errorf("buildVdev() = %q, want /dev/a3", got)
+	}
+}
+
+func TestBuildVdevRejectsExt4(t *testing.T) {
+	if _, err := buildVdev(DiskConfig{FS: FSExt4, Disks: disks(1, 40)}); err == nil {
+		t.Fatal("buildVdev accepted ext4")
+	}
+}
+
+func TestBuildVdevValidatesFirst(t *testing.T) {
+	// Three disks cannot form striped mirrors, and producing a two-disk mirror
+	// plus a silently dropped disk would be worse than an error.
+	if _, err := buildVdev(DiskConfig{FS: FSZFSRAID10, Disks: disks(3, 40)}); err == nil {
+		t.Fatal("buildVdev accepted an odd disk count for RAID10")
+	}
+}
+
+func TestDetectAshift(t *testing.T) {
+	tests := []struct {
+		name     string
+		logical  int
+		physical int
+		want     int
+	}{
+		// ashift cannot be changed after creation and one that is too small
+		// costs a read-modify-write per block forever, so 512e drives are
+		// deliberately treated as 4K.
+		{"512n drive is floored at 4K", 512, 512, 12},
+		{"512e drive reports 4K physical", 512, 4096, 12},
+		{"4Kn drive", 4096, 4096, 12},
+		{"8K native drive", 8192, 8192, 13},
+		{"16K native drive", 16384, 16384, 14},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			d := Disk{LogicalBlockSize: tt.logical, PhysicalBlockSize: tt.physical}
+			if got := detectAshift([]Disk{d}); got != tt.want {
+				t.Errorf("detectAshift() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+// A pool cannot have two sector sizes, so the widest member wins — the reverse
+// would make every write to the large-sector disk a read-modify-write.
+func TestDetectAshiftTakesWidestMember(t *testing.T) {
+	got := detectAshift([]Disk{
+		{LogicalBlockSize: 512, PhysicalBlockSize: 512},
+		{LogicalBlockSize: 512, PhysicalBlockSize: 8192},
+	})
+	if got != 13 {
+		t.Errorf("detectAshift() = %d, want 13", got)
+	}
+}
+
+func TestDefaultARCMaxMiB(t *testing.T) {
+	tests := []struct {
+		name    string
+		totalMi int
+		want    int
+	}{
+		{"tiny VM gets 10% of very little", 2048, 205},
+		{"8GB node gets 10%", 8192, 819},
+		{"64GB node gets 10%", 65536, 6554},
+		// The point of the cap: guest RAM is what this machine sells, and half
+		// of a 256GB node is 128GB of instance capacity the ARC would swallow.
+		{"256GB node is capped at 16GiB", 262144, 16384},
+		{"1TB node is still capped at 16GiB", 1048576, 16384},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := defaultARCMaxMiB(tt.totalMi); got != tt.want {
+				t.Errorf("defaultARCMaxMiB(%d) = %d, want %d", tt.totalMi, got, tt.want)
+			}
+		})
+	}
+}
+
+// An unreadable /proc/meminfo must not produce a zero or negative ceiling,
+// which ZFS would either reject or read as "no limit".
+func TestDefaultARCMaxMiBNeverGoesBelowFloor(t *testing.T) {
+	for _, total := range []int{0, 128, 512} {
+		if got := defaultARCMaxMiB(total); got < arcMinMiB {
+			t.Errorf("defaultARCMaxMiB(%d) = %d, below the floor of %d", total, got, arcMinMiB)
+		}
+	}
+}
+
+func TestResolveZFSOptsFillsDefaults(t *testing.T) {
+	got := resolveZFSOpts(DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)})
+	if got.Ashift != 12 {
+		t.Errorf("Ashift = %d, want 12", got.Ashift)
+	}
+	if got.Compress != "lz4" {
+		t.Errorf("Compress = %q, want lz4", got.Compress)
+	}
+	if got.Checksum != "on" {
+		t.Errorf("Checksum = %q, want on", got.Checksum)
+	}
+	if got.Copies != 1 {
+		t.Errorf("Copies = %d, want 1", got.Copies)
+	}
+	if got.ARCMaxMiB <= 0 {
+		t.Errorf("ARCMaxMiB = %d, want a positive ceiling", got.ARCMaxMiB)
+	}
+}
+
+func TestResolveZFSOptsKeepsOperatorChoices(t *testing.T) {
+	in := DiskConfig{
+		FS:    FSZFSRAIDZ1,
+		Disks: disks(3, 40),
+		ZFS:   ZFSOpts{Ashift: 13, Compress: "zstd", Checksum: "blake3", Copies: 2, ARCMaxMiB: 2048},
+	}
+	got := resolveZFSOpts(in)
+	if got != in.ZFS {
+		t.Errorf("resolveZFSOpts overrode explicit options: got %+v, want %+v", got, in.ZFS)
+	}
+}
+
+func TestParseImportablePools(t *testing.T) {
+	// Captured from `zpool import` on a machine with a pool from a previous
+	// install: the config tree below each header must not be mistaken for one.
+	const out = `   pool: rpool
+     id: 8543219876543210987
+  state: ONLINE
+ action: The pool can be imported using its name or numeric identifier.
+ config:
+
+	rpool                     ONLINE
+	  raidz1-0                ONLINE
+	    virtio-spinifex-disk1-part3  ONLINE
+	    virtio-spinifex-disk2-part3  ONLINE
+
+   pool: tank
+     id: 1111111111111111111
+  state: ONLINE
+ config:
+
+	tank                      ONLINE
+`
+	got := parseImportablePools(out)
+	want := []string{"rpool", "tank"}
+	if len(got) != len(want) {
+		t.Fatalf("parseImportablePools() = %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("pool %d = %q, want %q", i, got[i], want[i])
+		}
+	}
+}
+
+func TestParseImportablePoolsEmpty(t *testing.T) {
+	if got := parseImportablePools("no pools available to import\n"); len(got) != 0 {
+		t.Errorf("parseImportablePools() = %v, want none", got)
+	}
+}
+
+// Every dataset must be creatable in order: a child cannot be created before
+// its parent, and getting this wrong only shows up on real hardware.
+func TestDatasetsAreOrderedParentFirst(t *testing.T) {
+	seen := map[string]bool{ZFSPoolName: true}
+	for _, ds := range datasets {
+		parent := ds.name[:strings.LastIndex(ds.name, "/")]
+		if !seen[parent] {
+			t.Errorf("dataset %s is created before its parent %s", ds.name, parent)
+		}
+		seen[ds.name] = true
+	}
+}
+
+// The root dataset has to be the one the bootloader names, or the machine
+// mounts an empty filesystem and panics.
+func TestRootDatasetIsCreated(t *testing.T) {
+	for _, ds := range datasets {
+		if ds.name == ZFSRootDataset {
+			if ds.mountpoint != "/" {
+				t.Fatalf("%s mountpoint = %q, want /", ZFSRootDataset, ds.mountpoint)
+			}
+			return
+		}
+	}
+	t.Fatalf("%s is never created", ZFSRootDataset)
+}
+
+// rsync --delete tries to rmdir a dataset mountpoint the live source does not
+// have, which fails on a busy mount and aborts the whole copy. Every dataset
+// with a real mountpoint must be protected.
+func TestDatasetProtectFilters(t *testing.T) {
+	got := datasetProtectFilters(DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)})
+	for _, ds := range datasets {
+		if ds.mountpoint == "/" || ds.mountpoint == "none" {
+			continue
+		}
+		want := "--filter=P " + ds.mountpoint
+		if !slices.Contains(got, want) {
+			t.Errorf("missing protect rule for %s (%s)", ds.name, ds.mountpoint)
+		}
+	}
+	// The root dataset is the destination itself, and "none" datasets are never
+	// mounted, so protecting either would be meaningless.
+	for _, f := range got {
+		if f == "--filter=P /" || f == "--filter=P none" {
+			t.Errorf("unexpected protect rule %q", f)
+		}
+	}
+}
+
+func TestDatasetProtectFiltersSkipsExt4(t *testing.T) {
+	if got := datasetProtectFilters(DiskConfig{FS: FSExt4, Disks: disks(1, 40)}); got != nil {
+		t.Errorf("datasetProtectFilters() = %v on ext4, want none", got)
+	}
+}

cmd/installer/netprobe/netprobe.go

@@ -0,0 +1,426 @@
+// Package netprobe identifies physical network interfaces by hardware rather
+// than kernel name, so an operator choosing between eno1, ens1f0np0 and
+// ens1f1np1 can tell the 1gbe Broadcom from the 25gbe Mellanox.
+package netprobe
+
+import (
+	"cmp"
+	"fmt"
+	"log/slog"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// sysClassNet is the kernel's per-interface attribute tree. A variable, not a
+// constant, so tests can point the probe at a fixture instead of the running
+// machine's hardware.
+var sysClassNet = "/sys/class/net"
+
+// The external helpers the probe leans on, indirected so tests can exercise it
+// without the host's real tooling.
+var (
+	listInterfaces = net.Interfaces
+	commandOutput  = func(name string, args ...string) ([]byte, error) {
+		return exec.Command(name, args...).Output()
+	}
+)
+
+// NIC describes one physical interface as presented in the installer's
+// selection table.
+type NIC struct {
+	Name string
+
+	// Vendor and Model come from the udev hardware database, e.g. "Mellanox
+	// Technologies" and "MT27710 Family [ConnectX-4 Lx]". Empty when the
+	// database carries no entry for the card.
+	Vendor string
+	Model  string
+
+	// Driver is the kernel module that claimed the port, e.g. "mlx5_core". It
+	// comes straight from sysfs, so it still identifies a card the hardware
+	// database misses.
+	Driver string
+
+	// DeviceID is the PCI or USB "vendor:device" pair, the last-resort identity
+	// for hardware neither the database nor a driver name pins down.
+	DeviceID string
+
+	// AltName is the first alternative name systemd assigns, which is what
+	// operators often see in switch and cabling documentation.
+	AltName string
+
+	// Speed is the negotiated link speed rendered for display ("25 Gbps") and
+	// SpeedMbps the raw figure behind it. Both are zero on a port with no
+	// carrier, which is why Probe brings links up before reading.
+	Speed     string
+	SpeedMbps int
+
+	// Carrier reports a cable with link. It orders the picker so the port the
+	// operator has just plugged in is the one offered first.
+	Carrier bool
+
+	// State is the link summary shown in the table: "online", "no cable" or
+	// "down".
+	State string
+
+	MTU    int
+	IsWiFi bool
+}
+
+// Label renders the NIC's identity line: kernel name, negotiated speed and link
+// state. The hardware detail is deliberately absent — it goes on its own line
+// via Hardware, where there is room to read it.
+func (n NIC) Label() string {
+	speed := n.Speed
+	if speed == "" {
+		speed = "—"
+	}
+	state := n.State
+	if state == "" {
+		state = "unknown"
+	}
+	return fmt.Sprintf("%s  %s  %s", n.Name, speed, state)
+}
+
+// Hardware renders the card's identity for the detail line, degrading from the
+// database name to the driver and device id rather than to "unknown". An
+// operator needs something to match against the card in their hand.
+func (n NIC) Hardware() string {
+	if hw := strings.TrimSpace(n.Vendor + " " + n.Model); hw != "" {
+		return hw
+	}
+	switch {
+	case n.Driver != "" && n.DeviceID != "":
+		return n.Driver + " [" + n.DeviceID + "]"
+	case n.Driver != "":
+		return n.Driver
+	case n.DeviceID != "":
+		return n.DeviceID
+	}
+	return "no hardware detail"
+}
+
+// linkUpTimeout bounds how long Probe waits after bringing links up for speed
+// and carrier to settle. A 25gbe port negotiating against a cold switch can
+// take a second or two to report a speed.
+var linkUpTimeout = 2 * time.Second
+
+// Probe enumerates non-loopback physical interfaces and enriches each from
+// sysfs and the udev hardware database, ordered so the ports worth assigning
+// come first. Links are brought up first: a down port reports no speed and no
+// carrier, which would leave every fast port unidentified at exactly the moment
+// the operator needs to pick one.
+//
+// Nothing the probe depends on needs a running systemd. The live installer
+// boots straight into spinifex-init with only udevd started, so networkctl —
+// which has to reach systemd-networkd — enriches gaps and is never the source.
+func Probe() ([]NIC, error) {
+	ifaces, err := listInterfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	var names, physical []string
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagLoopback != 0 {
+			continue
+		}
+		if len(iface.HardwareAddr) == 0 {
+			continue
+		}
+		names = append(names, iface.Name)
+		if hasDevice(iface.Name) {
+			physical = append(physical, iface.Name)
+		}
+	}
+
+	// Bridges, bonds and veth pairs carry a MAC like any port but have no
+	// device behind them, and offering them would let a plane be bound to
+	// something that does not exist until after the install. Keep them only if
+	// the filter found nothing, so an unusual platform still gets a list.
+	if len(physical) > 0 {
+		names = physical
+	}
+
+	brought := false
+	for _, name := range names {
+		if _, err := commandOutput("ip", "link", "set", "dev", name, "up"); err != nil {
+			slog.Debug("netprobe: could not bring link up", "iface", name, "err", err)
+			continue
+		}
+		brought = true
+	}
+	if brought {
+		time.Sleep(linkUpTimeout)
+	}
+
+	nics := make([]NIC, 0, len(names))
+	for _, name := range names {
+		nics = append(nics, describe(name))
+	}
+	slices.SortFunc(nics, byUsefulness)
+	return nics, nil
+}
+
+// describe reads everything about one interface that needs no daemon, then lets
+// networkctl fill the gaps if it happens to be answering.
+func describe(name string) NIC {
+	nic := NIC{
+		Name:     name,
+		Driver:   sysLinkBase(name, "device/driver"),
+		DeviceID: deviceID(name),
+		MTU:      sysInt(name, "mtu"),
+		IsWiFi:   isWiFi(name),
+	}
+
+	// The kernel reports -1 for a port that cannot state its speed; treat that
+	// as unknown rather than letting it sort below every real figure.
+	nic.SpeedMbps = max(sysInt(name, "speed"), 0)
+	nic.Speed = formatSpeed(nic.SpeedMbps)
+	nic.Carrier = sysAttr(name, "carrier") == "1"
+	nic.State = linkState(name, nic.Carrier)
+
+	props := udevProperties(name)
+	nic.Vendor = props["ID_VENDOR_FROM_DATABASE"]
+	nic.Model = props["ID_MODEL_FROM_DATABASE"]
+
+	// udev's predictable names are the ones that appear in cabling notes; the
+	// path-based name is the one networkctl lists first.
+	for _, key := range []string{"ID_NET_NAME_PATH", "ID_NET_NAME_ONBOARD", "ID_NET_NAME_SLOT"} {
+		if alt := props[key]; alt != "" && alt != name {
+			nic.AltName = alt
+			break
+		}
+	}
+
+	if out, err := commandOutput("networkctl", "status", "--no-pager", name); err == nil {
+		fill(&nic, ParseStatus(string(out)))
+	} else {
+		slog.Debug("netprobe: networkctl unavailable, using sysfs only", "iface", name, "err", err)
+	}
+	return nic
+}
+
+// fill copies parsed hardware detail onto a NIC without overwriting anything
+// sysfs already established. State is never taken: the carrier bit is a truer
+// answer to "is this cable live" than networkd's configuration status.
+func fill(dst *NIC, src NIC) {
+	if dst.Vendor == "" {
+		dst.Vendor = src.Vendor
+	}
+	if dst.Model == "" {
+		dst.Model = src.Model
+	}
+	if dst.AltName == "" {
+		dst.AltName = src.AltName
+	}
+	if dst.Speed == "" {
+		dst.Speed = src.Speed
+	}
+	if dst.MTU == 0 {
+		dst.MTU = src.MTU
+	}
+}
+
+// byUsefulness orders the picker so the interfaces an operator would actually
+// assign come first: a live cable, then wired over wireless, then the fastest
+// port. The rest still appear, just below the plausible choices.
+func byUsefulness(a, b NIC) int {
+	if a.Carrier != b.Carrier {
+		return boolOrder(a.Carrier)
+	}
+	if a.IsWiFi != b.IsWiFi {
+		return boolOrder(b.IsWiFi)
+	}
+	if a.SpeedMbps != b.SpeedMbps {
+		return cmp.Compare(b.SpeedMbps, a.SpeedMbps)
+	}
+	return cmp.Compare(a.Name, b.Name)
+}
+
+// boolOrder sorts true ahead of false.
+func boolOrder(first bool) int {
+	if first {
+		return -1
+	}
+	return 1
+}
+
+// formatSpeed renders the kernel's Mbit/s figure the way a datasheet does.
+func formatSpeed(mbps int) string {
+	switch {
+	case mbps <= 0:
+		return ""
+	case mbps%1000 == 0:
+		return fmt.Sprintf("%d Gbps", mbps/1000)
+	default:
+		return fmt.Sprintf("%d Mbps", mbps)
+	}
+}
+
+// linkState collapses the kernel's flags into the three outcomes an operator
+// acts on: the port is live, the port is enabled but unplugged, or the port
+// never came up at all. operstate reports "down" for the latter two alike.
+func linkState(name string, carrier bool) string {
+	switch {
+	case carrier:
+		return "online"
+	case adminUp(name):
+		return "no cable"
+	default:
+		return "down"
+	}
+}
+
+// adminUp reads the IFF_UP bit out of the interface flags.
+func adminUp(name string) bool {
+	flags, err := strconv.ParseUint(strings.TrimPrefix(sysAttr(name, "flags"), "0x"), 16, 64)
+	return err == nil && flags&0x1 != 0
+}
+
+// deviceID reads the PCI or USB "vendor:device" pair. A NIC's device link
+// points at the PCI function directly; on USB it points at the interface, whose
+// parent carries the ids.
+func deviceID(name string) string {
+	dev := filepath.Join(sysClassNet, name, "device")
+	if id := idPair(dev, "vendor", "device"); id != "" {
+		return id
+	}
+	return idPair(filepath.Join(dev, ".."), "idVendor", "idProduct")
+}
+
+func idPair(dir, vendorAttr, productAttr string) string {
+	vendor := readTrimmed(filepath.Join(dir, vendorAttr))
+	product := readTrimmed(filepath.Join(dir, productAttr))
+	if vendor == "" || product == "" {
+		return ""
+	}
+	return strings.TrimPrefix(vendor, "0x") + ":" + strings.TrimPrefix(product, "0x")
+}
+
+// udevProperties reads the interface's udev database entry, which is where the
+// hardware-database vendor and model names live. udevd populates it during the
+// installer's device enumeration, so this works with no systemd running.
+func udevProperties(name string) map[string]string {
+	out, err := commandOutput("udevadm", "info", "--query=property", "--path="+filepath.Join(sysClassNet, name))
+	if err != nil {
+		slog.Debug("netprobe: udevadm info failed", "iface", name, "err", err)
+		return nil
+	}
+
+	props := make(map[string]string)
+	for line := range strings.SplitSeq(string(out), "\n") {
+		if key, value, ok := strings.Cut(strings.TrimSpace(line), "="); ok {
+			props[key] = value
+		}
+	}
+	return props
+}
+
+// ParseStatus extracts the hardware fields from `networkctl status <iface>`
+// output. The format is a set of right-aligned "Key: value" lines, with
+// Alternative Names continuing across subsequent indented lines.
+func ParseStatus(out string) NIC {
+	var nic NIC
+	var inAltNames bool
+
+	for line := range strings.SplitSeq(out, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" {
+			inAltNames = false
+			continue
+		}
+
+		key, value, found := strings.Cut(trimmed, ": ")
+		if !found {
+			// A continuation line under "Alternative Names:" — take the first
+			// only, which is the one operators see in cabling notes.
+			if inAltNames && nic.AltName == "" {
+				nic.AltName = trimmed
+			}
+			continue
+		}
+		inAltNames = false
+
+		key = strings.TrimSpace(key)
+		value = strings.TrimSpace(value)
+		switch key {
+		case "Vendor":
+			nic.Vendor = value
+		case "Model":
+			nic.Model = value
+		case "Speed":
+			nic.Speed = value
+		case "Online state":
+			// Rendered as "online" or "online (configured)".
+			if f := strings.Fields(value); len(f) > 0 {
+				nic.State = f[0]
+			}
+		case "MTU":
+			// Rendered as "9000 (min: 68, max: 9978)".
+			if f := strings.Fields(value); len(f) > 0 {
+				if mtu, err := strconv.Atoi(f[0]); err == nil {
+					nic.MTU = mtu
+				}
+			}
+		case "Alternative Names":
+			inAltNames = true
+			if f := strings.Fields(value); len(f) > 0 && nic.AltName == "" {
+				nic.AltName = f[0]
+			}
+		}
+	}
+	return nic
+}
+
+// hasDevice reports whether the interface is backed by real hardware, which
+// sysfs shows as a device link to the PCI, USB or platform node behind it.
+func hasDevice(name string) bool {
+	_, err := os.Stat(filepath.Join(sysClassNet, name, "device"))
+	return err == nil
+}
+
+// isWiFi reports whether the interface has a wireless subdirectory in sysfs.
+func isWiFi(name string) bool {
+	_, err := os.Stat(filepath.Join(sysClassNet, name, "wireless"))
+	return err == nil
+}
+
+// sysAttr reads one sysfs attribute. A missing or unreadable attribute is not
+// an error worth surfacing: speed and carrier both fail on a port with no link,
+// which the state column already conveys.
+func sysAttr(name, attr string) string {
+	return readTrimmed(filepath.Join(sysClassNet, name, attr))
+}
+
+func sysInt(name, attr string) int {
+	n, err := strconv.Atoi(sysAttr(name, attr))
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+// sysLinkBase resolves a sysfs symlink and returns its final element, which is
+// how the driver name is exposed.
+func sysLinkBase(name, attr string) string {
+	target, err := os.Readlink(filepath.Join(sysClassNet, name, attr))
+	if err != nil {
+		return ""
+	}
+	return filepath.Base(target)
+}
+
+func readTrimmed(path string) string {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(b))
+}

cmd/installer/netprobe/netprobe_test.go

@@ -0,0 +1,292 @@
+package netprobe
+
+import (
+	"errors"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Captured from hydrogen (HP DL325 Gen10, dual 25gbe Mellanox) in us-west-1-az1.
+const mellanox25g = `● 6: ens1f0np0
+                   Link File: /usr/lib/systemd/network/99-default.link
+                Network File: /etc/systemd/network/12-spinifex-lan-nic.network
+                       State: enslaved (configured)
+                Online state: online
+                        Type: ether
+                        Path: pci-0000:84:00.0
+                      Driver: mlx5_core
+                      Vendor: Mellanox Technologies
+                       Model: MT27710 Family [ConnectX-4 Lx]
+           Alternative Names: enp132s0f0np0
+                              enx040973e5ce60
+            Hardware Address: 04:09:73:e5:ce:60 (Hewlett Packard Enterprise)
+                         MTU: 9000 (min: 68, max: 9978)
+                       QDisc: mq
+                      Master: br-lan
+IPv6 Address Generation Mode: none
+    Number of Queues (Tx/Rx): 384/48
+            Auto negotiation: yes
+                       Speed: 25Gbps
+                      Duplex: full
+           Activation Policy: up
+         Required For Online: yes
+                Connected To: MikroTik (MikroTik RouterOS 7.11.3 (stable) Sep/27/2023 13:09:44 CRS504-4XQ) on port bridge/qsfp28-3-3
+`
+
+// A link with no carrier reports neither Speed nor an online state, which is
+// the case Probe brings links up to avoid.
+const linkDown = `● 3: eno6
+                   Link File: /usr/lib/systemd/network/99-default.link
+                       State: off
+                Online state: off
+                        Type: ether
+                      Driver: bnxt_en
+                      Vendor: Broadcom Inc. and subsidiaries
+                       Model: NetXtreme BCM5720 Gigabit Ethernet PCIe
+            Hardware Address: 04:09:73:e5:ce:5f
+                         MTU: 1500 (min: 60, max: 9000)
+`
+
+func TestParseStatusMellanox(t *testing.T) {
+	nic := ParseStatus(mellanox25g)
+
+	if nic.Vendor != "Mellanox Technologies" {
+		t.Errorf("Vendor = %q, want %q", nic.Vendor, "Mellanox Technologies")
+	}
+	if nic.Model != "MT27710 Family [ConnectX-4 Lx]" {
+		t.Errorf("Model = %q, want %q", nic.Model, "MT27710 Family [ConnectX-4 Lx]")
+	}
+	if nic.Speed != "25Gbps" {
+		t.Errorf("Speed = %q, want %q", nic.Speed, "25Gbps")
+	}
+	if nic.State != "online" {
+		t.Errorf("State = %q, want %q", nic.State, "online")
+	}
+	if nic.MTU != 9000 {
+		t.Errorf("MTU = %d, want 9000", nic.MTU)
+	}
+	// Only the first alternative name is taken; enx040973e5ce60 is ignored.
+	if nic.AltName != "enp132s0f0np0" {
+		t.Errorf("AltName = %q, want %q", nic.AltName, "enp132s0f0np0")
+	}
+}
+
+func TestParseStatusLinkDown(t *testing.T) {
+	nic := ParseStatus(linkDown)
+
+	if nic.Speed != "" {
+		t.Errorf("Speed = %q, want empty for a down link", nic.Speed)
+	}
+	if nic.State != "off" {
+		t.Errorf("State = %q, want %q", nic.State, "off")
+	}
+	if nic.Model != "NetXtreme BCM5720 Gigabit Ethernet PCIe" {
+		t.Errorf("Model = %q, want the Broadcom model", nic.Model)
+	}
+	if nic.MTU != 1500 {
+		t.Errorf("MTU = %d, want 1500", nic.MTU)
+	}
+}
+
+func TestParseStatusEmpty(t *testing.T) {
+	// A truncated or unparseable status must not panic or invent values.
+	nic := ParseStatus("● 2: eth0\n     Online state:\n     Alternative Names:\n")
+	if nic.State != "" || nic.AltName != "" {
+		t.Errorf("expected empty fields from valueless keys, got State=%q AltName=%q", nic.State, nic.AltName)
+	}
+}
+
+func TestLabelFallsBackWhenSpeedUnknown(t *testing.T) {
+	got := NIC{Name: "eth0", State: "no cable"}.Label()
+	want := "eth0  —  no cable"
+	if got != want {
+		t.Errorf("Label() = %q, want %q", got, want)
+	}
+}
+
+// Hardware must never render "unknown": an operator standing at the console
+// needs something to match against the card in their hand, and sysfs always has
+// at least a driver or a device id.
+func TestHardwareDegradesRatherThanUnknown(t *testing.T) {
+	tests := []struct {
+		name string
+		nic  NIC
+		want string
+	}{
+		{
+			name: "database name wins",
+			nic:  NIC{Vendor: "Mellanox Technologies", Model: "MT27710", Driver: "mlx5_core", DeviceID: "15b3:1015"},
+			want: "Mellanox Technologies MT27710",
+		},
+		{"vendor only", NIC{Vendor: "Intel Corporation"}, "Intel Corporation"},
+		{"driver and id", NIC{Driver: "igb", DeviceID: "8086:1533"}, "igb [8086:1533]"},
+		{"driver only", NIC{Driver: "virtio_net"}, "virtio_net"},
+		{"id only", NIC{DeviceID: "0bda:8153"}, "0bda:8153"},
+		{"nothing at all", NIC{Name: "eth0"}, "no hardware detail"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.nic.Hardware(); got != tt.want {
+				t.Errorf("Hardware() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestFormatSpeed(t *testing.T) {
+	tests := []struct {
+		mbps int
+		want string
+	}{
+		{25000, "25 Gbps"},
+		{1000, "1 Gbps"},
+		{100, "100 Mbps"},
+		{2500, "2500 Mbps"},
+		{0, ""},
+		{-1, ""},
+	}
+
+	for _, tt := range tests {
+		if got := formatSpeed(tt.mbps); got != tt.want {
+			t.Errorf("formatSpeed(%d) = %q, want %q", tt.mbps, got, tt.want)
+		}
+	}
+}
+
+// A live cable outranks everything, then wired over wireless, then speed. This
+// is what puts the ports an operator would actually assign at indices 0..2,
+// where newModel pre-fills wan, lan and vpc from.
+func TestByUsefulnessOrdersPluggedPortsFirst(t *testing.T) {
+	nics := []NIC{
+		{Name: "eno1", SpeedMbps: 1000},
+		{Name: "wlp2s0", Carrier: true, IsWiFi: true, SpeedMbps: 866},
+		{Name: "ens1f1np1", SpeedMbps: 25000},
+		{Name: "ens1f0np0", Carrier: true, SpeedMbps: 25000},
+		{Name: "eno2", Carrier: true, SpeedMbps: 1000},
+	}
+	slices.SortFunc(nics, byUsefulness)
+
+	want := []string{"ens1f0np0", "eno2", "wlp2s0", "ens1f1np1", "eno1"}
+	for i, name := range want {
+		if nics[i].Name != name {
+			t.Fatalf("order = %v, want %v", names(nics), want)
+		}
+	}
+}
+
+func names(nics []NIC) []string {
+	out := make([]string, len(nics))
+	for i, n := range nics {
+		out[i] = n.Name
+	}
+	return out
+}
+
+func TestProbeReadsSysfsWithoutSystemd(t *testing.T) {
+	root := t.TempDir()
+	sysClassNet = root
+	linkUpTimeout = 0
+	t.Cleanup(func() {
+		sysClassNet = "/sys/class/net"
+		linkUpTimeout = 2 * time.Second
+	})
+
+	// ens1f0np0: a 25gbe Mellanox with a live cable, identified by the hardware
+	// database. eno1: a 1gbe port that is up but unplugged, and absent from the
+	// database, so only its driver and PCI id identify it.
+	writeIface(t, root, "ens1f0np0", map[string]string{
+		"speed": "25000", "carrier": "1", "mtu": "9000", "flags": "0x1003",
+		"device/vendor": "0x15b3", "device/device": "0x1015",
+	}, "mlx5_core")
+	writeIface(t, root, "eno1", map[string]string{
+		"speed": "1000", "carrier": "0", "mtu": "1500", "flags": "0x1003",
+		"device/vendor": "0x8086", "device/device": "0x1533",
+	}, "igb")
+
+	listInterfaces = func() ([]net.Interface, error) {
+		return []net.Interface{
+			{Name: "lo", Flags: net.FlagLoopback},
+			{Name: "eno1", HardwareAddr: net.HardwareAddr{4, 9, 0x73, 0xe5, 0xce, 0x5f}},
+			{Name: "ens1f0np0", HardwareAddr: net.HardwareAddr{4, 9, 0x73, 0xe5, 0xce, 0x60}},
+			{Name: "veth0"}, // no hardware address, so not a physical port
+			{Name: "br-lan", HardwareAddr: net.HardwareAddr{2, 0, 0, 0, 0, 1}}, // a bridge, so no device behind it
+		}, nil
+	}
+	t.Cleanup(func() { listInterfaces = net.Interfaces })
+
+	// Only udevadm answers. networkctl fails the way it does in the live
+	// installer, where there is no systemd-networkd for it to reach.
+	commandOutput = func(name string, args ...string) ([]byte, error) {
+		switch {
+		case name == "ip":
+			return nil, nil
+		case name == "udevadm" && strings.HasSuffix(args[len(args)-1], "ens1f0np0"):
+			return []byte("ID_VENDOR_FROM_DATABASE=Mellanox Technologies\n" +
+				"ID_MODEL_FROM_DATABASE=MT27710 Family [ConnectX-4 Lx]\n" +
+				"ID_NET_NAME_PATH=enp132s0f0np0\n"), nil
+		case name == "udevadm":
+			return []byte("ID_NET_NAME_PATH=enp1s0f0\n"), nil
+		default:
+			return nil, errors.New("Failed to connect to bus: No such file or directory")
+		}
+	}
+	t.Cleanup(func() {
+		commandOutput = func(name string, args ...string) ([]byte, error) {
+			return exec.Command(name, args...).Output()
+		}
+	})
+
+	nics, err := Probe()
+	if err != nil {
+		t.Fatalf("Probe: %v", err)
+	}
+	if got := names(nics); !slices.Equal(got, []string{"ens1f0np0", "eno1"}) {
+		t.Fatalf("probed %v, want the two physical ports with the plugged one first", got)
+	}
+
+	mlx := nics[0]
+	if mlx.Label() != "ens1f0np0  25 Gbps  online" {
+		t.Errorf("Label() = %q", mlx.Label())
+	}
+	if want := "Mellanox Technologies MT27710 Family [ConnectX-4 Lx]"; mlx.Hardware() != want {
+		t.Errorf("Hardware() = %q, want %q", mlx.Hardware(), want)
+	}
+	if mlx.AltName != "enp132s0f0np0" || mlx.MTU != 9000 || !mlx.Carrier {
+		t.Errorf("unexpected detail: %+v", mlx)
+	}
+
+	// Up but unplugged, and unknown to the database: the driver and PCI id are
+	// all the operator gets, which is still enough to identify the card.
+	igb := nics[1]
+	if igb.Label() != "eno1  1 Gbps  no cable" {
+		t.Errorf("Label() = %q", igb.Label())
+	}
+	if igb.Hardware() != "igb [8086:1533]" {
+		t.Errorf("Hardware() = %q", igb.Hardware())
+	}
+}
+
+// writeIface lays out the sysfs attributes Probe reads for one interface,
+// including the driver symlink it resolves the module name from.
+func writeIface(t *testing.T, root, name string, attrs map[string]string, driver string) {
+	t.Helper()
+	for attr, value := range attrs {
+		path := filepath.Join(root, name, attr)
+		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(path, []byte(value+"\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := os.Symlink("../../../bus/pci/drivers/"+driver, filepath.Join(root, name, "device/driver")); err != nil {
+		t.Fatal(err)
+	}
+}

cmd/installer/systemd/systemd.go

@@ -6,44 +6,38 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
-// WriteLANBridgeUnit installs a non-critical oneshot service that activates
-// br-lan via networkctl *after* network-online.target. br-lan has
-// ActivationPolicy=manual in its networkd .network file, so networkd creates
-// the bridge at boot but does not bring it up automatically. This service
-// triggers the activation once the WAN path is confirmed online, ensuring a
-// missing LAN cable or DHCP timeout on the secondary bridge never stalls
+// WriteBridgeUnit installs a non-critical oneshot service that activates an
+// east-west bridge (br-lan, br-vpc) via networkctl *after* network-online.target.
+// Those bridges carry ActivationPolicy=manual in their networkd .network file,
+// so networkd creates them at boot but does not bring them up automatically.
+// This service triggers the activation once the WAN path is confirmed online,
+// ensuring a missing cable or DHCP timeout on a secondary bridge never stalls
 // the management interface or firstboot.
-func WriteLANBridgeUnit(root string) error {
-	unit := `[Unit]
-Description=Spinifex LAN bridge (non-critical)
+func WriteBridgeUnit(root, bridge string) error {
+	name := "spinifex-" + strings.TrimPrefix(bridge, "br-") + "-bridge.service"
+	unit := fmt.Sprintf(`[Unit]
+Description=Spinifex %s bridge (non-critical)
 After=network-online.target
 Wants=network-online.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/networkctl up br-lan
+ExecStart=/usr/bin/networkctl up %s
 RemainAfterExit=yes
 # Failure is non-critical — cable unplugged or switch not ready at boot.
 SuccessExitStatus=0 1
 
 [Install]
 WantedBy=multi-user.target
-`
-	unitPath := filepath.Join(root, "etc/systemd/system/spinifex-lan-bridge.service")
+`, bridge, bridge)
+	unitPath := filepath.Join(root, "etc/systemd/system", name)
 	if err := os.WriteFile(unitPath, []byte(unit), 0o644); err != nil {
 		return err
 	}
-	wantsDir := filepath.Join(root, "etc/systemd/system/multi-user.target.wants")
-	if err := os.MkdirAll(wantsDir, 0o755); err != nil {
-		return err
-	}
-	link := filepath.Join(wantsDir, "spinifex-lan-bridge.service")
-	if err := os.Remove(link); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("remove stale symlink %s: %w", link, err)
-	}
-	return os.Symlink("/etc/systemd/system/spinifex-lan-bridge.service", link)
+	return EnableUnit(root, name)
 }
 
 // WriteFirstbootUnit writes the spinifex-firstboot.service oneshot unit that
@@ -74,6 +68,8 @@ ExecStopPost=/bin/sh -c 'if [ "$EXIT_STATUS" != "0" ]; then printf "\n*** Spinif
 # ovn-central startup cannot wedge multi-user.target and keep getty from
 # ever reaching the login prompt.
 TimeoutStartSec=300s
+# Install diagnostics land in the journal, which persists for the life of the
+# node — nothing this script runs may print secrets to stdout or stderr.
 StandardOutput=journal
 StandardError=journal
 

cmd/installer/ui/network.go

@@ -0,0 +1,696 @@
+package ui
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	"github.com/charmbracelet/bubbles/textinput"
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/charmbracelet/lipgloss"
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+	"github.com/mulgadc/spinifex/cmd/installer/netprobe"
+)
+
+// foldedNIC marks a role with no interface of its own, which collapses onto
+// the plane above it (vpc <- lan <- wan).
+const foldedNIC = -1
+
+// roleField identifies one editable element on the per-role screen. Which of
+// these are actually shown depends on the plane, the addressing mode, whether
+// the NIC is wireless, and whether Advanced is on.
+type roleField int
+
+const (
+	roleFieldNIC roleField = iota
+	roleFieldMethod
+	roleFieldIP
+	roleFieldMask
+	roleFieldGateway
+	roleFieldDNS
+	roleFieldVLAN
+	roleFieldMTU
+	roleFieldSSID
+	roleFieldWiFiPass
+)
+
+// roleForm is the editable state for one plane. The three forms are the whole
+// of the installer's network configuration.
+type roleForm struct {
+	plane install.Plane
+
+	// nic indexes into model.nics, or foldedNIC when the role is folded.
+	nic   int
+	dhcp  bool
+	focus roleField
+
+	ip       textinput.Model
+	mask     textinput.Model
+	gateway  textinput.Model
+	dns      textinput.Model
+	vlan     textinput.Model
+	mtu      textinput.Model
+	ssid     textinput.Model
+	wifiPass textinput.Model
+}
+
+func newRoleForm(plane install.Plane, nic int) roleForm {
+	text := func(placeholder string, limit int) textinput.Model {
+		in := textinput.New()
+		in.Prompt = ""
+		in.Placeholder = placeholder
+		if limit > 0 {
+			in.CharLimit = limit
+		}
+		return in
+	}
+
+	f := roleForm{
+		plane:    plane,
+		nic:      nic,
+		dhcp:     plane == install.PlaneWAN,
+		ip:       text(defaultIPPlaceholder(plane), 0),
+		mask:     text("255.255.255.0", 0),
+		gateway:  text("192.168.1.1", 0),
+		dns:      text("1.1.1.1, 8.8.8.8", 0),
+		vlan:     text("untagged", 4),
+		mtu:      text("1500", 5),
+		ssid:     text("Network SSID", 64),
+		wifiPass: text("WiFi password", 128),
+	}
+	f.wifiPass.EchoMode = textinput.EchoPassword
+	return f
+}
+
+// defaultIPPlaceholder hints the addressing convention for each plane so an
+// operator filling in a rack does not have to remember it.
+func defaultIPPlaceholder(plane install.Plane) string {
+	switch plane {
+	case install.PlaneLAN:
+		return "10.0.0.3"
+	case install.PlaneVPC:
+		return "10.1.0.3"
+	default:
+		return "192.168.1.10"
+	}
+}
+
+func (f *roleForm) folded() bool { return f.nic == foldedNIC }
+
+// canFold reports whether this role may be left unbound. wan always needs an
+// uplink, so only lan and vpc can fold.
+func (f *roleForm) canFold() bool { return f.plane != install.PlaneWAN }
+
+// visibleFields returns the fields shown for this form, in tab order. Hiding
+// rather than disabling keeps the default screen narrow: a DHCP role shows two
+// lines, and VLAN/MTU only appear once Advanced is on.
+func (f *roleForm) visibleFields(isWiFi, advanced bool) []roleField {
+	if f.folded() {
+		return []roleField{roleFieldNIC}
+	}
+	fields := []roleField{roleFieldNIC, roleFieldMethod}
+	if !f.dhcp {
+		fields = append(fields, roleFieldIP, roleFieldMask)
+		// Default route and resolvers are properties of the node, not of each
+		// plane, and wan is the only one that carries either: it holds the
+		// default route, so it is the only plane that can reach an upstream
+		// resolver. On DHCP the lease supplies both.
+		if f.plane == install.PlaneWAN {
+			fields = append(fields, roleFieldGateway, roleFieldDNS)
+		}
+	}
+	if advanced {
+		fields = append(fields, roleFieldVLAN, roleFieldMTU)
+	}
+	if isWiFi {
+		fields = append(fields, roleFieldSSID, roleFieldWiFiPass)
+	}
+	return fields
+}
+
+// input returns the textinput backing a field, or nil for the non-text fields.
+func (f *roleForm) input(field roleField) *textinput.Model {
+	switch field {
+	case roleFieldIP:
+		return &f.ip
+	case roleFieldMask:
+		return &f.mask
+	case roleFieldGateway:
+		return &f.gateway
+	case roleFieldDNS:
+		return &f.dns
+	case roleFieldVLAN:
+		return &f.vlan
+	case roleFieldMTU:
+		return &f.mtu
+	case roleFieldSSID:
+		return &f.ssid
+	case roleFieldWiFiPass:
+		return &f.wifiPass
+	default:
+		return nil
+	}
+}
+
+// blurAll drops focus from every input so only the focused field has a cursor.
+func (f *roleForm) blurAll() {
+	for _, field := range []roleField{
+		roleFieldIP, roleFieldMask, roleFieldGateway, roleFieldDNS,
+		roleFieldVLAN, roleFieldMTU, roleFieldSSID, roleFieldWiFiPass,
+	} {
+		f.input(field).Blur()
+	}
+}
+
+// focusCurrent gives the cursor to the focused field, if it is a text field.
+func (f *roleForm) focusCurrent() {
+	f.blurAll()
+	if in := f.input(f.focus); in != nil {
+		in.Focus()
+	}
+}
+
+// toRole converts the form into the install package's role model.
+func (f *roleForm) toRole(nics []netprobe.NIC) install.NetworkRole {
+	if f.folded() || f.nic >= len(nics) {
+		return install.NetworkRole{}
+	}
+	role := install.NetworkRole{
+		Interface: nics[f.nic].Name,
+		DHCPMode:  f.dhcp,
+		VLAN:      atoiOr(f.vlan.Value(), 0),
+		MTU:       atoiOr(f.mtu.Value(), 0),
+	}
+	if !f.dhcp {
+		role.Address = strings.TrimSpace(f.ip.Value())
+		role.Mask = strings.TrimSpace(f.mask.Value())
+		role.DNS = parseDNS(f.dns.Value())
+		if f.plane == install.PlaneWAN {
+			role.Gateway = strings.TrimSpace(f.gateway.Value())
+		}
+	}
+	if nics[f.nic].IsWiFi {
+		role.WiFiSSID = strings.TrimSpace(f.ssid.Value())
+		role.WiFiPass = f.wifiPass.Value()
+	}
+	return role
+}
+
+func atoiOr(s string, fallback int) int {
+	n, err := strconv.Atoi(strings.TrimSpace(s))
+	if err != nil {
+		return fallback
+	}
+	return n
+}
+
+// ── Roles overview screen ─────────────────────────────────────────────────────
+
+// continueRow is the cursor position below the three role rows.
+const continueRow = 3
+
+func (m model) handleRolesKey(key string) (tea.Model, tea.Cmd) {
+	switch key {
+	case "up", "k":
+		if m.roleCursor > 0 {
+			m.roleCursor--
+		}
+	case "down", "j":
+		if m.roleCursor < continueRow {
+			m.roleCursor++
+		}
+	case "a":
+		// Advanced reveals VLAN and MTU on every role at once, rather than
+		// per-role, so the operator flips it on for the whole node.
+		m.advanced = !m.advanced
+		m.validationErr = ""
+	case "enter":
+		if m.roleCursor == continueRow {
+			if err := m.buildConfig().Validate(); err != nil {
+				m.validationErr = capitalise(err.Error())
+				return m, nil
+			}
+			m.validationErr = ""
+			m.screen = screenIdentity
+			m.hostnameInput.Focus()
+			return m, nil
+		}
+		m.validationErr = ""
+		m.screen = screenNetworkRole
+		m.roles[m.roleCursor].focus = roleFieldNIC
+		m.roles[m.roleCursor].blurAll()
+	case "esc":
+		m.screen = screenDiskConfirm
+		m.eraseInput.Focus()
+	}
+	return m, nil
+}
+
+func (m model) viewNetworkRoles(w int) string {
+	title := styleTitle.Render("Network Planes")
+	subtitle := styleMuted.Render("A role left unbound folds onto the plane above it.")
+
+	adv := styleMuted.Render("Advanced: off")
+	if m.advanced {
+		adv = styleSuccess.Render("Advanced: on")
+	}
+
+	header := fmt.Sprintf("  %-*s%-*s", colRole, "ROLE", colIface, "INTERFACE")
+	if m.advanced {
+		header += fmt.Sprintf("%-*s%-*s", colVLAN, "VLAN", colMTU, "MTU")
+	}
+	header += "ADDRESS"
+
+	lines := []string{title, subtitle, "", adv, "", styleMuted.Render(header)}
+
+	for i := range m.roles {
+		row := m.renderRoleRow(i)
+		if i == m.roleCursor {
+			lines = append(lines, styleSelected.Render("> "+row))
+		} else {
+			lines = append(lines, "  "+row)
+		}
+		if detail := m.roleHardware(i, bodyWidth(w)); detail != "" {
+			lines = append(lines, detail)
+		}
+	}
+
+	lines = append(lines, "")
+	cont := "  Continue"
+	if m.roleCursor == continueRow {
+		cont = styleSelected.Render("> Continue")
+	}
+	lines = append(lines, cont)
+
+	if m.validationErr != "" {
+		lines = append(lines, "", styleError.Render(m.validationErr))
+	}
+	lines = append(lines, styleHelp.Render("↑/↓ move • Enter edit • a advanced • Esc back"))
+
+	body := lipgloss.JoinVertical(lipgloss.Left, lines...)
+	return lipgloss.Place(w, m.height, lipgloss.Center, lipgloss.Center,
+		styleBox.Width(min(w-4, 78)).Render(body),
+	)
+}
+
+// renderRoleRow renders one line of the overview table. A folded role names
+// the plane it collapsed onto rather than showing a blank, so the operator can
+// always see where the traffic will actually go.
+func (m model) renderRoleRow(i int) string {
+	f := &m.roles[i]
+	name := fmt.Sprintf("%-5s", f.plane)
+
+	if f.folded() {
+		_, landed := m.buildConfig().Resolve(f.plane)
+		return fmt.Sprintf("%-*s%s", colRole, name, styleMuted.Render("folds onto "+string(landed)))
+	}
+
+	iface := styleMuted.Render("—")
+	if f.nic < len(m.nics) {
+		iface = nicSummary(m.nics[f.nic])
+	}
+
+	addr := "DHCP"
+	if !f.dhcp {
+		addr = formatCIDR(f.ip.Value(), f.mask.Value())
+		if addr == "" {
+			addr = styleWarning.Render("not set")
+		}
+	}
+
+	row := padCell(name, colRole) + padCell(iface, colIface)
+	if m.advanced {
+		row += padCell(valueOr(f.vlan.Value(), "—"), colVLAN) + padCell(valueOr(f.mtu.Value(), "—"), colMTU)
+	}
+	return row + addr
+}
+
+// roleHardware is the vendor/model line beneath a role row, indented under the
+// interface column. It is what tells an operator that wan really is the 25gbe
+// Mellanox and not the onboard 1gbe.
+func (m model) roleHardware(i, width int) string {
+	f := &m.roles[i]
+	if f.folded() || f.nic >= len(m.nics) {
+		return ""
+	}
+	indent := 2 + colRole
+	return strings.Repeat(" ", indent) + styleMuted.Render(nicHardware(m.nics[f.nic], width-indent))
+}
+
+// nicSummary renders name, speed and link state, colouring the state so a port
+// with no cable stands out rather than having to be read for.
+func nicSummary(n netprobe.NIC) string {
+	state := styleWarning.Render(n.State)
+	if n.Carrier {
+		state = styleSuccess.Render(n.State)
+	}
+	speed := n.Speed
+	if speed == "" {
+		speed = "—"
+	}
+	return fmt.Sprintf("%s  %s  %s", n.Name, speed, state)
+}
+
+// nicHardware is the detail line: what the card is, plus the predictable name
+// it also answers to, which is the one written on cabling notes. The alt name
+// is dropped rather than truncated when the two will not fit together.
+func nicHardware(n netprobe.NIC, width int) string {
+	hw := n.Hardware()
+	const separator = "  ·  "
+	if n.AltName != "" && len([]rune(hw))+len(separator)+len([]rune(n.AltName)) <= width {
+		return hw + separator + n.AltName
+	}
+	return truncate(hw, width)
+}
+
+// bodyWidth is the text width inside the framed body. Detail lines are trimmed
+// to it rather than left to wrap, which would push the table out of alignment.
+func bodyWidth(w int) int {
+	return min(w-4, 78) - 4
+}
+
+// padCell pads a cell to width by display width, which the %-*s verb cannot do
+// once a cell carries the ANSI escapes of a style.
+func padCell(s string, width int) string {
+	return lipgloss.PlaceHorizontal(width, lipgloss.Left, s)
+}
+
+// Column widths for the overview table, shared by the header and the rows.
+const (
+	colRole  = 6
+	colIface = 32
+	colVLAN  = 7
+	colMTU   = 7
+)
+
+// focusMark gives the focused text field a visible caret without every field
+// rendering its own prompt.
+func focusMark(focused bool) string {
+	if focused {
+		return styleLabel.Render("> ")
+	}
+	return "  "
+}
+
+// formatCIDR renders an address and mask the way an operator writes them,
+// falling back to the raw mask when it is not dotted-decimal.
+func formatCIDR(addr, mask string) string {
+	addr = strings.TrimSpace(addr)
+	mask = strings.TrimSpace(mask)
+	if addr == "" {
+		return ""
+	}
+	if mask == "" {
+		return addr
+	}
+	if ip := net.ParseIP(mask); ip != nil && ip.To4() != nil {
+		if ones, bits := net.IPMask(ip.To4()).Size(); bits != 0 {
+			return fmt.Sprintf("%s/%d", addr, ones)
+		}
+	}
+	return addr + "/" + strings.TrimPrefix(mask, "/")
+}
+
+func valueOr(v, fallback string) string {
+	if v = strings.TrimSpace(v); v != "" {
+		return v
+	}
+	return fallback
+}
+
+// ── Per-role editor screen ────────────────────────────────────────────────────
+
+func (m model) handleRoleKey(key string, msg tea.KeyMsg) (tea.Model, tea.Cmd) {
+	f := &m.roles[m.roleCursor]
+	fields := f.visibleFields(m.nicIsWiFi(f.nic), m.advanced)
+	pos := fieldIndex(fields, f.focus)
+
+	switch key {
+	case "esc":
+		f.blurAll()
+		m.screen = screenNetworkRoles
+		m.validationErr = ""
+		return m, nil
+
+	case "enter":
+		if err := m.validateRole(m.roleCursor); err != "" {
+			m.validationErr = err
+			return m, nil
+		}
+		f.blurAll()
+		m.screen = screenNetworkRoles
+		m.validationErr = ""
+		return m, nil
+
+	case "tab", "down":
+		pos = (pos + 1) % len(fields)
+		f.focus = fields[pos]
+		f.focusCurrent()
+		return m, nil
+
+	case "shift+tab", "up":
+		pos = (pos - 1 + len(fields)) % len(fields)
+		f.focus = fields[pos]
+		f.focusCurrent()
+		return m, nil
+
+	case "left", "right":
+		// Only the two selector fields consume arrows; text fields need them
+		// for cursor movement.
+		switch f.focus {
+		case roleFieldNIC:
+			return m.cycleNIC(m.roleCursor, key == "right"), nil
+		case roleFieldMethod:
+			f.dhcp = key == "left"
+			f.focusCurrent()
+			return m, nil
+		}
+	}
+
+	if in := f.input(f.focus); in != nil {
+		var cmd tea.Cmd
+		*in, cmd = in.Update(msg)
+		return m, cmd
+	}
+	return m, nil
+}
+
+// cycleNIC steps the interface selection, including the fold option for lan
+// and vpc. Wrapping keeps a single key usable on a node with one NIC.
+func (m model) cycleNIC(idx int, forward bool) model {
+	f := &m.roles[idx]
+	low := 0
+	if f.canFold() {
+		low = foldedNIC
+	}
+	high := len(m.nics) - 1
+
+	if forward {
+		if f.nic >= high {
+			f.nic = low
+		} else {
+			f.nic++
+		}
+		return m
+	}
+	if f.nic <= low {
+		f.nic = high
+	} else {
+		f.nic--
+	}
+	return m
+}
+
+func (m model) nicIsWiFi(idx int) bool {
+	return idx >= 0 && idx < len(m.nics) && m.nics[idx].IsWiFi
+}
+
+func (m model) viewNetworkRole(w int) string {
+	f := &m.roles[m.roleCursor]
+	title := styleTitle.Render(fmt.Sprintf("Configure %s plane", strings.ToUpper(string(f.plane))))
+	subtitle := styleMuted.Render(planeDescription(f.plane))
+
+	lines := []string{title, subtitle, ""}
+	for _, field := range f.visibleFields(m.nicIsWiFi(f.nic), m.advanced) {
+		lines = append(lines, m.renderRoleField(f, field, bodyWidth(w)), "")
+	}
+
+	if m.validationErr != "" {
+		lines = append(lines, styleError.Render(m.validationErr), "")
+	}
+	lines = append(lines, styleHelp.Render("Tab/↑↓ move • ←/→ change • Enter done • Esc cancel"))
+
+	body := lipgloss.JoinVertical(lipgloss.Left, lines...)
+	return lipgloss.Place(w, m.height, lipgloss.Center, lipgloss.Center,
+		styleBox.Width(min(w-4, 78)).Render(body),
+	)
+}
+
+// planeDescription explains what rides each plane, so the choice is not just a
+// three-letter name.
+func planeDescription(p install.Plane) string {
+	switch p {
+	case install.PlaneWAN:
+		return "Public ingress and egress, EIPs and NAT (br-wan)."
+	case install.PlaneLAN:
+		return "Storage replication, NATS mesh and OVN control (br-lan)."
+	default:
+		return "Geneve tunnels carrying EC2 traffic between nodes (br-vpc)."
+	}
+}
+
+// roleLabelWidth is the field-label column on the editor screen, shared so a
+// wrapped value can indent to line up underneath it.
+const roleLabelWidth = 14
+
+func (m model) renderRoleField(f *roleForm, field roleField, width int) string {
+	focused := f.focus == field
+	label := styleLabel.Render(fmt.Sprintf("%-*s", roleLabelWidth, roleFieldLabel(field)))
+
+	switch field {
+	case roleFieldNIC:
+		return label + m.renderNICChoice(f, focused, width)
+
+	case roleFieldMethod:
+		return label + renderToggle([]string{"DHCP (automatic)", "Static"}, boolToIdx(!f.dhcp), focused)
+
+	default:
+		if in := f.input(field); in != nil {
+			return label + focusMark(focused) + in.View()
+		}
+		return label
+	}
+}
+
+func roleFieldLabel(field roleField) string {
+	switch field {
+	case roleFieldNIC:
+		return "Interface"
+	case roleFieldMethod:
+		return "IP method"
+	case roleFieldIP:
+		return "IP address"
+	case roleFieldMask:
+		return "Subnet mask"
+	case roleFieldGateway:
+		return "Gateway"
+	case roleFieldDNS:
+		return "DNS"
+	case roleFieldVLAN:
+		return "VLAN id"
+	case roleFieldMTU:
+		return "MTU"
+	case roleFieldSSID:
+		return "WiFi SSID"
+	default:
+		return "WiFi password"
+	}
+}
+
+// renderNICChoice shows the selected interface on the field line and its
+// hardware identity on a line of its own beneath. Vendor and model are what
+// make a port identifiable, and they do not fit alongside the selector.
+func (m model) renderNICChoice(f *roleForm, focused bool, width int) string {
+	var text, detail string
+	switch {
+	case f.folded():
+		_, landed := m.buildConfig().Resolve(f.plane)
+		text = fmt.Sprintf("(fold onto %s)", landed)
+	case f.nic < len(m.nics):
+		text = nicSummary(m.nics[f.nic])
+		detail = nicHardware(m.nics[f.nic], width-roleLabelWidth-2)
+	default:
+		text = "(none detected)"
+	}
+
+	if focused {
+		text = styleSelected.Render(" ← " + text + " → ")
+	} else {
+		text = styleLabel.Render("[ " + text + " ]")
+	}
+	if detail == "" {
+		return text
+	}
+	return text + "\n" + strings.Repeat(" ", roleLabelWidth+2) + styleMuted.Render(detail)
+}
+
+func renderToggle(options []string, selected int, focused bool) string {
+	var parts []string
+	for i, opt := range options {
+		switch {
+		case i == selected && focused:
+			parts = append(parts, styleSelected.Render(" "+opt+" "))
+		case i == selected:
+			parts = append(parts, styleLabel.Render("["+opt+"]"))
+		default:
+			parts = append(parts, styleMuted.Render(opt))
+		}
+	}
+	return strings.Join(parts, "  ")
+}
+
+func boolToIdx(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+func fieldIndex(fields []roleField, want roleField) int {
+	for i, f := range fields {
+		if f == want {
+			return i
+		}
+	}
+	return 0
+}
+
+// validateRole checks one plane's entries. Cross-role rules (a shared NIC
+// without distinct VLAN ids, an unbound wan) belong to install.Config.Validate
+// and run when the operator continues.
+func (m model) validateRole(idx int) string {
+	f := &m.roles[idx]
+	if f.folded() {
+		return ""
+	}
+	if len(m.nics) == 0 {
+		return "No network interfaces detected"
+	}
+
+	if !f.dhcp {
+		if net.ParseIP(strings.TrimSpace(f.ip.Value())) == nil {
+			return "Enter a valid IP address"
+		}
+		if !validSubnetMask(f.mask.Value()) {
+			return "Enter a dotted-decimal subnet mask, e.g. 255.255.255.0 or 255.255.0.0"
+		}
+		if f.plane == install.PlaneWAN && net.ParseIP(strings.TrimSpace(f.gateway.Value())) == nil {
+			return "Enter a valid gateway address"
+		}
+	}
+
+	if v := strings.TrimSpace(f.vlan.Value()); v != "" {
+		id, err := strconv.Atoi(v)
+		if err != nil || id < 1 || id > 4094 {
+			return "VLAN id must be between 1 and 4094"
+		}
+	}
+	if v := strings.TrimSpace(f.mtu.Value()); v != "" {
+		mtu, err := strconv.Atoi(v)
+		if err != nil || mtu < 68 || mtu > 9216 {
+			return "MTU must be between 68 and 9216"
+		}
+	}
+	if m.nicIsWiFi(f.nic) && strings.TrimSpace(f.ssid.Value()) == "" {
+		return "Enter the WiFi network SSID"
+	}
+	return ""
+}
+
+func capitalise(s string) string {
+	if s == "" {
+		return s
+	}
+	return strings.ToUpper(s[:1]) + s[1:]
+}

cmd/installer/ui/network_test.go

@@ -0,0 +1,357 @@
+package ui
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+	"github.com/mulgadc/spinifex/cmd/installer/netprobe"
+)
+
+func nics(n int) []netprobe.NIC {
+	out := make([]netprobe.NIC, 0, n)
+	names := []string{"eno1", "ens1f0np0", "ens1f1np1", "ens2f0np0"}
+	for i := range n {
+		out = append(out, netprobe.NIC{Name: names[i], Speed: "25Gbps", Vendor: "Mellanox", State: "online"})
+	}
+	return out
+}
+
+// The pre-fill is the whole of the simple path: an operator with a
+// conventional node should be able to press Continue without editing anything.
+func TestNewModelPrefillsRolesByNICCount(t *testing.T) {
+	tests := []struct {
+		name             string
+		count            int
+		wantLAN, wantVPC int
+	}{
+		{"one NIC folds everything onto wan", 1, foldedNIC, foldedNIC},
+		{"two NICs dedicate the second to lan, vpc folds", 2, 1, foldedNIC},
+		{"three NICs give each plane its own", 3, 1, 2},
+		{"four NICs still only bind three", 4, 1, 2},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(tt.count))
+
+			if got := m.roles[0].nic; got != 0 {
+				t.Errorf("wan nic = %d, want 0", got)
+			}
+			if got := m.roles[1].nic; got != tt.wantLAN {
+				t.Errorf("lan nic = %d, want %d", got, tt.wantLAN)
+			}
+			if got := m.roles[2].nic; got != tt.wantVPC {
+				t.Errorf("vpc nic = %d, want %d", got, tt.wantVPC)
+			}
+		})
+	}
+}
+
+// Hiding rather than disabling is what keeps the default screen narrow.
+func TestVisibleFields(t *testing.T) {
+	tests := []struct {
+		name     string
+		mutate   func(*roleForm)
+		isWiFi   bool
+		advanced bool
+		want     []roleField
+	}{
+		{
+			name:   "wan on DHCP asks two questions",
+			mutate: func(f *roleForm) { f.dhcp = true },
+			want:   []roleField{roleFieldNIC, roleFieldMethod},
+		},
+		{
+			name:   "wan static adds a gateway",
+			mutate: func(f *roleForm) { f.dhcp = false },
+			want:   []roleField{roleFieldNIC, roleFieldMethod, roleFieldIP, roleFieldMask, roleFieldGateway, roleFieldDNS},
+		},
+		{
+			name:     "advanced reveals VLAN and MTU",
+			mutate:   func(f *roleForm) { f.dhcp = true },
+			advanced: true,
+			want:     []roleField{roleFieldNIC, roleFieldMethod, roleFieldVLAN, roleFieldMTU},
+		},
+		{
+			name:   "wireless adds credentials",
+			mutate: func(f *roleForm) { f.dhcp = true },
+			isWiFi: true,
+			want:   []roleField{roleFieldNIC, roleFieldMethod, roleFieldSSID, roleFieldWiFiPass},
+		},
+		{
+			name:   "a folded role has nothing to configure",
+			mutate: func(f *roleForm) { f.nic = foldedNIC },
+			want:   []roleField{roleFieldNIC},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newRoleForm(install.PlaneWAN, 0)
+			tt.mutate(&f)
+			got := f.visibleFields(tt.isWiFi, tt.advanced)
+
+			if len(got) != len(tt.want) {
+				t.Fatalf("visibleFields() = %v, want %v", got, tt.want)
+			}
+			for i := range got {
+				if got[i] != tt.want[i] {
+					t.Fatalf("visibleFields() = %v, want %v", got, tt.want)
+				}
+			}
+		})
+	}
+}
+
+// A non-wan role must be able to reach the folded state, and wan must not.
+func TestCycleNICFoldReachableOnlyOffWAN(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(2))
+
+	// lan: 1 -> 0 -> folded -> wraps to the last NIC.
+	m.roles[1].nic = 1
+	for _, want := range []int{0, foldedNIC, 1} {
+		m = m.cycleNIC(1, false)
+		if m.roles[1].nic != want {
+			t.Fatalf("lan nic = %d, want %d", m.roles[1].nic, want)
+		}
+	}
+
+	// wan cycles only through real interfaces.
+	m.roles[0].nic = 0
+	for range 4 {
+		m = m.cycleNIC(0, false)
+		if m.roles[0].nic == foldedNIC {
+			t.Fatal("wan reached the folded state, but it always needs an uplink")
+		}
+	}
+}
+
+func TestToRoleReadsVLANAndMTU(t *testing.T) {
+	f := newRoleForm(install.PlaneLAN, 0)
+	f.dhcp = false
+	f.ip.SetValue("10.0.0.2")
+	f.mask.SetValue("255.255.0.0")
+	f.vlan.SetValue("10")
+	f.mtu.SetValue("9000")
+
+	role := f.toRole(nics(1))
+
+	if role.VLAN != 10 || role.MTU != 9000 {
+		t.Errorf("VLAN/MTU = %d/%d, want 10/9000", role.VLAN, role.MTU)
+	}
+	if role.Link() != "eno1.10" {
+		t.Errorf("Link() = %q, want eno1.10", role.Link())
+	}
+	// Only wan carries a default route, even if a gateway was typed earlier.
+	f.gateway.SetValue("10.0.0.1")
+	if got := f.toRole(nics(1)).Gateway; got != "" {
+		t.Errorf("lan Gateway = %q, want empty", got)
+	}
+}
+
+func TestToRoleFoldedIsEmpty(t *testing.T) {
+	f := newRoleForm(install.PlaneVPC, foldedNIC)
+	if !f.toRole(nics(2)).Folded() {
+		t.Error("a folded form must produce a folded role")
+	}
+}
+
+func TestValidateRole(t *testing.T) {
+	tests := []struct {
+		name    string
+		plane   install.Plane
+		mutate  func(*roleForm)
+		wantErr string
+	}{
+		{"valid static lan", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("10.0.0.2")
+			f.mask.SetValue("255.255.0.0")
+		}, ""},
+		{"bad ip", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("not-an-ip")
+			f.mask.SetValue("255.255.0.0")
+		}, "valid IP"},
+		{"wan needs a gateway", install.PlaneWAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("216.218.163.99")
+			f.mask.SetValue("255.255.255.224")
+		}, "gateway"},
+		{"VLAN out of range", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = true
+			f.vlan.SetValue("9000")
+		}, "VLAN id must be"},
+		{"MTU out of range", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = true
+			f.mtu.SetValue("70000")
+		}, "MTU must be"},
+		{"folded roles skip validation", install.PlaneVPC, func(f *roleForm) {
+			f.nic = foldedNIC
+			f.ip.SetValue("nonsense")
+		}, ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(3))
+			m.roles[0] = newRoleForm(tt.plane, 0)
+			tt.mutate(&m.roles[0])
+
+			got := m.validateRole(0)
+			if tt.wantErr == "" {
+				if got != "" {
+					t.Fatalf("validateRole() = %q, want no error", got)
+				}
+				return
+			}
+			if !strings.Contains(got, tt.wantErr) {
+				t.Fatalf("validateRole() = %q, want it to mention %q", got, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Continue must refuse a layout install.Config.Validate would reject, rather
+// than failing later during the install itself.
+func TestContinueBlocksOnSharedNICWithoutVLANs(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(2))
+	m.screen = screenNetworkRoles
+	m.roleCursor = continueRow
+	m.roles[1].nic = 0 // lan onto the wan NIC, both untagged
+
+	next, _ := m.handleRolesKey("enter")
+	got := next.(model)
+
+	if got.screen != screenNetworkRoles {
+		t.Error("expected to stay on the roles screen when the layout is invalid")
+	}
+	if !strings.Contains(got.validationErr, "both bind eno1") {
+		t.Errorf("validationErr = %q, want it to name the shared interface", got.validationErr)
+	}
+
+	// Distinct VLAN ids make the same pairing legal.
+	got.roles[0].vlan.SetValue("30")
+	got.roles[1].vlan.SetValue("10")
+	next, _ = got.handleRolesKey("enter")
+	if next.(model).screen != screenIdentity {
+		t.Error("distinct VLAN ids should allow two roles to share one NIC")
+	}
+}
+
+func TestAdvancedToggle(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(3))
+	m.screen = screenNetworkRoles
+
+	next, _ := m.handleRolesKey("a")
+	if !next.(model).advanced {
+		t.Fatal("'a' should turn Advanced on")
+	}
+	next, _ = next.(model).handleRolesKey("a")
+	if next.(model).advanced {
+		t.Fatal("'a' should turn Advanced back off")
+	}
+}
+
+// The default route and the resolvers are node-level, and wan is the only plane
+// that carries either, so the east-west editors must not ask for them. Their
+// bridges are link-local and activate manually — a resolver pinned there is
+// unreachable, and a gateway would fight the real default route.
+func TestVisibleFieldsOmitsRoutingOffWAN(t *testing.T) {
+	for _, plane := range []install.Plane{install.PlaneLAN, install.PlaneVPC} {
+		t.Run(string(plane), func(t *testing.T) {
+			f := newRoleForm(plane, 1)
+			f.dhcp = false
+			got := f.visibleFields(false, false)
+
+			want := []roleField{roleFieldNIC, roleFieldMethod, roleFieldIP, roleFieldMask}
+			if len(got) != len(want) {
+				t.Fatalf("visibleFields() = %v, want %v", got, want)
+			}
+			for i := range got {
+				if got[i] != want[i] {
+					t.Fatalf("visibleFields() = %v, want %v", got, want)
+				}
+			}
+		})
+	}
+}
+
+// The mask field must accept only what netgen can consume. It previously
+// advertised and accepted a prefix length, which passed the TUI and then failed
+// the install itself with "invalid mask: 24".
+func TestValidSubnetMaskRejectsPrefixLength(t *testing.T) {
+	for _, bad := range []string{"24", "/24", "8", "", "255.255.0", "255.0.255.0", "::1"} {
+		if validSubnetMask(bad) {
+			t.Errorf("validSubnetMask(%q) = true, want false", bad)
+		}
+	}
+	for _, good := range []string{"255.255.255.0", "255.255.0.0", "255.0.0.0", "255.255.255.252"} {
+		if !validSubnetMask(good) {
+			t.Errorf("validSubnetMask(%q) = false, want true", good)
+		}
+	}
+}
+
+// The hardware detail is the point of the NIC table: an operator has to be able
+// to tell which physical port each plane landed on. It gets a line of its own,
+// under the interface column, rather than being squeezed into the row.
+func TestRoleHardwareRendersOnItsOwnLine(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, []netprobe.NIC{{
+		Name: "ens1f0np0", Vendor: "Mellanox Technologies", Model: "MT27710",
+		AltName: "enp132s0f0np0", Speed: "25 Gbps", Carrier: true, State: "online",
+	}})
+
+	view := m.viewNetworkRoles(100)
+	if !strings.Contains(view, "ens1f0np0  25 Gbps  online") {
+		t.Errorf("expected the identity line in:\n%s", view)
+	}
+	detail := m.roleHardware(0, bodyWidth(100))
+	if !strings.Contains(detail, "Mellanox Technologies MT27710") {
+		t.Errorf("roleHardware = %q, want the vendor and model", detail)
+	}
+	if !strings.Contains(detail, "enp132s0f0np0") {
+		t.Errorf("roleHardware = %q, want the alternative name", detail)
+	}
+	// Every line has to fit the box, or lipgloss wraps it and the columns shear.
+	for line := range strings.SplitSeq(view, "\n") {
+		if len([]rune(line)) > 100 {
+			t.Errorf("line exceeds the terminal width: %q", line)
+		}
+	}
+}
+
+// A narrow box drops the alternative name before it truncates the model, since
+// half a model name identifies nothing.
+func TestNICHardwareDropsAltNameBeforeTruncating(t *testing.T) {
+	n := netprobe.NIC{Vendor: "Broadcom Inc. and subsidiaries", Model: "NetXtreme BCM5720", AltName: "enp1s0f1"}
+
+	if got := nicHardware(n, 60); got != "Broadcom Inc. and subsidiaries NetXtreme BCM5720" {
+		t.Errorf("nicHardware(60) = %q, want the name without the alt name", got)
+	}
+	if got := nicHardware(n, 20); got != "Broadcom Inc. and s…" {
+		t.Errorf("nicHardware(20) = %q, want a marked truncation", got)
+	}
+}
+
+// A role bound to a port with no hardware database entry still gets something
+// to identify the card by, never the bare word "unknown".
+func TestRoleHardwareFallsBackToDriver(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, []netprobe.NIC{{
+		Name: "eno1", Driver: "bnxt_en", DeviceID: "14e4:165f", State: "no cable",
+	}})
+
+	if got := m.roleHardware(0, bodyWidth(100)); !strings.Contains(got, "bnxt_en [14e4:165f]") {
+		t.Errorf("roleHardware = %q, want the driver and device id", got)
+	}
+}
+
+// A folded role has no interface, so it must not render an orphan detail line
+// under the plane it collapsed onto.
+func TestRoleHardwareEmptyWhenFolded(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(1))
+	if got := m.roleHardware(1, bodyWidth(100)); got != "" {
+		t.Errorf("roleHardware for a folded role = %q, want empty", got)
+	}
+}

cmd/installer/ui/storage_test.go

@@ -0,0 +1,139 @@
+package ui
+
+import (
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+)
+
+// diskModel is a storage screen with n unselected disks and ext4 chosen, which
+// is where the wizard starts.
+func diskModel(n int) model {
+	m := model{fsCursor: slices.Index(install.AllFSTypes, install.FSExt4)}
+	for i := range n {
+		m.disks = append(m.disks, install.Disk{
+			Path:              string(rune('a'+i)) + "-disk",
+			Bytes:             200 << 30,
+			LogicalBlockSize:  512,
+			PhysicalBlockSize: 512,
+		})
+	}
+	return m
+}
+
+func rolePaths(cfg install.DiskConfig) map[install.DiskRole]string {
+	out := map[install.DiskRole]string{}
+	for _, rm := range cfg.Roles {
+		out[rm.Role] = rm.Disk.Path
+	}
+	return out
+}
+
+// A single-disk machine should need no storage input at all, so the disk that
+// newModel preselects has to arrive already holding the os role.
+func TestPreselectedDiskIsAValidConfiguration(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda", Bytes: 200 << 30, LogicalBlockSize: 512}}, nil)
+	cfg := m.storage()
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("the preselected disk must be installable as-is: %v", err)
+	}
+	if cfg.Primary().Path != "/dev/sda" {
+		t.Errorf("Primary() = %q, want the preselected disk", cfg.Primary().Path)
+	}
+}
+
+// Picking disks in order gives the layout an operator almost always wants, so
+// the role key is only needed for the exceptions.
+func TestSelectingDisksAssignsRolesInOrder(t *testing.T) {
+	m := diskModel(3)
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	got := rolePaths(m.storage())
+	for role, want := range map[install.DiskRole]string{
+		install.RoleOS: "a-disk", install.RoleSpinifex: "b-disk", install.RolePredastore: "c-disk",
+	} {
+		if got[role] != want {
+			t.Errorf("%s = %q, want %q", role, got[role], want)
+		}
+	}
+	if err := m.storage().Validate(); err != nil {
+		t.Errorf("the default assignment must be valid: %v", err)
+	}
+}
+
+// Deselecting has to drop the role with the disk, or every later role shifts
+// onto the wrong drive.
+func TestDeselectingADiskReleasesItsRole(t *testing.T) {
+	m := diskModel(3)
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	m = m.toggleDisk(1) // drop the spinifex drive
+
+	got := rolePaths(m.storage())
+	if got[install.RoleOS] != "a-disk" || got[install.RolePredastore] != "c-disk" {
+		t.Fatalf("remaining roles moved: %v", got)
+	}
+	if _, ok := got[install.RoleSpinifex]; ok {
+		t.Errorf("the spinifex role should be free again: %v", got)
+	}
+}
+
+// The two-drive predastore layout is only reachable by reassigning, so cycling
+// swaps rather than refusing a role another disk already holds.
+func TestCycleRoleSwapsWithTheDiskThatHoldsIt(t *testing.T) {
+	m := diskModel(2)
+	m = m.toggleDisk(0).toggleDisk(1)
+	// b-disk: spinifex → predastore, which no disk holds, so no swap.
+	m = m.cycleRole(1)
+
+	got := rolePaths(m.storage())
+	if got[install.RoleOS] != "a-disk" || got[install.RolePredastore] != "b-disk" {
+		t.Fatalf("roles = %v, want os on a-disk and predastore on b-disk", got)
+	}
+
+	// Cycling past the end wraps onto os, which a-disk holds, so they trade.
+	m = m.cycleRole(1)
+	got = rolePaths(m.storage())
+	if got[install.RoleOS] != "b-disk" || got[install.RolePredastore] != "a-disk" {
+		t.Fatalf("roles = %v, want the two disks to have swapped", got)
+	}
+	if err := m.storage().Validate(); err != nil {
+		t.Errorf("a swap must always leave a valid assignment: %v", err)
+	}
+}
+
+// ZFS pairs mirrors by selection order and has no roles to assign.
+func TestStorageLeavesRolesUnsetOnZFS(t *testing.T) {
+	m := diskModel(2)
+	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1)
+	m = m.toggleDisk(0).toggleDisk(1)
+
+	cfg := m.storage()
+	if len(cfg.Roles) != 0 {
+		t.Errorf("Roles = %v, want none on a pool", cfg.Roles)
+	}
+	if !slices.Equal(cfg.Paths(), []string{"a-disk", "b-disk"}) {
+		t.Errorf("Paths() = %v, want the selection order preserved", cfg.Paths())
+	}
+}
+
+// A three-drive selection looks like an array and is not one, so the mapping
+// and the absence of redundancy both have to be on screen before the erase.
+func TestConfirmScreenStatesTheLayoutAndTheRisk(t *testing.T) {
+	m := diskModel(3)
+	m.height = 40
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	got := m.viewDiskConfirm(100)
+
+	for _, want := range []string{"/var/lib/spinifex/predastore", "c-disk", "losing one loses"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("confirm screen is missing %q:\n%s", want, got)
+		}
+	}
+}

cmd/lb-agent/main.go

@@ -19,11 +19,17 @@ import (
 	"syscall"
 
 	"github.com/mulgadc/spinifex/spinifex/lbagent"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 
 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 func main() {
+	// Without this the agent falls back to slog's default text handler, which folds
+	// attributes into the message so the console scraper recovers a level and nothing
+	// else. No OTLP export: the agent's only egress is the gateway it is watching.
+	otelsetup.SetDefaultJSONLogger(slog.LevelInfo)
+
 	var (
 		lbID       string
 		gatewayURL string

cmd/rds-agent/acknowledge.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mulgadc/spinifex/internal/rdsgw"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+)
+
+// The completion receipt rds-init writes once the master role is durably
+// applied. Deliberately outside PGDATA: clear_unfinished_datadir does an rm -rf
+// of PGDATA, so keeping the receipt out of it means no initdb-adjacent path can
+// touch it and the datadir stays a byte-for-byte stock one.
+const (
+	receiptRelDir  = ".spinifex-rds/bootstrap"
+	receiptFile    = "receipt.env"
+	receiptPayload = "RDS_RECEIPT_PAYLOAD_ID"
+	receiptDBID    = "RDS_RECEIPT_DB_INSTANCE_IDENTIFIER"
+)
+
+func (a *Agent) receiptPath() string {
+	return filepath.Join(a.cfg.DataMount, receiptRelDir, receiptFile)
+}
+
+// Confirms the initial bootstrap, which is what destroys the staged ciphertext.
+// A no-op on the attach path, and deliberately synchronous: it inherits the
+// cancellation semantics register, bootstrap and the data-mount wait already
+// share, and blocking the stateful loops behind it is correct because on a first
+// boot the engine is not bootstrapped until rds-init has finished anyway.
+//
+// It does not wait for the engine to be healthy. The receipt already proves the
+// master role was durably applied, and acknowledging early shortens the window
+// in which the ciphertext exists.
+func (a *Agent) acknowledgeBootstrap(ctx context.Context) error {
+	pending := a.pending
+	if pending == nil {
+		return nil
+	}
+
+	// Polled rather than waited on: rds-init runs after this agent registered,
+	// so the file appears some time into the boot. There is no give-up deadline
+	// here — the control plane's bootstrap timeout already fails the instance,
+	// and a second one in the guest would only give the two a way to disagree.
+	if err := retryObserved(ctx, "bootstrap receipt", func(context.Context) error {
+		return a.checkBootstrapReceipt(pending)
+	}, func(err error) {
+		// An absent receipt is the expected state for the whole initdb window,
+		// so reporting it would put a failure on every healthy create's beat and
+		// then into the reason a timed-out create records.
+		if errors.Is(err, fs.ErrNotExist) {
+			return
+		}
+		a.hb.setBootstrapFailure("bootstrap receipt", err)
+	}); err != nil {
+		return err
+	}
+	a.hb.clearBootstrapFailure()
+
+	if err := retryObserved(ctx, "bootstrap acknowledgement", func(ctx context.Context) error {
+		return terminalOnDenial(a.cp.AcknowledgeBootstrap(ctx, a.id, pending))
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap acknowledgement", err)
+	}); err != nil {
+		if !errors.Is(err, errRetryTerminal) {
+			return err
+		}
+		// The staged payload will never be delivered to this VM, and blocking the
+		// command poller and the parameter guard on that would leave a serving
+		// database no one can administer. The reason rides the heartbeat instead.
+		slog.ErrorContext(ctx, "rds-agent: the initial bootstrap cannot be acknowledged",
+			"payloadId", pending.payloadID, "err", err)
+		return nil
+	}
+	a.hb.clearBootstrapFailure()
+
+	// AcknowledgedAt on the record is the durable audit trail, so the receipt is
+	// removed rather than archived: keeping it would only grow the surface of
+	// stale receipts riding along in data-volume snapshots. A removal failure is
+	// harmless, since a later boot fetches attach and never consults one.
+	if err := os.Remove(a.receiptPath()); err != nil && !os.IsNotExist(err) {
+		slog.WarnContext(ctx, "rds-agent: could not remove the bootstrap receipt",
+			"receipt", a.receiptPath(), "err", err)
+	}
+	slog.InfoContext(ctx, "rds-agent: initial bootstrap acknowledged", "payloadId", pending.payloadID)
+	return nil
+}
+
+// The control plane resolved these from its own record — a superseded
+// generation, a payload it never staged, a data volume that is not this
+// instance's. Asking again cannot change any of them.
+func terminalOnDenial(err error) error {
+	var apiErr *rdsgw.APIError
+	if !errors.As(err, &apiErr) {
+		return err
+	}
+	switch apiErr.Code {
+	case awserrors.ErrorAccessDenied, awserrors.ErrorInvalidParameterValue:
+		return fmt.Errorf("%w: %w", errRetryTerminal, err)
+	}
+	return err
+}
+
+// A receipt naming another payload or another DB instance is treated as absent.
+// The receipt lives on the data volume, so it rides along in every snapshot of
+// it and a restored instance's volume carries the source instance's receipt.
+func (a *Agent) checkBootstrapReceipt(pending *pendingBootstrap) error {
+	path := a.receiptPath()
+	values, err := readReceipt(path)
+	if err != nil {
+		return err
+	}
+	if got := values[receiptPayload]; got != pending.payloadID {
+		return fmt.Errorf("%s names bootstrap payload %q, not %q", path, got, pending.payloadID)
+	}
+	if got := values[receiptDBID]; got != a.id.DBInstanceIdentifier {
+		return fmt.Errorf("%s names DB instance %q, not %q", path, got, a.id.DBInstanceIdentifier)
+	}
+	return nil
+}
+
+// KEY=value, the one format both readers can parse: rds-init in POSIX shell
+// with no guaranteed jq, and this.
+func readReceipt(path string) (map[string]string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	values := make(map[string]string)
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, value, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		values[strings.TrimSpace(key)] = strings.TrimSpace(value)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	return values, nil
+}

cmd/rds-agent/agent.go

@@ -0,0 +1,423 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/gwsign"
+	"github.com/mulgadc/spinifex/internal/rdsgw"
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Overridable via -ldflags "-X main.version=...".
+var version = "dev"
+
+// The gateway resolves the authoritative DB instance from the request's
+// credentials, so DBInstanceIdentifier is an assertion to be checked, not
+// trusted.
+type identity struct {
+	DBInstanceIdentifier string
+	AgentVersion         string
+	EngineVersion        string
+}
+
+// Every method is a SigV4-signed Query request, so the NATS bus stays
+// host-internal.
+type controlPlane interface {
+	Register(ctx context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error)
+	SubmitState(ctx context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error)
+	// Returns Mode=initialize with the master password for as long as a payload
+	// staged for this VM generation is unacknowledged, and Mode=attach after.
+	GetBootstrapConfig(ctx context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error)
+	// Reports that PostgreSQL durably applied the staged password, which is what
+	// lets the control plane destroy it.
+	AcknowledgeBootstrap(ctx context.Context, id identity, pending *pendingBootstrap) error
+	PollCommands(ctx context.Context, id identity, replies []handlers_rds.CommandReply, wait time.Duration) ([]handlers_rds.Command, error)
+}
+
+// Bounds a register, heartbeat or bootstrap request. The long poll sets its own
+// deadline from the wait it asked the gateway to hold.
+const callTimeout = 15 * time.Second
+
+// Keeps the poll's deadline past the requested wait, so a gateway answering at
+// the end of its window is not cut off by the client.
+const pollSlack = 10 * time.Second
+
+type gatewayControlPlane struct {
+	client *rdsgw.Client
+}
+
+var _ controlPlane = (*gatewayControlPlane)(nil)
+
+// Signs with the instance-role credentials the SDK chain resolves from IMDS,
+// against the pinned gateway CA.
+func newGatewayControlPlane(cfg config) (*gatewayControlPlane, error) {
+	signer, err := gwsign.NewIMDS(context.Background(), cfg.Region)
+	if err != nil {
+		return nil, fmt.Errorf("build IMDS signer: %w", err)
+	}
+	// The client timeout is the ceiling for the longest call, the long poll;
+	// shorter calls carry their own deadline.
+	client, err := rdsgw.New(cfg.GatewayURL, cfg.GatewayCA, signer, cfg.Region, cfg.PollWait+pollSlack)
+	if err != nil {
+		return nil, err
+	}
+	return &gatewayControlPlane{client: client}, nil
+}
+
+func (g *gatewayControlPlane) Register(ctx context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error) {
+	params := url.Values{}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	setIfPresent(params, "AgentVersion", id.AgentVersion)
+	setIfPresent(params, "EngineVersion", id.EngineVersion)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.RegisterDBInstanceOutput
+	if err := g.client.Call(ctx, "RegisterDBInstance", params, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func (g *gatewayControlPlane) SubmitState(ctx context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error) {
+	params := url.Values{"EngineHealth": {string(health)}}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	setIfPresent(params, "EngineVersion", id.EngineVersion)
+	setIfPresent(params, "Message", message)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.SubmitDBStateChangeOutput
+	if err := g.client.Call(ctx, "SubmitDBStateChange", params, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func (g *gatewayControlPlane) GetBootstrapConfig(ctx context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error) {
+	params := url.Values{}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.GetDBBootstrapConfigOutput
+	if err := g.client.Call(ctx, "GetDBBootstrapConfig", params, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func (g *gatewayControlPlane) AcknowledgeBootstrap(ctx context.Context, id identity, pending *pendingBootstrap) error {
+	params := url.Values{
+		"PayloadId":    {pending.payloadID},
+		"VMGeneration": {strconv.FormatInt(pending.vmGeneration, 10)},
+	}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	setIfPresent(params, "DataVolumeId", pending.dataVolumeID)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.AcknowledgeDBBootstrapOutput
+	if err := g.client.Call(ctx, "AcknowledgeDBBootstrap", params, &out); err != nil {
+		return err
+	}
+	if !out.Acknowledged {
+		return fmt.Errorf("the control plane did not acknowledge bootstrap payload %s", pending.payloadID)
+	}
+	return nil
+}
+
+// Declared here so the in-guest binary does not link the server side of the API.
+type pollDBCommandsOutput struct {
+	Commands []handlers_rds.Command `xml:"Commands>member"`
+}
+
+func (g *gatewayControlPlane) PollCommands(ctx context.Context, id identity, replies []handlers_rds.CommandReply, wait time.Duration) ([]handlers_rds.Command, error) {
+	params := url.Values{"WaitTimeSeconds": {strconv.FormatInt(int64(wait.Seconds()), 10)}}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	// Replies ride the poll rather than a separate action, so replying and
+	// asking for the next command are one round trip.
+	for i, reply := range replies {
+		member := fmt.Sprintf("Replies.member.%d.", i+1)
+		params.Set(member+"CommandId", reply.CommandID)
+		params.Set(member+"Status", reply.Status)
+		setIfPresent(params, member+"Message", reply.Message)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, wait+pollSlack)
+	defer cancel()
+
+	var out pollDBCommandsOutput
+	if err := g.client.Call(ctx, "PollDBCommands", params, &out); err != nil {
+		return nil, err
+	}
+	return out.Commands, nil
+}
+
+// Only sets a parameter that has a value, so the gateway can tell "no
+// assertion" from "asserted as blank".
+func setIfPresent(params url.Values, key, value string) {
+	if value != "" {
+		params.Set(key, value)
+	}
+}
+
+type Agent struct {
+	cfg             config
+	id              identity
+	cp              controlPlane
+	probe           *engineProbe
+	engine          engine
+	handoffWriter   func(string, *handlers_rds.GetDBBootstrapConfigOutput) error
+	dataMountWaiter func(context.Context, string, string) error
+
+	// Set by bootstrap when the fetch replayed a staged payload; nil leaves the
+	// acknowledgement step a no-op, which is every attach boot.
+	pending *pendingBootstrap
+
+	hb    *heartbeater
+	cmd   *commander
+	guard *paramGuard
+}
+
+// Assembles from already-built parts, so tests can pass fakes; New builds the
+// production control plane and delegates here.
+func newAgent(cfg config, cp controlPlane, probe *engineProbe) (*Agent, error) {
+	if cfg.MountsFile == "" {
+		cfg.MountsFile = defaultMountsFile
+	}
+	id := identity{
+		DBInstanceIdentifier: cfg.DBInstanceIdentifier,
+		AgentVersion:         version,
+		EngineVersion:        cfg.EngineVersion,
+	}
+	a := &Agent{
+		cfg: cfg, id: id, cp: cp, probe: probe,
+		handoffWriter: writeHandoff, dataMountWaiter: waitForDataMount,
+	}
+	eng, err := newEngine(cfg, execCommandRunner, execSessionRunner, probe)
+	if err != nil {
+		return nil, err
+	}
+	a.engine = eng
+	a.hb = newHeartbeater(cp, probe, a.engine, handlers_rds.HeartbeatInterval)
+	a.cmd = newCommander(cp, newCommandRegistry(a.engine, newGuestStorage(cfg, execCommandRunner)), cfg.PollWait)
+	a.guard = newParamGuard(a.engine, probe, cp)
+	return a, nil
+}
+
+// Does not wait for IMDS: the register loop rides out a datapath still coming up.
+func New(cfg config) (*Agent, error) {
+	if cfg.GatewayURL == "" {
+		return nil, fmt.Errorf("no gateway URL configured (RDS_GATEWAY_URL)")
+	}
+	cp, err := newGatewayControlPlane(cfg)
+	if err != nil {
+		return nil, err
+	}
+	probe, err := newProbe(cfg, execProbeRunner)
+	if err != nil {
+		return nil, err
+	}
+	return newAgent(cfg, cp, probe)
+}
+
+func (a *Agent) Run(ctx context.Context) error {
+	// Named before the first dial. When register cannot reach the gateway the
+	// retry loop is all that survives, and the address it is dialing is what
+	// separates a mgmt NIC that came up without an address from a control plane
+	// that is genuinely down.
+	slog.Info("rds-agent: starting",
+		"gatewayURL", a.cfg.GatewayURL, "dbInstanceIdentifier", a.id.DBInstanceIdentifier,
+		"agentVersion", a.id.AgentVersion)
+
+	// Registration comes first: its response is the agent's identity.
+	if err := a.register(ctx); err != nil {
+		return err
+	}
+	a.hb.id, a.cmd.id, a.guard.id = a.id, a.id, a.id
+
+	// Ahead of the handoff, so rds-init never unblocks and no datadir is touched.
+	// An agent running on the wrong image would initialise one engine over
+	// another's data, coming up healthy and empty beside data nothing references.
+	if err := a.refuseEngineMismatch(ctx); err != nil {
+		return err
+	}
+
+	// Beating before the bootstrap keeps a stuck boot visible as a live VM with
+	// a down engine rather than as silence.
+	go a.hb.Run(ctx)
+
+	if err := a.bootstrap(ctx); err != nil {
+		return err
+	}
+
+	// The handoff must land before this wait: rds-datadir needs it to identify
+	// and authorize the volume. Commands and the parameter guard wait because
+	// their durable state belongs on that mount, never on the disposable boot disk.
+	if err := a.dataMountWaiter(ctx, a.cfg.MountsFile, a.cfg.DataMount); err != nil {
+		// A shutdown cancels this wait; main treats the wrapped context.Canceled
+		// as a clean stop, exactly as it does for register and bootstrap above.
+		return fmt.Errorf("wait for data mount %s: %w", a.cfg.DataMount, err)
+	}
+
+	// The receipt it waits on lives on that mount, so this cannot run any earlier.
+	// A cancelled wait is a clean stop here too.
+	if err := a.acknowledgeBootstrap(ctx); err != nil {
+		return err
+	}
+
+	// Directives are only meaningful against a bootstrapped, mounted engine.
+	go a.cmd.Run(ctx)
+
+	// Started after the handoff, so the window it measures covers the engine's
+	// own start rather than the time rds-init spent waiting for this agent.
+	go a.guard.Run(ctx)
+
+	<-ctx.Done()
+	return nil
+}
+
+// The engine cloud-init says this VM was launched as, against the engine the
+// image bakes. An empty expectation is no assertion rather than a disagreement:
+// a VM launched before the control plane emitted RDS_ENGINE carries none.
+func (a *Agent) refuseEngineMismatch(ctx context.Context) error {
+	expected := strings.ToLower(strings.TrimSpace(a.cfg.Engine))
+	if expected == "" || expected == a.cfg.BakedEngine {
+		return nil
+	}
+
+	message := fmt.Sprintf("this VM was launched as engine %q but its image bakes %q; refusing to bootstrap",
+		expected, a.cfg.BakedEngine)
+	slog.ErrorContext(ctx, "rds-agent: "+message)
+	// Reported before returning, so the control plane learns why the instance
+	// never came up rather than only that it never did.
+	if _, err := a.cp.SubmitState(ctx, a.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
+		slog.ErrorContext(ctx, "rds-agent: reporting the engine mismatch failed", "err", err)
+	}
+	return errors.New(message)
+}
+
+const dataMountPollInterval = time.Second
+
+// waitForDataMount blocks the stateful agent loops until the kernel reports the
+// configured data mount. Registration, heartbeat, and bootstrap run before it.
+func waitForDataMount(ctx context.Context, mountsFile, target string) error {
+	ticker := time.NewTicker(dataMountPollInterval)
+	defer ticker.Stop()
+
+	for {
+		mounted, err := mountTableContains(mountsFile, target)
+		if err != nil {
+			slog.WarnContext(ctx, "rds-agent: could not read mount table while waiting for data volume",
+				"mountsFile", mountsFile, "dataMount", target, "err", err)
+		} else if mounted {
+			slog.InfoContext(ctx, "rds-agent: data volume mount is ready", "dataMount", target)
+			return nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+	}
+}
+
+func mountTableContains(path, target string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		if len(fields) >= 2 && unescapeMountField(fields[1]) == target {
+			return true, nil
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return false, err
+	}
+	return false, nil
+}
+
+// Linux mount tables encode whitespace and backslashes as octal escapes.
+func unescapeMountField(field string) string {
+	replacer := strings.NewReplacer(`\134`, `\`, `\040`, " ", `\011`, "\t", `\012`, "\n")
+	return replacer.Replace(field)
+}
+
+// Boot-retry bounds. Tight to start, since the usual cause is a control plane
+// still creating this instance's record, and capped so an outage does not turn
+// a booting fleet into a retry storm.
+const (
+	retryMin = 1 * time.Second
+	retryMax = 30 * time.Second
+
+	// With the production backoff, attempt five is about 15 seconds into the
+	// failure and still precedes the first 30-second heartbeat.
+	retryErrorAttempt = 5
+)
+
+// Wrapping an error in this stops retryObserved. For the failures the control
+// plane decides from its own record, no retry can change the answer, and looping
+// on one only keeps the caller blocked.
+var errRetryTerminal = errors.New("terminal failure")
+
+// Never gives up on its own: an agent that stopped trying would leave the
+// control plane with no signal at all.
+func retry(ctx context.Context, what string, fn func(context.Context) error) error {
+	return retryObserved(ctx, what, fn, nil)
+}
+
+// onFailure lets boot-critical callers publish the latest error while the retry
+// remains blocked. It must not block because it runs on the retry path itself.
+func retryObserved(ctx context.Context, what string, fn func(context.Context) error, onFailure func(error)) error {
+	delay := retryMin
+	for attempt := 1; ; attempt++ {
+		err := fn(ctx)
+		if err == nil {
+			return nil
+		}
+		if ctx.Err() != nil {
+			return fmt.Errorf("%s: %w", what, ctx.Err())
+		}
+		if onFailure != nil {
+			onFailure(err)
+		}
+		if errors.Is(err, errRetryTerminal) {
+			slog.ErrorContext(ctx, "rds-agent: "+what+" failed terminally, not retrying", "err", err)
+			return fmt.Errorf("%s: %w", what, err)
+		}
+		if attempt >= retryErrorAttempt {
+			slog.ErrorContext(ctx, "rds-agent: "+what+" still failing, retrying",
+				"attempt", attempt, "retryIn", delay, "err", err)
+		} else {
+			slog.WarnContext(ctx, "rds-agent: "+what+" failed, retrying",
+				"attempt", attempt, "retryIn", delay, "err", err)
+		}
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("%s: %w", what, ctx.Err())
+		case <-time.After(delay):
+		}
+		delay = min(delay*2, retryMax)
+	}
+}

cmd/rds-agent/agent_test.go

@@ -0,0 +1,864 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/gwsign"
+	"github.com/mulgadc/spinifex/internal/rdsgw"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	gateway_rds "github.com/mulgadc/spinifex/spinifex/gateway/rds"
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeControlPlane is a scriptable controlPlane. Every call records what the
+// agent sent, so a test asserts on the request as well as the reaction.
+type fakeControlPlane struct {
+	mu sync.Mutex
+
+	registerOut  *handlers_rds.RegisterDBInstanceOutput
+	registerErrs []error
+	registerReqs []identity
+
+	bootstrapOut  *handlers_rds.GetDBBootstrapConfigOutput
+	bootstrapErrs []error
+	bootstrapReqs []identity
+	// Scripted responses ahead of bootstrapOut, so a test can answer the first
+	// fetch differently from the ones the agent retries with.
+	bootstrapQueue []*handlers_rds.GetDBBootstrapConfigOutput
+
+	ackErrs []error
+	ackReqs []pendingBootstrap
+
+	states []submittedState
+
+	pollReplies [][]handlers_rds.CommandReply
+	pollQueue   [][]handlers_rds.Command
+	pollErr     error
+	polled      chan struct{}
+}
+
+type submittedState struct {
+	id      identity
+	health  handlers_rds.EngineHealth
+	message string
+}
+
+var _ controlPlane = (*fakeControlPlane)(nil)
+
+func newFakeControlPlane() *fakeControlPlane {
+	return &fakeControlPlane{
+		registerOut: &handlers_rds.RegisterDBInstanceOutput{
+			DBInstanceIdentifier: "db-resolved", HeartbeatIntervalSeconds: 30,
+		},
+		polled: make(chan struct{}, 64),
+	}
+}
+
+// nextErr pops the next scripted error, so a test can fail a call n times and
+// then let it succeed.
+func nextErr(errs *[]error) error {
+	if len(*errs) == 0 {
+		return nil
+	}
+	err := (*errs)[0]
+	*errs = (*errs)[1:]
+	return err
+}
+
+func (f *fakeControlPlane) Register(_ context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.registerReqs = append(f.registerReqs, id)
+	if err := nextErr(&f.registerErrs); err != nil {
+		return nil, err
+	}
+	return f.registerOut, nil
+}
+
+func (f *fakeControlPlane) SubmitState(_ context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.states = append(f.states, submittedState{id: id, health: health, message: message})
+	// Zero means "keep the cadence you have", so a test's fast interval is not
+	// reset by every beat.
+	return &handlers_rds.SubmitDBStateChangeOutput{Acknowledged: true}, nil
+}
+
+func (f *fakeControlPlane) GetBootstrapConfig(_ context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.bootstrapReqs = append(f.bootstrapReqs, id)
+	if err := nextErr(&f.bootstrapErrs); err != nil {
+		return nil, err
+	}
+	if len(f.bootstrapQueue) > 0 {
+		out := f.bootstrapQueue[0]
+		f.bootstrapQueue = f.bootstrapQueue[1:]
+		return out, nil
+	}
+	return f.bootstrapOut, nil
+}
+
+func (f *fakeControlPlane) AcknowledgeBootstrap(_ context.Context, _ identity, pending *pendingBootstrap) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.ackReqs = append(f.ackReqs, *pending)
+	return nextErr(&f.ackErrs)
+}
+
+func (f *fakeControlPlane) PollCommands(ctx context.Context, _ identity, replies []handlers_rds.CommandReply, _ time.Duration) ([]handlers_rds.Command, error) {
+	f.mu.Lock()
+	f.pollReplies = append(f.pollReplies, slices.Clone(replies))
+	err := f.pollErr
+	var out []handlers_rds.Command
+	if len(f.pollQueue) > 0 {
+		out, f.pollQueue = f.pollQueue[0], f.pollQueue[1:]
+	}
+	f.mu.Unlock()
+
+	select {
+	case f.polled <- struct{}{}:
+	default:
+	}
+	if err != nil {
+		return nil, err
+	}
+	if len(out) > 0 {
+		return out, nil
+	}
+	// An idle window: block until the agent hangs up, as the gateway's long poll
+	// does, rather than spinning the caller's loop.
+	<-ctx.Done()
+	return nil, ctx.Err()
+}
+
+func (f *fakeControlPlane) ackRequests() []pendingBootstrap {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return slices.Clone(f.ackReqs)
+}
+
+func (f *fakeControlPlane) snapshotStates() []submittedState {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]submittedState(nil), f.states...)
+}
+
+func (f *fakeControlPlane) snapshotReplies() [][]handlers_rds.CommandReply {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([][]handlers_rds.CommandReply, len(f.pollReplies))
+	for i := range f.pollReplies {
+		out[i] = slices.Clone(f.pollReplies[i])
+	}
+	return out
+}
+
+// testConfig stamps the engine a real rds-postgres image bakes, so the agent
+// resolves the same guest layout it would in a guest, and points its file output
+// at a temp dir.
+func testConfig(t *testing.T) config {
+	t.Helper()
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.GatewayURL = "https://gw.test"
+	cfg.HandoffDir = filepath.Join(t.TempDir(), "spinifex-rds")
+	cfg.PollWait = time.Second
+	return cfg
+}
+
+// newTestAgent builds the agent over a probe the test drives directly.
+func newTestAgent(t *testing.T, cfg config, cp controlPlane, run probeRunner) *Agent {
+	t.Helper()
+	a, err := newAgent(cfg, cp, newPostgresProbe(cfg, run))
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
+	}
+	return a
+}
+
+// staticProbe is a probeRunner that always reports the same exit code.
+func staticProbe(code int) probeRunner {
+	return func(context.Context, string, ...string) (int, error) { return code, nil }
+}
+
+func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput {
+	mode := handlers_rds.BootstrapModeAttach
+	if password != nil {
+		mode = handlers_rds.BootstrapModeInitialize
+	}
+	return &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:                 mode,
+		DBInstanceIdentifier: "db-resolved",
+		Engine:               "postgres",
+		MasterUsername:       "master",
+		MasterUserPassword:   password,
+		Port:                 5432,
+		DataVolumeID:         "vol-data-01",
+		DataVolumeSerial:     "voldata01",
+		VMGeneration:         1,
+		PayloadID:            testPayloadID,
+		BootstrapPending:     password != nil,
+	}
+}
+
+const testPayloadID = "bp-0123456789abcdef"
+
+// Writes the completion receipt rds-init leaves behind, so the acknowledgement
+// step finds what a finished initialization would have produced.
+func writeReceipt(t *testing.T, a *Agent, payloadID, dbInstanceIdentifier string) {
+	t.Helper()
+	path := a.receiptPath()
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		t.Fatalf("mkdir receipt dir: %v", err)
+	}
+	body := fmt.Sprintf("%s=%s\n%s=%s\n", receiptPayload, payloadID, receiptDBID, dbInstanceIdentifier)
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatalf("write receipt: %v", err)
+	}
+}
+
+// runAgent runs the agent until its bootstrap has landed, then cancels. It
+// returns once Run has exited so no goroutine outlives the test.
+func runAgent(t *testing.T, a *Agent) error {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool {
+		_, err := os.Stat(filepath.Join(a.cfg.HandoffDir, handoffEnvFile))
+		return err == nil
+	}, "handoff file to be written")
+
+	cancel()
+	select {
+	case err := <-done:
+		// The cancellation above is this helper's own, so Run reporting it is the
+		// expected clean stop rather than a failure, exactly as main treats it.
+		if errors.Is(err, context.Canceled) {
+			return nil
+		}
+		return err
+	case <-time.After(5 * time.Second):
+		t.Fatal("Run did not exit after cancellation")
+		return nil
+	}
+}
+
+// waitFor polls cond until it holds or the test times out.
+func waitFor(t *testing.T, cond func() bool, what string) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatalf("timed out waiting for %s", what)
+}
+
+func TestRun_RegistersThenDeliversBootstrap(t *testing.T) {
+	password := "s3cret"
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(&password)
+
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	if err := runAgent(t, a); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	if len(cp.registerReqs) != 1 {
+		t.Fatalf("registered %d times, want 1", len(cp.registerReqs))
+	}
+	// The agent had no configured identifier, so the gateway's resolution is
+	// what every later call must carry.
+	if got := cp.bootstrapReqs[0].DBInstanceIdentifier; got != "db-resolved" {
+		t.Errorf("bootstrap sent identifier %q, want the registered db-resolved", got)
+	}
+
+	env := readFile(t, filepath.Join(cfg.HandoffDir, handoffEnvFile))
+	if !strings.Contains(env, "RDS_MODE='initialize'") || !strings.Contains(env, "RDS_MASTER_PASSWORD='s3cret'") {
+		t.Errorf("handoff env = %q, want initialize mode carrying the password", env)
+	}
+	// The probe follows the port the control plane assigned, not the default.
+	if got := a.probe.port.Load(); got != 5432 {
+		t.Errorf("probe port = %d, want 5432", got)
+	}
+}
+
+// A reboot re-fetches in attach mode. The agent must write that payload as it
+// stands rather than refusing it or inventing a password.
+func TestRun_AttachModeNeedsNoPassword(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	env := readFile(t, filepath.Join(cfg.HandoffDir, handoffEnvFile))
+	if !strings.Contains(env, "RDS_MODE='attach'") {
+		t.Errorf("handoff env = %q, want attach mode", env)
+	}
+	if strings.Contains(env, "RDS_MASTER_PASSWORD") {
+		t.Errorf("handoff env = %q, want no password key at all in attach mode", env)
+	}
+}
+
+// The record may not exist yet when a freshly launched VM first calls, so both
+// boot-critical calls have to ride out failures instead of exiting.
+func TestRun_RetriesRegisterAndBootstrap(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cp.registerErrs = []error{errors.New("DBInstanceNotFound")}
+	cp.bootstrapErrs = []error{errors.New("ServerInternal")}
+
+	cfg := testConfig(t)
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	if len(cp.registerReqs) != 2 {
+		t.Errorf("register attempts = %d, want 2 (one failure, one success)", len(cp.registerReqs))
+	}
+	if len(cp.bootstrapReqs) != 2 {
+		t.Errorf("bootstrap attempts = %d, want 2 (one failure, one success)", len(cp.bootstrapReqs))
+	}
+}
+
+// The fetch mutates nothing, so a handoff that cannot be written is retried
+// against the payload already in hand rather than costing a second fetch.
+func TestBootstrap_RetriesHandoffWithoutRefetching(t *testing.T) {
+	cp := newFakeControlPlane()
+	password := "staged-secret"
+	cp.bootstrapOut = bootstrapOutput(&password)
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	writes := 0
+	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
+		writes++
+		if got.MasterUserPassword == nil || *got.MasterUserPassword != password {
+			t.Fatalf("handoff attempt %d lost the initialize password", writes)
+		}
+		if writes == 1 {
+			return errors.New("temporary tmpfs write error")
+		}
+		return nil
+	}
+
+	if err := a.bootstrap(t.Context()); err != nil {
+		t.Fatalf("bootstrap: %v", err)
+	}
+	if len(cp.bootstrapReqs) != 1 {
+		t.Errorf("bootstrap fetches = %d, want 1", len(cp.bootstrapReqs))
+	}
+	if writes != 2 {
+		t.Errorf("handoff writes = %d, want 2", writes)
+	}
+	if a.pending == nil || a.pending.payloadID != testPayloadID {
+		t.Errorf("pending bootstrap = %+v, want the staged payload %s", a.pending, testPayloadID)
+	}
+}
+
+// A daemon that predates the encrypted payload answers initialize with nothing
+// to initialise with. Writing that handoff would let rds-init run initdb with an
+// empty master password, so it is retried until an upgraded node answers.
+func TestBootstrap_RetriesInitializeWithNoPassword(t *testing.T) {
+	cp := newFakeControlPlane()
+	empty := ""
+	password := "staged-secret"
+	cp.bootstrapQueue = []*handlers_rds.GetDBBootstrapConfigOutput{{
+		Mode: handlers_rds.BootstrapModeInitialize, DBInstanceIdentifier: "db-resolved",
+		MasterUsername: "master", MasterUserPassword: &empty,
+	}}
+	cp.bootstrapOut = bootstrapOutput(&password)
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	var handed []*handlers_rds.GetDBBootstrapConfigOutput
+	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
+		handed = append(handed, got)
+		return nil
+	}
+
+	if err := a.bootstrap(t.Context()); err != nil {
+		t.Fatalf("bootstrap: %v", err)
+	}
+	if len(handed) != 1 {
+		t.Fatalf("handoff writes = %d, want 1 once a usable payload arrived", len(handed))
+	}
+	if handed[0].MasterUserPassword == nil || *handed[0].MasterUserPassword != password {
+		t.Error("the agent wrote a handoff for a fetch that carried no master password")
+	}
+}
+
+func TestAcknowledgeBootstrap_WaitsForTheReceiptThenRetiresIt(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1, dataVolumeID: "vol-data-01"}
+
+	// rds-init runs after the agent registers, so the receipt is absent when the
+	// acknowledgement step first looks and appears some way into the boot.
+	if err := a.checkBootstrapReceipt(a.pending); err == nil {
+		t.Fatal("a missing receipt must not read as a completed bootstrap")
+	}
+	writeReceipt(t, a, testPayloadID, "db-resolved")
+
+	if err := a.acknowledgeBootstrap(t.Context()); err != nil {
+		t.Fatalf("acknowledgeBootstrap: %v", err)
+	}
+	if len(cp.ackRequests()) != 1 {
+		t.Fatalf("acknowledgements = %d, want 1", len(cp.ackRequests()))
+	}
+	if got := cp.ackRequests()[0]; got != *a.pending {
+		t.Errorf("acknowledged %+v, want %+v", got, *a.pending)
+	}
+	if _, err := os.Stat(a.receiptPath()); !os.IsNotExist(err) {
+		t.Error("the receipt must be removed once the record holds the audit trail")
+	}
+}
+
+// The receipt rides along in every snapshot of the data volume, so a restored
+// instance's volume carries the source instance's receipt. Acknowledging on one
+// would destroy the ciphertext this instance still needs.
+func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
+	tests := map[string]struct{ payloadID, dbID string }{
+		"another payload":  {payloadID: "bp-somebody-else", dbID: "db-resolved"},
+		"another instance": {payloadID: testPayloadID, dbID: "db-the-snapshot-source"},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			cp := newFakeControlPlane()
+			cfg := testConfig(t)
+			cfg.DataMount = t.TempDir()
+			a := newTestAgent(t, cfg, cp, staticProbe(0))
+			a.id = identity{DBInstanceIdentifier: "db-resolved"}
+			a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+			writeReceipt(t, a, tc.payloadID, tc.dbID)
+
+			ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
+			defer cancel()
+			if err := a.acknowledgeBootstrap(ctx); err == nil {
+				t.Fatal("acknowledgeBootstrap returned nil, want it still waiting on a real receipt")
+			}
+			if n := len(cp.ackRequests()); n != 0 {
+				t.Errorf("acknowledgements = %d, want 0", n)
+			}
+		})
+	}
+}
+
+// An attach boot has nothing staged, so it must not go looking for a receipt the
+// datadir will never grow.
+func TestAcknowledgeBootstrap_IsANoOpOnAttach(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	if err := a.acknowledgeBootstrap(t.Context()); err != nil {
+		t.Fatalf("acknowledgeBootstrap: %v", err)
+	}
+	if n := len(cp.ackRequests()); n != 0 {
+		t.Errorf("acknowledgements = %d, want 0 when nothing was staged", n)
+	}
+}
+
+// The image is the authority on which engine the agent runs. A VM launched as
+// another engine has to stop ahead of the handoff: rds-init would otherwise
+// initialise one engine over the other's datadir and come up healthy and empty
+// beside data nothing references.
+func TestRun_RefusesAnEngineTheImageDoesNotBake(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = "mariadb"
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	err := a.Run(t.Context())
+	if err == nil {
+		t.Fatal("Run returned nil for a VM launched as an engine this image does not bake")
+	}
+	if !strings.Contains(err.Error(), "mariadb") {
+		t.Errorf("error = %v, want it to name the engine the VM was launched as", err)
+	}
+	if _, statErr := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); !os.IsNotExist(statErr) {
+		t.Error("the agent wrote a bootstrap handoff for an engine it refused to run")
+	}
+	if n := len(cp.bootstrapReqs); n != 0 {
+		t.Errorf("bootstrap fetches = %d, want none before the refusal", n)
+	}
+	// Reported rather than only logged, so the control plane learns why the
+	// instance never came up rather than only that it never did.
+	states := cp.snapshotStates()
+	if len(states) != 1 || states[0].health != handlers_rds.EngineHealthUnhealthy {
+		t.Fatalf("submitted states = %+v, want a single unhealthy report", states)
+	}
+}
+
+// The same assertion when it agrees is not a gate: the boot goes on exactly as
+// it does for a VM launched before the control plane sent one at all.
+func TestRun_ProceedsWhenTheDeliveredEngineMatchesTheImage(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = enginePostgres
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}
+
+// A configured identifier is asserted on the wire so a mis-provisioned VM is
+// rejected by the gateway instead of adopting whatever it resolves to.
+func TestRun_SendsConfiguredIdentifier(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.DBInstanceIdentifier = "db-configured"
+	cfg.EngineVersion = "18.1"
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	sent := cp.registerReqs[0]
+	if sent.DBInstanceIdentifier != "db-configured" || sent.EngineVersion != "18.1" {
+		t.Errorf("register sent %+v, want the configured identifier and engine version", sent)
+	}
+}
+
+func TestRun_WaitsForDataMountBeforePollingCommands(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
+
+	waiting := make(chan struct{})
+	release := make(chan struct{})
+	a.dataMountWaiter = func(ctx context.Context, mountsFile, target string) error {
+		if mountsFile != defaultMountsFile || target != cfg.DataMount {
+			t.Errorf("mount waiter got %q/%q, want configured defaults", mountsFile, target)
+		}
+		close(waiting)
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-release:
+			return nil
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	select {
+	case <-waiting:
+	case <-ctx.Done():
+		t.Fatal("agent did not reach the data-mount wait after writing the handoff")
+	}
+	if _, err := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); err != nil {
+		t.Fatalf("handoff was not present before mount wait: %v", err)
+	}
+	select {
+	case <-cp.polled:
+		t.Fatal("command poll started before the data volume was mounted")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	close(release)
+	select {
+	case <-cp.polled:
+	case <-ctx.Done():
+		t.Fatal("command poll did not start after the data mount became ready")
+	}
+	cancel()
+	if err := <-done; err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}
+
+func TestMountTableContains(t *testing.T) {
+	mounts := filepath.Join(t.TempDir(), "mounts")
+	require.NoError(t, os.WriteFile(mounts, []byte(
+		"/dev/vda1 / ext4 rw 0 0\n/dev/vdb /var/lib/postgresql ext4 rw 0 0\n"+
+			"/dev/vdc /path\\040with\\040spaces xfs rw 0 0\nmalformed\n"), 0o600))
+
+	tests := []struct {
+		target string
+		want   bool
+	}{
+		{target: "/var/lib/postgresql", want: true},
+		{target: "/path with spaces", want: true},
+		{target: "/missing", want: false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.target, func(t *testing.T) {
+			got, err := mountTableContains(mounts, tc.target)
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestRun_HeartbeatIncludesBootstrapFailure(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapErrs = []error{errors.New("403 AccessDenied: instance profile is not authorised")}
+	cp.registerOut.HeartbeatIntervalSeconds = 0
+
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
+	a.hb.interval = 10 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool {
+		for _, state := range cp.snapshotStates() {
+			if strings.Contains(state.message, "bootstrap fetch failed") &&
+				strings.Contains(state.message, "403 AccessDenied") {
+				return true
+			}
+		}
+		return false
+	}, "the bootstrap failure to reach a heartbeat")
+	cancel()
+	<-done
+}
+
+func TestRun_HeartbeatsEngineHealth(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	// Declining to set a cadence leaves the agent's own, which the test shortens
+	// rather than waiting out the 30s production interval.
+	cp.registerOut.HeartbeatIntervalSeconds = 0
+
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	a.hb.interval = 10 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool { return len(cp.snapshotStates()) > 0 }, "a heartbeat")
+	cancel()
+	<-done
+
+	beat := cp.snapshotStates()[0]
+	if beat.health != handlers_rds.EngineHealthHealthy {
+		t.Errorf("heartbeat health = %q, want healthy", beat.health)
+	}
+	if beat.id.DBInstanceIdentifier != "db-resolved" {
+		t.Errorf("heartbeat identifier = %q, want db-resolved", beat.id.DBInstanceIdentifier)
+	}
+}
+
+// The cadence is the control plane's to set: it feeds the staleness window the
+// recovery reconciler judges an instance against, so the guest adopts it rather
+// than keeping its own.
+func TestRegister_AdoptsControlPlaneCadence(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cp.registerOut.HeartbeatIntervalSeconds = 45
+
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	if err := runAgent(t, a); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if a.hb.interval != 45*time.Second {
+		t.Errorf("heartbeat interval = %v, want the 45s the control plane returned", a.hb.interval)
+	}
+}
+
+// The agent being up says nothing about the engine. A down engine has to reach
+// the control plane as such, or auto-recovery has nothing to act on.
+func TestRun_HeartbeatReportsDownEngine(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cp.registerOut.HeartbeatIntervalSeconds = 0
+
+	cfg := testConfig(t)
+	probe := newPostgresProbe(cfg, staticProbe(2))
+	// The engine has served before, so a silent one now is a failure rather
+	// than a boot still in progress.
+	probe.seenHealthy = true
+
+	a, err := newAgent(cfg, cp, probe)
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
+	}
+	a.hb.interval = 10 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool { return len(cp.snapshotStates()) > 0 }, "a heartbeat")
+	cancel()
+	<-done
+
+	beat := cp.snapshotStates()[0]
+	if beat.health != handlers_rds.EngineHealthUnhealthy {
+		t.Errorf("heartbeat health = %q, want unhealthy while the agent is up", beat.health)
+	}
+	if beat.message == "" {
+		t.Error("unhealthy heartbeat carried no message explaining the probe result")
+	}
+}
+
+// The mirror the agent decodes poll responses into has to stay aligned with the
+// gateway type that produces them; nothing else in the build ties the two.
+func TestPollDBCommandsOutput_MatchesGatewayShape(t *testing.T) {
+	issued := time.Date(2026, 7, 28, 1, 2, 3, 0, time.UTC)
+	// Rendered exactly as the gateway's typed adapter renders it, so a change to
+	// either side's tags fails here.
+	payload := utils.GenerateIAMXMLPayload("PollDBCommands", &gateway_rds.PollDBCommandsOutput{
+		Commands: []handlers_rds.Command{{
+			CommandID:  "cmd-1",
+			Type:       "reload-parameters",
+			Parameters: []handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}},
+			IssuedAt:   &issued,
+		}},
+	})
+	body, err := utils.MarshalToXML(payload)
+	if err != nil {
+		t.Fatalf("marshal poll result: %v", err)
+	}
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/xml")
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	client, err := rdsgw.New(srv.URL, "", gwsign.NewStatic("AKIATEST", "secret"), "us-east-1", time.Second)
+	if err != nil {
+		t.Fatalf("rdsgw.New: %v", err)
+	}
+
+	var got pollDBCommandsOutput
+	if err := client.Call(context.Background(), "PollDBCommands", nil, &got); err != nil {
+		t.Fatalf("Call: %v", err)
+	}
+	if len(got.Commands) != 1 {
+		t.Fatalf("decoded %d commands, want 1", len(got.Commands))
+	}
+	cmd := got.Commands[0]
+	if cmd.CommandID != "cmd-1" || cmd.Type != "reload-parameters" {
+		t.Errorf("command = %+v, want cmd-1/reload-parameters", cmd)
+	}
+	if len(cmd.Parameters) != 1 || cmd.Parameters[0].Value != "8MB" {
+		t.Errorf("Parameters = %+v, want the seeded member", cmd.Parameters)
+	}
+	if cmd.IssuedAt == nil || !cmd.IssuedAt.Equal(issued) {
+		t.Errorf("IssuedAt = %v, want %v", cmd.IssuedAt, issued)
+	}
+}
+
+func readFile(t *testing.T, path string) string {
+	t.Helper()
+	body, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	return string(body)
+}
+
+// AccessDenied is decided by the control plane's own record, so no retry can
+// change it. Looping on one would leave the command poller and the parameter
+// guard unstarted for the life of a VM whose engine is serving normally.
+func TestAcknowledgeBootstrap_StopsOnATerminalDenial(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.ackErrs = []error{&rdsgw.APIError{
+		Action: "AcknowledgeDBBootstrap", StatusCode: 403,
+		Code:    awserrors.ErrorAccessDenied,
+		Message: "bootstrap payload bp-x at generation 1 does not match the current state",
+	}}
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+	writeReceipt(t, a, testPayloadID, "db-resolved")
+
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+	defer cancel()
+	if err := a.acknowledgeBootstrap(ctx); err != nil {
+		t.Fatalf("acknowledgeBootstrap = %v, want nil so the stateful loops still start", err)
+	}
+	if n := len(cp.ackRequests()); n != 1 {
+		t.Errorf("acknowledgements = %d, want 1 attempt and no retry", n)
+	}
+	if a.hb.bootstrapFailure.Load() == nil {
+		t.Error("a terminal denial must still reach the operator through the heartbeat")
+	}
+	if _, err := os.Stat(a.receiptPath()); err != nil {
+		t.Error("a denied acknowledgement must leave the receipt for a later attempt")
+	}
+}
+
+// rds-init runs after this agent, so the receipt is absent for the whole initdb
+// window. Reporting that would put a failure on every healthy create's beat, and
+// then into the reason a create that timed out for some other cause records.
+func TestAcknowledgeBootstrap_DoesNotReportAnAbsentReceiptAsAFailure(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+
+	ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
+	defer cancel()
+	if err := a.acknowledgeBootstrap(ctx); err == nil {
+		t.Fatal("acknowledgeBootstrap returned nil, want it still waiting for rds-init")
+	}
+	if got := a.hb.bootstrapFailure.Load(); got != nil {
+		t.Errorf("bootstrap failure = %q, want none while the receipt has simply not been written", *got)
+	}
+
+	// A receipt that exists but names another instance is a real mismatch and is
+	// reported, which is what separates the two cases.
+	writeReceipt(t, a, testPayloadID, "db-the-snapshot-source")
+	mismatch, cancelMismatch := context.WithTimeout(t.Context(), 50*time.Millisecond)
+	defer cancelMismatch()
+	if err := a.acknowledgeBootstrap(mismatch); err == nil {
+		t.Fatal("a foreign receipt must not be acknowledged")
+	}
+	if a.hb.bootstrapFailure.Load() == nil {
+		t.Error("a receipt naming another instance must be reported")
+	}
+}

cmd/rds-agent/bootstrap.go

@@ -0,0 +1,232 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The staged payload this boot has to acknowledge once PostgreSQL has applied
+// it. Nil on an attach boot, which makes the acknowledgement step a no-op.
+type pendingBootstrap struct {
+	payloadID    string
+	vmGeneration int64
+	dataVolumeID string
+}
+
+// Fetches the boot material and writes the handoff rds-init is blocked waiting
+// for, then points the health probe at the assigned port.
+func (a *Agent) bootstrap(ctx context.Context) error {
+	var cfg *handlers_rds.GetDBBootstrapConfigOutput
+	if err := retryObserved(ctx, "bootstrap fetch", func(ctx context.Context) error {
+		fetched, err := a.cp.GetBootstrapConfig(ctx, a.id)
+		if err != nil {
+			return err
+		}
+		if fetched == nil {
+			return fmt.Errorf("bootstrap fetch returned no config")
+		}
+		// A control plane that predates the encrypted payload answers initialize
+		// with nothing to initialise with. Retrying re-dials the worker queue
+		// group until an upgraded node answers, so a mixed-version fleet heals
+		// within one boot rather than needing a reboot.
+		if fetched.Mode == handlers_rds.BootstrapModeInitialize &&
+			(fetched.MasterUserPassword == nil || *fetched.MasterUserPassword == "") {
+			return fmt.Errorf("bootstrap fetch returned mode=%s with no master password", fetched.Mode)
+		}
+		cfg = fetched
+		return nil
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap fetch", err)
+	}); err != nil {
+		return err
+	}
+	a.hb.clearBootstrapFailure()
+
+	// The fetch mutates nothing, so a handoff that cannot be written is retried
+	// against the same staged payload rather than against an attach response.
+	if err := retryObserved(ctx, "bootstrap handoff", func(context.Context) error {
+		return a.handoffWriter(a.cfg.HandoffDir, cfg)
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap handoff", err)
+	}); err != nil {
+		return err
+	}
+	a.hb.clearBootstrapFailure()
+	if cfg.Port > 0 {
+		a.probe.setPort(int(cfg.Port))
+		a.engine.setPort(int(cfg.Port))
+	}
+	if cfg.BootstrapPending && cfg.PayloadID != "" {
+		a.pending = &pendingBootstrap{
+			payloadID:    cfg.PayloadID,
+			vmGeneration: cfg.VMGeneration,
+			dataVolumeID: cfg.DataVolumeID,
+		}
+	}
+	// Mode is logged but never branched on: rds-init decides whether to initdb
+	// from the state of the datadir.
+	slog.Info("rds-agent: bootstrap config delivered",
+		"mode", cfg.Mode, "port", cfg.Port, "parameters", len(cfg.Parameters),
+		"bootstrapPending", cfg.BootstrapPending)
+	return nil
+}
+
+// The handoff files rds-init reads. They live on tmpfs, so nothing survives a
+// reboot and the next boot re-fetches rather than reusing a stale password or a
+// re-minted cert.
+const (
+	handoffEnvFile    = "bootstrap.env"
+	handoffParamsFile = "parameters.conf"
+	handoffCertFile   = "server.crt"
+	handoffKeyFile    = "server.key"
+
+	// Root-only: only rds-init reads these, and one holds the master password.
+	handoffMode    = 0o600
+	handoffDirMode = 0o700
+)
+
+// bootstrap.env is written last and renamed into place, so its appearance —
+// which rds-init waits on — means the whole handoff is complete.
+func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) error {
+	if cfg.MasterUsername == "" {
+		return fmt.Errorf("bootstrap config carries no master username")
+	}
+	if err := os.MkdirAll(dir, handoffDirMode); err != nil {
+		return fmt.Errorf("create handoff dir %s: %w", dir, err)
+	}
+	// MkdirAll leaves an existing directory's mode alone, and this one may have
+	// been created more permissively earlier in the boot.
+	if err := os.Chmod(dir, handoffDirMode); err != nil {
+		return fmt.Errorf("secure handoff dir %s: %w", dir, err)
+	}
+
+	params, err := renderParameters(cfg.Engine, cfg.Parameters)
+	if err != nil {
+		return err
+	}
+	if err := writeHandoffFile(dir, handoffParamsFile, params); err != nil {
+		return err
+	}
+	// Cert and key are written as a pair; half of one would start the engine
+	// with TLS configured against a key it does not have.
+	if cfg.ServingCertificate != "" && cfg.ServingPrivateKey != "" {
+		if err := writeHandoffFile(dir, handoffCertFile, cfg.ServingCertificate); err != nil {
+			return err
+		}
+		if err := writeHandoffFile(dir, handoffKeyFile, cfg.ServingPrivateKey); err != nil {
+			return err
+		}
+	}
+
+	return writeHandoffFile(dir, handoffEnvFile, renderBootstrapEnv(cfg))
+}
+
+// Every value is single-quoted because rds-init sources this with `.`: an
+// unquoted password holding a space, `$` or `;` would be word-split or executed.
+func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
+	var b strings.Builder
+	b.WriteString("# Written by rds-agent. Regenerated on every boot; edits are lost.\n")
+	writeEnvLine(&b, "RDS_MODE", cfg.Mode)
+	writeEnvLine(&b, "RDS_DB_INSTANCE_IDENTIFIER", cfg.DBInstanceIdentifier)
+	writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
+	writeEnvLine(&b, "RDS_DATA_VOLUME_ID", cfg.DataVolumeID)
+	writeEnvLine(&b, "RDS_DATA_VOLUME_SERIAL", cfg.DataVolumeSerial)
+	writeEnvLine(&b, "RDS_VM_GENERATION", strconv.FormatInt(cfg.VMGeneration, 10))
+	writeEnvLine(&b, "RDS_FORMAT_AUTHORIZED", strconv.FormatBool(cfg.FormatAuthorized))
+	// Neither needs volume access, which is why the assertion is made here: the
+	// agent runs before rds-datadir mounts the volume, so rds-init is the first
+	// component that can compare it against the completion receipt on disk.
+	writeEnvLine(&b, "RDS_BOOTSTRAP_PENDING", boolFlag(cfg.BootstrapPending))
+	writeEnvLine(&b, "RDS_PAYLOAD_ID", cfg.PayloadID)
+	// Present only in initialize mode: an attach fetch has no password to write
+	// and rds-init must not find a stale one.
+	if cfg.MasterUserPassword != nil {
+		writeEnvLine(&b, "RDS_MASTER_PASSWORD", *cfg.MasterUserPassword)
+	}
+	if cfg.DBName != "" {
+		writeEnvLine(&b, "RDS_DB_NAME", cfg.DBName)
+	}
+	if cfg.Port > 0 {
+		writeEnvLine(&b, "RDS_PORT", strconv.FormatInt(cfg.Port, 10))
+	}
+	return b.String()
+}
+
+// rds-init compares against 1 rather than sourcing a Go boolean, so the two
+// halves of the guard cannot drift on spelling.
+func boolFlag(v bool) string {
+	if v {
+		return "1"
+	}
+	return "0"
+}
+
+func writeEnvLine(b *strings.Builder, key, value string) {
+	b.WriteString(key)
+	b.WriteString("=")
+	b.WriteString(shellQuote(value))
+	b.WriteString("\n")
+}
+
+// Single quotes suppress every shell expansion; an embedded quote is closed,
+// escaped and reopened.
+func shellQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+// postgresql.conf syntax, which MariaDB's option files also parse. Values are
+// quoted so a setting with a space or unit suffix survives; the engine accepts
+// quoted numerics and booleans too.
+//
+// Names are the engine's rather than the customer's: a setting the server takes
+// only through SET has a different spelling at startup, and writing the wrong one
+// leaves the engine refusing to boot on a file already on the data volume.
+func renderParameters(engineName string, params []handlers_rds.Parameter) (string, error) {
+	engine, err := handlers_rds.LookupEngine(engineName)
+	if err != nil {
+		return "", fmt.Errorf("render the resolved parameters: %w", err)
+	}
+	var b strings.Builder
+	b.WriteString("# Resolved parameter group, written by rds-agent.\n")
+	for _, p := range params {
+		if p.Name == "" {
+			continue
+		}
+		fmt.Fprintf(&b, "%s = '%s'\n", engine.OptionFileName(p.Name), strings.ReplaceAll(p.Value, "'", "''"))
+	}
+	return b.String(), nil
+}
+
+// Atomic: a temp file in the same directory, renamed over the target. The mode
+// is set at creation, so content is never briefly readable at the process umask.
+func writeHandoffFile(dir, name, content string) error {
+	path := filepath.Join(dir, name)
+	tmp := path + ".new"
+
+	f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, handoffMode)
+	if err != nil {
+		return fmt.Errorf("create %s: %w", tmp, err)
+	}
+	// A failed write must not leave the temp file behind holding a password.
+	if _, err := f.WriteString(content); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return fmt.Errorf("write %s: %w", tmp, err)
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("close %s: %w", tmp, err)
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("install %s: %w", path, err)
+	}
+	return nil
+}

cmd/rds-agent/bootstrap_test.go

@@ -0,0 +1,270 @@
+package main
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	password := "s3cret"
+
+	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
+		MasterUsername:     "master",
+		MasterUserPassword: &password,
+		DBName:             "appdb",
+		Port:               5433,
+		DataVolumeID:       "vol-data-01",
+		DataVolumeSerial:   "voldata01",
+		VMGeneration:       4,
+		FormatAuthorized:   true,
+		Parameters: []handlers_rds.Parameter{
+			{Name: "max_connections", Value: "200"},
+			{Name: "shared_buffers", Value: "256MB"},
+		},
+		ServingCertificate: "CERT",
+		ServingPrivateKey:  "KEY",
+	})
+	if err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	env := readFile(t, filepath.Join(dir, handoffEnvFile))
+	for _, want := range []string{
+		"RDS_MODE='initialize'",
+		"RDS_MASTER_USERNAME='master'",
+		"RDS_MASTER_PASSWORD='s3cret'",
+		"RDS_DB_NAME='appdb'",
+		"RDS_PORT='5433'",
+		"RDS_DATA_VOLUME_ID='vol-data-01'",
+		"RDS_DATA_VOLUME_SERIAL='voldata01'",
+		"RDS_VM_GENERATION='4'",
+		"RDS_FORMAT_AUTHORIZED='true'",
+	} {
+		if !strings.Contains(env, want) {
+			t.Errorf("handoff env missing %s:\n%s", want, env)
+		}
+	}
+
+	params := readFile(t, filepath.Join(dir, handoffParamsFile))
+	if !strings.Contains(params, "max_connections = '200'") || !strings.Contains(params, "shared_buffers = '256MB'") {
+		t.Errorf("parameters.conf = %q, want both resolved settings", params)
+	}
+	if got := readFile(t, filepath.Join(dir, handoffCertFile)); got != "CERT" {
+		t.Errorf("server.crt = %q, want CERT", got)
+	}
+	if got := readFile(t, filepath.Join(dir, handoffKeyFile)); got != "KEY" {
+		t.Errorf("server.key = %q, want KEY", got)
+	}
+
+	// The handoff holds a master password and a private key; anything readable
+	// beyond root is a disclosure to every process in the guest.
+	if mode := statMode(t, dir); mode != handoffDirMode {
+		t.Errorf("handoff dir mode = %#o, want %#o", mode, handoffDirMode)
+	}
+	for _, name := range []string{handoffEnvFile, handoffParamsFile, handoffCertFile, handoffKeyFile} {
+		if mode := statMode(t, filepath.Join(dir, name)); mode != handoffMode {
+			t.Errorf("%s mode = %#o, want %#o", name, mode, handoffMode)
+		}
+	}
+}
+
+// Attach mode carries no password, and the key must be absent rather than
+// present and empty — rds-init reads an empty value as "no password" only by
+// accident, and a later boot would inherit it.
+func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
+		MasterUsername: "master",
+		Port:           5432,
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	env := readFile(t, filepath.Join(dir, handoffEnvFile))
+	if strings.Contains(env, "RDS_MASTER_PASSWORD") {
+		t.Errorf("handoff env = %q, want no RDS_MASTER_PASSWORD key", env)
+	}
+	for _, want := range []string{
+		"RDS_DATA_VOLUME_ID=''",
+		"RDS_DATA_VOLUME_SERIAL=''",
+		"RDS_VM_GENERATION='0'",
+		"RDS_FORMAT_AUTHORIZED='false'",
+	} {
+		if !strings.Contains(env, want) {
+			t.Errorf("handoff env missing fail-closed field %s: %q", want, env)
+		}
+	}
+}
+
+// The env file is sourced by a shell, so a password holding shell metacharacters
+// must survive as data rather than being split, expanded or executed.
+func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	password := `a b$(id);'"\` + "`whoami`"
+
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
+		MasterUsername:     "master",
+		MasterUserPassword: &password,
+		Port:               5432,
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	env := readFile(t, filepath.Join(dir, handoffEnvFile))
+	line := findLine(t, env, "RDS_MASTER_PASSWORD=")
+	if got := shellSource(t, line, "RDS_MASTER_PASSWORD"); got != password {
+		t.Errorf("sourced password = %q, want %q (line was %s)", got, password, line)
+	}
+}
+
+// A serving cert with no key would have rds-init configure TLS against a key
+// that is not there; the pair is delivered together or not at all.
+func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:               handlers_rds.BootstrapModeAttach,
+		Engine:             "postgres",
+		MasterUsername:     "master",
+		Port:               5432,
+		ServingCertificate: "CERT",
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+
+	if _, err := os.Stat(filepath.Join(dir, handoffCertFile)); !os.IsNotExist(err) {
+		t.Errorf("server.crt exists without a key; stat err = %v", err)
+	}
+}
+
+// A config with no master username cannot bootstrap anything, and writing it
+// would only move the failure into rds-init after the boot has committed to it.
+func TestWriteHandoff_RejectsConfigWithNoMasterUser(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:   handlers_rds.BootstrapModeAttach,
+		Engine: "postgres",
+		Port:   5432,
+	})
+	if err == nil {
+		t.Fatal("writeHandoff accepted a config with no master username, want an error")
+	}
+	if _, statErr := os.Stat(filepath.Join(dir, handoffEnvFile)); !os.IsNotExist(statErr) {
+		t.Errorf("handoff env was written despite the rejection; stat err = %v", statErr)
+	}
+}
+
+// A directory left over from an earlier boot at a looser mode must be tightened,
+// not accepted as it is found.
+func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "handoff")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatalf("seed dir: %v", err)
+	}
+
+	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
+		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
+		MasterUsername: "master",
+		Port:           5432,
+	}); err != nil {
+		t.Fatalf("writeHandoff: %v", err)
+	}
+	if mode := statMode(t, dir); mode != handoffDirMode {
+		t.Errorf("handoff dir mode = %#o, want it tightened to %#o", mode, handoffDirMode)
+	}
+}
+
+func TestShellQuote_EscapesEmbeddedQuotes(t *testing.T) {
+	for _, in := range []string{"", "plain", "it's", `''`, "new\nline", `$HOME`} {
+		if got := shellSource(t, "V="+shellQuote(in), "V"); got != in {
+			t.Errorf("shellQuote(%q) sourced back as %q", in, got)
+		}
+	}
+}
+
+// shellSource evaluates one KEY=value assignment in a real shell and returns
+// what the variable ends up holding. The quoting is only correct if a shell
+// agrees, so the test asks one instead of re-implementing its rules.
+func shellSource(t *testing.T, line, name string) string {
+	t.Helper()
+	out, err := exec.Command("/bin/sh", "-c", line+"\nprintf %s \"$"+name+"\"\n").Output()
+	if err != nil {
+		t.Fatalf("sourcing %q: %v", line, err)
+	}
+	return string(out)
+}
+
+func statMode(t *testing.T, path string) os.FileMode {
+	t.Helper()
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatalf("stat %s: %v", path, err)
+	}
+	return info.Mode().Perm()
+}
+
+func findLine(t *testing.T, body, prefix string) string {
+	t.Helper()
+	for line := range strings.SplitSeq(body, "\n") {
+		if strings.HasPrefix(line, prefix) {
+			return line
+		}
+	}
+	t.Fatalf("no line starting %q in:\n%s", prefix, body)
+	return ""
+}
+
+// The include rds-init installs is what the engine starts on, so the spelling
+// written here is the one that has to boot.
+func TestRenderParameters_UsesTheEngineStartupSpelling(t *testing.T) {
+	params := []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "SYSTEM"},
+		{Name: "max_connections", Value: "85"},
+	}
+
+	rendered, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "default_time_zone = 'SYSTEM'\n") {
+		t.Errorf("time_zone was not written under its startup spelling:\n%s", rendered)
+	}
+	if strings.Contains(rendered, "\ntime_zone = ") {
+		t.Errorf("time_zone reached the option file, which mariadbd refuses:\n%s", rendered)
+	}
+	if !strings.Contains(rendered, "max_connections = '85'\n") {
+		t.Errorf("an unmapped parameter was not written unchanged:\n%s", rendered)
+	}
+}
+
+func TestRenderParameters_PostgresKeepsEveryName(t *testing.T) {
+	rendered, err := renderParameters("postgres", []handlers_rds.Parameter{
+		{Name: "shared_buffers", Value: "262144"},
+	})
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "shared_buffers = '262144'\n") {
+		t.Errorf("postgres name was rewritten:\n%s", rendered)
+	}
+}
+
+// Failing closed beats writing a file the engine may refuse: the bad include
+// would already be on the data volume by the time the server rejected it.
+func TestRenderParameters_RefusesAnUnknownEngine(t *testing.T) {
+	if _, err := renderParameters("", nil); err == nil {
+		t.Fatal("renderParameters accepted a config naming no engine, want an error")
+	}
+}

cmd/rds-agent/command.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strconv"
+	"strings"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Executes one control-plane directive inside the guest and returns the message
+// reported back to the issuer.
+type commandHandler func(ctx context.Context, cmd handlers_rds.Command) (string, error)
+
+// A registry rather than a dispatch switch, so guest operations — password
+// apply, parameter reload, storage grow, snapshot quiesce — add an entry each.
+type commandRegistry map[string]commandHandler
+
+// An unregistered type is replied to as unsupported, so a control plane ahead
+// of the guest gets a clear answer rather than a timeout.
+func newCommandRegistry(engine engineOps, storage storageOps) commandRegistry {
+	return commandRegistry{
+		handlers_rds.CommandSetPassword: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			params := commandParams(cmd)
+			return "", engine.SetPassword(ctx,
+				params[handlers_rds.CommandParamMasterUsername],
+				params[handlers_rds.CommandParamMasterUserPassword])
+		},
+		handlers_rds.CommandApplyParams: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			pending, err := engine.ApplyParameters(ctx, cmd.Parameters)
+			if err != nil {
+				return "", err
+			}
+			// The reply carries no structured payload, so the settings still
+			// awaiting a restart come back as the message itself.
+			return strings.Join(pending, ","), nil
+		},
+		handlers_rds.CommandStopEngine: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			return "", engine.Stop(ctx)
+		},
+		handlers_rds.CommandGrowFilesystem: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			return storage.GrowFilesystem(ctx)
+		},
+		handlers_rds.CommandQuiesce: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			params := commandParams(cmd)
+			hold, err := quiesceHoldFrom(params[handlers_rds.CommandParamQuiesceDeadlineSeconds])
+			if err != nil {
+				return "", err
+			}
+			return "", engine.Quiesce(ctx, params[handlers_rds.CommandParamQuiesceLabel], hold)
+		},
+		handlers_rds.CommandUnquiesce: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
+			return "", engine.Unquiesce(ctx)
+		},
+	}
+}
+
+// The control plane's hold deadline. A missing or unreadable one is refused
+// rather than defaulted: a hold the guest chose the length of would not be the
+// bound the control plane is relying on.
+func quiesceHoldFrom(value string) (time.Duration, error) {
+	seconds, err := strconv.Atoi(strings.TrimSpace(value))
+	if err != nil || seconds <= 0 {
+		return 0, fmt.Errorf("%s must be a positive number of seconds, got %q",
+			handlers_rds.CommandParamQuiesceDeadlineSeconds, value)
+	}
+	return time.Duration(seconds) * time.Second, nil
+}
+
+// A later value wins, so a malformed command carrying a parameter twice cannot
+// leave the handler acting on the first of two conflicting values.
+func commandParams(cmd handlers_rds.Command) map[string]string {
+	params := make(map[string]string, len(cmd.Parameters))
+	for _, p := range cmd.Parameters {
+		params[p.Name] = p.Value
+	}
+	return params
+}
+
+// Spaces retries after a failed poll, so a broken channel is not re-polled at
+// line rate for as long as it stays broken.
+const pollErrorBackoff = 5 * time.Second
+
+// The agent's command channel: a long poll that carries back the replies for
+// whatever the previous poll delivered.
+type commander struct {
+	cp       controlPlane
+	id       identity
+	registry commandRegistry
+	wait     time.Duration
+	// Cleared only once a poll has carried them, so a failed poll re-delivers
+	// rather than dropping the result of work the guest actually did.
+	pending []handlers_rds.CommandReply
+}
+
+func newCommander(cp controlPlane, registry commandRegistry, wait time.Duration) *commander {
+	if wait <= 0 {
+		wait = defaultPollWait
+	}
+	return &commander{cp: cp, registry: registry, wait: wait}
+}
+
+func (c *commander) Run(ctx context.Context) {
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+		commands, err := c.cp.PollCommands(ctx, c.id, c.pending, c.wait)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			slog.Warn("rds-agent: command poll failed", "err", err)
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(pollErrorBackoff):
+			}
+			continue
+		}
+		c.pending = nil
+
+		for _, cmd := range commands {
+			c.pending = append(c.pending, c.execute(ctx, cmd))
+		}
+	}
+}
+
+// Every command gets a reply: the issuer is blocked on this command ID, so a
+// silent drop costs it a full timeout.
+func (c *commander) execute(ctx context.Context, cmd handlers_rds.Command) handlers_rds.CommandReply {
+	handler, ok := c.registry[cmd.Type]
+	if !ok {
+		slog.Warn("rds-agent: unsupported command", "commandID", cmd.CommandID, "type", cmd.Type)
+		return failedReply(cmd, fmt.Sprintf("unsupported command type %q", cmd.Type))
+	}
+
+	slog.Info("rds-agent: executing command", "commandID", cmd.CommandID, "type", cmd.Type)
+	message, err := handler(ctx, cmd)
+	if err != nil {
+		slog.Error("rds-agent: command failed", "commandID", cmd.CommandID, "type", cmd.Type, "err", err)
+		return failedReply(cmd, err.Error())
+	}
+	return handlers_rds.CommandReply{
+		CommandID: cmd.CommandID,
+		Status:    handlers_rds.CommandStatusSucceeded,
+		Message:   message,
+	}
+}
+
+func failedReply(cmd handlers_rds.Command, message string) handlers_rds.CommandReply {
+	return handlers_rds.CommandReply{
+		CommandID: cmd.CommandID,
+		Status:    handlers_rds.CommandStatusFailed,
+		Message:   message,
+	}
+}

cmd/rds-agent/command_test.go

@@ -0,0 +1,173 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCommander_DispatchesByNameAndReplies(t *testing.T) {
+	var ran handlers_rds.Command
+	registry := commandRegistry{
+		"reload-parameters": func(_ context.Context, cmd handlers_rds.Command) (string, error) {
+			ran = cmd
+			return "reloaded", nil
+		},
+	}
+
+	reply := newCommander(nil, registry, time.Second).execute(context.Background(), handlers_rds.Command{
+		CommandID:  "cmd-1",
+		Type:       "reload-parameters",
+		Parameters: []handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}},
+	})
+
+	if ran.CommandID != "cmd-1" || len(ran.Parameters) != 1 {
+		t.Errorf("handler received %+v, want the command with its parameters", ran)
+	}
+	if reply.Status != handlers_rds.CommandStatusSucceeded || reply.Message != "reloaded" {
+		t.Errorf("reply = %+v, want succeeded/reloaded", reply)
+	}
+	if reply.CommandID != "cmd-1" {
+		t.Errorf("reply command ID = %q, want cmd-1", reply.CommandID)
+	}
+}
+
+func TestCommander_FailedHandlerRepliesWithItsError(t *testing.T) {
+	registry := commandRegistry{
+		"grow-storage": func(context.Context, handlers_rds.Command) (string, error) {
+			return "", errors.New("no space left on device")
+		},
+	}
+
+	reply := newCommander(nil, registry, time.Second).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-2", Type: "grow-storage"})
+
+	if reply.Status != handlers_rds.CommandStatusFailed {
+		t.Errorf("reply status = %q, want failed", reply.Status)
+	}
+	if reply.Message != "no space left on device" {
+		t.Errorf("reply message = %q, want the handler's error", reply.Message)
+	}
+}
+
+// A control plane ahead of the guest issues types this build has no handler
+// for. The issuer is blocked on the command ID, so it gets an answer rather
+// than a timeout.
+func TestCommander_UnknownTypeRepliesFailedNotDropped(t *testing.T) {
+	reply := newCommander(nil, newCommandRegistry(&fakeEngine{}, &fakeStorage{}), time.Second).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-3", Type: "quiesce"})
+
+	if reply.CommandID != "cmd-3" || reply.Status != handlers_rds.CommandStatusFailed {
+		t.Errorf("reply = %+v, want a failed reply for cmd-3", reply)
+	}
+	if reply.Message == "" {
+		t.Error("unsupported-command reply carried no message")
+	}
+}
+
+// The reply for work the guest actually did rides the next poll. A poll that
+// fails must not consume it, or the issuer never learns the outcome.
+func TestCommander_RunCarriesRepliesOnTheNextPoll(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollQueue = [][]handlers_rds.Command{
+		{{CommandID: "cmd-1", Type: "reload-parameters"}},
+	}
+	registry := commandRegistry{
+		"reload-parameters": func(context.Context, handlers_rds.Command) (string, error) { return "done", nil },
+	}
+
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	waitFor(t, func() bool {
+		replies := cp.snapshotReplies()
+		return len(replies) >= 2 && len(replies[1]) == 1
+	}, "the reply to ride the second poll")
+	cancel()
+	<-done
+
+	replies := cp.snapshotReplies()
+	if len(replies[0]) != 0 {
+		t.Errorf("first poll carried replies %+v, want none", replies[0])
+	}
+	if got := replies[1][0]; got.CommandID != "cmd-1" || got.Status != handlers_rds.CommandStatusSucceeded {
+		t.Errorf("second poll carried %+v, want the succeeded reply for cmd-1", got)
+	}
+}
+
+func TestCommander_RetainsAndRetriesTheCompletePendingBatch(t *testing.T) {
+	cp := newFakeControlPlane()
+	pending := []handlers_rds.CommandReply{
+		{CommandID: "cmd-1", Status: handlers_rds.CommandStatusSucceeded},
+		{CommandID: "cmd-2", Status: handlers_rds.CommandStatusFailed, Message: "disk full"},
+	}
+	c := newCommander(cp, commandRegistry{
+		"reload-parameters": func(context.Context, handlers_rds.Command) (string, error) {
+			return "reloaded", nil
+		},
+	}, 10*time.Millisecond)
+	c.pending = append([]handlers_rds.CommandReply(nil), pending...)
+	cp.pollErr = errors.New("reply publication failed")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+	<-cp.polled
+	cancel()
+	<-done
+
+	assert.Equal(t, pending, c.pending)
+	replies := cp.snapshotReplies()
+	require.Len(t, replies, 1)
+	assert.Equal(t, pending, replies[0])
+
+	cp.mu.Lock()
+	cp.pollErr = nil
+	cp.pollQueue = [][]handlers_rds.Command{{{
+		CommandID: "cmd-3",
+		Type:      "reload-parameters",
+	}}}
+	cp.mu.Unlock()
+
+	ctx, cancel = context.WithCancel(context.Background())
+	done = make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+	waitFor(t, func() bool {
+		return len(cp.snapshotReplies()) >= 3
+	}, "the retained batch to succeed and the new reply to reach the following poll")
+	cancel()
+	<-done
+
+	replies = cp.snapshotReplies()
+	assert.Equal(t, pending, replies[1], "the complete failed batch must be retried in order")
+	require.Len(t, replies[2], 1)
+	assert.Equal(t, "cmd-3", replies[2][0].CommandID)
+	assert.Equal(t, handlers_rds.CommandStatusSucceeded, replies[2][0].Status)
+}
+
+// A poll error must not spin the loop: without a backoff a gateway that is down
+// would be re-polled at line rate for as long as it stayed down.
+func TestCommander_BacksOffAfterAFailedPoll(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollErr = errors.New("gateway unreachable")
+
+	c := newCommander(cp, newCommandRegistry(&fakeEngine{}, &fakeStorage{}), 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+	<-done
+
+	// pollErrorBackoff is 5s, so the window admits the first poll and no more.
+	if polls := len(cp.snapshotReplies()); polls > 1 {
+		t.Errorf("polled %d times in 200ms against a broken channel, want 1", polls)
+	}
+}

cmd/rds-agent/config.go

@@ -0,0 +1,183 @@
+package main
+
+import (
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/guestenv"
+)
+
+const (
+	defaultEnvFile    = "/etc/spinifex-rds/agent.env"
+	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
+	defaultHandoffDir = "/run/spinifex-rds"
+	defaultEngineHost = "127.0.0.1"
+	// Where setup.sh stamps the engine the image bakes. The agent builds its
+	// engine implementation from this file rather than from anything delivered.
+	defaultEngineFile         = "/etc/spinifex-rds/engine"
+	defaultRCService          = "rc-service"
+	defaultEngineProbeTimeout = 5 * time.Second
+	// The long-poll window the agent asks the gateway to hold a request open
+	// for. The gateway caps it at 20s.
+	defaultPollWait = 20 * time.Second
+)
+
+// Static settings delivered per-instance by cloud-init. It carries no secrets:
+// IMDS is readable by anything in the guest, so the master password only
+// arrives via GetDBBootstrapConfig.
+type config struct {
+	GatewayURL string
+	GatewayCA  string
+	Region     string
+	// Optional — the gateway resolves the instance from the caller's
+	// credentials. When set it is sent, so a mis-provisioned VM is rejected.
+	DBInstanceIdentifier string
+	// What this image actually ships. Empty leaves the control plane's recorded
+	// version alone rather than clearing it.
+	EngineVersion string
+	HandoffDir    string
+	EngineHost    string
+	EnginePort    int
+	PollWait      time.Duration
+
+	// The engine the control plane launched this VM as, delivered by cloud-init,
+	// and the engine the image itself bakes. Two independent statements of the
+	// same fact: the agent refuses to bootstrap when they disagree.
+	Engine      string
+	EngineFile  string
+	BakedEngine string
+
+	// Where the command handlers reach the engine: the client binaries, the
+	// datadir the parameter file is installed into, the socket they connect
+	// over, the OS user they drop to, and the service the stop goes through.
+	EngineBinDir  string
+	EngineDataDir string
+	SocketDir     string
+	EngineUser    string
+	RCService     string
+	EngineService string
+	// Where the engine records the pid its probe checks for liveness, for an
+	// engine whose own client cannot tell a server still recovering from one that
+	// is not running at all.
+	EnginePidFile      string
+	EngineProbeTimeout time.Duration
+
+	// Where the data volume is mounted, and the two kernel surfaces a storage
+	// grow resolves its device from. Overridable so a test can point them at
+	// fixtures rather than at the running host's.
+	DataMount  string
+	MountsFile string
+	SysBlock   string
+}
+
+// Reads the cloud-init env file, then lets real env vars override.
+func loadConfig(envFile string) config {
+	get := guestenv.Load(envFile).Get
+
+	cfg := config{
+		GatewayURL:           get("RDS_GATEWAY_URL"),
+		GatewayCA:            get("RDS_GATEWAY_CA"),
+		Region:               get("RDS_REGION"),
+		DBInstanceIdentifier: get("RDS_DB_INSTANCE_IDENTIFIER"),
+		Engine:               get("RDS_ENGINE"),
+		EngineFile:           get("RDS_ENGINE_FILE"),
+		EngineVersion:        get("RDS_ENGINE_VERSION"),
+		HandoffDir:           get("RDS_HANDOFF_DIR"),
+		EngineHost:           get("RDS_ENGINE_HOST"),
+		EngineBinDir:         get("RDS_ENGINE_BIN"),
+		EngineDataDir:        get("RDS_ENGINE_DATA"),
+		SocketDir:            get("RDS_SOCKET_DIR"),
+		EngineUser:           get("RDS_ENGINE_USER"),
+		RCService:            get("RDS_RC_SERVICE"),
+		EngineService:        get("RDS_ENGINE_SERVICE"),
+		EnginePidFile:        get("RDS_ENGINE_PIDFILE"),
+		DataMount:            get("RDS_DATA_MOUNT"),
+		PollWait:             defaultPollWait,
+		EngineProbeTimeout:   defaultEngineProbeTimeout,
+	}
+	if cfg.GatewayCA == "" {
+		cfg.GatewayCA = defaultGatewayCA
+	}
+	if cfg.HandoffDir == "" {
+		cfg.HandoffDir = defaultHandoffDir
+	}
+	if cfg.EngineHost == "" {
+		cfg.EngineHost = defaultEngineHost
+	}
+	if cfg.EngineFile == "" {
+		cfg.EngineFile = defaultEngineFile
+	}
+	if cfg.RCService == "" {
+		cfg.RCService = defaultRCService
+	}
+	// The image's own stamp rather than the delivered engine: an agent that took
+	// its layout from the payload could run one engine's implementation against
+	// another engine's datadir, which is the failure the stamp exists to prevent.
+	cfg.BakedEngine = readBakedEngine(cfg.EngineFile)
+	cfg.applyLayout(engineLayouts[cfg.BakedEngine])
+
+	// rds-datadir reads the same RDS_DATA_MOUNT, so the two agree on where the
+	// volume landed without either asserting it to the other.
+	cfg.MountsFile = defaultMountsFile
+	cfg.SysBlock = defaultSysBlock
+	// The authoritative port comes from the bootstrap config; this is only what
+	// the probe uses until that first fetch lands.
+	if v := get("RDS_ENGINE_PORT"); v != "" {
+		if p, err := strconv.Atoi(v); err == nil && p > 0 {
+			cfg.EnginePort = p
+		}
+	}
+	if v := get("RDS_POLL_WAIT"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.PollWait = d
+		}
+	}
+	if v := get("RDS_ENGINE_PROBE_TIMEOUT"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.EngineProbeTimeout = d
+		}
+	}
+	return cfg
+}
+
+// Fills in whatever the delivered configuration left unset. An override always
+// wins, so a test can point any of these at a fixture. An unrecognised engine
+// leaves the layout empty, and New refuses rather than guessing at one.
+func (c *config) applyLayout(layout engineLayout) {
+	if c.EngineBinDir == "" {
+		c.EngineBinDir = layout.binDir
+	}
+	if c.EngineDataDir == "" {
+		c.EngineDataDir = layout.dataDir
+	}
+	if c.SocketDir == "" {
+		c.SocketDir = layout.socketDir
+	}
+	if c.EngineUser == "" {
+		c.EngineUser = layout.osUser
+	}
+	if c.EngineService == "" {
+		c.EngineService = layout.service
+	}
+	if c.EnginePidFile == "" {
+		c.EnginePidFile = layout.pidFile
+	}
+	if c.DataMount == "" {
+		c.DataMount = layout.dataMount
+	}
+	if c.EnginePort == 0 {
+		c.EnginePort = layout.port
+	}
+}
+
+// The engine the image bakes. An absent or unreadable stamp reads as empty,
+// which New refuses: an agent that guessed would defeat the stamp.
+func readBakedEngine(path string) string {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.ToLower(strings.TrimSpace(string(raw)))
+}

cmd/rds-agent/config_test.go

@@ -0,0 +1,162 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Stamps the engine a real image bakes and loads the configuration against it,
+// which is what selects the guest layout the agent's defaults come from.
+func testLoadConfig(t *testing.T, engine string) config {
+	t.Helper()
+	stamp := filepath.Join(t.TempDir(), "engine")
+	if err := os.WriteFile(stamp, []byte(engine+"\n"), 0o444); err != nil {
+		t.Fatalf("write the engine stamp: %v", err)
+	}
+	t.Setenv("RDS_ENGINE_FILE", stamp)
+	return loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+}
+
+func TestLoadConfig_Defaults(t *testing.T) {
+	cfg := testLoadConfig(t, enginePostgres)
+
+	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
+		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
+	}
+	if cfg.EngineHost != defaultEngineHost || cfg.PollWait != defaultPollWait {
+		t.Errorf("engine host/pollWait = %q/%v, want the built-in defaults", cfg.EngineHost, cfg.PollWait)
+	}
+	if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
+		t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)
+	}
+	// Identity is optional: the gateway resolves it from the caller's
+	// credentials, so an agent with no configured identifier is normal.
+	if cfg.DBInstanceIdentifier != "" {
+		t.Errorf("DBInstanceIdentifier = %q, want empty when unconfigured", cfg.DBInstanceIdentifier)
+	}
+}
+
+// The guest layout is the image's to state, not the control plane's: everything
+// the agent reaches the engine through comes from the stamp it was built with.
+func TestLoadConfig_TakesTheLayoutFromTheBakedEngine(t *testing.T) {
+	cfg := testLoadConfig(t, enginePostgres)
+
+	if cfg.BakedEngine != enginePostgres {
+		t.Fatalf("baked engine = %q, want %q", cfg.BakedEngine, enginePostgres)
+	}
+	layout := engineLayouts[enginePostgres]
+	if cfg.EngineBinDir != layout.binDir || cfg.EngineDataDir != layout.dataDir {
+		t.Errorf("bin/data = %q/%q, want the PostgreSQL layout", cfg.EngineBinDir, cfg.EngineDataDir)
+	}
+	if cfg.EngineUser != layout.osUser || cfg.EngineService != layout.service {
+		t.Errorf("user/service = %q/%q, want the PostgreSQL layout", cfg.EngineUser, cfg.EngineService)
+	}
+	if cfg.SocketDir != layout.socketDir || cfg.DataMount != layout.dataMount {
+		t.Errorf("socket/mount = %q/%q, want the PostgreSQL layout", cfg.SocketDir, cfg.DataMount)
+	}
+	if cfg.EnginePort != layout.port {
+		t.Errorf("port = %d, want the PostgreSQL default %d", cfg.EnginePort, layout.port)
+	}
+}
+
+// An image with no stamp cannot say which engine it is, and an agent that
+// guessed would be the failure the stamp exists to prevent.
+func TestNewEngine_RefusesAnImageWithNoEngineStamp(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+
+	if _, err := newProbe(cfg, staticProbe(0)); err == nil {
+		t.Error("newProbe built a probe for an image carrying no engine stamp")
+	}
+	if _, err := newEngine(cfg, nil, nil, nil); err == nil {
+		t.Error("newEngine built an engine for an image carrying no engine stamp")
+	}
+}
+
+func TestNewEngine_RefusesAnEngineItDoesNotImplement(t *testing.T) {
+	cfg := testLoadConfig(t, "oracle")
+
+	_, err := newEngine(cfg, nil, nil, nil)
+	if err == nil {
+		t.Fatal("newEngine built an engine this agent does not implement")
+	}
+	if !strings.Contains(err.Error(), "oracle") {
+		t.Errorf("error = %v, want it to name the engine the image bakes", err)
+	}
+}
+
+// The implementation is the agent's, but the definition it validates names and
+// classifies parameters against is the control plane's, which now offers MariaDB.
+func TestNewEngine_BuildsMariaDBForAMariaDBImage(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+	probe, err := newProbe(cfg, staticProbe(0))
+	if err != nil {
+		t.Fatalf("newProbe: %v", err)
+	}
+
+	built, err := newEngine(cfg, nil, nil, probe)
+	if err != nil {
+		t.Fatalf("newEngine: %v", err)
+	}
+	if _, ok := built.(*mariadbEngine); !ok {
+		t.Errorf("newEngine built %T for a MariaDB image", built)
+	}
+}
+
+// The probe reads the pidfile Alpine's packaged service starts mariadbd with,
+// because nothing in the generated configuration can move it: the service passes
+// --pid-file on the command line, which beats an option file.
+func TestLoadConfig_TakesTheMariaDBPidfileFromTheLayout(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+
+	if cfg.EnginePidFile != engineLayouts[engineMariaDB].pidFile {
+		t.Errorf("pidfile = %q, want the MariaDB layout's", cfg.EnginePidFile)
+	}
+	if cfg.EnginePidFile == "" {
+		t.Error("an engine whose probe needs a pidfile was given none")
+	}
+}
+
+func TestLoadConfig_ReadsEnvFile(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
+	path := filepath.Join(t.TempDir(), "agent.env")
+	body := "RDS_GATEWAY_URL=https://gw.internal:9999\n" +
+		"RDS_REGION=ap-southeast-2\n" +
+		"RDS_DB_INSTANCE_IDENTIFIER=db-1\n" +
+		"RDS_ENGINE=postgres\n" +
+		"RDS_ENGINE_VERSION=18.1\n" +
+		"RDS_ENGINE_PORT=6543\n" +
+		"RDS_POLL_WAIT=5s\n"
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatalf("write env file: %v", err)
+	}
+
+	cfg := loadConfig(path)
+	if cfg.GatewayURL != "https://gw.internal:9999" || cfg.Region != "ap-southeast-2" {
+		t.Errorf("gateway/region = %q/%q, want the delivered values", cfg.GatewayURL, cfg.Region)
+	}
+	if cfg.DBInstanceIdentifier != "db-1" || cfg.EngineVersion != "18.1" {
+		t.Errorf("identity = %q/%q, want db-1/18.1", cfg.DBInstanceIdentifier, cfg.EngineVersion)
+	}
+	if cfg.Engine != enginePostgres {
+		t.Errorf("delivered engine = %q, want the one cloud-init carried", cfg.Engine)
+	}
+	if cfg.EnginePort != 6543 || cfg.PollWait != 5*time.Second {
+		t.Errorf("port/pollWait = %d/%v, want 6543/5s", cfg.EnginePort, cfg.PollWait)
+	}
+}
+
+// An unparseable override falls back to the layout's own default rather than
+// leaving the agent with a zero port or a zero-length long poll.
+func TestLoadConfig_IgnoresUnusableOverrides(t *testing.T) {
+	t.Setenv("RDS_ENGINE_PORT", "not-a-port")
+	t.Setenv("RDS_POLL_WAIT", "forever")
+
+	cfg := testLoadConfig(t, enginePostgres)
+	if cfg.EnginePort != engineLayouts[enginePostgres].port || cfg.PollWait != defaultPollWait {
+		t.Errorf("port/pollWait = %d/%v, want the defaults", cfg.EnginePort, cfg.PollWait)
+	}
+}

cmd/rds-agent/defaultparams_test.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The include the image build validates against a real mariadbd. It is checked
+// in rather than generated during the build because the build runs inside the
+// libguestfs appliance, where there is no Go toolchain to resolve the catalog.
+const mariadbDefaultParametersFixture = "../../scripts/images/rds-mariadb/default-parameters.cnf"
+
+// Byte-for-byte what rds-init installs on a smallest-class instance that
+// overrides nothing. Regenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/.
+//
+// Keeping it in step matters because setup.sh feeds exactly this file to
+// mariadbd: a catalog that grew a name the server refuses at startup would
+// otherwise reach a customer as an instance that never boots.
+func TestMariaDBDefaultParametersFixture_MatchesTheCatalog(t *testing.T) {
+	engine, err := handlers_rds.LookupEngine("mariadb")
+	if err != nil {
+		t.Fatalf("LookupEngine: %v", err)
+	}
+	class := handlers_rds.SmallestInstanceClass()
+	params, err := engine.ResolveEffectiveParameters(class, nil)
+	if err != nil {
+		t.Fatalf("ResolveEffectiveParameters(%s): %v", class, err)
+	}
+	body, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	want := mariadbParametersHeader + body
+
+	path := filepath.FromSlash(mariadbDefaultParametersFixture)
+	if os.Getenv("RDS_UPDATE_FIXTURES") == "1" {
+		if err := os.WriteFile(path, []byte(want), 0o644); err != nil {
+			t.Fatalf("update %s: %v", path, err)
+		}
+		return
+	}
+
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	if string(got) != want {
+		t.Errorf("%s is stale for class %s.\nRegenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/\n\ngot:\n%s\nwant:\n%s",
+			path, class, got, want)
+	}
+}
+
+// The one thing the fixture exists to prove cannot regress without the image
+// build catching it, asserted here too so a plain preflight names it.
+func TestMariaDBDefaultParametersFixture_CarriesNoSetOnlyName(t *testing.T) {
+	got, err := os.ReadFile(filepath.FromSlash(mariadbDefaultParametersFixture))
+	if err != nil {
+		t.Fatalf("read the fixture: %v", err)
+	}
+	for _, name := range []string{"\ntime_zone ", "\ntx_isolation ", "\ninnodb_buffer_pool_instances "} {
+		if strings.Contains(string(got), name) {
+			t.Errorf("the default include carries %q, which mariadbd refuses at startup", name)
+		}
+	}
+}

cmd/rds-agent/engine.go

@@ -0,0 +1,244 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The in-guest half of the engine seam. Everything engine-shaped that a
+// control-plane command needs lives behind this, so the command registry stays
+// engine-agnostic.
+type engineOps interface {
+	// Rotates the master role's password live. Never persisted anywhere in the
+	// guest: it exists only for the length of this call.
+	SetPassword(ctx context.Context, username, password string) error
+	// Installs the resolved parameter set and reloads. Returns the settings the
+	// engine accepted but will not honour until it restarts.
+	ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error)
+	// Shuts the engine down cleanly, so the data volume is checkpointed before
+	// the VM is stopped or snapshotted.
+	Stop(ctx context.Context) error
+	// Puts the engine into backup mode so the datadir can be snapshotted at a
+	// checkpoint. Releases itself after hold, whatever happens to the caller.
+	Quiesce(ctx context.Context, label string, hold time.Duration) error
+	// Takes the engine back out of backup mode. Fails when no hold is active,
+	// which is how a hold that expired mid-snapshot becomes visible.
+	Unquiesce(ctx context.Context) error
+}
+
+// The rollback half of the parameter path, kept off engineOps because it is not
+// a control-plane directive: nothing issues it, the agent runs it when the
+// engine fails to come up after a parameter change.
+type parameterRecovery interface {
+	// Reports whether the installed set differed and was replaced.
+	RestoreLastKnownGoodParameters(ctx context.Context) (bool, error)
+	Restart(ctx context.Context) error
+}
+
+// Records the parameter file only after the probe observes the engine serving
+// it. This is separate from apply, where static values are installed but not
+// adopted until a later restart.
+type servingParameterRecorder interface {
+	RecordServingParameters(ctx context.Context) error
+}
+
+// Everything one engine implementation owns. The narrow interfaces above are
+// what the heartbeat, the commander and the rollback guard each hold; this is
+// only what the agent has to construct.
+type engine interface {
+	engineOps
+	parameterRecovery
+	servingParameterRecorder
+	// Follows the port the control plane assigned, once the bootstrap fetch
+	// lands. Called from a different goroutine than the commands that read it.
+	setPort(port int)
+}
+
+// The guest layout an engine preset lays down, keyed by the engine the image
+// bakes. Every default that belongs to one engine lives here, so nothing
+// outside an implementation names its paths, service, port or probe.
+type engineLayout struct {
+	binDir    string
+	dataDir   string
+	socketDir string
+	osUser    string
+	service   string
+	dataMount string
+	port      int
+	// Where the engine records the pid the probe checks for liveness. Empty for
+	// an engine whose probe answers without one.
+	pidFile string
+	// The engine's own liveness signal, which is the only part of engineProbe
+	// that is not shared.
+	newProbe func(cfg config, run probeRunner) *engineProbe
+	// The implementation every control-plane directive is served by. It fails
+	// when the control plane this agent was built against does not offer the
+	// engine the image bakes, rather than serving one it has no definition for.
+	newEngine func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error)
+}
+
+// The names the control plane knows these implementations by, which are also
+// what setup.sh stamps into each image.
+const (
+	enginePostgres = "postgres"
+	engineMariaDB  = "mariadb"
+)
+
+var engineLayouts = map[string]engineLayout{
+	enginePostgres: {
+		binDir:    "/usr/libexec/postgresql18",
+		dataDir:   "/var/lib/postgresql/18/data",
+		socketDir: "/run/postgresql",
+		osUser:    "postgres",
+		service:   "postgresql",
+		dataMount: "/var/lib/postgresql",
+		port:      5432,
+		newProbe:  newPostgresProbe,
+		newEngine: func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+			return newPostgresEngine(cfg, run, startSess, probe), nil
+		},
+	},
+	engineMariaDB: {
+		binDir:    "/usr/bin",
+		dataDir:   "/var/lib/mysql/data",
+		socketDir: "/run/mysqld",
+		osUser:    "mysql",
+		service:   "mariadb",
+		dataMount: "/var/lib/mysql",
+		port:      3306,
+		// Alpine's packaged service starts mysqld_safe with
+		// --pid-file=/run/mysqld/$RC_SVCNAME.pid, and a command-line option beats
+		// the configuration file, so this is the path rather than anything the
+		// platform drop-in could name. setup.sh asserts it against the packaged
+		// init script at build time.
+		pidFile:   "/run/mysqld/mariadb.pid",
+		newProbe:  newMariaDBProbe,
+		newEngine: newMariaDBEngineFromCatalog,
+	},
+}
+
+// Builds the health probe the image's engine supplies.
+func newProbe(cfg config, run probeRunner) (*engineProbe, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newProbe(cfg, run), nil
+}
+
+// Builds the implementation the image bakes. Nothing here consults the control
+// plane for which engine to be: the agent runs the engine it is made of, and the
+// delivered engine is only ever checked against it.
+func newEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newEngine(cfg, run, startSess, probe)
+}
+
+func layoutFor(cfg config) (engineLayout, error) {
+	if cfg.BakedEngine == "" {
+		return engineLayout{}, fmt.Errorf("this image carries no engine stamp at %s", cfg.EngineFile)
+	}
+	layout, ok := engineLayouts[cfg.BakedEngine]
+	if !ok {
+		return engineLayout{}, fmt.Errorf("this image bakes engine %q, which this agent does not implement", cfg.BakedEngine)
+	}
+	return layout, nil
+}
+
+// One child process. Env replaces the agent's own environment rather than
+// extending it, so a secret placed here reaches only the process that needs it.
+type command struct {
+	Name  string
+	Args  []string
+	Env   []string
+	Stdin string
+	// The OS user to drop to before exec. Empty runs as the agent itself.
+	User string
+	// The statement that makes this client print the session sentinel. Read only
+	// when the command is started as a held session, whose reader has to know
+	// when the engine has finished each script it was fed.
+	SentinelStatement string
+}
+
+// Returns stdout. stderr is folded into the error, because both clients report
+// the actual SQL failure there while exiting with a bare status.
+type commandRunner func(ctx context.Context, c command) (string, error)
+
+func execCommandRunner(ctx context.Context, c command) (string, error) {
+	cmd := exec.CommandContext(ctx, c.Name, c.Args...)
+	cmd.Env = c.Env
+	if c.Stdin != "" {
+		cmd.Stdin = strings.NewReader(c.Stdin)
+	}
+	if c.User != "" {
+		credential, err := lookupCredential(c.User)
+		if err != nil {
+			return "", err
+		}
+		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: credential}
+	}
+
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout, cmd.Stderr = &stdout, &stderr
+	if err := cmd.Run(); err != nil {
+		message := strings.TrimSpace(stderr.String())
+		if message == "" {
+			message = err.Error()
+		}
+		return stdout.String(), fmt.Errorf("%s: %s", filepath.Base(c.Name), message)
+	}
+	return stdout.String(), nil
+}
+
+func lookupCredential(username string) (*syscall.Credential, error) {
+	u, err := user.Lookup(username)
+	if err != nil {
+		return nil, fmt.Errorf("resolve user %s: %w", username, err)
+	}
+	uid, err := strconv.ParseUint(u.Uid, 10, 32)
+	if err != nil {
+		return nil, fmt.Errorf("parse uid of %s: %w", username, err)
+	}
+	gid, err := strconv.ParseUint(u.Gid, 10, 32)
+	if err != nil {
+		return nil, fmt.Errorf("parse gid of %s: %w", username, err)
+	}
+	return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil
+}
+
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the state and does not restart the engine underneath a VM
+// that is going down.
+func serviceAction(ctx context.Context, run commandRunner, rcService, service, action string) error {
+	if _, err := run(ctx, command{
+		Name: rcService,
+		Args: []string{service, action},
+		Env:  []string{"PATH=" + defaultGuestPath},
+	}); err != nil {
+		return fmt.Errorf("%s the %s service: %w", action, service, err)
+	}
+	return nil
+}
+
+// A minimal environment for a child that is not carrying secrets.
+const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+func redact(s, secret string) string {
+	if secret == "" {
+		return s
+	}
+	return strings.ReplaceAll(s, secret, "[REDACTED]")
+}

cmd/rds-agent/engine_contract_test.go

@@ -0,0 +1,329 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// A probe the test drives directly, in place of whichever binaries the engine's
+// own state function would have run. Each implementation's state function is
+// covered on its own; what these cases exercise is what the engine does with the
+// answer.
+func drivenProbe(state *engineState) *engineProbe {
+	return newEngineProbe(0, func(context.Context, int64) (engineState, string) {
+		return *state, "driven by the test"
+	})
+}
+
+// One engine behind the interfaces the heartbeat, the commander and the rollback
+// guard hold, plus the few facts a shared case needs to say what it means.
+type engineHarness struct {
+	engine  engine
+	runner  *recordingRunner
+	params  parameterStore
+	service string
+	// The state the probe reports, moved by the test rather than by the engine.
+	state *engineState
+	// Points the next quiesce at a session the test can inspect, and returns the
+	// commands those sessions were opened with.
+	useSession func(*fakeSession) *[]command
+	// A setting this engine's catalog really carries, so a shared case installs
+	// something the implementation would genuinely be handed.
+	parameter handlers_rds.Parameter
+	// What the engine's own include directive globs. Neither the rollback copy
+	// nor the serving copy may end in it.
+	includeSuffix string
+}
+
+func engineHarnesses() []struct {
+	name  string
+	build func(*testing.T) *engineHarness
+} {
+	return []struct {
+		name  string
+		build func(*testing.T) *engineHarness
+	}{
+		{name: enginePostgres, build: buildPostgresHarness},
+		{name: engineMariaDB, build: buildMariaDBHarness},
+	}
+}
+
+func forEachEngine(t *testing.T, run func(*testing.T, *engineHarness)) {
+	t.Helper()
+	for _, fixture := range engineHarnesses() {
+		t.Run(fixture.name, func(t *testing.T) { run(t, fixture.build(t)) })
+	}
+}
+
+func buildPostgresHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "work_mem", Value: "4096"},
+		includeSuffix: ".conf",
+	}
+}
+
+func buildMariaDBHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestMariaDBEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "max_connections", Value: "200"},
+		includeSuffix: ".cnf",
+	}
+}
+
+// The parameters land where rds-init installs them, so the next boot overwrites
+// the file rather than shadowing it with a second copy.
+func TestEngineContract_ApplyParametersInstallsWhereRdsInitDoes(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), h.parameter.Name+" = '"+h.parameter.Value+"'") {
+			t.Errorf("installed parameters = %q, want the resolved value", installed)
+		}
+		if !strings.HasSuffix(h.params.installedPath(), h.includeSuffix) {
+			t.Errorf("the installed set at %s is not a file the engine includes", h.params.installedPath())
+		}
+		// Comparison material, not a second set of settings: an include glob that
+		// read either of these would apply a withdrawn configuration.
+		copies := []string{h.params.lastGoodPath()}
+		if h.params.serving != "" {
+			copies = append(copies, h.params.servingPath())
+		}
+		for _, path := range copies {
+			if strings.HasSuffix(path, h.includeSuffix) {
+				t.Errorf("the copy at %s would be included by the engine", path)
+			}
+		}
+	})
+}
+
+// An engine still starting or replaying has not finished adopting the set it is
+// on, so replacing that set underneath it would be installing over a moving
+// target.
+func TestEngineContract_ApplyParametersRefusesWhileTheEngineIsRecovering(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		*h.state = engineRecovering
+
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err == nil {
+			t.Fatal("ApplyParameters succeeded against an engine that was still coming up")
+		}
+		if _, err := os.Stat(h.params.installedPath()); !os.IsNotExist(err) {
+			t.Errorf("the refused set was installed anyway (stat err = %v)", err)
+		}
+	})
+}
+
+// The boot-time half of the parameter safety net: the last set the engine
+// accepted goes back only when the engine is genuinely down, and only once.
+func TestEngineContract_RestoreLastKnownGoodOnlyRollsBackADownEngine(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		accepted := h.parameter
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{accepted}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+		if err := h.engine.RecordServingParameters(t.Context()); err != nil {
+			t.Fatalf("RecordServingParameters: %v", err)
+		}
+		if _, err := h.engine.ApplyParameters(t.Context(),
+			[]handlers_rds.Parameter{{Name: accepted.Name, Value: accepted.Value + "0"}}); err != nil {
+			t.Fatalf("second ApplyParameters: %v", err)
+		}
+
+		// A guard whose deadline expired while an apply brought the engine back must
+		// not reverse that repair.
+		restored, err := h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters against a serving engine: %v", err)
+		}
+		if restored {
+			t.Fatal("rolled the parameters back while the engine was serving")
+		}
+
+		*h.state = engineAbsent
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if !restored {
+			t.Fatal("did not roll back to a set that differs from the last accepted one")
+		}
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), accepted.Name+" = '"+accepted.Value+"'") {
+			t.Errorf("installed = %q, want the last accepted set", installed)
+		}
+
+		// Repeating a rollback would only churn a cluster that is failing for
+		// another reason.
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if restored {
+			t.Error("rolled back again when the installed set already matched")
+		}
+	})
+}
+
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the engine as stopped and does not restart it underneath a
+// VM that is going down.
+func TestEngineContract_StopGoesThroughTheServiceManager(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if err := h.engine.Stop(t.Context()); err != nil {
+			t.Fatalf("Stop: %v", err)
+		}
+		if len(h.runner.calls) != 1 {
+			t.Fatalf("ran %d commands, want the service stop", len(h.runner.calls))
+		}
+		call := h.runner.calls[0]
+		if !slices.Equal(call.Args, []string{h.service, "stop"}) {
+			t.Errorf("stop ran %s %v, want the service manager", call.Name, call.Args)
+		}
+	})
+}
+
+// One backup at a time, released by Unquiesce or by its own deadline, whichever
+// comes first. The deadline is what keeps a control plane that dies mid-snapshot
+// from leaving the engine held.
+func TestEngineContract_QuiesceHoldsUntilReleased(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		started := h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		if len(*started) != 1 {
+			t.Fatalf("started %d sessions, want 1", len(*started))
+		}
+		if session.closes() != 0 {
+			t.Fatal("the session was closed while the backup was meant to be held")
+		}
+
+		err := h.engine.Quiesce(t.Context(), "second", time.Minute)
+		if err == nil {
+			t.Fatal("a second quiesce was accepted while the first was held")
+		}
+		if !strings.Contains(err.Error(), "orders-db-pre-upgrade") {
+			t.Errorf("error %q does not name the backup already holding the engine", err)
+		}
+		if len(*started) != 1 {
+			t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
+		}
+
+		if err := h.engine.Unquiesce(t.Context()); err != nil {
+			t.Fatalf("Unquiesce: %v", err)
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the session ended exactly once", session.closes())
+		}
+	})
+}
+
+// A hold that expired mid-snapshot means the snapshot was not taken against a
+// held checkpoint, so the release reports it rather than succeeding silently.
+func TestEngineContract_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		deadline := time.Now().Add(2 * time.Second)
+		for session.closes() == 0 && time.Now().Before(deadline) {
+			time.Sleep(time.Millisecond)
+		}
+		if session.closes() != 1 {
+			t.Fatal("the backup session outlived its deadline")
+		}
+
+		err := h.engine.Unquiesce(t.Context())
+		if err == nil {
+			t.Fatal("Unquiesce succeeded after the hold expired")
+		}
+		if !strings.Contains(err.Error(), "expired") {
+			t.Errorf("error %q does not say the hold had expired", err)
+		}
+	})
+}
+
+// A failed start leaves nothing held, so a retry is not refused by a hold that
+// never took effect.
+func TestEngineContract_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{execErr: errors.New("the engine refused the backup")}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err == nil {
+			t.Fatal("Quiesce succeeded against an engine that refused the backup")
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the failed session ended", session.closes())
+		}
+		if err := h.engine.Unquiesce(t.Context()); err == nil {
+			t.Error("a failed quiesce left a hold behind")
+		}
+	})
+}
+
+// Every command the control plane can issue is served by both implementations,
+// so a directive cannot be answered as unsupported on one engine and honoured on
+// the other.
+func TestEngineContract_ServesEveryRegisteredCommand(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		registry := newCommandRegistry(h.engine, &fakeStorage{})
+		for _, name := range []string{
+			handlers_rds.CommandSetPassword, handlers_rds.CommandApplyParams, handlers_rds.CommandStopEngine,
+			handlers_rds.CommandGrowFilesystem, handlers_rds.CommandQuiesce, handlers_rds.CommandUnquiesce,
+		} {
+			if _, ok := registry[name]; !ok {
+				t.Errorf("command %q has no handler", name)
+			}
+		}
+	})
+}

cmd/rds-agent/engine_mariadb.go

@@ -0,0 +1,638 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The control plane's own statements about this engine, resolved once at
+// startup. Held as functions rather than as the Engine value so nothing here
+// re-derives a rule the API already owns.
+type controlPlaneRules struct {
+	// The whole master-username rule rather than only the reserved set. MariaDB's
+	// client has no identifier quoting, so the name is interpolated into SQL by
+	// this process and every part of the rule is load-bearing here.
+	validateUsername func(username string) error
+	// Whether a setting takes effect only at a restart. Already customer-facing
+	// and authoritative, since the API refuses ApplyMethod=immediate on one.
+	isStatic func(name string) bool
+	// The catalog name behind a spelling read back out of an option file. Only a
+	// setting whose startup spelling differs from the customer's moves.
+	catalogName func(optionFileName string) string
+}
+
+func controlPlaneRulesFrom(meta handlers_rds.Engine) controlPlaneRules {
+	// Built once per engine: everything read back out of a generated file has to
+	// return to the catalog's namespace before it is classified or reported, or a
+	// startup spelling would read as an unknown name.
+	catalogNames := map[string]string{}
+	for _, name := range meta.CatalogParameterNames() {
+		if optionFileName := meta.OptionFileName(name); optionFileName != name {
+			catalogNames[optionFileName] = name
+		}
+	}
+
+	return controlPlaneRules{
+		validateUsername: meta.ValidateMasterUsername,
+		isStatic: func(name string) bool {
+			spec, ok := meta.LookupParameter(name)
+			// A name the catalog does not carry cannot be shown to have been adopted
+			// without a restart, and is never issued as a live SET GLOBAL either.
+			return !ok || spec.ApplyType == handlers_rds.ApplyTypeStatic
+		},
+		catalogName: func(optionFileName string) string {
+			if name, ok := catalogNames[optionFileName]; ok {
+				return name
+			}
+			return optionFileName
+		},
+	}
+}
+
+// The engine is reached over its unix socket as root, which the datadir's
+// unix_socket plugin authenticates from the connecting process' own uid — the
+// direct analogue of PostgreSQL's peer auth, and the reason the agent holds no
+// password of its own.
+type mariadbEngine struct {
+	quiesceState
+
+	rules     controlPlaneRules
+	run       commandRunner
+	startSess sessionRunner
+	client    string
+	admin     string
+	socket    string
+	rcService string
+	service   string
+	probe     *engineProbe
+
+	params parameterStore
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
+}
+
+var _ engine = (*mariadbEngine)(nil)
+
+const (
+	mariadbClientBinary = "mariadb"
+	mariadbAdminBinary  = "mariadb-admin"
+	// The account mariadb-install-db creates for the unix_socket plugin, which
+	// rds-init keeps for the platform and never hands to the customer.
+	mariadbSuperuser                  = "root"
+	mariadbProbeConnectTimeoutSeconds = 3
+	// Named by the platform drop-in rds-init writes, so both halves reach the
+	// same socket without either asserting it to the other.
+	mariadbSocketFile = "mysqld.sock"
+	// The customer reaches the instance over its customer ENI, so the master is
+	// created and rotated on the wildcard host rather than on localhost.
+	mariadbMasterHost = "%"
+)
+
+// The generated drop-ins on the data volume. The rollback and serving copies
+// deliberately do not end in .cnf: !includedir reads *.cnf, and neither is a
+// second set of settings.
+const (
+	mariadbParametersFile = "10-rds-parameters.cnf"
+	mariadbLastGoodFile   = "10-rds-parameters.last-good"
+	mariadbServingFile    = "10-rds-parameters.serving"
+	// MariaDB treats a setting before any group as a fatal parsing error, so
+	// every generated file carries the header rds-init writes.
+	mariadbParametersHeader = "[mysqld]\n"
+)
+
+// The layout's factory. The rules come from the control plane's own definition
+// of this engine, so a build whose control plane does not offer MariaDB refuses
+// to run this implementation rather than inventing a definition for it.
+func newMariaDBEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	meta, err := handlers_rds.LookupEngine(engineMariaDB)
+	if err != nil {
+		return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", engineMariaDB, err)
+	}
+	return newMariaDBEngine(cfg, controlPlaneRulesFrom(meta), run, startSess, probe), nil
+}
+
+func newMariaDBEngine(cfg config, rules controlPlaneRules, run commandRunner, startSess sessionRunner, probe *engineProbe) *mariadbEngine {
+	return &mariadbEngine{
+		rules:     rules,
+		run:       run,
+		startSess: startSess,
+		client:    filepath.Join(cfg.EngineBinDir, mariadbClientBinary),
+		admin:     filepath.Join(cfg.EngineBinDir, mariadbAdminBinary),
+		socket:    mariadbSocketPath(cfg),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		probe:     probe,
+		params: parameterStore{
+			// The mount point rather than the datadir one level inside it: the
+			// include directory has to outlive the sweep a failed bootstrap runs.
+			dir:       filepath.Join(cfg.DataMount, "conf.d"),
+			installed: mariadbParametersFile,
+			lastGood:  mariadbLastGoodFile,
+			serving:   mariadbServingFile,
+			header:    mariadbParametersHeader,
+			osUser:    cfg.EngineUser,
+			engine:    engineMariaDB,
+		},
+		repairTimeout: parameterRepairTimeout,
+		repairPoll:    parameterRepairPoll,
+	}
+}
+
+func mariadbSocketPath(cfg config) string {
+	return filepath.Join(cfg.SocketDir, mariadbSocketFile)
+}
+
+// Nothing this implementation runs needs the assigned port: every connection it
+// makes is over the engine's unix socket, and the platform drop-in rds-init
+// writes is what puts the server on the port. The probe follows it separately.
+func (e *mariadbEngine) setPort(int) {}
+
+// Reports whether a pid is a live process. EPERM is a process this agent may not
+// signal rather than one that is gone, which matters because a stale pidfile
+// read as "alive" is far less dangerous than a live engine read as "absent".
+type processLivenessFn func(pid int) bool
+
+func processAlive(pid int) bool {
+	if pid <= 0 {
+		return false
+	}
+	err := syscall.Kill(pid, 0)
+	return err == nil || errors.Is(err, syscall.EPERM)
+}
+
+func newMariaDBProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg, run, processAlive))
+}
+
+// Three stages, because MariaDB has no single signal that separates an engine
+// still coming up from one that is not there at all: during InnoDB crash
+// recovery mariadbd opens neither its socket nor its port, so a ping fails
+// exactly as it would against nothing.
+//
+// Losing that distinction would break the rollback guard, which treats a
+// recovering engine as a reason to reset its deadline. An instance killed hard
+// mid-write can spend minutes replaying its redo log; read as absent, the guard
+// would roll the parameter file back and restart the server mid-recovery,
+// discarding the work — a rollback that exists to break a boot loop would create
+// one, on an instance whose parameters were never at fault.
+//
+// The port is not consulted. Both stages below reach the server over its unix
+// socket, whose path does not move with the port.
+func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) probeStateFn {
+	pidFile, socket := cfg.EnginePidFile, mariadbSocketPath(cfg)
+	admin, client := filepath.Join(cfg.EngineBinDir, mariadbAdminBinary), filepath.Join(cfg.EngineBinDir, mariadbClientBinary)
+	connect := []string{
+		"--no-defaults", "--protocol=socket", "--socket=" + socket,
+		"--user=" + mariadbSuperuser,
+		fmt.Sprintf("--connect-timeout=%d", mariadbProbeConnectTimeoutSeconds),
+	}
+	probeTimeout := cfg.EngineProbeTimeout
+	if probeTimeout <= 0 {
+		probeTimeout = defaultEngineProbeTimeout
+	}
+	runBounded := func(ctx context.Context, name string, args ...string) (int, error) {
+		probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)
+		defer cancel()
+		return run(probeCtx, name, args...)
+	}
+
+	return func(ctx context.Context, _ int64) (engineState, string) {
+		pid, err := readPidFile(pidFile)
+		switch {
+		case os.IsNotExist(err):
+			return engineAbsent, fmt.Sprintf("the engine has written no pidfile at %s", pidFile)
+		case err != nil:
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s could not be read: %v", pidFile, err)
+		case !alive(pid):
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s names pid %d, which is not running", pidFile, pid)
+		}
+
+		// The process is up from here on, so every remaining failure is an engine
+		// that is not serving yet rather than one that is gone — including a probe
+		// binary that will not run at all. Reporting absent against a live server
+		// would have the rollback guard restart one that is making progress.
+		switch code, err := runBounded(ctx, admin, append(slices.Clone(connect), "ping")...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			return engineRecovering, "engine is not answering on its socket yet (startup or crash recovery)"
+		}
+
+		// ping answers successfully even on ER_ACCESS_DENIED, so on its own it
+		// certifies a server that may be unable to execute anything at all. The
+		// statement is a literal in the argv rather than on stdin because it
+		// carries nothing secret and a probe reads no result back.
+		query := append(slices.Clone(connect), "--batch", "--skip-column-names", "--execute=SELECT 1")
+		switch code, err := runBounded(ctx, client, query...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			return engineRecovering, "engine answered its socket but would not execute a statement"
+		}
+		return engineServing, ""
+	}
+}
+
+func readPidFile(path string) (int, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return 0, err
+	}
+	pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
+	if err != nil {
+		return 0, fmt.Errorf("%s does not hold a pid: %w", path, err)
+	}
+	return pid, nil
+}
+
+func (e *mariadbEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback and a failed apply call this: a restart the
+// control plane wants goes through RebootDBInstance, which cycles the VM.
+//
+// The serving copy is refreshed first, because the server is about to start on
+// whatever is installed. That is exactly what rds-init does on every boot, and
+// it is what keeps the two equal by construction whenever the engine starts.
+func (e *mariadbEngine) Restart(ctx context.Context) error {
+	if err := e.recordServingCopy(); err != nil {
+		return err
+	}
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// The installed set copied to the name the pending-restart comparison reads. A
+// live SET GLOBAL apply deliberately does not touch it: that value was adopted
+// without a restart, which is what "not pending" means.
+func (e *mariadbEngine) recordServingCopy() error {
+	content, err := os.ReadFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("read the installed parameters: %w", err)
+	}
+	return e.params.write(e.params.servingPath(), content)
+}
+
+// A value safe to interpolate into a single-quoted SQL literal. The quote is
+// doubled, which holds in every sql_mode, and the backslash is doubled, which is
+// why every statement built here pins a mode with backslash escapes on.
+func sqlLiteral(value string) string {
+	return strings.ReplaceAll(strings.ReplaceAll(value, `\`, `\\`), `'`, `''`)
+}
+
+// Turns off the two logs that would otherwise copy a statement's text, and pins
+// the mode the escaping above assumes. All three are session-scoped: a rotation
+// must not leave the customer's own general log switched off behind it.
+const mariadbSessionGuard = `SET SESSION sql_log_off = 1;
+SET SESSION slow_query_log = 0;
+SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';
+`
+
+// Rotates the master's password. The statement is built here rather than by the
+// client, which has neither psql's identifier interpolation nor its parameter
+// quoting, and it rides stdin so it is never visible in the process table.
+func (e *mariadbEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// This runs as the server's own superuser over the socket, so a reserved name
+	// in the command payload would rotate the platform's account rather than the
+	// customer's. The rest of the rule matters just as much here, because the name
+	// is interpolated into the statement below rather than quoted by the client.
+	if err := e.rules.validateUsername(username); err != nil {
+		return fmt.Errorf("refusing to set the password of role %q: %w", username, err)
+	}
+
+	escaped := sqlLiteral(password)
+	sql := fmt.Sprintf("%sALTER USER '%s'@'%s' IDENTIFIED BY '%s';\n",
+		mariadbSessionGuard, username, mariadbMasterHost, escaped)
+	if _, err := e.clientRun(ctx, sql); err != nil {
+		// The client echoes the failing statement, which carries the escaped form
+		// of the password rather than the raw one — so redacting only the raw one
+		// would leak exactly the passwords that needed escaping.
+		return fmt.Errorf("apply the master password: %s", redact(redact(err.Error(), password), escaped))
+	}
+	return nil
+}
+
+// Installs the resolved set and applies the half of it a running server can
+// adopt. MariaDB re-reads no configuration file while it is up, so SET GLOBAL is
+// the only way an immediate apply reaches it; the static half waits in the
+// installed file for the next start, and is what comes back as pending.
+//
+// There is no offline parser to check the set with first — MariaDB ships no
+// equivalent of postgres -C — so the safety net for a static value the server
+// will not start on is the boot-time rollback to the last accepted set.
+func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	restore, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
+	}
+
+	if err := e.applyDynamicParameters(ctx, params); err != nil {
+		// The engine may have gone down after the first probe. In that case start it
+		// on the newly installed set rather than restoring one that already left it
+		// unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+// The dynamic half of the set, in one invocation that stops at the first refusal
+// so the server's own message names the setting it would not take.
+//
+// Only names the catalog carries are emitted, which is also what keeps the
+// statement safe to build: a name is an identifier here, and the classification
+// that selects it comes from the same table the API validated the request
+// against. MariaDB cannot set several globals atomically, so a value set before
+// a refusal stays live until the next restart, when the restored file wins.
+func (e *mariadbEngine) applyDynamicParameters(ctx context.Context, params []handlers_rds.Parameter) error {
+	var b strings.Builder
+	b.WriteString("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\n")
+	applied := 0
+	for _, p := range params {
+		if p.Name == "" || e.rules.isStatic(p.Name) {
+			continue
+		}
+		fmt.Fprintf(&b, "SET GLOBAL %s = '%s';\n", p.Name, sqlLiteral(p.Value))
+		applied++
+	}
+	if applied == 0 {
+		return nil
+	}
+	if _, err := e.clientRun(ctx, b.String()); err != nil {
+		return fmt.Errorf("apply the dynamic parameters: %w", err)
+	}
+	return nil
+}
+
+func (e *mariadbEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// MariaDB has no pending_restart. Nothing in information_schema reports a
+// setting the server has stored but not adopted, so the answer is computed from
+// the files instead: the catalog-static keys whose value in the installed
+// drop-in differs from the one the server actually started on.
+//
+// Comparing live values against the desired set was rejected. mysqld silently
+// rewrites values at startup — rounding the buffer pool and log file sizes, and
+// downgrading max_connections when it cannot obtain enough file descriptors —
+// and each rewrite would read as a permanent pending restart: un-clearable
+// pending-reboot state, permanent drift, and a last-known-good copy frozen for
+// good, silently disabling a recovery mechanism only exercised during an
+// incident.
+func (e *mariadbEngine) pendingRestartParameters(context.Context) ([]string, error) {
+	installed, err := readOptionFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Nothing is installed, so nothing is waiting to be adopted.
+			return nil, nil
+		}
+		return nil, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	serving, err := readOptionFile(e.params.servingPath())
+	if err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("read the parameters the engine started on: %w", err)
+	}
+	// Back to the catalog's names before anything is classified or reported: the
+	// files are written in the engine's startup spellings, and the answer is a
+	// customer-facing list the API refuses ApplyMethod=immediate against.
+	installed = e.catalogKeyed(installed)
+	serving = e.catalogKeyed(serving)
+
+	// An absent serving copy is a set the engine has not started on: rds-init
+	// writes the two together on every boot, so the whole static half counts as
+	// pending rather than being promoted on the strength of a missing file.
+	var pending []string
+	for name, value := range installed {
+		if !e.rules.isStatic(name) {
+			continue
+		}
+		if served, ok := serving[name]; !ok || served != value {
+			pending = append(pending, name)
+		}
+	}
+	// A static setting the group stopped naming reverts to its default at the next
+	// start, which the running server has not adopted either.
+	for name := range serving {
+		if _, ok := installed[name]; !ok && e.rules.isStatic(name) {
+			pending = append(pending, name)
+		}
+	}
+	slices.Sort(pending)
+	return pending, nil
+}
+
+// Snapshots the installed set as the rollback target, but only when nothing in
+// it is still waiting for a restart. The parameter mutex keeps an apply from
+// replacing the file between that check and the copy.
+func (e *mariadbEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *mariadbEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a start that failed
+// after a parameter change.
+func (e *mariadbEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return restoreLastGood(ctx, e.params, e.probe)
+}
+
+// The same settings under the names the catalog and the API know them by, since
+// a file is written in the engine's startup spellings and those are not always
+// the customer's.
+func (e *mariadbEngine) catalogKeyed(values map[string]string) map[string]string {
+	out := make(map[string]string, len(values))
+	for name, value := range values {
+		out[e.rules.catalogName(name)] = value
+	}
+	return out
+}
+
+// The subset of MariaDB's option-file syntax the generated drop-ins are written
+// in: a group header, comments, and one `name = value` per line. Nothing else
+// reaches these two files, because only rds-init and this agent write them.
+func readOptionFile(path string) (map[string]string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	values := make(map[string]string)
+	for line := range strings.SplitSeq(string(raw), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "[") {
+			continue
+		}
+		// A bare option such as skip-log-bin carries no value; it is still a setting
+		// whose presence or absence differs.
+		name, value, _ := strings.Cut(line, "=")
+		values[normaliseOptionName(name)] = strings.Trim(strings.TrimSpace(value), `"'`)
+	}
+	return values, nil
+}
+
+// MariaDB reads - and _ as the same character in an option name, so the two
+// spellings of one setting must not compare as two settings.
+func normaliseOptionName(name string) string {
+	return strings.ReplaceAll(strings.TrimSpace(name), "-", "_")
+}
+
+// How long any one BACKUP STAGE waits for the metadata locks it needs. Well
+// under the control plane's own quiesce timeout, so a stage that cannot take its
+// lock fails and is reported rather than being abandoned mid-wait with the
+// request still queued in front of live traffic.
+const mariadbQuiesceLockWait = 20 * time.Second
+
+// MariaDB's own backup API rather than FLUSH TABLES WITH READ LOCK. FTWRL turns
+// a snapshot into a write outage: the whole database is read-only until it is
+// released, so a control plane that died after issuing quiesce would leave the
+// customer read-only for the full hold, on a schedule and unattended. Its
+// acquisition phase is unbounded too, with writes queueing behind a pending lock
+// request. BACKUP STAGE blocks commits rather than all writes and its flush
+// stage covers Aria and MyISAM, so it yields a better consistency point while
+// being far less disruptive — and it is what mariadb-backup itself uses.
+func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.client,
+		Args:              e.clientArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath},
+		SentinelStatement: "SELECT '" + sessionSentinel + "';\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// The stages run in order and on one connection: MariaDB releases the whole
+	// hold with the session that took it, which is what bounds a control plane
+	// that dies mid-snapshot. Fed one at a time so a stage that cannot take its
+	// locks is named rather than reported as a line number.
+	stages := []string{
+		fmt.Sprintf("SET SESSION lock_wait_timeout = %d;\n", int(mariadbQuiesceLockWait.Seconds())),
+		"BACKUP STAGE START;\n",
+		"BACKUP STAGE FLUSH;\n",
+		"BACKUP STAGE BLOCK_DDL;\n",
+		"BACKUP STAGE BLOCK_COMMIT;\n",
+	}
+	for _, sql := range stages {
+		if err := session.Exec(ctx, sql); err != nil {
+			if closeErr := session.Close(); closeErr != nil {
+				slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+			}
+			return fmt.Errorf("put the engine into backup mode at %q: %w", strings.TrimSpace(sql), err)
+		}
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+// Ends the backup cleanly. A missing hold is an error rather than a silent
+// success: it means the deadline fired first, so the snapshot the control plane
+// just took was not taken against a held stage.
+func (e *mariadbEngine) Unquiesce(ctx context.Context) error {
+	held := e.takeHold()
+	if held == nil {
+		return errors.New("the engine is not quiesced; the backup hold had already expired")
+	}
+
+	// The release has to run on the session that took the stages, and the session
+	// is closed either way — the engine ends an unreleased backup with it.
+	execErr := held.session.Exec(ctx, "BACKUP STAGE END;\n")
+	closeErr := held.session.Close()
+	if execErr != nil {
+		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
+	}
+	if closeErr != nil {
+		return fmt.Errorf("close the backup session: %w", closeErr)
+	}
+	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
+	return nil
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never visible
+// in the process table.
+func (e *mariadbEngine) clientRun(ctx context.Context, sql string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.client,
+		Args:  e.clientArgs(),
+		Env:   []string{"PATH=" + defaultGuestPath},
+		Stdin: sql,
+	})
+}
+
+// --no-defaults so no option file can move the connection this agent makes, and
+// --unbuffered so a held session's output reaches the reader as each statement
+// finishes rather than when a pipe buffer fills. No User: the agent is root, and
+// root is what the datadir's unix_socket plugin authenticates.
+func (e *mariadbEngine) clientArgs() []string {
+	return []string{
+		"--no-defaults", "--batch", "--skip-column-names", "--unbuffered",
+		"--protocol=socket", "--socket=" + e.socket, "--user=" + mariadbSuperuser,
+	}
+}

cmd/rds-agent/engine_mariadb_test.go

@@ -0,0 +1,796 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Stands in for the control plane's own definition of the engine. It mirrors the
+// production wiring exactly, including that a name the catalog does not carry
+// counts as static: a rule the fake inverted would hide the failure it exists to
+// prevent.
+func testMariaDBRules() controlPlaneRules {
+	applyTypes := map[string]string{
+		"innodb_buffer_pool_size": handlers_rds.ApplyTypeStatic,
+		"innodb_log_file_size":    handlers_rds.ApplyTypeStatic,
+		"max_connections":         handlers_rds.ApplyTypeDynamic,
+		"long_query_time":         handlers_rds.ApplyTypeDynamic,
+		"time_zone":               handlers_rds.ApplyTypeDynamic,
+	}
+	reserved := []string{"root", "mysql", "mariadb.sys", "rdsadmin", "public"}
+
+	return controlPlaneRules{
+		validateUsername: func(username string) error {
+			lower := strings.ToLower(username)
+			if slices.Contains(reserved, lower) || strings.HasPrefix(lower, "mysql.") || strings.HasPrefix(lower, "mariadb.") {
+				return fmt.Errorf("MasterUsername %q is reserved by mariadb", username)
+			}
+			for _, r := range username {
+				if !isTestIdentifierRune(r) {
+					return fmt.Errorf("MasterUsername may contain only letters, digits and underscores")
+				}
+			}
+			return nil
+		},
+		isStatic: func(name string) bool {
+			applyType, ok := applyTypes[name]
+			return !ok || applyType == handlers_rds.ApplyTypeStatic
+		},
+		catalogName: func(optionFileName string) string {
+			if optionFileName == "default_time_zone" {
+				return "time_zone"
+			}
+			return optionFileName
+		},
+	}
+}
+
+func isTestIdentifierRune(r rune) bool {
+	return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_'
+}
+
+func newTestMariaDBEngine(t *testing.T, run commandRunner) *mariadbEngine {
+	t.Helper()
+	cfg := testLoadConfig(t, engineMariaDB)
+	cfg.DataMount = t.TempDir()
+	if err := os.MkdirAll(filepath.Join(cfg.DataMount, "conf.d"), 0o750); err != nil {
+		t.Fatalf("create conf.d: %v", err)
+	}
+	// The guest's mysql user does not exist here, and the chown of the installed
+	// parameter file has to resolve to something real.
+	current, err := user.Current()
+	if err != nil {
+		t.Fatalf("resolve the current user: %v", err)
+	}
+	cfg.EngineUser = current.Username
+
+	state := engineServing
+	return newMariaDBEngine(cfg, testMariaDBRules(), run, nil, drivenProbe(&state))
+}
+
+// The wiring the guest takes its classification from, checked against a catalog
+// that is registered: what a spec says about a parameter is what the guest must
+// see, and an unlisted name must read as static rather than as dynamic.
+func TestControlPlaneRulesFrom_TakesTheClassificationFromTheCatalog(t *testing.T) {
+	rules := controlPlaneRulesFrom(postgresEngineMeta)
+
+	for name, want := range map[string]bool{"shared_buffers": true, "work_mem": false} {
+		spec, ok := postgresEngineMeta.LookupParameter(name)
+		if !ok {
+			t.Fatalf("the catalog no longer carries %s", name)
+		}
+		if (spec.ApplyType == handlers_rds.ApplyTypeStatic) != want {
+			t.Fatalf("%s is %s in the catalog, so this test is asserting the wrong way round", name, spec.ApplyType)
+		}
+		if got := rules.isStatic(name); got != want {
+			t.Errorf("isStatic(%s) = %v, want %v", name, got, want)
+		}
+	}
+	if !rules.isStatic("not_a_parameter_any_catalog_carries") {
+		t.Error("a name the catalog does not carry read as dynamic, so it would be issued as a live SET GLOBAL")
+	}
+	if rules.validateUsername("postgres") == nil {
+		t.Error("the username rule does not refuse a reserved role")
+	}
+}
+
+// The client has no parameter quoting of its own, so the password is built into
+// the statement here. It must reach the process on stdin and nowhere else.
+func TestMariaDBEngine_SetPasswordKeepsTheSecretOutOfArgvAndEnv(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	call := runner.calls[0]
+	for _, arg := range call.Args {
+		if strings.Contains(arg, "n3w-pw") {
+			t.Fatalf("argv %v carries the password", call.Args)
+		}
+	}
+	for _, env := range call.Env {
+		if strings.Contains(env, "n3w-pw") {
+			t.Fatalf("env %v carries the password, which the client would not read anyway", call.Env)
+		}
+	}
+	if !strings.Contains(call.Stdin, "ALTER USER 'mulgamaster'@'%' IDENTIFIED BY 'n3w-pw';") {
+		t.Errorf("SQL %q does not rotate the master on the wildcard host", call.Stdin)
+	}
+}
+
+// ValidateMasterUserPassword permits both of these, and the client offers no
+// equivalent of psql's parameter quoting, so the escaping here is what keeps a
+// password from ending the literal it is inside.
+func TestMariaDBEngine_SetPasswordEscapesQuotesAndBackslashes(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", `a'b\c`); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	sql := runner.calls[0].Stdin
+	if !strings.Contains(sql, `IDENTIFIED BY 'a''b\\c';`) {
+		t.Errorf("SQL %q does not escape both the quote and the backslash", sql)
+	}
+	// The escaping assumes a mode where backslash escapes are on, so the statement
+	// cannot inherit whatever sql_mode the customer's parameter group set.
+	if !strings.Contains(sql, "SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';") {
+		t.Errorf("SQL %q does not pin the mode its escaping assumes", sql)
+	}
+}
+
+// The password is interpolated into the statement, so a parameter group that
+// turned the general or slow log on would copy it into a log file every snapshot
+// then carries. Both are silenced for this session only: a rotation must not
+// leave the customer's own logging switched off behind it.
+func TestMariaDBEngine_SetPasswordSilencesStatementLoggingForItsSessionOnly(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	sql := runner.calls[0].Stdin
+	alter := strings.Index(sql, "ALTER USER")
+	if alter < 0 {
+		t.Fatalf("SQL %q does not alter the user", sql)
+	}
+	for _, guard := range []string{"SET SESSION sql_log_off = 1;", "SET SESSION slow_query_log = 0;"} {
+		at := strings.Index(sql, guard)
+		if at < 0 {
+			t.Errorf("SQL %q is missing %q", sql, guard)
+			continue
+		}
+		if at > alter {
+			t.Errorf("SQL %q sets %q only after the statement has already been logged", sql, guard)
+		}
+	}
+	if strings.Contains(sql, "SET GLOBAL general_log") || strings.Contains(sql, "SET GLOBAL slow_query_log") {
+		t.Errorf("SQL %q turns logging off globally, which outlives the rotation", sql)
+	}
+}
+
+// The client echoes the statement it failed on, which carries the escaped form
+// of the password rather than the raw one — so redacting only the raw one would
+// leak exactly the passwords that needed escaping.
+func TestMariaDBEngine_SetPasswordRedactsBothFormsOfTheSecret(t *testing.T) {
+	secret := `a'b\c`
+	runner := &recordingRunner{err: errors.New(`ERROR 1064 near "IDENTIFIED BY 'a''b\\c'"`)}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	err := engine.SetPassword(context.Background(), "mulgamaster", secret)
+	if err == nil {
+		t.Fatal("SetPassword succeeded against a failing client")
+	}
+	if strings.Contains(err.Error(), secret) || strings.Contains(err.Error(), sqlLiteral(secret)) {
+		t.Errorf("error %q leaks the password", err)
+	}
+	if !strings.Contains(err.Error(), "[REDACTED]") {
+		t.Errorf("error %q does not mark the redaction", err)
+	}
+}
+
+// The apply runs as the server's own superuser over the socket, and the name is
+// interpolated into the statement rather than quoted by the client, so both
+// halves of the rule are refused before any SQL is built.
+func TestMariaDBEngine_SetPasswordRefusesNamesTheEngineWillNotTake(t *testing.T) {
+	for _, username := range []string{"root", "mysql", "mariadb.sys", "rdsadmin", "PUBLIC", "mysql.infoschema", `ma'ster`} {
+		t.Run(username, func(t *testing.T) {
+			runner := &recordingRunner{}
+			engine := newTestMariaDBEngine(t, runner.run)
+
+			if err := engine.SetPassword(context.Background(), username, "n3w-pw"); err == nil {
+				t.Fatalf("SetPassword accepted %q", username)
+			}
+			if len(runner.calls) != 0 {
+				t.Errorf("ran %d commands for a refused name, want 0", len(runner.calls))
+			}
+		})
+	}
+}
+
+func TestMariaDBEngine_SetPasswordRejectsAnIncompleteCommand(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", ""); err == nil {
+		t.Error("SetPassword accepted an empty password")
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands for an incomplete request, want 0", len(runner.calls))
+	}
+}
+
+// A running server re-reads no configuration file, so the dynamic half has to
+// reach it as SET GLOBAL. The static half is installed and waits for a restart,
+// which is what comes back as pending.
+func TestMariaDBEngine_ApplyParametersSetsOnlyTheDynamicHalfLive(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	pending, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "max_connections", Value: "200"},
+		{Name: "innodb_buffer_pool_size", Value: "1073741824"},
+	})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.HasPrefix(string(installed), "[mysqld]\n") {
+		t.Errorf("installed parameters = %q, want the group header rds-init writes", installed)
+	}
+	for _, want := range []string{"max_connections = '200'", "innodb_buffer_pool_size = '1073741824'"} {
+		if !strings.Contains(string(installed), want) {
+			t.Errorf("installed parameters = %q, want %q", installed, want)
+		}
+	}
+
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want the one live apply", len(runner.calls))
+	}
+	sql := runner.calls[0].Stdin
+	if !strings.Contains(sql, "SET GLOBAL max_connections = '200';") {
+		t.Errorf("SQL %q does not set the dynamic value live", sql)
+	}
+	if strings.Contains(sql, "innodb_buffer_pool_size") {
+		t.Errorf("SQL %q sets a static value live, which the server would refuse", sql)
+	}
+	if !slices.Equal(pending, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the static setting the engine has not adopted", pending)
+	}
+}
+
+// A set with nothing dynamic in it must not open a connection at all: the
+// installed file is the whole of the change.
+func TestMariaDBEngine_ApplyParametersRunsNoStatementForAStaticOnlySet(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "innodb_log_file_size", Value: "268435456"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands for a static-only set, want 0", len(runner.calls))
+	}
+}
+
+// A value the server refuses leaves the installed file back where it was, so it
+// does not sit on the data volume turning the next restart into a boot loop.
+func TestMariaDBEngine_ApplyParametersRollsBackAValueTheServerRefuses(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "200"}}); err != nil {
+		t.Fatalf("first ApplyParameters: %v", err)
+	}
+
+	runner.err = errors.New("ERROR 1231 (42000): Variable 'max_connections' can't be set to the value of '0'")
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "0"}}); err == nil {
+		t.Fatal("ApplyParameters succeeded against a value the server refused")
+	}
+
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "max_connections = '200'") {
+		t.Errorf("installed parameters = %q, want the previous accepted set restored", installed)
+	}
+}
+
+// The whole of the pending-restart answer, since MariaDB has none of its own:
+// the installed drop-in against the one the server actually started on, over the
+// static keys only.
+func TestMariaDBEngine_PendingRestartComparesTheInstalledSetAgainstTheServingCopy(t *testing.T) {
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	write := func(path, body string) {
+		t.Helper()
+		if err := os.WriteFile(path, []byte("[mysqld]\n"+body), 0o600); err != nil {
+			t.Fatalf("write %s: %v", path, err)
+		}
+	}
+	pending := func() []string {
+		t.Helper()
+		names, err := engine.pendingRestartParameters(t.Context())
+		if err != nil {
+			t.Fatalf("pendingRestartParameters: %v", err)
+		}
+		return names
+	}
+
+	// A fresh boot: rds-init wrote the two together, so nothing is waiting.
+	write(engine.params.installedPath(), "innodb_buffer_pool_size = '1073741824'\nmax_connections = '100'\n")
+	write(engine.params.servingPath(), "innodb_buffer_pool_size = '1073741824'\nmax_connections = '100'\n")
+	if names := pending(); len(names) != 0 {
+		t.Errorf("pending = %v, want nothing on a set the engine started on", names)
+	}
+
+	// A dynamic value the server took live is not pending: it was adopted without
+	// a restart, which is exactly what the serving copy not moving records.
+	write(engine.params.installedPath(), "innodb_buffer_pool_size = '1073741824'\nmax_connections = '200'\n")
+	if names := pending(); len(names) != 0 {
+		t.Errorf("pending = %v, want nothing for a value adopted live", names)
+	}
+
+	// A static value is.
+	write(engine.params.installedPath(), "innodb_buffer_pool_size = '2147483648'\nmax_connections = '200'\n")
+	if names := pending(); !slices.Equal(names, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the static setting", names)
+	}
+
+	// So is a static setting the group stopped naming, which reverts to its
+	// default at the next start.
+	write(engine.params.installedPath(), "max_connections = '200'\n")
+	if names := pending(); !slices.Equal(names, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the withdrawn static setting", names)
+	}
+
+	// MariaDB reads - and _ as the same character, so one setting spelled two ways
+	// is not two settings.
+	write(engine.params.installedPath(), "innodb-buffer-pool-size = '1073741824'\nmax_connections = '200'\n")
+	if names := pending(); len(names) != 0 {
+		t.Errorf("pending = %v, want the two spellings of one setting to compare equal", names)
+	}
+}
+
+// rds-init writes the serving copy beside the installed one on every boot, so a
+// set with no copy beside it is one the engine has not started on. Promoting it
+// would point the rollback at a configuration that has never served.
+func TestMariaDBEngine_PendingRestartTreatsAMissingServingCopyAsUnadopted(t *testing.T) {
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	if err := os.WriteFile(engine.params.installedPath(),
+		[]byte("[mysqld]\ninnodb_buffer_pool_size = '1073741824'\nmax_connections = '100'\n"), 0o600); err != nil {
+		t.Fatalf("write installed parameters: %v", err)
+	}
+
+	names, err := engine.pendingRestartParameters(t.Context())
+	if err != nil {
+		t.Fatalf("pendingRestartParameters: %v", err)
+	}
+	if !slices.Equal(names, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the static half of a set the engine never started on", names)
+	}
+}
+
+// The restart the agent performs is the only thing that moves the serving copy,
+// which is what makes a fresh start clear whatever was pending.
+func TestMariaDBEngine_RestartRefreshesTheServingCopy(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	installed := []byte("[mysqld]\ninnodb_buffer_pool_size = '2147483648'\n")
+	if err := os.WriteFile(engine.params.installedPath(), installed, 0o600); err != nil {
+		t.Fatalf("write installed parameters: %v", err)
+	}
+	if err := os.WriteFile(engine.params.servingPath(), []byte("[mysqld]\ninnodb_buffer_pool_size = '1073741824'\n"), 0o600); err != nil {
+		t.Fatalf("write serving parameters: %v", err)
+	}
+
+	if err := engine.Restart(context.Background()); err != nil {
+		t.Fatalf("Restart: %v", err)
+	}
+
+	serving, err := os.ReadFile(engine.params.servingPath())
+	if err != nil {
+		t.Fatalf("read the serving copy: %v", err)
+	}
+	if string(serving) != string(installed) {
+		t.Errorf("serving copy = %q, want the set the engine was just started on", serving)
+	}
+	names, err := engine.pendingRestartParameters(t.Context())
+	if err != nil {
+		t.Fatalf("pendingRestartParameters: %v", err)
+	}
+	if len(names) != 0 {
+		t.Errorf("pending = %v, want a restart to have cleared it", names)
+	}
+	// The copy the comparison reads must never be parsed as a second set of
+	// settings by the engine's own include glob.
+	if strings.HasSuffix(engine.params.servingPath(), ".cnf") {
+		t.Errorf("the serving copy at %s would be included by the engine", engine.params.servingPath())
+	}
+}
+
+// The rollback target advances only once a restart has proved the installed
+// static values can serve, not when a dynamic apply merely set them live.
+func TestMariaDBEngine_LastKnownGoodWaitsForARestartToAdoptAStaticSet(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(),
+		[]handlers_rds.Parameter{{Name: "innodb_buffer_pool_size", Value: "1073741824"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+	if _, err := os.Stat(engine.params.lastGoodPath()); !os.IsNotExist(err) {
+		t.Errorf("a set the engine has not restarted onto became the rollback target (stat err = %v)", err)
+	}
+
+	if err := engine.Restart(t.Context()); err != nil {
+		t.Fatalf("Restart: %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters after the restart: %v", err)
+	}
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read the last known good parameters: %v", err)
+	}
+	if !strings.Contains(string(lastGood), "innodb_buffer_pool_size = '1073741824'") {
+		t.Errorf("last known good = %q, want the set the engine restarted onto", lastGood)
+	}
+}
+
+// A dynamic apply is adopted without a restart, so it may advance the rollback
+// target on the next healthy beat rather than waiting for one.
+func TestMariaDBEngine_LastKnownGoodAdvancesOnADynamicApply(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	if err := os.WriteFile(engine.params.servingPath(), []byte("[mysqld]\n"), 0o600); err != nil {
+		t.Fatalf("write serving parameters: %v", err)
+	}
+
+	if _, err := engine.ApplyParameters(t.Context(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "200"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read the last known good parameters: %v", err)
+	}
+	if !strings.Contains(string(lastGood), "max_connections = '200'") {
+		t.Errorf("last known good = %q, want the set the engine is already running", lastGood)
+	}
+}
+
+// A probe that reports what the test hands it, in place of whichever binaries
+// the engine's own state function would have run.
+func mariadbTestProbeConfig(t *testing.T) config {
+	t.Helper()
+	cfg := testLoadConfig(t, engineMariaDB)
+	cfg.EnginePidFile = filepath.Join(t.TempDir(), "mariadb.pid")
+	return cfg
+}
+
+func writePid(t *testing.T, path string, pid int) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(strconv.Itoa(pid)+"\n"), 0o644); err != nil {
+		t.Fatalf("write pidfile: %v", err)
+	}
+}
+
+// The three stages, and what each one separates. During InnoDB crash recovery
+// mariadbd opens neither its socket nor its port, so nothing the client can ask
+// distinguishes a server replaying its redo log from no server at all.
+func TestMariaDBProbe_SeparatesRecoveringFromAbsent(t *testing.T) {
+	tests := []struct {
+		name     string
+		writePid bool
+		alive    bool
+		pingCode int
+		pingErr  error
+		execCode int
+		want     engineState
+	}{
+		{name: "no pidfile at all", want: engineAbsent},
+		{name: "pidfile names a dead process", writePid: true, want: engineAbsent},
+		{name: "process alive but socket silent", writePid: true, alive: true, pingCode: 1, want: engineRecovering},
+		{name: "process alive but probe cannot run", writePid: true, alive: true, pingErr: errors.New("no such file"), want: engineRecovering},
+		{name: "answers ping but executes nothing", writePid: true, alive: true, execCode: 1, want: engineRecovering},
+		{name: "serving", writePid: true, alive: true, want: engineServing},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := mariadbTestProbeConfig(t)
+			if tt.writePid {
+				writePid(t, cfg.EnginePidFile, 4242)
+			}
+			run := func(_ context.Context, name string, args ...string) (int, error) {
+				if strings.HasSuffix(name, mariadbAdminBinary) {
+					return tt.pingCode, tt.pingErr
+				}
+				if !slices.Contains(args, "--execute=SELECT 1") {
+					t.Errorf("the second stage ran %s %v, want a statement the server has to execute", name, args)
+				}
+				return tt.execCode, nil
+			}
+			stateFn := mariadbProbeState(cfg, run, func(int) bool { return tt.alive })
+
+			got, message := stateFn(context.Background(), int64(cfg.EnginePort))
+			if got != tt.want {
+				t.Errorf("state = %v, want %v (%s)", got, tt.want, message)
+			}
+			if got != engineServing && message == "" {
+				t.Error("a non-serving result carried no message explaining it")
+			}
+		})
+	}
+}
+
+// The failure the guard exists for still behaves correctly: a static value
+// mariadbd will not accept makes it exit during startup, so the pidfile names a
+// process that is gone and the deadline runs.
+func TestMariaDBProbe_BoundsAStalledClient(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineProbeTimeout = 20 * time.Millisecond
+	writePid(t, cfg.EnginePidFile, 4242)
+
+	started := time.Now()
+	state, message := mariadbProbeState(cfg,
+		func(ctx context.Context, _ string, args ...string) (int, error) {
+			if !slices.Contains(args, "--connect-timeout=3") {
+				t.Errorf("probe args = %v, want a client connection timeout", args)
+			}
+			<-ctx.Done()
+			return -1, ctx.Err()
+		},
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering after a live engine's probe timed out", state)
+	}
+	if !strings.Contains(message, context.DeadlineExceeded.Error()) {
+		t.Errorf("message = %q, want the probe deadline", message)
+	}
+	if elapsed := time.Since(started); elapsed > time.Second {
+		t.Errorf("probe took %v, want the configured deadline to bound it", elapsed)
+	}
+}
+
+func TestMariaDBProbe_MapsToTheHealthTheControlPlaneReads(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	writePid(t, cfg.EnginePidFile, 4242)
+	alive := true
+	serving := true
+	probe := newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, error) {
+			if serving {
+				return 0, nil
+			}
+			return 1, nil
+		},
+		func(int) bool { return alive }))
+
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthHealthy {
+		t.Fatalf("health = %q, want healthy", got)
+	}
+	serving = false
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthStarting {
+		t.Errorf("health while recovering = %q, want starting", got)
+	}
+	alive = false
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthUnhealthy {
+		t.Errorf("health after the engine went away = %q, want unhealthy", got)
+	}
+}
+
+// An instance killed hard mid-write can spend minutes replaying its redo log. If
+// that read as absent, the guard would roll the parameter file back and restart
+// the server mid-recovery — a rollback that exists to break a boot loop would
+// create one, on an instance whose parameters were never at fault.
+func TestMariaDBProbe_LiveButUnreachableEngineResetsTheRollbackDeadline(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	writePid(t, cfg.EnginePidFile, 4242)
+	probe := newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, error) { return 1, nil },
+		func(int) bool { return true }))
+
+	recovery := &fakeRecovery{restored: true}
+	guard := newTestGuard(recovery, probe)
+	// Bounded so the deadline-resetting loop cannot run forever if it stops
+	// resetting; the assertion is that the rollback was never reached.
+	ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+	defer cancel()
+	guard.Run(ctx)
+
+	if recovery.restarts != 0 {
+		t.Errorf("restarts = %d, want none while the engine is still recovering", recovery.restarts)
+	}
+}
+
+// MariaDB releases a BACKUP STAGE with the connection that took it, so the whole
+// sequence has to run on one held session — which is also what bounds a control
+// plane that dies mid-snapshot.
+func TestMariaDBEngine_QuiesceTakesTheBackupStagesOnAHeldSession(t *testing.T) {
+	session := &fakeSession{}
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	var started []command
+	engine.startSess = func(_ context.Context, c command) (engineSession, error) {
+		started = append(started, c)
+		return session, nil
+	}
+
+	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	if len(started) != 1 {
+		t.Fatalf("started %d sessions, want 1", len(started))
+	}
+	if !slices.Contains(started[0].Args, "--unbuffered") {
+		t.Errorf("session argv %v does not flush after each statement, so the sentinel could sit in a pipe buffer", started[0].Args)
+	}
+	statements := session.statements()
+	want := []string{
+		"SET SESSION lock_wait_timeout = 20;",
+		"BACKUP STAGE START;",
+		"BACKUP STAGE FLUSH;",
+		"BACKUP STAGE BLOCK_DDL;",
+		"BACKUP STAGE BLOCK_COMMIT;",
+	}
+	if len(statements) != len(want) {
+		t.Fatalf("statements = %q, want the bounded stage sequence %q", statements, want)
+	}
+	for i, sql := range want {
+		if strings.TrimSpace(statements[i]) != sql {
+			t.Errorf("statement %d = %q, want %q", i, statements[i], sql)
+		}
+	}
+	// FLUSH TABLES WITH READ LOCK would make the whole database read-only for the
+	// length of the hold rather than blocking commits.
+	if strings.Contains(strings.Join(statements, " "), "FLUSH TABLES") {
+		t.Error("the quiesce took a read lock over the whole database")
+	}
+	if session.closes() != 0 {
+		t.Error("the session was closed while the backup was meant to be held")
+	}
+}
+
+// The stage bound has to leave the control plane's own quiesce timeout room for
+// the whole sequence, or a stage abandoned mid-wait leaves its lock request
+// queued in front of live traffic.
+func TestMariaDBEngine_QuiesceLockWaitFitsInsideTheControlPlaneTimeout(t *testing.T) {
+	const stages = 4
+	if stages*mariadbQuiesceLockWait >= 2*time.Minute {
+		t.Errorf("%d stages of %s can outlast the control plane's quiesce timeout", stages, mariadbQuiesceLockWait)
+	}
+}
+
+func TestMariaDBEngine_UnquiesceEndsTheBackupAndTheSession(t *testing.T) {
+	session := &fakeSession{}
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	engine.startSess = func(context.Context, command) (engineSession, error) { return session, nil }
+	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	if err := engine.Unquiesce(context.Background()); err != nil {
+		t.Fatalf("Unquiesce: %v", err)
+	}
+
+	statements := session.statements()
+	if last := strings.TrimSpace(statements[len(statements)-1]); last != "BACKUP STAGE END;" {
+		t.Errorf("last statement = %q, want the stage released", last)
+	}
+	if session.closes() != 1 {
+		t.Errorf("closed %d times, want the session ended exactly once", session.closes())
+	}
+}
+
+// The two halves of a time_zone apply use different names: SET GLOBAL takes the
+// customer's, the option file the server's startup spelling. Getting either
+// backwards is a statement the server refuses or a boot it refuses.
+func TestMariaDBEngine_TimeZoneAppliesLiveByItsCustomerFacingName(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "+10:00"},
+	}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want the one live apply", len(runner.calls))
+	}
+	sql := runner.calls[0].Stdin
+	if !strings.Contains(sql, "SET GLOBAL time_zone = '+10:00';") {
+		t.Errorf("SQL %q does not set time_zone under the name MariaDB accepts for SET", sql)
+	}
+	if strings.Contains(sql, "default_time_zone") {
+		t.Errorf("SQL %q names the startup spelling, which is not a settable variable", sql)
+	}
+
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "default_time_zone = '+10:00'") {
+		t.Errorf("installed parameters = %q, want the startup spelling", installed)
+	}
+}
+
+// A dynamic setting applied live deliberately leaves the serving copy alone, so
+// the two files differ by design. Read back under the startup spelling the
+// catalog does not carry, that difference would classify as static and report a
+// pending restart that nothing could ever clear.
+func TestMariaDBEngine_ALiveTimeZoneChangeLeavesNothingPendingRestart(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := os.WriteFile(engine.params.servingPath(),
+		[]byte("[mysqld]\ndefault_time_zone = 'SYSTEM'\n"), 0o600); err != nil {
+		t.Fatalf("seed the serving copy: %v", err)
+	}
+
+	pending, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "+10:00"},
+	})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if len(pending) != 0 {
+		t.Errorf("pending = %v, want nothing: time_zone is dynamic and was applied live", pending)
+	}
+}
+
+// The list is customer-facing, so a static setting still has to be reported
+// under the name the customer set rather than the engine's startup spelling.
+func TestMariaDBEngine_PendingRestartReportsCatalogNames(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := os.WriteFile(engine.params.servingPath(),
+		[]byte("[mysqld]\ninnodb_buffer_pool_size = '536870912'\ndefault_time_zone = 'SYSTEM'\n"), 0o600); err != nil {
+		t.Fatalf("seed the serving copy: %v", err)
+	}
+
+	pending, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "innodb_buffer_pool_size", Value: "1073741824"},
+		{Name: "time_zone", Value: "SYSTEM"},
+	})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if !slices.Equal(pending, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want only the static setting under its catalog name", pending)
+	}
+}

cmd/rds-agent/engine_postgres.go

@@ -0,0 +1,380 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The engine is reached over its unix socket under peer authentication, so the
+// agent drops to the postgres OS user rather than holding a password of its own.
+type postgresEngine struct {
+	quiesceState
+
+	// The control plane's own metadata for this engine, resolved once at
+	// startup: the live password apply runs as the cluster superuser, so it
+	// re-checks the role name against the same reserved set the API validates.
+	meta      handlers_rds.Engine
+	run       commandRunner
+	startSess sessionRunner
+	psql      string
+	rcService string
+	service   string
+	pgData    string
+	socketDir string
+	osUser    string
+	probe     *engineProbe
+	// Set from the bootstrap config, on a different goroutine than the commands
+	// that read it.
+	port atomic.Int64
+
+	params parameterStore
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
+}
+
+var _ engine = (*postgresEngine)(nil)
+
+// pg_isready is what gives PostgreSQL three states for free: exit 0 is serving,
+// exit 1 is a postmaster up and rejecting connections, and anything else is an
+// engine that is not there at all. Resolved on PATH, where the client package
+// puts it.
+const postgresProbeBinary = "pg_isready"
+
+func newPostgresProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.EngineHost, run))
+}
+
+func postgresProbeState(host string, run probeRunner) probeStateFn {
+	return func(ctx context.Context, port int64) (engineState, string) {
+		portArg := strconv.FormatInt(port, 10)
+		code, err := run(ctx, postgresProbeBinary, "-h", host, "-p", portArg, "-q")
+		switch {
+		case err != nil:
+			// A missing binary or broken image. Reporting healthy on the strength of
+			// nothing would hide it, so this reads as absent like an engine that did
+			// not answer.
+			return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
+		case code == 0:
+			return engineServing, ""
+		case code == 1:
+			return engineRecovering, "engine is rejecting connections (startup or recovery)"
+		default:
+			return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", host, portArg)
+		}
+	}
+}
+
+// Resolved once, under the name the control plane knows this implementation by.
+// A lookup failure is a mismatch between the agent and the control plane it was
+// built against, so it fails at startup rather than at the first rotation.
+var postgresEngineMeta = mustLookupEngine(enginePostgres)
+
+func mustLookupEngine(name string) handlers_rds.Engine {
+	engine, err := handlers_rds.LookupEngine(name)
+	if err != nil {
+		panic("rds-agent: " + err.Error())
+	}
+	return engine
+}
+
+// The include the resolved parameter set is rendered to, and the copy of the
+// last one the engine accepted. Both live beside the data rather than in /etc,
+// matching rds-init: a class change boots a fresh root volume, which would
+// otherwise revert them.
+const (
+	postgresParametersFile = "10-rds-parameters.conf"
+	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
+	// copy must not be read as a second set of settings.
+	postgresLastGoodFile = "10-rds-parameters.last-good"
+)
+
+func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
+	e := &postgresEngine{
+		meta:      postgresEngineMeta,
+		run:       run,
+		startSess: startSess,
+		psql:      filepath.Join(cfg.EngineBinDir, "psql"),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		pgData:    cfg.EngineDataDir,
+		socketDir: cfg.SocketDir,
+		osUser:    cfg.EngineUser,
+		probe:     probe,
+		params: parameterStore{
+			dir:       filepath.Join(cfg.EngineDataDir, "conf.d"),
+			installed: postgresParametersFile,
+			lastGood:  postgresLastGoodFile,
+			osUser:    cfg.EngineUser,
+			engine:    enginePostgres,
+		},
+		repairTimeout: parameterRepairTimeout,
+		repairPoll:    parameterRepairPoll,
+	}
+	e.port.Store(int64(cfg.EnginePort))
+	return e
+}
+
+func (e *postgresEngine) setPort(port int) {
+	e.port.Store(int64(port))
+}
+
+// The role name and the password ride the environment and are re-quoted by
+// psql, so neither reaches a shell word or an argv another process can read.
+func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// The statement below runs as the cluster superuser over the socket under
+	// peer auth, so a reserved name in the command payload would hand the
+	// customer the bootstrap superuser rather than rotate their own role.
+	if err := e.meta.ValidateUsernameNotReserved(username); err != nil {
+		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
+	}
+	// psql interpolates the password into the ALTER ROLE before the server sees
+	// it, and these three are what would write it to the log — the last on any
+	// failure at its own default. SUSET, so the parameter group cannot win.
+	const sql = `SET log_statement = 'none';
+SET log_min_duration_statement = -1;
+SET log_min_error_statement = 'panic';
+\getenv master RDS_MASTER_USERNAME
+\getenv password RDS_MASTER_PASSWORD
+ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
+`
+	_, err := e.psqlRun(ctx, sql,
+		"RDS_MASTER_USERNAME="+username,
+		"RDS_MASTER_PASSWORD="+password,
+	)
+	if err != nil {
+		// psql echoes the failing statement, which here would carry the new
+		// password back to the control plane and into the event ring.
+		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
+	}
+	return nil
+}
+
+// Installs the resolved set, validates it with the engine's own config parser
+// before the engine ever adopts it, and reloads. A value the engine refuses is
+// rolled back here rather than left on the data volume, where it would survive
+// every VM replace and turn the next restart into a boot loop.
+func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	// The check has to run against the file in place, because the engine parses
+	// the datadir's own include_dir. The window that leaves is closed from both
+	// ends: the rollback below, and the last-known-good restore the agent runs at
+	// boot when the engine does not come up.
+	restore, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := e.checkConfig(ctx); err != nil {
+		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restore())
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
+	}
+
+	if _, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n"); err != nil {
+		// The engine may have gone down after the first probe. In that case start
+		// it on the new, parser-checked repair set rather than restoring the set
+		// that already left it unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(fmt.Errorf("reload the engine configuration: %w", err), restore())
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+func (e *postgresEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// The postmaster's own answer: a static setting it has stored but not adopted is
+// what it reports here, so nothing in the guest has to classify one.
+func (e *postgresEngine) pendingRestartParameters(ctx context.Context) ([]string, error) {
+	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
+	if err != nil {
+		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
+	}
+	var pending []string
+	for line := range strings.SplitSeq(out, "\n") {
+		if name := strings.TrimSpace(line); name != "" {
+			pending = append(pending, name)
+		}
+	}
+	return pending, nil
+}
+
+// Snapshots the include only when the running postmaster has no settings still
+// pending a restart. The parameter mutex keeps an apply from replacing the file
+// between that check and the copy.
+func (e *postgresEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *postgresEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change.
+func (e *postgresEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return restoreLastGood(ctx, e.params, e.probe)
+}
+
+// The engine's own parser, run offline against the datadir. Reading one setting
+// back is enough: postgres parses postgresql.conf and every include first, and
+// exits non-zero naming the file and line of an unknown parameter or a value
+// outside its range.
+func (e *postgresEngine) checkConfig(ctx context.Context) error {
+	if _, err := e.run(ctx, command{
+		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
+		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
+		Env:  []string{"PATH=" + defaultGuestPath},
+		User: e.osUser,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (e *postgresEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback calls this: a restart the control plane wants goes
+// through RebootDBInstance, which cycles the VM.
+func (e *postgresEngine) Restart(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never
+// visible in the process table.
+func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.psql,
+		Args:  e.psqlArgs(),
+		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
+		Stdin: sql,
+		User:  e.osUser,
+	})
+}
+
+// Reading the script from stdin is what lets one invocation serve both a
+// one-shot run and a session held open across several statements.
+func (e *postgresEngine) psqlArgs() []string {
+	return []string{
+		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
+		"-v", "ON_ERROR_STOP=1",
+		"-h", e.socketDir,
+		"-p", strconv.FormatInt(e.port.Load(), 10),
+		"-U", e.osUser,
+		"-d", "postgres",
+		"-f", "-",
+	}
+}
+
+// Puts the engine into backup mode: the datadir is checkpointed and the engine
+// stops writing over the pages a snapshot is about to read. The hold is released
+// by Unquiesce, or by its own deadline, whichever comes first.
+func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.psql,
+		Args:              e.psqlArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
+		User:              e.osUser,
+		SentinelStatement: `\echo ` + sessionSentinel + "\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// fast forces an immediate checkpoint rather than spreading it over the
+	// checkpoint interval, which would hold the snapshot open for minutes.
+	const sql = `\getenv label RDS_BACKUP_LABEL
+SELECT pg_backup_start(:'label', fast => true);
+`
+	if err := session.Exec(ctx, sql); err != nil {
+		if closeErr := session.Close(); closeErr != nil {
+			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+		}
+		return fmt.Errorf("put the engine into backup mode: %w", err)
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+// Ends the backup cleanly. A missing hold is an error rather than a silent
+// success: it means the deadline fired first, so the snapshot the control plane
+// just took was not taken against a held checkpoint.
+func (e *postgresEngine) Unquiesce(ctx context.Context) error {
+	held := e.takeHold()
+	if held == nil {
+		return errors.New("the engine is not quiesced; the backup hold had already expired")
+	}
+
+	// The stop has to run on the session that started the backup, and the session
+	// is closed either way — the engine aborts an unstopped backup with it.
+	execErr := held.session.Exec(ctx, "SELECT pg_backup_stop();\n")
+	closeErr := held.session.Close()
+	if execErr != nil {
+		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
+	}
+	if closeErr != nil {
+		return fmt.Errorf("close the backup session: %w", closeErr)
+	}
+	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
+	return nil
+}

cmd/rds-agent/engine_test.go

@@ -0,0 +1,402 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"os/user"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Stands in for the engine in the registry tests, where what matters is that a
+// command reaches the right method with the right arguments.
+type fakeEngine struct {
+	username string
+	password string
+	params   []handlers_rds.Parameter
+	pending  []string
+	stopped  bool
+	label    string
+	hold     time.Duration
+	released bool
+	err      error
+}
+
+var _ engineOps = (*fakeEngine)(nil)
+
+func (f *fakeEngine) SetPassword(_ context.Context, username, password string) error {
+	f.username, f.password = username, password
+	return f.err
+}
+
+func (f *fakeEngine) ApplyParameters(_ context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	f.params = params
+	return f.pending, f.err
+}
+
+func (f *fakeEngine) Stop(context.Context) error {
+	f.stopped = true
+	return f.err
+}
+
+func (f *fakeEngine) Quiesce(_ context.Context, label string, hold time.Duration) error {
+	f.label, f.hold = label, hold
+	return f.err
+}
+
+func (f *fakeEngine) Unquiesce(context.Context) error {
+	f.released = true
+	return f.err
+}
+
+func TestCommandRegistry_SetPasswordCarriesBothParameters(t *testing.T) {
+	engine := &fakeEngine{}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(), handlers_rds.Command{
+		CommandID: "cmd-1",
+		Type:      handlers_rds.CommandSetPassword,
+		Parameters: []handlers_rds.Parameter{
+			{Name: handlers_rds.CommandParamMasterUsername, Value: "mulgamaster"},
+			{Name: handlers_rds.CommandParamMasterUserPassword, Value: "n3w-pw"},
+		},
+	})
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if engine.username != "mulgamaster" || engine.password != "n3w-pw" {
+		t.Errorf("engine got %q/%q, want mulgamaster/n3w-pw", engine.username, engine.password)
+	}
+	// The reply is recorded by the control plane; the new password must not be
+	// in it.
+	if strings.Contains(reply.Message, "n3w-pw") {
+		t.Errorf("reply message %q carries the new password", reply.Message)
+	}
+}
+
+// CommandReply has no structured payload, so the settings still awaiting a
+// restart come back as the message the issuer parses.
+func TestCommandRegistry_ApplyParamsReportsPendingRestart(t *testing.T) {
+	engine := &fakeEngine{pending: []string{"shared_buffers", "max_connections"}}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(), handlers_rds.Command{
+		CommandID:  "cmd-2",
+		Type:       handlers_rds.CommandApplyParams,
+		Parameters: []handlers_rds.Parameter{{Name: "shared_buffers", Value: "256MB"}},
+	})
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if reply.Message != "shared_buffers,max_connections" {
+		t.Errorf("reply message = %q, want the pending-restart list", reply.Message)
+	}
+	if len(engine.params) != 1 || engine.params[0].Name != "shared_buffers" {
+		t.Errorf("engine got %+v, want the command's parameters", engine.params)
+	}
+}
+
+func TestCommandRegistry_StopEngineReachesTheEngine(t *testing.T) {
+	engine := &fakeEngine{}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-3", Type: handlers_rds.CommandStopEngine})
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if !engine.stopped {
+		t.Error("stop-engine did not reach the engine")
+	}
+}
+
+func TestCommandRegistry_FailureRepliesFailed(t *testing.T) {
+	engine := &fakeEngine{err: errors.New("the engine did not stop")}
+	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),
+		handlers_rds.Command{CommandID: "cmd-4", Type: handlers_rds.CommandStopEngine})
+
+	if reply.Status != handlers_rds.CommandStatusFailed {
+		t.Fatalf("reply = %+v, want failed", reply)
+	}
+	if !strings.Contains(reply.Message, "the engine did not stop") {
+		t.Errorf("reply message = %q, want the engine's error", reply.Message)
+	}
+}
+
+// Records what a postgresEngine would have run, so the SQL and the argv are
+// asserted without a live cluster.
+type recordingRunner struct {
+	calls []command
+	out   string
+	err   error
+}
+
+func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
+	r.calls = append(r.calls, c)
+	return r.out, r.err
+}
+
+func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
+	t.Helper()
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.EngineDataDir = t.TempDir()
+	if err := os.MkdirAll(filepath.Join(cfg.EngineDataDir, "conf.d"), 0o700); err != nil {
+		t.Fatalf("create conf.d: %v", err)
+	}
+	// The guest's postgres user does not exist here, and the chown of the
+	// installed parameter file has to resolve to something real.
+	current, err := user.Current()
+	if err != nil {
+		t.Fatalf("resolve the current user: %v", err)
+	}
+	cfg.EngineUser = current.Username
+	return newPostgresEngine(cfg, run, nil, newPostgresProbe(cfg, staticProbe(0)))
+}
+
+// The password reaches psql through the environment and is re-quoted there,
+// so it is never a shell word or an argv another process can read.
+func TestPostgresEngine_SetPasswordKeepsTheSecretOutOfArgv(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	call := runner.calls[0]
+	for _, arg := range call.Args {
+		if strings.Contains(arg, "n3w-pw") {
+			t.Fatalf("argv %v carries the password", call.Args)
+		}
+	}
+	if strings.Contains(call.Stdin, "n3w-pw") {
+		t.Errorf("SQL %q carries the password literally, want it read from the environment", call.Stdin)
+	}
+	if !slices.Contains(call.Env, "RDS_MASTER_PASSWORD=n3w-pw") {
+		t.Errorf("env %v does not carry the password", call.Env)
+	}
+	if !strings.Contains(call.Stdin, "ALTER ROLE") {
+		t.Errorf("SQL %q does not alter the role", call.Stdin)
+	}
+}
+
+// psql interpolates the password into the ALTER ROLE before the server sees it,
+// so a parameter group that turns statement logging on would write the rotated
+// password into a postmaster.log every snapshot then carries.
+func TestPostgresEngine_SetPasswordSilencesStatementLogging(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	sql := runner.calls[0].Stdin
+	alter := strings.Index(sql, "ALTER ROLE")
+	if alter < 0 {
+		t.Fatalf("SQL %q does not alter the role", sql)
+	}
+	for _, guard := range []string{
+		"SET log_statement = 'none';",
+		"SET log_min_duration_statement = -1;",
+		"SET log_min_error_statement = 'panic';",
+	} {
+		at := strings.Index(sql, guard)
+		if at < 0 {
+			t.Errorf("SQL %q is missing %q", sql, guard)
+			continue
+		}
+		if at > alter {
+			t.Errorf("SQL %q sets %q only after the ALTER ROLE has already been logged", sql, guard)
+		}
+	}
+}
+
+// psql echoes the failing statement, which here would carry the new password
+// back into a reply the control plane records.
+func TestPostgresEngine_SetPasswordRedactsTheSecretFromAFailure(t *testing.T) {
+	runner := &recordingRunner{err: errors.New(`ERROR: syntax error at or near "n3w-pw"`)}
+	engine := newTestEngine(t, runner.run)
+
+	err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw")
+	if err == nil {
+		t.Fatal("SetPassword succeeded against a failing psql")
+	}
+	if strings.Contains(err.Error(), "n3w-pw") {
+		t.Errorf("error %q leaks the password", err)
+	}
+	if !strings.Contains(err.Error(), "[REDACTED]") {
+		t.Errorf("error %q does not mark the redaction", err)
+	}
+}
+
+func TestPostgresEngine_SetPasswordRejectsAnIncompleteCommand(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", ""); err == nil {
+		t.Error("SetPassword accepted an empty password")
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands for an incomplete request, want 0", len(runner.calls))
+	}
+}
+
+// The apply runs as the cluster superuser under peer auth, so a reserved
+// name arriving in a command payload would rotate the bootstrap superuser's
+// password rather than the customer's own role.
+func TestPostgresEngine_SetPasswordRefusesReservedRoles(t *testing.T) {
+	for _, username := range []string{"postgres", "rds_superuser", "rdsadmin", "pg_toast_owner", "PostGres"} {
+		t.Run(username, func(t *testing.T) {
+			runner := &recordingRunner{}
+			engine := newTestEngine(t, runner.run)
+
+			err := engine.SetPassword(context.Background(), username, "n3w-pw")
+			if err == nil {
+				t.Fatalf("SetPassword accepted the reserved role %q", username)
+			}
+			if len(runner.calls) != 0 {
+				t.Errorf("ran %d commands for a reserved role, want 0", len(runner.calls))
+			}
+		})
+	}
+}
+
+// The parameters land where rds-init installs them, so the next boot overwrites
+// the file rather than shadowing it with a second copy.
+func TestPostgresEngine_ApplyParametersInstallsAndReloads(t *testing.T) {
+	runner := &recordingRunner{out: "shared_buffers\nmax_connections\n"}
+	engine := newTestEngine(t, runner.run)
+
+	pending, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "shared_buffers", Value: "256MB"}})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+
+	installed, err := os.ReadFile(filepath.Join(engine.pgData, "conf.d", "10-rds-parameters.conf"))
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "shared_buffers = '256MB'") {
+		t.Errorf("installed parameters = %q, want the resolved value", installed)
+	}
+	if len(runner.calls) != 3 {
+		t.Fatalf("ran %d commands, want a config check, a reload and a pending-restart read", len(runner.calls))
+	}
+	// The check runs before the reload, so a value the engine would refuse never
+	// reaches a running cluster.
+	if !slices.Contains(runner.calls[0].Args, "-C") {
+		t.Errorf("first command = %v, want the offline config check", runner.calls[0].Args)
+	}
+	if !strings.Contains(runner.calls[1].Stdin, "pg_reload_conf") {
+		t.Errorf("second statement = %q, want the reload", runner.calls[1].Stdin)
+	}
+	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
+		t.Errorf("third statement = %q, want the pending-restart read", runner.calls[2].Stdin)
+	}
+	if len(pending) != 2 || pending[0] != "shared_buffers" || pending[1] != "max_connections" {
+		t.Errorf("pending = %v, want both settings the engine reported", pending)
+	}
+}
+
+// A reload that failed leaves the engine on its old values, so reporting the
+// apply as successful would tell the customer a change took effect that did not.
+func TestPostgresEngine_ApplyParametersSurfacesAFailedReload(t *testing.T) {
+	runner := &reloadFailingRunner{reloadErr: errors.New("reload was rejected")}
+	engine := newTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}}); err == nil {
+		t.Fatal("ApplyParameters succeeded against a failing reload")
+	}
+}
+
+type reloadFailingRunner struct {
+	recordingRunner
+
+	reloadErr error
+}
+
+func (r *reloadFailingRunner) run(ctx context.Context, c command) (string, error) {
+	if strings.Contains(c.Stdin, "pg_reload_conf") {
+		r.calls = append(r.calls, c)
+		return "", r.reloadErr
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+// A failed instance cannot reload through psql. Its corrected, parser-checked
+// include must start the service and remain installed as the serving set.
+func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	codes := []int{2, 2, 0}
+	probeCall := 0
+	cfg := testProbeConfig()
+	engine.probe = newPostgresProbe(cfg, func(context.Context, string, ...string) (int, error) {
+		code := codes[min(probeCall, len(codes)-1)]
+		probeCall++
+		return code, nil
+	})
+	engine.repairTimeout = 100 * time.Millisecond
+	engine.repairPoll = time.Millisecond
+	params := []handlers_rds.Parameter{
+		{Name: "shared_buffers", Value: "32768"},
+		{Name: "work_mem", Value: "8192"},
+	}
+
+	pending, err := engine.ApplyParameters(t.Context(), params)
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if len(pending) != 0 {
+		t.Errorf("pending = %v, want the restarted engine's actual pending set", pending)
+	}
+	if len(runner.calls) != 3 {
+		t.Fatalf("calls = %+v, want config check, service restart and pending read", runner.calls)
+	}
+	if !slices.Contains(runner.calls[0].Args, "-C") {
+		t.Errorf("first call = %+v, want the offline config check", runner.calls[0])
+	}
+	if !slices.Equal(runner.calls[1].Args, []string{"postgresql", "restart"}) {
+		t.Errorf("second call args = %v, want the service restart", runner.calls[1].Args)
+	}
+	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
+		t.Errorf("third call = %+v, want the pending-restart read", runner.calls[2])
+	}
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "work_mem = '8192'") {
+		t.Errorf("installed parameters = %q, want the repair set retained", installed)
+	}
+}
+
+func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	engine.probe = stubProbe(t, 2, nil)
+	engine.repairTimeout = 10 * time.Millisecond
+	engine.repairPoll = time.Millisecond
+
+	_, err := engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{{Name: "work_mem", Value: "8192"}})
+	if err == nil || !strings.Contains(err.Error(), "wait for the engine") {
+		t.Fatalf("ApplyParameters error = %v, want the repair wait failure", err)
+	}
+	installed, readErr := os.ReadFile(engine.params.installedPath())
+	if readErr != nil {
+		t.Fatalf("read installed parameters: %v", readErr)
+	}
+	if !strings.Contains(string(installed), "work_mem = '8192'") {
+		t.Errorf("installed parameters = %q, want the checked repair set retained", installed)
+	}
+}

cmd/rds-agent/health.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os/exec"
+	"sync/atomic"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// A non-zero exit is a result, not a fault, so it comes back as a code; err is
+// reserved for the probe failing to run at all.
+type probeRunner func(ctx context.Context, name string, args ...string) (int, error)
+
+func execProbeRunner(ctx context.Context, name string, args ...string) (int, error) {
+	err := exec.CommandContext(ctx, name, args...).Run()
+	if err == nil {
+		return 0, nil
+	}
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return -1, ctxErr
+	}
+	var exitErr *exec.ExitError
+	if errors.As(err, &exitErr) {
+		return exitErr.ExitCode(), nil
+	}
+	return -1, err
+}
+
+// What the engine reports about itself, which is the only part of the probe
+// that differs between engines. The port is passed in rather than read from the
+// engine, so the atomic below stays the one place it is followed from.
+type probeStateFn func(ctx context.Context, port int64) (engineState, string)
+
+// Engine-neutral: the latch, the port, the degraded mapping and the translation
+// to a reported health are shared, and only the state function is supplied by
+// the engine the image bakes.
+type engineProbe struct {
+	stateFn probeStateFn
+	// Set from the bootstrap config once it lands, from a different goroutine
+	// than the heartbeat that reads it.
+	port atomic.Int64
+	// Latches once the engine has answered. The agent is up well before initdb
+	// finishes, so a down engine is "starting" until it has served once.
+	seenHealthy bool
+}
+
+func newEngineProbe(port int, stateFn probeStateFn) *engineProbe {
+	p := &engineProbe{stateFn: stateFn}
+	p.port.Store(int64(port))
+	return p
+}
+
+func (p *engineProbe) setPort(port int) {
+	p.port.Store(int64(port))
+}
+
+// What the engine is doing, before the seenHealthy latch collapses "never up"
+// and "was up and went away" into one health. The parameter rollback needs them
+// apart: a postmaster that is up and replaying WAL will come back on its own, one
+// that is not running at all after a parameter change will not.
+type engineState int
+
+const (
+	// Not answering at all, or the probe could not run.
+	engineAbsent engineState = iota
+	// The process is up but not yet serving: startup or crash recovery.
+	engineRecovering
+	engineServing
+)
+
+// Only the heartbeat calls this, so the seenHealthy latch is single-goroutine
+// state.
+func (p *engineProbe) Check(ctx context.Context) (handlers_rds.EngineHealth, string) {
+	state, message := p.state(ctx)
+	switch state {
+	case engineServing:
+		p.seenHealthy = true
+		return handlers_rds.EngineHealthHealthy, ""
+	case engineRecovering:
+		return handlers_rds.EngineHealthStarting, message
+	default:
+		return p.degraded(), message
+	}
+}
+
+// The raw probe result. Safe to call from a goroutine other than the
+// heartbeat's: it touches no latched state.
+func (p *engineProbe) state(ctx context.Context) (engineState, string) {
+	return p.stateFn(ctx, p.port.Load())
+}
+
+// A non-answering engine: starting until it has answered once, unhealthy after.
+func (p *engineProbe) degraded() handlers_rds.EngineHealth {
+	if p.seenHealthy {
+		return handlers_rds.EngineHealthUnhealthy
+	}
+	return handlers_rds.EngineHealthStarting
+}

cmd/rds-agent/health_test.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
+	tests := []struct {
+		name        string
+		code        int
+		runErr      error
+		seenHealthy bool
+		want        handlers_rds.EngineHealth
+	}{
+		{name: "accepting connections", code: 0, want: handlers_rds.EngineHealthHealthy},
+		{name: "rejecting during startup", code: 1, want: handlers_rds.EngineHealthStarting},
+		{name: "silent before first success", code: 2, want: handlers_rds.EngineHealthStarting},
+		{name: "silent after first success", code: 2, seenHealthy: true, want: handlers_rds.EngineHealthUnhealthy},
+		{name: "no attempt made", code: 3, want: handlers_rds.EngineHealthStarting},
+		{name: "probe missing before first success", runErr: errors.New("executable not found"), want: handlers_rds.EngineHealthStarting},
+		{name: "probe missing after first success", runErr: errors.New("executable not found"), seenHealthy: true, want: handlers_rds.EngineHealthUnhealthy},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
+				return tt.code, tt.runErr
+			})
+			probe.seenHealthy = tt.seenHealthy
+
+			got, message := probe.Check(context.Background())
+			if got != tt.want {
+				t.Errorf("health = %q, want %q", got, tt.want)
+			}
+			if got != handlers_rds.EngineHealthHealthy && message == "" {
+				t.Error("a non-healthy result carried no message explaining it")
+			}
+			if got == handlers_rds.EngineHealthHealthy && message != "" {
+				t.Errorf("a healthy result carried a message: %q", message)
+			}
+		})
+	}
+}
+
+// The latch is what separates a boot still in progress from a failure, and it
+// has to flip the first time the engine answers.
+func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
+	code := 0
+	probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
+		return code, nil
+	})
+
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthHealthy {
+		t.Fatalf("first check = %q, want healthy", got)
+	}
+	code = 2
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthUnhealthy {
+		t.Errorf("check after the engine went away = %q, want unhealthy", got)
+	}
+}
+
+// The port the engine listens on comes from the bootstrap config, so the probe
+// must follow it rather than keep asking the default.
+func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
+	var gotArgs []string
+	probe := newPostgresProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, error) {
+		gotArgs = args
+		return 0, nil
+	})
+
+	probe.setPort(6543)
+	probe.Check(context.Background())
+
+	if !strings.Contains(strings.Join(gotArgs, " "), "-p 6543") {
+		t.Errorf("probe args = %v, want the assigned port 6543", gotArgs)
+	}
+}
+
+func TestExecProbeRunner_PreservesTheProbeDeadline(t *testing.T) {
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+
+	code, err := execProbeRunner(ctx, "sh", "-c", "while :; do :; done")
+	if code != -1 {
+		t.Errorf("exit code = %d, want -1 for a timed-out probe", code)
+	}
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("error = %v, want the context deadline", err)
+	}
+}
+
+func testProbeConfig() config {
+	return config{EngineHost: defaultEngineHost, EnginePort: engineLayouts[enginePostgres].port}
+}

cmd/rds-agent/heartbeat.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"sync/atomic"
+	"time"
+	"unicode/utf8"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+const (
+	servingParameterRecordTimeout = 15 * time.Second
+	maxBootstrapFailureBytes      = 512
+)
+
+// The beat carries the probe's result, not the agent's own liveness: an agent
+// up while the engine is down is what the recovery reconciler must see.
+type heartbeater struct {
+	cp       controlPlane
+	probe    *engineProbe
+	recorder servingParameterRecorder
+	id       identity
+	interval time.Duration
+
+	// Bootstrap and heartbeat run concurrently. Keeping the latest failure here
+	// makes a VM stuck before engine startup explain why through its next beat.
+	bootstrapFailure atomic.Pointer[string]
+}
+
+func newHeartbeater(cp controlPlane, probe *engineProbe, recorder servingParameterRecorder, interval time.Duration) *heartbeater {
+	if interval <= 0 {
+		interval = handlers_rds.HeartbeatInterval
+	}
+	return &heartbeater{cp: cp, probe: probe, recorder: recorder, interval: interval}
+}
+
+// Called before Run starts and from Run's own goroutine after, so the interval
+// is never written concurrently with the loop reading it.
+func (h *heartbeater) setInterval(d time.Duration) {
+	if d > 0 {
+		h.interval = d
+	}
+}
+
+func (h *heartbeater) setBootstrapFailure(what string, err error) {
+	message := what + " failed: " + err.Error()
+	if len(message) > maxBootstrapFailureBytes {
+		limit := maxBootstrapFailureBytes - len("...")
+		for limit > 0 && !utf8.RuneStart(message[limit]) {
+			limit--
+		}
+		message = message[:limit] + "..."
+	}
+	h.bootstrapFailure.Store(&message)
+}
+
+func (h *heartbeater) clearBootstrapFailure() {
+	h.bootstrapFailure.Store(nil)
+}
+
+func (h *heartbeater) beat(ctx context.Context) {
+	health, message := h.probe.Check(ctx)
+	if failure := h.bootstrapFailure.Load(); failure != nil {
+		if message != "" {
+			message += "; "
+		}
+		message += *failure
+	}
+	if health == handlers_rds.EngineHealthHealthy && h.recorder != nil {
+		// Checking every healthy beat also observes a restart that completed
+		// between probes. The recorder skips unchanged and pending-restart sets.
+		recordCtx, cancel := context.WithTimeout(ctx, servingParameterRecordTimeout)
+		err := h.recorder.RecordServingParameters(recordCtx)
+		cancel()
+		if err != nil {
+			slog.Warn("rds-agent: recording the serving parameters failed", "err", err)
+		}
+	}
+
+	out, err := h.cp.SubmitState(ctx, h.id, health, message)
+	if err != nil {
+		// Not escalated: the control plane already treats a missing heartbeat as
+		// staleness, and the next tick retries.
+		slog.Warn("rds-agent: heartbeat failed", "health", health, "err", err)
+		return
+	}
+	slog.Debug("rds-agent: heartbeat", "health", health, "persisted", out.Persisted)
+	h.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
+}
+
+// A timer rather than a ticker, so a cadence change takes effect on the next
+// beat, not at a restart.
+func (h *heartbeater) Run(ctx context.Context) {
+	timer := time.NewTimer(h.interval)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			h.beat(ctx)
+			timer.Reset(h.interval)
+		}
+	}
+}