feat(installer): three-plane networking, ZFS root, distributed NAT and multi-node install (#663)

build/systemd/spinifex-daemon.service

@@ -7,6 +7,8 @@ Wants=openvswitch-ipsec.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
+# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see
+EnvironmentFile=-/etc/spinifex/host.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-daemon
@@ -19,8 +21,7 @@ KillMode=process
 TimeoutStopSec=15
 # RG-4: infra tier — reaped only after every guest
 OOMScoreAdjust=-500
-# Guest QEMU still runs inside this cgroup, so the ceiling tracks the slice
-# budget (just under 80%/90%), not just daemon overhead.
+# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead
 MemoryHigh=75%
 MemoryMax=85%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml

cmd/installer/autoinstall/autoinstall.go

@@ -4,12 +4,14 @@
 package autoinstall
 
 import (
+	"cmp"
 	"fmt"
 	"log/slog"
 	"net"
 	"os"
 	"os/exec"
-	"sort"
+	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"syscall"
@@ -18,6 +20,10 @@ import (
 	"github.com/mulgadc/spinifex/cmd/installer/install"
 )
 
+// listDisks is the block-device scan, indirected so tests can drive selection
+// without real hardware.
+var listDisks = install.ListDisks
+
 // Load returns an install.Config built from SPINIFEX_* environment variables,
 // or nil if SPINIFEX_AUTO is not set to "1" (interactive mode).
 func Load() (*install.Config, error) {
@@ -65,9 +71,9 @@ func buildConfig() (*install.Config, error) {
 		hostname = "spinifex-node"
 	}
 
-	disk, err := resolveDisk(os.Getenv("SPINIFEX_DISK"))
+	storage, err := resolveStorage()
 	if err != nil {
-		return nil, fmt.Errorf("disk: %w", err)
+		return nil, fmt.Errorf("storage: %w", err)
 	}
 
 	wanIface, err := resolveNIC(os.Getenv("SPINIFEX_WAN_IFACE"), "")
@@ -76,73 +82,167 @@ func buildConfig() (*install.Config, error) {
 	}
 
 	cfg := &install.Config{
-		Disk:         disk,
+		Storage:      storage,
 		Hostname:     hostname,
 		RootPassword: password,
 		// Optional on the headless path — empty means no --email passed to
 		// spx admin init, which omits the operator identity from telemetry.
-		Email:        strings.TrimSpace(os.Getenv("SPINIFEX_EMAIL")),
-		WANInterface: wanIface,
-		WANDHCPMode:  strings.ToLower(os.Getenv("SPINIFEX_WAN_MODE")) != "static",
+		Email: strings.TrimSpace(os.Getenv("SPINIFEX_EMAIL")),
 	}
 
-	if !cfg.WANDHCPMode {
-		ip := os.Getenv("SPINIFEX_WAN_IP")
-		mask := os.Getenv("SPINIFEX_WAN_MASK")
-		gw := os.Getenv("SPINIFEX_WAN_GW")
-		if ip == "" || mask == "" || gw == "" {
-			return nil, fmt.Errorf("SPINIFEX_WAN_IP, SPINIFEX_WAN_MASK, SPINIFEX_WAN_GW required for static mode")
-		}
-		cfg.WANAddress = ip
-		cfg.WANMask = mask
-		cfg.WANGateway = gw
-		if dns := os.Getenv("SPINIFEX_WAN_DNS"); dns != "" {
-			cfg.WANDNS = strings.Split(dns, ",")
-		}
+	cfg.WAN, err = parseRole("WAN", wanIface)
+	if err != nil {
+		return nil, err
+	}
+	if cfg.WAN.DHCPMode {
+		cfg.WAN.Gateway = ""
+	} else if cfg.WAN.Gateway == "" {
+		return nil, fmt.Errorf("SPINIFEX_WAN_IP, SPINIFEX_WAN_MASK, SPINIFEX_WAN_GW required for static mode")
 	}
 
-	if lanIface := os.Getenv("SPINIFEX_LAN_IFACE"); lanIface != "" {
-		lan, err := resolveNIC(lanIface, wanIface)
+	// lan and vpc are optional: an unset _IFACE leaves the role folded, which
+	// is how a single- or two-NIC node collapses onto the planes above it.
+	for _, r := range []struct {
+		plane string
+		dst   *install.NetworkRole
+	}{{"LAN", &cfg.LAN}, {"VPC", &cfg.VPC}} {
+		name := os.Getenv("SPINIFEX_" + r.plane + "_IFACE")
+		if name == "" {
+			continue
+		}
+		iface, err := resolveNIC(name, wanIface)
 		if err != nil {
-			return nil, fmt.Errorf("LAN NIC: %w", err)
+			return nil, fmt.Errorf("%s NIC: %w", r.plane, err)
 		}
-		cfg.LANInterface = lan
-		cfg.LANDHCPMode = strings.ToLower(os.Getenv("SPINIFEX_LAN_MODE")) != "static"
-		if !cfg.LANDHCPMode {
-			cfg.LANAddress = os.Getenv("SPINIFEX_LAN_IP")
-			cfg.LANMask = os.Getenv("SPINIFEX_LAN_MASK")
-			if dns := os.Getenv("SPINIFEX_LAN_DNS"); dns != "" {
-				cfg.LANDNS = strings.Split(dns, ",")
-			}
+		if *r.dst, err = parseRole(r.plane, iface); err != nil {
+			return nil, err
 		}
+		// Only the wan plane installs a default route.
+		r.dst.Gateway = ""
+	}
+
+	if err := cfg.Validate(); err != nil {
+		return nil, fmt.Errorf("network roles: %w", err)
 	}
 
 	if os.Getenv("SPINIFEX_GPU_PASSTHROUGH") == "1" {
 		cfg.GPUPassthrough = true
 	}
 
-	role := strings.ToLower(os.Getenv("SPINIFEX_ROLE"))
-	if role == "" {
-		role = "init"
-	}
-	cfg.ClusterRole = role
-	skipFormation := os.Getenv("SPINIFEX_SKIP_FORMATION") == "1"
-	if skipFormation {
+	if os.Getenv("SPINIFEX_SKIP_FORMATION") == "1" {
 		cfg.SkipFormation = true
 	}
 
-	if role == "join" && !skipFormation {
-		joinAddr := os.Getenv("SPINIFEX_JOIN_ADDR")
-		if joinAddr == "" {
-			return nil, fmt.Errorf("SPINIFEX_JOIN_ADDR required when SPINIFEX_ROLE=join")
-		}
-		cfg.JoinAddr = joinAddr
-	}
-
 	return cfg, nil
 }
 
-// resolveDisk maps the spx_disk value from grub.cfg to a block device path.
+// resolveStorage builds the disk configuration from the kernel cmdline.
+//
+//	SPINIFEX_FS     ext4 (default) or one of the zfs-* topologies
+//	SPINIFEX_DISKS  explicit ordered member list, required for multi-disk pools
+//	SPINIFEX_DISK   single-disk selector, as before
+func resolveStorage() (install.DiskConfig, error) {
+	cfg := install.DiskConfig{FS: install.FSExt4}
+	if v := strings.TrimSpace(os.Getenv("SPINIFEX_FS")); v != "" {
+		fs, err := install.ParseFSType(v)
+		if err != nil {
+			return cfg, err
+		}
+		cfg.FS = fs
+	}
+
+	var err error
+	switch list := strings.TrimSpace(os.Getenv("SPINIFEX_DISKS")); {
+	case list != "":
+		if cfg.Disks, err = disksByName(strings.Split(list, ",")); err != nil {
+			return cfg, err
+		}
+	case cfg.FS.MinDisks() > 1:
+		// No auto-selection for a multi-disk pool. Choosing which disks to
+		// erase is not a decision an unattended install gets to make, and the
+		// member order determines the RAID10 pairing.
+		return cfg, fmt.Errorf("%s needs at least %d disks — set SPINIFEX_DISKS to an explicit comma-separated list",
+			cfg.FS.Label(), cfg.FS.MinDisks())
+	default:
+		d, derr := resolveDisk(os.Getenv("SPINIFEX_DISK"))
+		if derr != nil {
+			return cfg, derr
+		}
+		cfg.Disks = []install.Disk{d}
+	}
+
+	if cfg.ZFS, err = parseZFSOpts(); err != nil {
+		return cfg, err
+	}
+	if err := cfg.Validate(); err != nil {
+		return cfg, err
+	}
+	slog.Info("autoinstall: storage selected", "fs", cfg.FS, "disks", cfg.Paths())
+	return cfg, nil
+}
+
+// parseZFSOpts reads the advanced pool tunables. Every one is optional; unset
+// values are computed from the hardware at install time.
+func parseZFSOpts() (install.ZFSOpts, error) {
+	var o install.ZFSOpts
+	ints := []struct {
+		env string
+		dst *int
+	}{
+		{"SPINIFEX_ZFS_ASHIFT", &o.Ashift},
+		{"SPINIFEX_ZFS_COPIES", &o.Copies},
+		{"SPINIFEX_ZFS_ARC_MAX_MIB", &o.ARCMaxMiB},
+	}
+	for _, f := range ints {
+		v := strings.TrimSpace(os.Getenv(f.env))
+		if v == "" {
+			continue
+		}
+		n, err := strconv.Atoi(v)
+		if err != nil {
+			return o, fmt.Errorf("%s: %w", f.env, err)
+		}
+		*f.dst = n
+	}
+	o.Compress = strings.TrimSpace(os.Getenv("SPINIFEX_ZFS_COMPRESS"))
+	o.Checksum = strings.TrimSpace(os.Getenv("SPINIFEX_ZFS_CHECKSUM"))
+	return o, nil
+}
+
+// disksByName resolves an explicit member list. Names may be kernel names
+// ("sdb"), device paths or by-id paths; the order given is preserved because
+// RAID10 pairs members in it.
+func disksByName(names []string) ([]install.Disk, error) {
+	available, err := listDisks()
+	if err != nil {
+		return nil, err
+	}
+	out := make([]install.Disk, 0, len(names))
+	for _, raw := range names {
+		name := strings.TrimSpace(raw)
+		if name == "" {
+			continue
+		}
+		// A by-id path or a symlink resolves to the kernel device the scan
+		// reported, so both spellings name the same disk.
+		resolved := name
+		if !strings.HasPrefix(resolved, "/dev/") {
+			resolved = "/dev/" + resolved
+		}
+		if target, err := filepath.EvalSymlinks(resolved); err == nil {
+			resolved = target
+		}
+
+		idx := slices.IndexFunc(available, func(d install.Disk) bool { return d.Path == resolved })
+		if idx < 0 {
+			return nil, fmt.Errorf("%q is not an available disk — found:\n%s", name, diskList(available))
+		}
+		out = append(out, available[idx])
+	}
+	return out, nil
+}
+
+// resolveDisk maps the SPINIFEX_DISK value from grub.cfg to a disk.
 //
 // Supported values:
 //
@@ -151,141 +251,78 @@ func buildConfig() (*install.Config, error) {
 //	smallest        — smallest non-removable disk (typical OS-on-SSD pattern)
 //	nvme            — the only NVMe disk; fail if multiple found
 //	/dev/sda (etc.) — exact device path; fail if not present
-func resolveDisk(target string) (string, error) {
-	switch strings.ToLower(target) {
+func resolveDisk(target string) (install.Disk, error) {
+	disks, err := candidateDisks()
+	if err != nil {
+		return install.Disk{}, err
+	}
+	switch strings.ToLower(strings.TrimSpace(target)) {
 	case "", "auto":
-		return singleDisk()
-	case "largest":
-		return diskBySize(true)
-	case "smallest":
-		return diskBySize(false)
-	case "nvme":
-		return nvmeDisk()
-	default:
-		if _, err := os.Stat(target); err != nil {
-			return "", fmt.Errorf("%q not found", target)
+		if len(disks) != 1 {
+			return install.Disk{}, fmt.Errorf(
+				"expected exactly one disk, found %d — set SPINIFEX_DISK to one of:\n%s\n"+
+					"  largest   (largest disk)\n"+
+					"  smallest  (smallest disk)\n"+
+					"  nvme      (NVMe only)\n"+
+					"  /dev/sdX  (exact path)",
+				len(disks), diskList(disks))
 		}
-		return target, nil
+		return disks[0], nil
+	case "largest":
+		return pickBySize(disks, true)
+	case "smallest":
+		return pickBySize(disks, false)
+	case "nvme":
+		nvmes := slices.DeleteFunc(slices.Clone(disks), func(d install.Disk) bool {
+			return !strings.HasPrefix(filepath.Base(d.Path), "nvme")
+		})
+		if len(nvmes) != 1 {
+			return install.Disk{}, fmt.Errorf("expected exactly one NVMe disk, found %d:\n%s",
+				len(nvmes), diskList(nvmes))
+		}
+		return nvmes[0], nil
+	default:
+		found, err := disksByName([]string{target})
+		if err != nil {
+			return install.Disk{}, err
+		}
+		return found[0], nil
 	}
 }
 
-type diskCandidate struct {
-	dev   string
-	bytes int64
-}
-
-// nonRemovableDisks returns all non-removable, non-virtual block devices.
-func nonRemovableDisks() ([]diskCandidate, error) {
-	entries, err := os.ReadDir("/sys/block")
+// candidateDisks lists the disks an unattended install may erase: fixed, and
+// not the installer's own media.
+func candidateDisks() ([]install.Disk, error) {
+	all, err := listDisks()
 	if err != nil {
 		return nil, err
 	}
-	var out []diskCandidate
-	for _, e := range entries {
-		dev := e.Name()
-		switch {
-		case strings.HasPrefix(dev, "loop"),
-			strings.HasPrefix(dev, "ram"),
-			strings.HasPrefix(dev, "dm-"),
-			strings.HasPrefix(dev, "sr"):
-			continue
-		}
-		removable, _ := os.ReadFile("/sys/block/" + dev + "/removable")
-		if strings.TrimSpace(string(removable)) == "1" {
-			continue
-		}
-		sizeRaw, _ := os.ReadFile("/sys/block/" + dev + "/size")
-		sectors, parseErr := strconv.ParseInt(strings.TrimSpace(string(sizeRaw)), 10, 64)
-		if parseErr != nil || sectors == 0 {
-			continue
-		}
-		out = append(out, diskCandidate{dev: dev, bytes: sectors * 512})
-	}
-	return out, nil
+	return slices.DeleteFunc(all, func(d install.Disk) bool {
+		return d.Removable || d.LiveMedia
+	}), nil
 }
 
-// diskList returns a human-readable list of candidates for error messages.
-func diskList(disks []diskCandidate) string {
+func pickBySize(disks []install.Disk, largest bool) (install.Disk, error) {
+	if len(disks) == 0 {
+		return install.Disk{}, fmt.Errorf("no non-removable disks found")
+	}
+	sorted := slices.Clone(disks)
+	slices.SortFunc(sorted, func(a, b install.Disk) int { return cmp.Compare(a.Bytes, b.Bytes) })
+	if largest {
+		return sorted[len(sorted)-1], nil
+	}
+	return sorted[0], nil
+}
+
+// diskList renders candidates for an error message.
+func diskList(disks []install.Disk) string {
 	lines := make([]string, len(disks))
 	for i, d := range disks {
-		lines[i] = fmt.Sprintf("  /dev/%s (%dG)", d.dev, d.bytes>>30)
+		lines[i] = fmt.Sprintf("  %s (%s, %s)", d.Path, d.SizeHuman(), d.Content)
 	}
 	return strings.Join(lines, "\n")
 }
 
-// singleDisk selects the only non-removable disk, or fails listing all found.
-func singleDisk() (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	switch len(disks) {
-	case 0:
-		return "", fmt.Errorf("no non-removable disks found")
-	case 1:
-		slog.Info("autoinstall: single disk selected", "disk", disks[0].dev, "size_gb", disks[0].bytes>>30)
-		return "/dev/" + disks[0].dev, nil
-	default:
-		return "", fmt.Errorf(
-			"multiple disks found — set SPINIFEX_DISK to one of:\n%s\n"+
-				"  largest   (largest disk)\n"+
-				"  smallest  (smallest disk)\n"+
-				"  nvme      (NVMe only)\n"+
-				"  /dev/sdX  (exact path)",
-			diskList(disks),
-		)
-	}
-}
-
-// diskBySize selects the largest (largest=true) or smallest non-removable disk.
-func diskBySize(largest bool) (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	if len(disks) == 0 {
-		return "", fmt.Errorf("no non-removable disks found")
-	}
-	sort.Slice(disks, func(i, j int) bool {
-		if largest {
-			return disks[i].bytes > disks[j].bytes
-		}
-		return disks[i].bytes < disks[j].bytes
-	})
-	label := "smallest"
-	if largest {
-		label = "largest"
-	}
-	slog.Info("autoinstall: disk selected by size", "mode", label, "disk", disks[0].dev, "size_gb", disks[0].bytes>>30)
-	return "/dev/" + disks[0].dev, nil
-}
-
-// nvmeDisk selects the only NVMe disk, or fails listing all NVMe disks found.
-func nvmeDisk() (string, error) {
-	disks, err := nonRemovableDisks()
-	if err != nil {
-		return "", err
-	}
-	var nvmes []diskCandidate
-	for _, d := range disks {
-		if strings.HasPrefix(d.dev, "nvme") {
-			nvmes = append(nvmes, d)
-		}
-	}
-	switch len(nvmes) {
-	case 0:
-		return "", fmt.Errorf("no NVMe disks found")
-	case 1:
-		slog.Info("autoinstall: NVMe disk selected", "disk", nvmes[0].dev, "size_gb", nvmes[0].bytes>>30)
-		return "/dev/" + nvmes[0].dev, nil
-	default:
-		return "", fmt.Errorf(
-			"multiple NVMe disks found — set SPINIFEX_DISK to one of:\n%s",
-			diskList(nvmes),
-		)
-	}
-}
-
 // virtualNICPrefixes lists interface name prefixes that identify non-physical
 // interfaces (bridges, tunnels, container/OVS veth pairs, etc.). These are
 // skipped when auto-selecting a NIC so we don't configure docker0 or
@@ -297,6 +334,42 @@ var virtualNICPrefixes = []string{
 
 // resolveNIC returns the interface name to use. "auto" or empty picks the
 // first physical (non-loopback, non-virtual) interface that is not exclude.
+// parseRole reads the SPINIFEX_<PLANE>_* kernel-cmdline vars for one plane.
+// iface is the already-resolved interface name; _VLAN and _MTU are optional
+// and default to untagged at the link MTU.
+func parseRole(plane, iface string) (install.NetworkRole, error) {
+	role := install.NetworkRole{
+		Interface: iface,
+		DHCPMode:  strings.ToLower(os.Getenv("SPINIFEX_"+plane+"_MODE")) != "static",
+	}
+	if v := os.Getenv("SPINIFEX_" + plane + "_VLAN"); v != "" {
+		id, err := strconv.Atoi(v)
+		if err != nil {
+			return role, fmt.Errorf("SPINIFEX_%s_VLAN: %w", plane, err)
+		}
+		role.VLAN = id
+	}
+	if v := os.Getenv("SPINIFEX_" + plane + "_MTU"); v != "" {
+		mtu, err := strconv.Atoi(v)
+		if err != nil {
+			return role, fmt.Errorf("SPINIFEX_%s_MTU: %w", plane, err)
+		}
+		role.MTU = mtu
+	}
+	if !role.DHCPMode {
+		role.Address = os.Getenv("SPINIFEX_" + plane + "_IP")
+		role.Mask = os.Getenv("SPINIFEX_" + plane + "_MASK")
+		role.Gateway = os.Getenv("SPINIFEX_" + plane + "_GW")
+		if role.Address == "" || role.Mask == "" {
+			return role, fmt.Errorf("SPINIFEX_%s_IP and SPINIFEX_%s_MASK required for static mode", plane, plane)
+		}
+	}
+	if dns := os.Getenv("SPINIFEX_" + plane + "_DNS"); dns != "" {
+		role.DNS = strings.Split(dns, ",")
+	}
+	return role, nil
+}
+
 func resolveNIC(name, exclude string) (string, error) {
 	if name != "" && name != "auto" {
 		return name, nil

cmd/installer/autoinstall/autoinstall_test.go

@@ -1,12 +1,57 @@
 package autoinstall
 
 import (
+	"errors"
+	"fmt"
+	"maps"
+	"net"
 	"os"
+	"slices"
+	"strings"
 	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
 )
 
+const gib = 1 << 30
+
+// setEnv clears every SPINIFEX_* variable inherited from the test runner before
+// applying want, so one case cannot leak a kernel-cmdline value into the next.
+func setEnv(t *testing.T, want map[string]string) {
+	t.Helper()
+	for _, kv := range os.Environ() {
+		if k, _, ok := strings.Cut(kv, "="); ok && strings.HasPrefix(k, "SPINIFEX_") {
+			t.Setenv(k, "")
+		}
+	}
+	for k, v := range want {
+		t.Setenv(k, v)
+	}
+}
+
+// withDisks replaces the block-device scan. The clone matters: candidateDisks
+// filters with slices.DeleteFunc, which would otherwise mutate the fixture.
+func withDisks(t *testing.T, disks []install.Disk, err error) {
+	t.Helper()
+	prev := listDisks
+	listDisks = func() ([]install.Disk, error) { return slices.Clone(disks), err }
+	t.Cleanup(func() { listDisks = prev })
+}
+
+func mkDisk(path string, bytes int64) install.Disk {
+	return install.Disk{
+		Path:              path,
+		ByID:              "/dev/disk/by-id/test-" + strings.TrimPrefix(path, "/dev/"),
+		Bytes:             bytes,
+		Model:             "TESTDISK",
+		LogicalBlockSize:  512,
+		PhysicalBlockSize: 512,
+		Content:           "empty",
+	}
+}
+
 func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
-	os.Unsetenv("SPINIFEX_AUTO")
+	setEnv(t, nil)
 	cfg, err := Load()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -15,3 +60,586 @@ func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
 		t.Fatal("expected nil config when SPINIFEX_AUTO is not set")
 	}
 }
+
+func TestLoadBuildsConfigFromEnv(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_AUTO":            "1",
+		"SPINIFEX_PASSWORD":        "s3cret",
+		"SPINIFEX_HOSTNAME":        "node-7",
+		"SPINIFEX_EMAIL":           "  ops@example.com  ",
+		"SPINIFEX_WAN_IFACE":       "eno1",
+		"SPINIFEX_WAN_DNS":         "1.1.1.1,8.8.8.8",
+		"SPINIFEX_LAN_IFACE":       "eno2",
+		"SPINIFEX_LAN_VLAN":        "100",
+		"SPINIFEX_LAN_MTU":         "9000",
+		"SPINIFEX_GPU_PASSTHROUGH": "1",
+	})
+
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.Hostname != "node-7" || cfg.RootPassword != "s3cret" {
+		t.Errorf("identity = %q/%q", cfg.Hostname, cfg.RootPassword)
+	}
+	if cfg.Email != "ops@example.com" {
+		t.Errorf("Email = %q, want trimmed", cfg.Email)
+	}
+	if cfg.Storage.FS != install.FSExt4 || len(cfg.Storage.Disks) != 1 {
+		t.Errorf("Storage = %v / %d disks", cfg.Storage.FS, len(cfg.Storage.Disks))
+	}
+	if cfg.WAN.Interface != "eno1" || !cfg.WAN.DHCPMode {
+		t.Errorf("WAN = %+v, want eno1 on DHCP", cfg.WAN)
+	}
+	if !slices.Equal(cfg.WAN.DNS, []string{"1.1.1.1", "8.8.8.8"}) {
+		t.Errorf("WAN.DNS = %v", cfg.WAN.DNS)
+	}
+	if cfg.LAN.Interface != "eno2" || cfg.LAN.VLAN != 100 || cfg.LAN.MTU != 9000 {
+		t.Errorf("LAN = %+v", cfg.LAN)
+	}
+	// Only the wan plane installs a default route.
+	if cfg.LAN.Gateway != "" {
+		t.Errorf("LAN.Gateway = %q, want cleared", cfg.LAN.Gateway)
+	}
+	if !cfg.VPC.Folded() {
+		t.Errorf("VPC = %+v, want folded when no _IFACE is set", cfg.VPC)
+	}
+	if !cfg.GPUPassthrough {
+		t.Error("GPUPassthrough not enabled")
+	}
+}
+
+func TestLoadDefaultsHostname(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_AUTO":      "1",
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+	})
+
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.Hostname != "spinifex-node" {
+		t.Errorf("Hostname = %q, want spinifex-node", cfg.Hostname)
+	}
+}
+
+func TestLoadRequiresPassword(t *testing.T) {
+	setEnv(t, map[string]string{"SPINIFEX_AUTO": "1"})
+
+	cfg, err := Load()
+	if err == nil {
+		t.Fatalf("expected error, got config %+v", cfg)
+	}
+	if !strings.Contains(err.Error(), "SPINIFEX_PASSWORD") {
+		t.Errorf("err = %v, want it to name SPINIFEX_PASSWORD", err)
+	}
+	if !strings.HasPrefix(err.Error(), "autoinstall config:") {
+		t.Errorf("err = %v, want the autoinstall wrapper", err)
+	}
+}
+
+func TestBuildConfigStaticWAN(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     map[string]string
+		wantErr string
+	}{
+		{
+			name: "full static",
+			env: map[string]string{
+				"SPINIFEX_WAN_MODE": "static",
+				"SPINIFEX_WAN_IP":   "10.0.0.5",
+				"SPINIFEX_WAN_MASK": "255.255.255.0",
+				"SPINIFEX_WAN_GW":   "10.0.0.1",
+			},
+		},
+		{
+			name: "missing gateway",
+			env: map[string]string{
+				"SPINIFEX_WAN_MODE": "static",
+				"SPINIFEX_WAN_IP":   "10.0.0.5",
+				"SPINIFEX_WAN_MASK": "255.255.255.0",
+			},
+			wantErr: "SPINIFEX_WAN_GW required",
+		},
+		{
+			name:    "missing address",
+			env:     map[string]string{"SPINIFEX_WAN_MODE": "static"},
+			wantErr: "SPINIFEX_WAN_IP and SPINIFEX_WAN_MASK required",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+			env := map[string]string{
+				"SPINIFEX_PASSWORD":  "s3cret",
+				"SPINIFEX_WAN_IFACE": "eno1",
+			}
+			maps.Copy(env, tt.env)
+			setEnv(t, env)
+
+			cfg, err := buildConfig()
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("buildConfig: %v", err)
+			}
+			if cfg.WAN.DHCPMode {
+				t.Error("WAN.DHCPMode = true, want static")
+			}
+			if cfg.WAN.Address != "10.0.0.5" || cfg.WAN.Gateway != "10.0.0.1" {
+				t.Errorf("WAN = %+v", cfg.WAN)
+			}
+		})
+	}
+}
+
+// SPINIFEX_ROLE and SPINIFEX_JOIN_ADDR no longer exist: the installer cannot
+// form a multi-node cluster, so a stale cmdline from an older ISO has to be
+// inert rather than producing a half-configured node.
+func TestBuildConfigSkipFormation(t *testing.T) {
+	tests := []struct {
+		name         string
+		env          map[string]string
+		wantSkipForm bool
+	}{
+		{
+			name:         "default initializes a single-node cluster",
+			wantSkipForm: false,
+		},
+		{
+			name:         "formation deferred to the provisioner",
+			env:          map[string]string{"SPINIFEX_SKIP_FORMATION": "1"},
+			wantSkipForm: true,
+		},
+		{
+			name:         "stale join cmdline is ignored",
+			env:          map[string]string{"SPINIFEX_ROLE": "join", "SPINIFEX_JOIN_ADDR": "10.0.0.1:9999"},
+			wantSkipForm: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+			env := map[string]string{
+				"SPINIFEX_PASSWORD":  "s3cret",
+				"SPINIFEX_WAN_IFACE": "eno1",
+			}
+			maps.Copy(env, tt.env)
+			setEnv(t, env)
+
+			cfg, err := buildConfig()
+			if err != nil {
+				t.Fatalf("buildConfig: %v", err)
+			}
+			if cfg.SkipFormation != tt.wantSkipForm {
+				t.Errorf("SkipFormation = %v, want %v", cfg.SkipFormation, tt.wantSkipForm)
+			}
+		})
+	}
+}
+
+func TestBuildConfigRejectsDuplicatePlaneBinding(t *testing.T) {
+	withDisks(t, []install.Disk{mkDisk("/dev/vdb", 40*gib)}, nil)
+	setEnv(t, map[string]string{
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+		"SPINIFEX_LAN_IFACE": "eno1",
+	})
+
+	if _, err := buildConfig(); err == nil || !strings.Contains(err.Error(), "network roles") {
+		t.Fatalf("err = %v, want a network roles validation failure", err)
+	}
+}
+
+func TestBuildConfigPropagatesStorageError(t *testing.T) {
+	setEnv(t, map[string]string{
+		"SPINIFEX_PASSWORD":  "s3cret",
+		"SPINIFEX_WAN_IFACE": "eno1",
+		"SPINIFEX_FS":        "btrfs",
+	})
+
+	_, err := buildConfig()
+	if err == nil || !strings.HasPrefix(err.Error(), "storage:") {
+		t.Fatalf("err = %v, want a storage: prefix", err)
+	}
+}
+
+func TestParseRole(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     map[string]string
+		want    install.NetworkRole
+		wantErr string
+	}{
+		{
+			name: "dhcp by default",
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true},
+		},
+		{
+			name: "vlan and mtu",
+			env:  map[string]string{"SPINIFEX_LAN_VLAN": "42", "SPINIFEX_LAN_MTU": "9000"},
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true, VLAN: 42, MTU: 9000},
+		},
+		{
+			name: "dns list is comma separated",
+			env:  map[string]string{"SPINIFEX_LAN_DNS": "1.1.1.1,9.9.9.9"},
+			want: install.NetworkRole{Interface: "eno1", DHCPMode: true, DNS: []string{"1.1.1.1", "9.9.9.9"}},
+		},
+		{
+			name: "static",
+			env: map[string]string{
+				"SPINIFEX_LAN_MODE": "STATIC",
+				"SPINIFEX_LAN_IP":   "192.168.1.2",
+				"SPINIFEX_LAN_MASK": "255.255.255.0",
+				"SPINIFEX_LAN_GW":   "192.168.1.1",
+			},
+			want: install.NetworkRole{
+				Interface: "eno1",
+				Address:   "192.168.1.2",
+				Mask:      "255.255.255.0",
+				Gateway:   "192.168.1.1",
+			},
+		},
+		{
+			name:    "static without mask",
+			env:     map[string]string{"SPINIFEX_LAN_MODE": "static", "SPINIFEX_LAN_IP": "192.168.1.2"},
+			wantErr: "SPINIFEX_LAN_IP and SPINIFEX_LAN_MASK required",
+		},
+		{
+			name:    "unparsable vlan",
+			env:     map[string]string{"SPINIFEX_LAN_VLAN": "trunk"},
+			wantErr: "SPINIFEX_LAN_VLAN",
+		},
+		{
+			name:    "unparsable mtu",
+			env:     map[string]string{"SPINIFEX_LAN_MTU": "jumbo"},
+			wantErr: "SPINIFEX_LAN_MTU",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			setEnv(t, tt.env)
+
+			got, err := parseRole("LAN", "eno1")
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("parseRole: %v", err)
+			}
+			if fmt.Sprintf("%+v", got) != fmt.Sprintf("%+v", tt.want) {
+				t.Errorf("role = %+v, want %+v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseZFSOpts(t *testing.T) {
+	t.Run("unset leaves the computed defaults", func(t *testing.T) {
+		setEnv(t, nil)
+		got, err := parseZFSOpts()
+		if err != nil {
+			t.Fatalf("parseZFSOpts: %v", err)
+		}
+		if got != (install.ZFSOpts{}) {
+			t.Errorf("opts = %+v, want zero", got)
+		}
+	})
+
+	t.Run("all tunables", func(t *testing.T) {
+		setEnv(t, map[string]string{
+			"SPINIFEX_ZFS_ASHIFT":      " 12 ",
+			"SPINIFEX_ZFS_COPIES":      "2",
+			"SPINIFEX_ZFS_ARC_MAX_MIB": "4096",
+			"SPINIFEX_ZFS_COMPRESS":    " lz4 ",
+			"SPINIFEX_ZFS_CHECKSUM":    "blake3",
+		})
+		got, err := parseZFSOpts()
+		if err != nil {
+			t.Fatalf("parseZFSOpts: %v", err)
+		}
+		want := install.ZFSOpts{Ashift: 12, Copies: 2, ARCMaxMiB: 4096, Compress: "lz4", Checksum: "blake3"}
+		if got != want {
+			t.Errorf("opts = %+v, want %+v", got, want)
+		}
+	})
+
+	for _, env := range []string{"SPINIFEX_ZFS_ASHIFT", "SPINIFEX_ZFS_COPIES", "SPINIFEX_ZFS_ARC_MAX_MIB"} {
+		t.Run("unparsable "+env, func(t *testing.T) {
+			setEnv(t, map[string]string{env: "auto"})
+			if _, err := parseZFSOpts(); err == nil || !strings.Contains(err.Error(), env) {
+				t.Fatalf("err = %v, want it to name %s", err, env)
+			}
+		})
+	}
+}
+
+func TestResolveStorage(t *testing.T) {
+	four := []install.Disk{
+		mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 40*gib),
+		mkDisk("/dev/vdd", 40*gib), mkDisk("/dev/vde", 40*gib),
+	}
+
+	tests := []struct {
+		name      string
+		disks     []install.Disk
+		env       map[string]string
+		wantFS    install.FSType
+		wantPaths []string
+		wantErr   string
+	}{
+		{
+			name:      "ext4 single disk by default",
+			disks:     four[:1],
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb"},
+		},
+		{
+			name:      "explicit member list preserves order",
+			disks:     four,
+			env:       map[string]string{"SPINIFEX_FS": "zfs-raid10", "SPINIFEX_DISKS": "vde,/dev/vdd,vdc, vdb "},
+			wantFS:    install.FSZFSRAID10,
+			wantPaths: []string{"/dev/vde", "/dev/vdd", "/dev/vdc", "/dev/vdb"},
+		},
+		{
+			name:    "unknown filesystem",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_FS": "btrfs"},
+			wantErr: "unknown filesystem",
+		},
+		{
+			// Choosing which disks to erase is not an unattended decision.
+			name:    "multi-disk pool without an explicit list",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_FS": "zfs-raid1"},
+			wantErr: "set SPINIFEX_DISKS to an explicit comma-separated list",
+		},
+		{
+			name:    "member that is not present",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_DISKS": "vdz"},
+			wantErr: `"vdz" is not an available disk`,
+		},
+		{
+			name:    "member list that fails validation",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_DISKS": "vdb,vdc"},
+			wantErr: "supports a single disk only",
+		},
+		{
+			name:    "bad zfs tunable",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_ZFS_ASHIFT": "big"},
+			wantErr: "SPINIFEX_ZFS_ASHIFT",
+		},
+		{
+			name:    "scan failure",
+			disks:   nil,
+			env:     map[string]string{"SPINIFEX_DISKS": "vdb"},
+			wantErr: "sysfs unavailable",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var scanErr error
+			if tt.disks == nil {
+				scanErr = errors.New("sysfs unavailable")
+			}
+			withDisks(t, tt.disks, scanErr)
+			setEnv(t, tt.env)
+
+			got, err := resolveStorage()
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveStorage: %v", err)
+			}
+			if got.FS != tt.wantFS {
+				t.Errorf("FS = %v, want %v", got.FS, tt.wantFS)
+			}
+			if !slices.Equal(got.Paths(), tt.wantPaths) {
+				t.Errorf("Paths = %v, want %v", got.Paths(), tt.wantPaths)
+			}
+		})
+	}
+}
+
+func TestResolveDisk(t *testing.T) {
+	small := mkDisk("/dev/vdb", 40*gib)
+	large := mkDisk("/dev/vdc", 400*gib)
+	nvme := mkDisk("/dev/nvme0n1", 200*gib)
+
+	tests := []struct {
+		name     string
+		disks    []install.Disk
+		target   string
+		wantPath string
+		wantErr  string
+	}{
+		{name: "auto with one candidate", disks: []install.Disk{small}, target: "auto", wantPath: "/dev/vdb"},
+		{name: "empty target is auto", disks: []install.Disk{small}, wantPath: "/dev/vdb"},
+		{
+			name:    "auto with several candidates",
+			disks:   []install.Disk{small, large},
+			target:  "auto",
+			wantErr: "expected exactly one disk, found 2",
+		},
+		{name: "largest", disks: []install.Disk{small, large, nvme}, target: "LARGEST", wantPath: "/dev/vdc"},
+		{name: "smallest", disks: []install.Disk{large, small, nvme}, target: " smallest ", wantPath: "/dev/vdb"},
+		{name: "nvme", disks: []install.Disk{small, large, nvme}, target: "nvme", wantPath: "/dev/nvme0n1"},
+		{
+			name:    "nvme with none present",
+			disks:   []install.Disk{small, large},
+			target:  "nvme",
+			wantErr: "expected exactly one NVMe disk, found 0",
+		},
+		{name: "exact path", disks: []install.Disk{small, large}, target: "/dev/vdc", wantPath: "/dev/vdc"},
+		{
+			name:    "path that is not present",
+			disks:   []install.Disk{small},
+			target:  "/dev/vdz",
+			wantErr: "is not an available disk",
+		},
+		{name: "no candidates at all", disks: []install.Disk{}, target: "largest", wantErr: "no non-removable disks found"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withDisks(t, tt.disks, nil)
+
+			got, err := resolveDisk(tt.target)
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("err = %v, want %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveDisk(%q): %v", tt.target, err)
+			}
+			if got.Path != tt.wantPath {
+				t.Errorf("path = %q, want %q", got.Path, tt.wantPath)
+			}
+		})
+	}
+}
+
+func TestCandidateDisksExcludesRemovableAndLiveMedia(t *testing.T) {
+	fixed := mkDisk("/dev/vdb", 40*gib)
+	usb := mkDisk("/dev/sdz", 16*gib)
+	usb.Removable = true
+	boot := mkDisk("/dev/sdy", 16*gib)
+	boot.LiveMedia = true
+	withDisks(t, []install.Disk{usb, fixed, boot}, nil)
+
+	got, err := candidateDisks()
+	if err != nil {
+		t.Fatalf("candidateDisks: %v", err)
+	}
+	if len(got) != 1 || got[0].Path != "/dev/vdb" {
+		t.Fatalf("candidates = %v, want only /dev/vdb", got)
+	}
+}
+
+func TestCandidateDisksPropagatesScanError(t *testing.T) {
+	withDisks(t, nil, errors.New("read /sys/block: denied"))
+	if _, err := candidateDisks(); err == nil {
+		t.Fatal("expected the scan error to propagate")
+	}
+}
+
+func TestPickBySizeRejectsEmptySet(t *testing.T) {
+	if _, err := pickBySize(nil, true); err == nil {
+		t.Fatal("expected an error for an empty disk set")
+	}
+}
+
+func TestPickBySizeLeavesInputOrderIntact(t *testing.T) {
+	disks := []install.Disk{mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 400*gib)}
+	if _, err := pickBySize(disks, false); err != nil {
+		t.Fatalf("pickBySize: %v", err)
+	}
+	if disks[0].Path != "/dev/vdb" {
+		t.Errorf("input reordered: %v", disks)
+	}
+}
+
+func TestDiskList(t *testing.T) {
+	got := diskList([]install.Disk{mkDisk("/dev/vdb", 40*gib), mkDisk("/dev/vdc", 2048*gib)})
+	want := "  /dev/vdb (40.0G, empty)\n  /dev/vdc (2.0T, empty)"
+	if got != want {
+		t.Errorf("diskList =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestResolveNICExplicitName(t *testing.T) {
+	got, err := resolveNIC("eno1", "")
+	if err != nil {
+		t.Fatalf("resolveNIC: %v", err)
+	}
+	if got != "eno1" {
+		t.Errorf("nic = %q, want eno1", got)
+	}
+}
+
+// The auto path reads the host's real interfaces, so assert the selection rules
+// rather than a specific name.
+func TestResolveNICAutoSkipsVirtualInterfaces(t *testing.T) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		t.Skipf("no interface list on this host: %v", err)
+	}
+
+	got, err := resolveNIC("auto", "")
+	if err != nil {
+		if len(ifaces) > 0 {
+			t.Logf("no physical NIC among %d interfaces: %v", len(ifaces), err)
+		}
+		return
+	}
+
+	idx := slices.IndexFunc(ifaces, func(i net.Interface) bool { return i.Name == got })
+	if idx < 0 {
+		t.Fatalf("resolveNIC returned %q, which is not a host interface", got)
+	}
+	sel := ifaces[idx]
+	if sel.Flags&net.FlagLoopback != 0 || sel.Flags&net.FlagBroadcast == 0 || len(sel.HardwareAddr) == 0 {
+		t.Errorf("selected %q with flags %v and MAC %q", got, sel.Flags, sel.HardwareAddr)
+	}
+	for _, pfx := range virtualNICPrefixes {
+		if strings.HasPrefix(got, pfx) {
+			t.Errorf("selected virtual interface %q (prefix %q)", got, pfx)
+		}
+	}
+}
+
+func TestResolveNICAutoHonoursExclude(t *testing.T) {
+	first, err := resolveNIC("auto", "")
+	if err != nil {
+		t.Skipf("no physical NIC on this host: %v", err)
+	}
+	got, err := resolveNIC("", first)
+	if err == nil && got == first {
+		t.Errorf("resolveNIC returned the excluded interface %q", first)
+	}
+}

cmd/installer/firstboot/firstboot.go

@@ -14,14 +14,23 @@ import (
 // Config holds the values the firstboot service needs to configure the node.
 type Config struct {
 	Hostname string
-	// EncapIP is the Geneve tunnel IP for OVN. Set to the LAN bridge IP when a
-	// dedicated LAN NIC is present, otherwise the WAN bridge IP. Empty when DHCP
-	// is used — setup-ovn.sh auto-detects the IP from the default route in that case.
+	// EncapIP is the Geneve tunnel IP for OVN, taken from the vpc plane after
+	// collapsing (vpc <- lan <- wan). Empty when that plane uses DHCP —
+	// setup-ovn.sh auto-detects the IP from the default route in that case.
 	EncapIP string
-	// ClusterRole is "init" or "join".
-	ClusterRole string
-	// JoinAddr is host:port of the primary node, only used when ClusterRole is "join".
-	JoinAddr string
+	// LANIP is the internal cluster address, taken from the lan plane after
+	// collapsing. Passed as --bind and --cluster-bind so predastore replication,
+	// the NATS mesh and OVN control traffic stay off the public interface.
+	// Empty leaves the 0.0.0.0 wildcard default, which is correct for a
+	// single-NIC node where lan folds onto wan.
+	LANIP string
+	// WANIP is the public address, taken from the wan plane. Passed as
+	// --advertise so northstar's :53 listener, the awsgw registry host and the
+	// dial target recorded for off-host clients stay on the public interface.
+	// Without it a concrete --bind is echoed back as the advertise address,
+	// which would move all three onto the internal plane. Empty when wan uses
+	// DHCP, leaving spx to auto-detect from the default route.
+	WANIP string
 	// Email is the operator email collected by the TUI or SPINIFEX_EMAIL on
 	// the headless path. Passed to `spx admin init --email=<value>` when set;
 	// omitted entirely when empty.
@@ -247,14 +256,67 @@ func buildClusterCmd(cfg Config) string {
 	if cfg.GPUPassthrough {
 		gpuFlag = " --gpu-passthrough"
 	}
-	switch cfg.ClusterRole {
-	case "join":
-		return fmt.Sprintf("spx admin join --node %s --host %s%s", cfg.Hostname, cfg.JoinAddr, emailFlag)
-	default:
-		return fmt.Sprintf("spx admin init --node %s --nodes 1%s%s", cfg.Hostname, emailFlag, gpuFlag)
+	// Without these the node takes the 0.0.0.0 wildcard default and every
+	// internal service resolves onto the auto-detected WAN address, which is
+	// exactly what the three-plane model exists to prevent.
+	bindFlags := ""
+	if cfg.LANIP != "" {
+		bindFlags = fmt.Sprintf(" --bind %s --cluster-bind %s", cfg.LANIP, cfg.LANIP)
 	}
+	// --advertise must be explicit whenever --bind is: spx echoes a concrete
+	// bind address straight back as the advertise address and never reaches its
+	// WAN auto-detection, which would silently publish the internal plane as
+	// this node's public dial target.
+	preamble := ""
+	switch {
+	case bindFlags == "":
+		// Nothing pinned, so spx auto-detects both and a guessed advertise
+		// address would only get in the way.
+	case cfg.WANIP != "":
+		bindFlags += " --advertise " + cfg.WANIP
+	default:
+		// A DHCP wan has no address at install time, but it does by the time
+		// firstboot runs, so read it off the bridge instead of shipping the
+		// bind address as the public one.
+		preamble = wanAdvertisePreamble
+		bindFlags += ` $SPX_ADVERTISE`
+	}
+
+	// Always a single-node cluster. The installer cannot form a multi-node one:
+	// cluster membership decides the OVN database topology and the join token
+	// only exists once the primary has booted, so neither is knowable while the
+	// nodes are still being installed. Multi-node is a post-install conversion,
+	// documented in the multi-node install guide.
+	cmd := fmt.Sprintf("spx admin init --node %s --nodes 1%s%s%s", cfg.Hostname, bindFlags, emailFlag, gpuFlag)
+	return preamble + cmd
 }
 
+// wanAdvertisePreamble resolves the wan plane's DHCP lease into $SPX_ADVERTISE
+// just before formation. br-wan is always the wan bridge — that plane is the one
+// role that cannot fold — so its address is the node's public identity.
+//
+// Falling through with SPX_ADVERTISE empty is deliberate: no lease means there
+// is no public address to advertise yet, and spx echoing the bind address is
+// still better than formation failing outright.
+const wanAdvertisePreamble = `# The wan plane leases its address, so it is only knowable at boot. Without this
+# spx would echo --bind back as the advertise address and publish the internal
+# plane as this node's public dial target.
+SPX_ADVERTISE=""
+echo "[firstboot] waiting for the wan plane to acquire an address..."
+for _i in $(seq 1 60); do
+    _wan_ip=$(ip -4 -o addr show br-wan scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1)
+    if [ -n "$_wan_ip" ]; then
+        SPX_ADVERTISE=" --advertise $_wan_ip"
+        echo "[firstboot] wan address $_wan_ip (${_i}s)"
+        break
+    fi
+    sleep 1
+done
+if [ -z "$SPX_ADVERTISE" ]; then
+    echo "[firstboot] warning: br-wan has no address after 60s — this node will advertise its bind address"
+fi
+`
+
 // shellEscapeSingle wraps s in single quotes with any embedded single
 // quotes escaped. Minimal — we only need this because the email value is
 // interpolated into a shell script written by Write().

cmd/installer/firstboot/firstboot_test.go

@@ -2,6 +2,7 @@ package firstboot
 
 import (
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -24,7 +25,7 @@ func TestWriteScriptNoCallbackWhenEmpty(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	cfg := Config{Hostname: "test-node"}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -45,7 +46,6 @@ func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
 	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	cfg := Config{
 		Hostname:        "test-node",
-		ClusterRole:     "init",
 		InstallCallback: callbackURL,
 	}
 	if err := Write(root, cfg); err != nil {
@@ -70,7 +70,7 @@ func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	cfg := Config{Hostname: "test-node"}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -88,7 +88,7 @@ func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
+	cfg := Config{Hostname: "test-node", SkipFormation: true}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -122,7 +122,7 @@ func TestWriteScriptEnablesNatsWatchdogTimer(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
 
-	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	cfg := Config{Hostname: "test-node"}
 	if err := Write(root, cfg); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
@@ -140,7 +140,6 @@ func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	cfg := Config{
 		Hostname:        "node1",
-		ClusterRole:     "init",
 		InstallCallback: callbackURL,
 	}
 	if err := Write(root, cfg); err != nil {
@@ -165,3 +164,121 @@ func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 		t.Error("curl must appear after done marker write")
 	}
 }
+
+// A multi-NIC node must keep cluster traffic on the internal plane while still
+// publishing the public one. --advertise has to be explicit to do that: spx
+// returns a concrete --bind verbatim as the advertise address and never reaches
+// its WAN auto-detection, so omitting it moves northstar's :53 listener and the
+// off-host dial target onto the internal plane.
+func TestBuildClusterCmdSeparatesBindFromAdvertise(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "hydrogen",
+		LANIP:    "10.0.0.3",
+		WANIP:    "216.218.163.99",
+	})
+	for _, want := range []string{
+		"--bind 10.0.0.3",
+		"--cluster-bind 10.0.0.3",
+		"--advertise 216.218.163.99",
+	} {
+		if !strings.Contains(cmd, want) {
+			t.Errorf("missing %q in:\n%s", want, cmd)
+		}
+	}
+}
+
+// Without a bind address there is nothing to correct: spx auto-detects both,
+// and a stray --advertise would pin the node to an address the installer only
+// guessed at. A single-NIC node also collapses wan and lan onto one address,
+// where the split is meaningless.
+func TestBuildClusterCmdOmitsAdvertiseWithoutBind(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "node1",
+		WANIP:    "216.218.163.99",
+	})
+	if strings.Contains(cmd, "--advertise") {
+		t.Errorf("advertise must not be passed without a bind address:\n%s", cmd)
+	}
+}
+
+// A DHCP wan has no address at install time, so the advertise value has to be
+// resolved at boot. Shipping --bind alone here would republish the internal
+// plane as the node's public dial target, which is the case the static path
+// already guards against.
+func TestBuildClusterCmdResolvesDHCPWanAtBoot(t *testing.T) {
+	cmd := buildClusterCmd(Config{
+		Hostname: "hydrogen",
+		LANIP:    "10.0.0.3",
+		// WANIP empty — the wan plane is on DHCP.
+	})
+	if !strings.Contains(cmd, "ip -4 -o addr show br-wan") {
+		t.Errorf("must read the wan address off the bridge:\n%s", cmd)
+	}
+	if !strings.Contains(cmd, "$SPX_ADVERTISE") {
+		t.Errorf("resolved address must reach the command:\n%s", cmd)
+	}
+	// The preamble has to run before the command that consumes it.
+	if strings.Index(cmd, "SPX_ADVERTISE=\"\"") > strings.Index(cmd, "spx admin init") {
+		t.Errorf("preamble must precede the spx invocation:\n%s", cmd)
+	}
+	if !strings.Contains(cmd, "--bind 10.0.0.3") {
+		t.Errorf("bind still comes from the lan plane:\n%s", cmd)
+	}
+}
+
+// With no bind address there is nothing to correct, so neither the literal flag
+// nor the boot-time lookup should appear — spx auto-detects both ends.
+func TestBuildClusterCmdSkipsWanLookupWithoutBind(t *testing.T) {
+	cmd := buildClusterCmd(Config{Hostname: "node1"})
+	if strings.Contains(cmd, "SPX_ADVERTISE") || strings.Contains(cmd, "--advertise") {
+		t.Errorf("no bind means no advertise handling at all:\n%s", cmd)
+	}
+}
+
+// The advertise handling injects a shell preamble into the generated script, so
+// the permutations that change its shape are syntax-checked rather than only
+// string-matched — a broken quote here bricks first boot with no installer error.
+func TestGeneratedScriptIsValidShell(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		cfg  Config
+	}{
+		{"dhcp wan resolves at boot", Config{Hostname: "hydrogen", LANIP: "10.0.0.3"}},
+		{"static wan is inlined", Config{Hostname: "hydrogen", LANIP: "10.0.0.3", WANIP: "216.218.163.99"}},
+		{"single nic, nothing pinned", Config{Hostname: "node1"}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			root := t.TempDir()
+			makeRootDirs(t, root)
+			if err := Write(root, tc.cfg); err != nil {
+				t.Fatalf("Write: %v", err)
+			}
+			path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
+			if out, err := exec.Command("bash", "-n", path).CombinedOutput(); err != nil {
+				script, _ := os.ReadFile(path)
+				t.Fatalf("generated script is not valid shell: %v\n%s\n---\n%s", err, out, script)
+			}
+		})
+	}
+}
+
+// The installer cannot form a multi-node cluster: membership decides the OVN
+// database topology, and the join token only exists once the primary has
+// booted. Neither is knowable at install time, so firstboot must only ever
+// initialize a single-node cluster and leave multi-node to a post-install
+// conversion.
+func TestBuildClusterCmdNeverJoins(t *testing.T) {
+	for _, cfg := range []Config{
+		{Hostname: "node1"},
+		{Hostname: "node1", LANIP: "10.0.0.3", WANIP: "216.218.163.99"},
+		{Hostname: "node1", LANIP: "10.0.0.3"},
+	} {
+		cmd := buildClusterCmd(cfg)
+		if strings.Contains(cmd, "spx admin join") {
+			t.Errorf("firstboot must never join a cluster:\n%s", cmd)
+		}
+		if !strings.Contains(cmd, "--nodes 1") {
+			t.Errorf("want a single-node init, got:\n%s", cmd)
+		}
+	}
+}

cmd/installer/install/assets/grub-install-wrapper

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Diversion wrapper for grub-install on a spinifex-boot-tool system.
+#
+# On ZFS root the bootloader lives on per-disk ESPs, not in /boot. A plain
+# grub-install writes a core image that tries to read the pool, which works
+# until the next OpenZFS feature flag lands and then silently does not. Worse,
+# the grub-pc and shim packages run grub-install from their own postinst, so an
+# ordinary `apt upgrade` would undo the layout with no visible error.
+#
+# Mirrors the wrapper Proxmox ships for the same reason.
+
+set -e
+
+REAL="/usr/sbin/grub-install.real"
+TOOL="/usr/sbin/spinifex-boot-tool"
+
+# Not a managed system: behave exactly as the real grub-install.
+if ! "$TOOL" status --quiet 2>/dev/null; then
+	exec "$REAL" "$@"
+fi
+
+# dpkg sets this when a maintainer script is running. The bootloader packages
+# reinstalling themselves is expected and is redirected, not refused.
+case "${DPKG_MAINTSCRIPT_PACKAGE:-}" in
+grub-* | shim-*)
+	echo "This system is managed by spinifex-boot-tool; redirecting grub-install to 'spinifex-boot-tool reinit'." >&2
+	exec "$TOOL" reinit
+	;;
+esac
+
+cat >&2 <<EOF
+E: This system's bootloader is managed by spinifex-boot-tool.
+
+Running grub-install directly would install GRUB somewhere the firmware does
+not read, and would leave the per-disk ESPs holding a stale kernel.
+
+Use instead:
+  spinifex-boot-tool refresh     re-sync kernels and grub.cfg to every ESP
+  spinifex-boot-tool reinit      reinstall GRUB on every ESP
+  spinifex-boot-tool status      list the ESPs under management
+
+To run the real grub-install anyway (you probably do not want this):
+  $REAL $*
+EOF
+exit 1

cmd/installer/install/assets/spinifex-boot-tool

@@ -0,0 +1,412 @@
+#!/bin/bash
+# spinifex-boot-tool — keep a bootloader on every disk of a ZFS root pool.
+#
+# GRUB cannot be trusted to read a ZFS pool: its ZFS reader lags OpenZFS by
+# years, and enabling a new feature flag on the pool can make the machine
+# unbootable with no warning. So /boot on the pool is never read by firmware.
+# Instead each pool member carries a small FAT32 ESP holding its own copy of
+# GRUB, the kernels and the initramfs images, and this tool keeps them in step.
+#
+# Modelled directly on Proxmox's proxmox-boot-tool, which has run this design
+# across their entire installed base for years.
+
+set -u
+set -o pipefail
+
+ESP_LIST="/etc/kernel/spinifex-boot-uuids"
+ESPMOUNTS="/var/tmp/espmounts"
+MARKER="spinifex-boot-uuids"
+
+# Kernels are only ever copied out of the real /boot, which lives on the pool.
+# Once an ESP is bind-mounted over /boot this variable is how we still find it.
+BOOTROOT="/boot"
+
+warn() { echo "W: $*" >&2; }
+die() { echo "E: $*" >&2; exit 1; }
+
+usage() {
+	cat <<EOF
+Usage: $(basename "$0") <command>
+
+  format <partition> [--force]  make <partition> a Spinifex ESP
+  init <partition>              register <partition> and install a bootloader on it
+  clean [--dry-run]             drop ESPs that are no longer present
+  refresh [--hook <name>]       re-sync kernels and grub.cfg to every ESP
+  reinit                        re-run refresh, reinstalling GRUB on every ESP
+  kernel <add|remove> <ver>     pin or unpin a kernel version
+  status [--quiet]              list registered ESPs
+EOF
+}
+
+# reexec_in_mountns re-runs this script inside a private mount namespace. Every
+# code path here bind-mounts an ESP over /boot; if the script is killed at the
+# wrong moment outside a namespace, the running system is left with its real
+# /boot masked and the next kernel upgrade writes into a FAT partition.
+reexec_in_mountns() {
+	if [ "${SPINIFEX_BOOT_TOOL_NS:-0}" = "1" ]; then
+		return 0
+	fi
+	export SPINIFEX_BOOT_TOOL_NS=1
+	# --propagation private stops the bind mounts leaking back to the host.
+	exec unshare --mount --propagation private -- "$0" "$@"
+}
+
+# --- ESP registry -----------------------------------------------------------
+
+# esp_list prints the registered ESP filesystem UUIDs.
+esp_list() {
+	[ -f "$ESP_LIST" ] || return 0
+	grep -v '^[[:space:]]*\(#\|$\)' "$ESP_LIST" || true
+}
+
+# uuid_of prints the FAT filesystem UUID of a partition.
+uuid_of() {
+	blkid "$1" -o value -s UUID 2>/dev/null
+}
+
+# path_for_uuid resolves a registered UUID back to a device, or nothing if the
+# disk has been pulled.
+path_for_uuid() {
+	local dev
+	dev="/dev/disk/by-uuid/$1"
+	[ -e "$dev" ] || return 1
+	readlink -f "$dev"
+}
+
+# mount_esp mounts the ESP for a UUID and echoes its mountpoint.
+mount_esp() {
+	local uuid="$1" dev mp
+	dev=$(path_for_uuid "$uuid") || return 1
+	mp="$ESPMOUNTS/$uuid"
+	mkdir -p "$mp"
+	mountpoint -q "$mp" || mount "$dev" "$mp" || return 1
+	echo "$mp"
+}
+
+umount_esp() {
+	local mp="$ESPMOUNTS/$1"
+	mountpoint -q "$mp" && umount "$mp"
+	rmdir "$mp" 2>/dev/null || true
+}
+
+# --- format / init ----------------------------------------------------------
+
+do_format() {
+	local part="$1" force="${2:-}"
+	[ -b "$part" ] || die "$part is not a block device"
+
+	if [ "$force" != "--force" ]; then
+		local existing
+		existing=$(blkid -o value -s TYPE "$part" 2>/dev/null || true)
+		[ -z "$existing" ] || die "$part already contains a $existing filesystem; pass --force to overwrite"
+	fi
+	mkfs.vfat -F 32 "$part" || die "mkfs.vfat failed on $part"
+}
+
+do_init() {
+	local part="$1"
+	[ -b "$part" ] || die "$part is not a block device"
+
+	local uuid
+	uuid=$(uuid_of "$part") || true
+	[ -n "$uuid" ] || die "$part has no filesystem UUID — run '$(basename "$0") format $part' first"
+
+	mkdir -p "$(dirname "$ESP_LIST")"
+	touch "$ESP_LIST"
+	if ! grep -qx "$uuid" "$ESP_LIST"; then
+		echo "$uuid" >>"$ESP_LIST"
+	fi
+	echo "Registered $part ($uuid)"
+
+	install_hooks
+	# Only the ESP just registered: the others are already current, and
+	# refreshing all of them here makes registering N disks an N-squared
+	# sequence of grub-installs.
+	refresh_one "$uuid" || die "could not make $part bootable"
+}
+
+do_clean() {
+	local dry="${1:-}" uuid keep=()
+	for uuid in $(esp_list); do
+		if path_for_uuid "$uuid" >/dev/null 2>&1; then
+			keep+=("$uuid")
+		else
+			echo "Removing stale ESP $uuid"
+		fi
+	done
+	if [ "$dry" = "--dry-run" ]; then
+		return 0
+	fi
+	printf '%s\n' "${keep[@]+"${keep[@]}"}" >"$ESP_LIST"
+}
+
+# --- refresh ----------------------------------------------------------------
+
+# kernel_versions lists the kernels present in the real /boot, newest first, so
+# the default menu entry is the newest kernel.
+kernel_versions() {
+	local img
+	for img in "$BOOTROOT"/vmlinuz-*; do
+		[ -e "$img" ] || continue
+		basename "$img" | sed 's/^vmlinuz-//'
+	done | sort -Vr
+}
+
+# sync_esp copies the current kernels and initramfs images onto one ESP and
+# removes anything left from a kernel that is no longer installed — an ESP is
+# small and a stale 100MB initramfs is what makes the next upgrade fail.
+sync_esp() {
+	local mp="$1" kver f base
+
+	local -A wanted=()
+	for kver in $(kernel_versions); do
+		wanted["vmlinuz-$kver"]=1
+		wanted["initrd.img-$kver"]=1
+	done
+
+	for f in "$mp"/vmlinuz-* "$mp"/initrd.img-*; do
+		[ -e "$f" ] || continue
+		base=$(basename "$f")
+		[ -n "${wanted[$base]:-}" ] || rm -f "$f"
+	done
+
+	for kver in $(kernel_versions); do
+		for base in "vmlinuz-$kver" "initrd.img-$kver"; do
+			[ -e "$BOOTROOT/$base" ] || continue
+			# Copy to a temp name and rename: a half-written kernel on the ESP
+			# is a machine that does not come back from a reboot.
+			cp -f "$BOOTROOT/$base" "$mp/$base.tmp" || return 1
+			mv -f "$mp/$base.tmp" "$mp/$base" || return 1
+		done
+	done
+
+	# GRUB reads its theme from $prefix, which is the ESP's own grub directory,
+	# so the branding has to be copied out of /boot alongside the kernels.
+	mkdir -p "$mp/grub/fonts"
+	for f in grub/theme.txt grub/splash.png grub/fonts/unicode.pf2; do
+		[ -e "$BOOTROOT/$f" ] && cp -f "$BOOTROOT/$f" "$mp/$f"
+	done
+
+	# Marks the partition as ours, so a later `format` on the wrong disk is
+	# obviously a mistake rather than a silent overwrite.
+	echo "$MARKER" >"$mp/$MARKER"
+	return 0
+}
+
+# grub_install_bin echoes the real grub-install. On a managed system the normal
+# name is a wrapper that refuses to run, and it must not intercept this tool's
+# own calls — that would make the refusal a self-deadlock.
+grub_install_bin() {
+	if [ -x /usr/sbin/grub-install.real ]; then
+		echo /usr/sbin/grub-install.real
+	else
+		echo grub-install
+	fi
+}
+
+# install_grub puts GRUB itself on one ESP, for both firmware types.
+#
+# --boot-directory points at the ESP, not at /boot, so the GRUB core image
+# embeds a FAT reader and a search for the ESP's own UUID. It never needs to
+# read the pool, which is the entire point of this tool.
+install_grub() {
+	local mp="$1" dev="$2" rc=0 grub
+	grub="$(grub_install_bin)"
+
+	if [ ! -d /sys/firmware/efi ]; then
+		"$grub" --target=i386-pc --boot-directory="$mp" --recheck "$dev" || rc=1
+		return $rc
+	fi
+
+	# NVRAM is written at install time only. A refresh runs on every kernel
+	# upgrade, and rewriting boot variables that often has bricked firmware on
+	# more than one vendor's boards.
+	local nvram="--no-nvram"
+	[ "${SPINIFEX_BOOT_TOOL_NVRAM:-0}" = "1" ] && nvram=""
+
+	# Twice: once as a named entry, and once at the removable path, which is all
+	# some firmware will ever look at and is what keeps the disk bootable after
+	# its NVRAM entry is lost or the drive is moved to another machine.
+	# shellcheck disable=SC2086 # $nvram is a single optional flag, not a path
+	"$grub" --target=x86_64-efi \
+		--efi-directory="$mp" --boot-directory="$mp" \
+		--bootloader-id=spinifex $nvram --recheck "$dev" || rc=1
+	"$grub" --target=x86_64-efi \
+		--efi-directory="$mp" --boot-directory="$mp" \
+		--bootloader-id=spinifex --removable --no-nvram --recheck "$dev" || rc=1
+	return $rc
+}
+
+# update_grub_on regenerates grub.cfg for one ESP.
+#
+# The bind mount is the trick that makes this work: grub-mkconfig probes the
+# device behind /boot to decide how to find the kernel at boot. With the ESP
+# mounted there it emits a FAT UUID search and paths relative to the ESP root,
+# so the generated config matches what the embedded core image can actually do.
+update_grub_on() {
+	local mp="$1" rc=0
+	mount --bind "$mp" "$BOOTROOT" || return 1
+	grub-mkconfig -o "$BOOTROOT/grub/grub.cfg" || rc=1
+	umount "$BOOTROOT" || warn "could not unmount the bind mount over $BOOTROOT"
+	return $rc
+}
+
+# refresh_one makes a single registered ESP independently bootable: kernels,
+# bootloader, and a grub.cfg generated against that ESP's own UUID.
+refresh_one() {
+	local uuid="$1" mp dev disk rc=0
+
+	if ! dev=$(path_for_uuid "$uuid"); then
+		warn "ESP $uuid is not present, skipping"
+		return 1
+	fi
+	if ! mp=$(mount_esp "$uuid"); then
+		warn "could not mount ESP $uuid"
+		return 1
+	fi
+
+	# The whole disk, not the partition: BIOS GRUB embeds into the boot
+	# sector and the BIOS boot partition, both of which live on the disk.
+	disk=$(lsblk -no pkname "$dev")
+	disk="/dev/$disk"
+
+	if sync_esp "$mp" && install_grub "$mp" "$disk" && update_grub_on "$mp"; then
+		echo "Synced $dev ($uuid)"
+	else
+		warn "failed to update ESP $uuid on $dev"
+		rc=1
+	fi
+	umount_esp "$uuid"
+	return $rc
+}
+
+do_refresh() {
+	local uuid failed=0 ok=0 total=0
+
+	local uuids
+	uuids=$(esp_list)
+	if [ -z "$uuids" ]; then
+		echo "No ESPs registered, nothing to do."
+		return 0
+	fi
+
+	for uuid in $uuids; do
+		total=$((total + 1))
+		if refresh_one "$uuid"; then
+			ok=$((ok + 1))
+		else
+			failed=$((failed + 1))
+		fi
+	done
+
+	if [ "$ok" -eq 0 ]; then
+		die "no ESP could be updated ($failed of $total failed) — this system may not boot"
+	fi
+	if [ "$failed" -gt 0 ]; then
+		# Strict mode is used by the installer, where a partly-updated set of
+		# ESPs means the install is broken and should fail loudly now rather
+		# than at the first reboot.
+		if [ "${SPINIFEX_BOOT_TOOL_STRICT:-0}" = "1" ]; then
+			die "$failed of $total ESPs failed to update"
+		fi
+		warn "$failed of $total ESPs failed to update; $ok succeeded, so the system will still boot"
+	fi
+	return 0
+}
+
+do_status() {
+	local quiet="${1:-}" uuid dev
+	if [ ! -s "$ESP_LIST" ]; then
+		[ "$quiet" = "--quiet" ] || echo "No ESPs registered — this system does not use spinifex-boot-tool."
+		return 1
+	fi
+	[ "$quiet" = "--quiet" ] && return 0
+	for uuid in $(esp_list); do
+		if dev=$(path_for_uuid "$uuid"); then
+			echo "$uuid  $dev"
+		else
+			echo "$uuid  MISSING"
+		fi
+	done
+}
+
+do_kernel() {
+	local action="$1" kver="${2:-}"
+	local pinfile="/etc/kernel/spinifex-boot-pin"
+	case "$action" in
+	add)
+		[ -n "$kver" ] || die "kernel add needs a version"
+		touch "$pinfile"
+		grep -qx "$kver" "$pinfile" || echo "$kver" >>"$pinfile"
+		;;
+	remove)
+		[ -n "$kver" ] || die "kernel remove needs a version"
+		[ -f "$pinfile" ] && sed -i "\|^$kver\$|d" "$pinfile"
+		;;
+	*) die "unknown kernel action: $action" ;;
+	esac
+	do_refresh
+}
+
+# --- hooks ------------------------------------------------------------------
+
+# install_hooks wires this tool into every package operation that would
+# otherwise write a bootloader to the wrong place.
+install_hooks() {
+	mkdir -p /etc/kernel/postinst.d /etc/kernel/postrm.d /etc/initramfs/post-update.d
+
+	# Debian's own hook runs update-grub against the real /boot, which is on the
+	# pool. Firmware never reads that, so it would produce a config nothing uses
+	# while the ESPs silently kept an old kernel.
+	if [ -e /etc/kernel/postinst.d/zz-update-grub ]; then
+		dpkg-divert --divert /etc/kernel/postinst.d/zz-update-grub.distrib \
+			--rename --add /etc/kernel/postinst.d/zz-update-grub >/dev/null 2>&1 || true
+	fi
+
+	for hook in /etc/kernel/postinst.d/zz-spinifex-boot-tool \
+		/etc/kernel/postrm.d/zz-spinifex-boot-tool \
+		/etc/initramfs/post-update.d/spinifex-boot-tool; do
+		cat >"$hook" <<'HOOK'
+#!/bin/sh
+# Re-sync the ESPs after a kernel or initramfs change.
+set -e
+exec /usr/sbin/spinifex-boot-tool refresh --hook "$(basename "$0")"
+HOOK
+		chmod 0755 "$hook"
+	done
+}
+
+# --- dispatch ---------------------------------------------------------------
+
+cmd="${1:-}"
+[ $# -gt 0 ] && shift
+
+case "$cmd" in
+format) do_format "$@" ;;
+init)
+	reexec_in_mountns init "$@"
+	do_init "$@"
+	;;
+refresh)
+	# --hook <name> is passed by the kernel hooks purely so the reason for a
+	# refresh is visible in the log.
+	if [ "${1:-}" = "--hook" ]; then shift 2 || shift; fi
+	reexec_in_mountns refresh
+	do_refresh
+	;;
+reinit)
+	reexec_in_mountns reinit
+	do_refresh
+	;;
+clean) do_clean "$@" ;;
+kernel)
+	reexec_in_mountns kernel "$@"
+	do_kernel "$@"
+	;;
+status) do_status "$@" ;;
+help | --help | -h | "") usage ;;
+*)
+	usage
+	exit 1
+	;;
+esac

cmd/installer/install/blockdev.go

@@ -0,0 +1,267 @@
+package install
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// Disk is one candidate block device with every attribute the installer needs
+// to validate a selection: sizes for the same-size rule, block sizes for ashift
+// and the 4Kn/BIOS check, rotational for autotrim, and ByID for pool members.
+type Disk struct {
+	Path  string // /dev/sda
+	ByID  string // /dev/disk/by-id/ata-… — stable across controller reordering
+	Bytes int64
+	Model string
+
+	LogicalBlockSize  int
+	PhysicalBlockSize int
+
+	Rotational bool
+	Removable  bool
+
+	// LiveMedia marks the device the installer itself booted from. Selecting it
+	// would erase the running system mid-install.
+	LiveMedia bool
+
+	// Content is a human-readable summary of what is on the disk today
+	// ("ext4 filesystem", "empty"), shown before the operator confirms erasure.
+	Content string
+}
+
+// skipPrefixes are kernel block devices that are never install targets:
+// virtual, mapped, or removable media. Mirrors Proxmox's hd_list filter.
+var skipPrefixes = []string{"ram", "loop", "md", "dm-", "fd", "sr", "zram", "nbd"}
+
+// The kernel-owned trees the scan reads, indirected so tests can point it at a
+// fixture instead of the running machine's hardware.
+var (
+	sysBlockDir      = "/sys/block"
+	sysClassBlockDir = "/sys/class/block"
+	devByIDDir       = "/dev/disk/by-id"
+	procMountsPath   = "/proc/mounts"
+)
+
+// ListDisks enumerates install-candidate disks from /sys/block. It reads sysfs
+// and blkid only — pool detection is separate (ImportablePools) because it is
+// far slower and not every caller needs it.
+func ListDisks() ([]Disk, error) {
+	entries, err := os.ReadDir(sysBlockDir)
+	if err != nil {
+		return nil, fmt.Errorf("read /sys/block: %w", err)
+	}
+
+	byID := buildByIDIndex()
+	live := liveMediaDevices()
+
+	var disks []Disk
+	for _, e := range entries {
+		name := e.Name()
+		if hasAnyPrefix(name, skipPrefixes) {
+			continue
+		}
+		// A zero-sized device is an empty card reader slot, not a disk.
+		sectors := sysfsInt(name, "size")
+		if sectors == 0 {
+			continue
+		}
+		d := Disk{
+			Path:              "/dev/" + name,
+			ByID:              byID[name],
+			Bytes:             sectors * 512,
+			Model:             sysfsString(name, "device/model"),
+			LogicalBlockSize:  sysfsIntDefault(name, "queue/logical_block_size", 512),
+			PhysicalBlockSize: sysfsIntDefault(name, "queue/physical_block_size", 512),
+			Rotational:        sysfsInt(name, "queue/rotational") == 1,
+			Removable:         sysfsInt(name, "removable") == 1,
+			LiveMedia:         live[name],
+		}
+		d.Content = describeContent(d.Path)
+		disks = append(disks, d)
+	}
+	slices.SortFunc(disks, func(a, b Disk) int { return strings.Compare(a.Path, b.Path) })
+	return disks, nil
+}
+
+// SizeHuman renders the disk size for display, e.g. "1.7T".
+func (d Disk) SizeHuman() string {
+	switch {
+	case d.Bytes >= 1<<40:
+		return fmt.Sprintf("%.1fT", float64(d.Bytes)/(1<<40))
+	case d.Bytes >= 1<<30:
+		return fmt.Sprintf("%.1fG", float64(d.Bytes)/(1<<30))
+	case d.Bytes >= 1<<20:
+		return fmt.Sprintf("%.1fM", float64(d.Bytes)/(1<<20))
+	default:
+		return fmt.Sprintf("%dB", d.Bytes)
+	}
+}
+
+// Stable returns the path to use when building a pool: the by-id path when one
+// exists, else the kernel name. Pools referencing /dev/sdX break when a
+// controller enumerates disks in a different order after a reboot.
+func (d Disk) Stable() string {
+	if d.ByID != "" {
+		return d.ByID
+	}
+	return d.Path
+}
+
+// PartitionPath returns the device path for partition n of this disk. NVMe and
+// other devices whose name ends in a digit take a 'p' separator.
+func (d Disk) PartitionPath(n int) string { return partitionPath(d.Path, n) }
+
+// StablePartitionPath is PartitionPath against the by-id path, which udev
+// publishes with a "-partN" suffix rather than the kernel's 'p' convention.
+func (d Disk) StablePartitionPath(n int) string {
+	if d.ByID == "" {
+		return d.PartitionPath(n)
+	}
+	return fmt.Sprintf("%s-part%d", d.ByID, n)
+}
+
+func partitionPath(disk string, n int) string {
+	if len(disk) > 0 && disk[len(disk)-1] >= '0' && disk[len(disk)-1] <= '9' {
+		return fmt.Sprintf("%sp%d", disk, n)
+	}
+	return fmt.Sprintf("%s%d", disk, n)
+}
+
+// byIDPreference orders the /dev/disk/by-id/ namespaces we prefer to name a
+// pool member by. wwn- ids are last: they are stable but opaque, so a degraded
+// pool reports a number nobody can map to a drive bay.
+var byIDPreference = []string{"nvme-", "ata-", "scsi-", "virtio-", "usb-", "wwn-"}
+
+// buildByIDIndex maps kernel device name to its preferred by-id path.
+func buildByIDIndex() map[string]string {
+	entries, err := os.ReadDir(devByIDDir)
+	if err != nil {
+		return map[string]string{}
+	}
+	// Track the chosen preference rank per device so a later, less-preferred
+	// id cannot displace a better one already found.
+	rank := map[string]int{}
+	out := map[string]string{}
+	for _, e := range entries {
+		id := e.Name()
+		// Partition links describe a slice of the disk, not the disk.
+		if strings.Contains(id, "-part") {
+			continue
+		}
+		target, err := filepath.EvalSymlinks(filepath.Join(devByIDDir, id))
+		if err != nil {
+			continue
+		}
+		dev := filepath.Base(target)
+		r := prefixRank(id)
+		if existing, ok := rank[dev]; ok && existing <= r {
+			continue
+		}
+		rank[dev] = r
+		out[dev] = filepath.Join(devByIDDir, id)
+	}
+	return out
+}
+
+func prefixRank(id string) int {
+	for i, p := range byIDPreference {
+		if strings.HasPrefix(id, p) {
+			return i
+		}
+	}
+	return len(byIDPreference)
+}
+
+// liveMediaDevices returns the set of parent disk names backing the live
+// installer's own media, so they can never be selected as install targets.
+func liveMediaDevices() map[string]bool {
+	out := map[string]bool{}
+	data, err := os.ReadFile(procMountsPath)
+	if err != nil {
+		return out
+	}
+	liveMounts := []string{"/cdrom", "/run/live/medium", "/lib/live/mount/medium"}
+	for line := range strings.SplitSeq(string(data), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) < 2 || !strings.HasPrefix(fields[0], "/dev/") {
+			continue
+		}
+		if !slices.Contains(liveMounts, fields[1]) {
+			continue
+		}
+		if parent := parentDisk(filepath.Base(fields[0])); parent != "" {
+			out[parent] = true
+		}
+	}
+	return out
+}
+
+// parentDisk resolves a partition's kernel name to its whole-disk name by
+// walking /sys/class/block/<part>/.. — the sysfs parent of a partition is its
+// disk. Returns the input unchanged when it is already a whole disk.
+func parentDisk(dev string) string {
+	link, err := filepath.EvalSymlinks(filepath.Join(sysClassBlockDir, dev))
+	if err != nil {
+		return dev
+	}
+	// A whole disk lives directly under a bus path; a partition sits one level
+	// deeper, inside its disk's directory.
+	if _, err := os.Stat(filepath.Join(link, "partition")); err != nil {
+		return dev
+	}
+	return filepath.Base(filepath.Dir(link))
+}
+
+// describeContent summarises what is currently on a disk for the confirmation
+// screen. Best-effort: an unreadable disk is reported as unknown, not empty,
+// because "empty" invites an operator to erase something that is not.
+func describeContent(dev string) string {
+	out, err := exec.Command("blkid", "-p", "-o", "value", "-s", "TYPE", dev).Output()
+	if err == nil {
+		if t := strings.TrimSpace(string(out)); t != "" {
+			return t + " filesystem"
+		}
+	}
+	// No filesystem on the whole device: report the partition table instead,
+	// since that is what tells the operator the disk is in use.
+	out, err = exec.Command("blkid", "-p", "-o", "value", "-s", "PTTYPE", dev).Output()
+	if err != nil {
+		return "unknown"
+	}
+	if t := strings.TrimSpace(string(out)); t != "" {
+		return t + " partition table"
+	}
+	return "empty"
+}
+
+func sysfsString(dev, file string) string {
+	data, err := os.ReadFile(filepath.Join(sysBlockDir, dev, file))
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(data))
+}
+
+func sysfsInt(dev, file string) int64 {
+	n, err := strconv.ParseInt(sysfsString(dev, file), 10, 64)
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+func sysfsIntDefault(dev, file string, def int) int {
+	if n := sysfsInt(dev, file); n > 0 {
+		return int(n)
+	}
+	return def
+}
+
+func hasAnyPrefix(s string, prefixes []string) bool {
+	return slices.ContainsFunc(prefixes, func(p string) bool { return strings.HasPrefix(s, p) })
+}

cmd/installer/install/blockdev_test.go

@@ -0,0 +1,354 @@
+package install
+
+import (
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"testing"
+)
+
+// fakeDisk describes one device to lay down in the fixture sysfs tree.
+type fakeDisk struct {
+	name       string
+	sectors    int64
+	model      string
+	logical    int
+	physical   int
+	rotational bool
+	removable  bool
+	partitions []string
+}
+
+// fakeSysfs builds a /sys/block, /sys/class/block, /dev/disk/by-id and
+// /proc/mounts fixture and points the scan at it for the duration of the test.
+type fakeSysfs struct {
+	root string
+	t    *testing.T
+}
+
+func newFakeSysfs(t *testing.T, disks ...fakeDisk) *fakeSysfs {
+	t.Helper()
+	f := &fakeSysfs{root: t.TempDir(), t: t}
+
+	swap := func(dst *string, v string) {
+		prev := *dst
+		*dst = v
+		t.Cleanup(func() { *dst = prev })
+	}
+	swap(&sysBlockDir, filepath.Join(f.root, "sys/block"))
+	swap(&sysClassBlockDir, filepath.Join(f.root, "sys/class/block"))
+	swap(&devByIDDir, filepath.Join(f.root, "dev/disk/by-id"))
+	swap(&procMountsPath, filepath.Join(f.root, "proc/mounts"))
+
+	f.mkdirAll(sysBlockDir)
+	f.mkdirAll(sysClassBlockDir)
+	for _, d := range disks {
+		f.addDisk(d)
+	}
+	return f
+}
+
+func (f *fakeSysfs) mkdirAll(dir string) {
+	f.t.Helper()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		f.t.Fatalf("mkdir %s: %v", dir, err)
+	}
+}
+
+func (f *fakeSysfs) write(path, content string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Dir(path))
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		f.t.Fatalf("write %s: %v", path, err)
+	}
+}
+
+// addDisk lays out one device the way the kernel does: the attribute files
+// under /sys/block/<dev>, and a /sys/class/block symlink per disk and
+// partition, since parentDisk walks the latter.
+func (f *fakeSysfs) addDisk(d fakeDisk) {
+	f.t.Helper()
+	dir := filepath.Join(sysBlockDir, d.name)
+	f.write(filepath.Join(dir, "size"), strconv.FormatInt(d.sectors, 10)+"\n")
+	if d.model != "" {
+		f.write(filepath.Join(dir, "device/model"), d.model+"\n")
+	}
+	if d.logical > 0 {
+		f.write(filepath.Join(dir, "queue/logical_block_size"), strconv.Itoa(d.logical)+"\n")
+	}
+	if d.physical > 0 {
+		f.write(filepath.Join(dir, "queue/physical_block_size"), strconv.Itoa(d.physical)+"\n")
+	}
+	f.write(filepath.Join(dir, "queue/rotational"), boolAttr(d.rotational))
+	f.write(filepath.Join(dir, "removable"), boolAttr(d.removable))
+
+	f.symlink(dir, filepath.Join(sysClassBlockDir, d.name))
+	for _, part := range d.partitions {
+		partDir := filepath.Join(dir, part)
+		f.write(filepath.Join(partDir, "partition"), "1\n")
+		f.symlink(partDir, filepath.Join(sysClassBlockDir, part))
+	}
+}
+
+func (f *fakeSysfs) symlink(target, link string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Dir(link))
+	if err := os.Symlink(target, link); err != nil {
+		f.t.Fatalf("symlink %s -> %s: %v", link, target, err)
+	}
+}
+
+// addByID creates a by-id alias pointing at a kernel device name. The link
+// target must exist for EvalSymlinks, so a placeholder /dev tree is used.
+func (f *fakeSysfs) addByID(id, dev string) {
+	f.t.Helper()
+	devPath := filepath.Join(f.root, "dev", dev)
+	f.write(devPath, "")
+	f.symlink(devPath, filepath.Join(devByIDDir, id))
+}
+
+func (f *fakeSysfs) setMounts(content string) {
+	f.t.Helper()
+	f.write(procMountsPath, content)
+}
+
+func boolAttr(b bool) string {
+	if b {
+		return "1\n"
+	}
+	return "0\n"
+}
+
+func TestListDisksReadsSysfsAttributes(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{
+		name: "sdb", sectors: 2 << 20, model: "SAMSUNG MZ7", logical: 512, physical: 4096, rotational: true,
+	})
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	if len(disks) != 1 {
+		t.Fatalf("got %d disks, want 1: %+v", len(disks), disks)
+	}
+	d := disks[0]
+	if d.Path != "/dev/sdb" || d.Bytes != (2<<20)*512 {
+		t.Errorf("path/size = %s/%d", d.Path, d.Bytes)
+	}
+	if d.Model != "SAMSUNG MZ7" {
+		t.Errorf("Model = %q", d.Model)
+	}
+	if d.LogicalBlockSize != 512 || d.PhysicalBlockSize != 4096 {
+		t.Errorf("block sizes = %d/%d", d.LogicalBlockSize, d.PhysicalBlockSize)
+	}
+	if !d.Rotational || d.Removable || d.LiveMedia {
+		t.Errorf("flags = rot %v, rem %v, live %v", d.Rotational, d.Removable, d.LiveMedia)
+	}
+	if d.Content == "" {
+		t.Error("Content is empty; an unreadable device must report something")
+	}
+}
+
+func TestListDisksSkipsVirtualAndEmptyDevices(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "loop0", sectors: 1 << 20},
+		fakeDisk{name: "dm-0", sectors: 1 << 20},
+		fakeDisk{name: "zram0", sectors: 1 << 20},
+		fakeDisk{name: "sr0", sectors: 1 << 20},
+		fakeDisk{name: "nbd0", sectors: 1 << 20},
+		// An empty card-reader slot reports zero sectors.
+		fakeDisk{name: "mmcblk0", sectors: 0},
+	)
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	var paths []string
+	for _, d := range disks {
+		paths = append(paths, d.Path)
+	}
+	if !slices.Equal(paths, []string{"/dev/sda"}) {
+		t.Errorf("paths = %v, want only /dev/sda", paths)
+	}
+}
+
+func TestListDisksSortsByPath(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sdc", sectors: 1 << 20},
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "sdb", sectors: 1 << 20},
+	)
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	var paths []string
+	for _, d := range disks {
+		paths = append(paths, d.Path)
+	}
+	if !slices.Equal(paths, []string{"/dev/sda", "/dev/sdb", "/dev/sdc"}) {
+		t.Errorf("paths = %v, want sorted", paths)
+	}
+}
+
+func TestListDisksDefaultsBlockSizesWhenSysfsIsSilent(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "vda", sectors: 1 << 20})
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	if disks[0].LogicalBlockSize != 512 || disks[0].PhysicalBlockSize != 512 {
+		t.Errorf("block sizes = %d/%d, want the 512 default",
+			disks[0].LogicalBlockSize, disks[0].PhysicalBlockSize)
+	}
+}
+
+func TestListDisksErrorsWhenSysfsIsUnreadable(t *testing.T) {
+	newFakeSysfs(t)
+	sysBlockDir = filepath.Join(t.TempDir(), "does-not-exist")
+
+	if _, err := ListDisks(); err == nil {
+		t.Fatal("expected an error when /sys/block cannot be read")
+	}
+}
+
+func TestListDisksMarksTheLiveMedium(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 20},
+		fakeDisk{name: "sdz", sectors: 1 << 20, removable: true, partitions: []string{"sdz1"}},
+	)
+	// The installer boots from a partition; the whole disk must be excluded.
+	f.setMounts("/dev/sdz1 /run/live/medium iso9660 ro 0 0\n" +
+		"proc /proc proc rw 0 0\n" +
+		"/dev/sda1 / ext4 rw 0 0\n")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	byPath := map[string]Disk{}
+	for _, d := range disks {
+		byPath[d.Path] = d
+	}
+	if !byPath["/dev/sdz"].LiveMedia {
+		t.Error("/dev/sdz backs the live medium and was not marked")
+	}
+	if byPath["/dev/sda"].LiveMedia {
+		t.Error("/dev/sda is not the live medium but was marked")
+	}
+}
+
+func TestLiveMediaDevicesIsEmptyWithoutProcMounts(t *testing.T) {
+	newFakeSysfs(t)
+	if got := liveMediaDevices(); len(got) != 0 {
+		t.Errorf("liveMediaDevices = %v, want empty when /proc/mounts is missing", got)
+	}
+}
+
+func TestParentDiskLeavesWholeDisksAlone(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20, partitions: []string{"sda1"}})
+
+	if got := parentDisk("sda"); got != "sda" {
+		t.Errorf("parentDisk(sda) = %q, want sda", got)
+	}
+	if got := parentDisk("sda1"); got != "sda" {
+		t.Errorf("parentDisk(sda1) = %q, want sda", got)
+	}
+	// An unknown name cannot be resolved and is returned unchanged.
+	if got := parentDisk("sdq9"); got != "sdq9" {
+		t.Errorf("parentDisk(sdq9) = %q, want it unchanged", got)
+	}
+}
+
+func TestBuildByIDIndexPrefersTheMostLegibleAlias(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20})
+	// wwn- is stable but opaque, so ata- must win regardless of scan order.
+	f.addByID("wwn-0x5000c500", "sda")
+	f.addByID("ata-SAMSUNG_MZ7_S1", "sda")
+	f.addByID("ata-SAMSUNG_MZ7_S1-part1", "sda")
+
+	idx := buildByIDIndex()
+	want := filepath.Join(devByIDDir, "ata-SAMSUNG_MZ7_S1")
+	if idx["sda"] != want {
+		t.Errorf("by-id[sda] = %q, want %q", idx["sda"], want)
+	}
+}
+
+func TestBuildByIDIndexIsEmptyWhenTheDirIsAbsent(t *testing.T) {
+	newFakeSysfs(t)
+	if got := buildByIDIndex(); len(got) != 0 {
+		t.Errorf("buildByIDIndex = %v, want empty", got)
+	}
+}
+
+func TestListDisksAttachesByIDPath(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20})
+	f.addByID("ata-TEST_SERIAL0", "sda")
+	f.setMounts("")
+
+	disks, err := ListDisks()
+	if err != nil {
+		t.Fatalf("ListDisks: %v", err)
+	}
+	want := filepath.Join(devByIDDir, "ata-TEST_SERIAL0")
+	if disks[0].ByID != want {
+		t.Errorf("ByID = %q, want %q", disks[0].ByID, want)
+	}
+	if disks[0].Stable() != want {
+		t.Errorf("Stable() = %q, want the by-id path", disks[0].Stable())
+	}
+}
+
+func TestPrefixRank(t *testing.T) {
+	if prefixRank("nvme-Foo") >= prefixRank("wwn-0x1") {
+		t.Error("nvme- must outrank wwn-")
+	}
+	if prefixRank("md-uuid-x") != len(byIDPreference) {
+		t.Error("an unknown namespace must rank last")
+	}
+}
+
+func TestSizeHuman(t *testing.T) {
+	tests := []struct {
+		bytes int64
+		want  string
+	}{
+		{2 << 40, "2.0T"},
+		{500 << 30, "500.0G"},
+		{64 << 20, "64.0M"},
+		{512, "512B"},
+	}
+	for _, tt := range tests {
+		if got := (Disk{Bytes: tt.bytes}).SizeHuman(); got != tt.want {
+			t.Errorf("SizeHuman(%d) = %q, want %q", tt.bytes, got, tt.want)
+		}
+	}
+}
+
+func TestStableFallsBackToKernelName(t *testing.T) {
+	d := Disk{Path: "/dev/sda"}
+	if d.Stable() != "/dev/sda" {
+		t.Errorf("Stable() = %q", d.Stable())
+	}
+	if d.StablePartitionPath(2) != "/dev/sda2" {
+		t.Errorf("StablePartitionPath(2) = %q", d.StablePartitionPath(2))
+	}
+}
+
+func TestHasAnyPrefix(t *testing.T) {
+	if !hasAnyPrefix("loop3", skipPrefixes) {
+		t.Error("loop3 should match")
+	}
+	if hasAnyPrefix("sda", skipPrefixes) {
+		t.Error("sda should not match")
+	}
+}

cmd/installer/install/boottool.go

@@ -0,0 +1,202 @@
+package install
+
+import (
+	_ "embed"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+//go:embed assets/spinifex-boot-tool
+var bootToolScript []byte
+
+//go:embed assets/grub-install-wrapper
+var grubInstallWrapper []byte
+
+const (
+	bootToolPath     = "/usr/sbin/spinifex-boot-tool"
+	grubInstallPath  = "/usr/sbin/grub-install"
+	grubInstallReal  = "/usr/sbin/grub-install.real"
+	bootToolUUIDList = "/etc/kernel/spinifex-boot-uuids"
+)
+
+// installBootToolZFS sets up per-disk bootloaders on a ZFS root system.
+//
+// Nothing in the boot path is allowed to read the pool: GRUB's ZFS support
+// trails OpenZFS, so a pool feature flag enabled years from now could make the
+// machine unbootable. Each disk instead carries a self-contained ESP with its
+// own GRUB, kernels and initramfs.
+func installBootToolZFS(cfg DiskConfig) error {
+	if err := writeTargetFile(bootToolPath, bootToolScript, 0o755); err != nil {
+		return err
+	}
+	if err := writeGrubDefaults(cfg); err != nil {
+		return err
+	}
+
+	if err := bindChrootMounts(); err != nil {
+		return err
+	}
+	defer unbindChrootMounts()
+
+	// Diverted before the first init: init runs a refresh, and refresh calls
+	// grub-install. With the wrapper already in place that call has to reach
+	// the real binary, which is what the diversion arranges.
+	if err := divertGrubInstall(); err != nil {
+		return err
+	}
+
+	// A partially-updated set of ESPs is an install that boots today and fails
+	// after the next kernel upgrade, so every disk must succeed here.
+	env := []string{"SPINIFEX_BOOT_TOOL_STRICT=1"}
+	for _, d := range cfg.Disks {
+		esp := d.PartitionPath(espPartNum)
+		if err := runEnv(env, "chroot", mountRoot, bootToolPath, "init", esp); err != nil {
+			return fmt.Errorf("spinifex-boot-tool init %s: %w", esp, err)
+		}
+	}
+
+	// The one moment firmware boot variables are written. Refreshes run on
+	// every kernel upgrade and deliberately leave NVRAM alone: repeatedly
+	// rewriting boot entries has bricked more than one vendor's boards. Losing
+	// the entry is survivable anyway, because each ESP also carries the
+	// removable-path loader that firmware falls back to.
+	if err := runEnv(append(env, "SPINIFEX_BOOT_TOOL_NVRAM=1"),
+		"chroot", mountRoot, bootToolPath, "refresh"); err != nil {
+		return fmt.Errorf("spinifex-boot-tool refresh: %w", err)
+	}
+	return verifyBootConfig(cfg)
+}
+
+// divertGrubInstall moves the real grub-install aside and puts the wrapper in
+// its place. dpkg-divert makes the move survive package upgrades, which is the
+// point: grub-pc's postinst runs grub-install and would otherwise quietly
+// replace a working boot layout with one that reads the pool.
+func divertGrubInstall() error {
+	if err := run("chroot", mountRoot, "dpkg-divert", "--divert", grubInstallReal,
+		"--rename", "--add", grubInstallPath); err != nil {
+		return fmt.Errorf("divert grub-install: %w", err)
+	}
+	return writeTargetFile(grubInstallPath, grubInstallWrapper, 0o755)
+}
+
+// verifyBootConfig asserts that the config the machine will actually boot from
+// names the ZFS root. grub-mkconfig degrades quietly when grub-probe cannot
+// read the pool, emitting a config that gets as far as the GRUB menu and then
+// panics with "unable to mount root fs" — which looks like a hardware fault.
+func verifyBootConfig(cfg DiskConfig) error {
+	uuids, err := os.ReadFile(filepath.Join(mountRoot, strings.TrimPrefix(bootToolUUIDList, "/")))
+	if err != nil {
+		return fmt.Errorf("read ESP list: %w", err)
+	}
+	if len(strings.Fields(string(uuids))) != len(cfg.Disks) {
+		return fmt.Errorf("expected %d registered ESPs, found %d — some disks have no bootloader",
+			len(cfg.Disks), len(strings.Fields(string(uuids))))
+	}
+
+	// One ESP is enough to prove grub-mkconfig produced the right thing; they
+	// are all generated by the same code path in the same run.
+	mp := filepath.Join(mountRoot, "var/tmp/verify-esp")
+	if err := os.MkdirAll(mp, 0o755); err != nil {
+		return err
+	}
+	defer func() { _ = os.Remove(mp) }()
+
+	esp := cfg.Disks[0].PartitionPath(espPartNum)
+	if err := run("mount", esp, mp); err != nil {
+		return fmt.Errorf("mount %s for verification: %w", esp, err)
+	}
+	defer func() { _ = run("umount", mp) }()
+
+	grubCfg, err := os.ReadFile(filepath.Join(mp, "grub/grub.cfg"))
+	if err != nil {
+		return fmt.Errorf("read grub.cfg from %s: %w", esp, err)
+	}
+	want := "root=ZFS=" + ZFSRootDataset
+	if !strings.Contains(string(grubCfg), want) {
+		return fmt.Errorf("grub.cfg on %s does not contain %q — the installed system would not find its root filesystem", esp, want)
+	}
+	if !strings.Contains(string(grubCfg), "vmlinuz-") {
+		return fmt.Errorf("grub.cfg on %s references no kernel", esp)
+	}
+	return nil
+}
+
+// writeTargetFile writes a file into the install target.
+func writeTargetFile(path string, content []byte, mode os.FileMode) error {
+	full := filepath.Join(mountRoot, strings.TrimPrefix(path, "/"))
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		return err
+	}
+	if err := os.WriteFile(full, content, mode); err != nil {
+		return fmt.Errorf("write %s: %w", path, err)
+	}
+	// WriteFile does not apply the mode to a file that already exists, and this
+	// path is re-run on a retried install.
+	return os.Chmod(full, mode)
+}
+
+// writeGrubDefaults writes /etc/default/grub and the branding snippet.
+//
+// Both consoles are listed: tty0 stays primary for a local install while ttyS0
+// mirrors kernel output for headless racks and CI. The last console= on the
+// cmdline becomes the system console, so serial-only hardware still works.
+func writeGrubDefaults(cfg DiskConfig) error {
+	cmdline := "console=tty0 console=ttyS0,115200 systemd.show_status=1"
+	if cfg.FS.IsZFS() {
+		// Stated explicitly rather than relying on grub-mkconfig's probe. It
+		// appears after the root= that 10_linux computes, so it wins if the
+		// probe came back with something wrong.
+		cmdline = fmt.Sprintf("root=ZFS=%s boot=zfs %s", ZFSRootDataset, cmdline)
+	}
+	grubDefault := fmt.Sprintf(`GRUB_DEFAULT=0
+GRUB_TIMEOUT=5
+GRUB_DISTRIBUTOR=Spinifex
+GRUB_CMDLINE_LINUX_DEFAULT=""
+GRUB_CMDLINE_LINUX="%s"
+`, cmdline)
+	if err := os.WriteFile(filepath.Join(mountRoot, "etc/default/grub"), []byte(grubDefault), 0o644); err != nil {
+		return fmt.Errorf("write /etc/default/grub: %w", err)
+	}
+
+	// gfxterm must be activated before serial is appended — background_image
+	// silently does nothing in text mode. $prefix is used rather than a literal
+	// /boot/grub so the same snippet is correct whether GRUB's root is the ext4
+	// filesystem or a per-disk ESP.
+	grubTheme := `#!/bin/sh
+exec tail -n +3 $0
+insmod all_video
+insmod font
+if loadfont $prefix/fonts/unicode.pf2; then
+  set gfxmode=auto
+  insmod gfxterm
+  terminal_output gfxterm
+fi
+insmod serial
+if serial --unit=0 --speed=115200 --timeout=1; then
+  terminal_input  --append serial
+  terminal_output --append serial
+fi
+
+# --- Branding ---
+insmod png
+set theme=$prefix/theme.txt
+export theme
+`
+	if err := os.WriteFile(filepath.Join(mountRoot, "etc/grub.d/06_spinifex"), []byte(grubTheme), 0o755); err != nil {
+		return fmt.Errorf("write /etc/grub.d/06_spinifex: %w", err)
+	}
+	return nil
+}
+
+// refreshBootToolNote logs the command an operator needs after replacing a
+// disk. Nothing in the installer can do this for them later.
+func refreshBootToolNote(cfg DiskConfig) {
+	if cfg.FS.IsZFS() {
+		slog.Info("bootloader installed on every pool member",
+			"disks", len(cfg.Disks),
+			"after_disk_replacement", "spinifex-boot-tool format <esp> && spinifex-boot-tool init <esp>")
+	}
+}

cmd/installer/install/config.go

@@ -2,42 +2,107 @@
 // collected configuration.
 package install
 
+import (
+	"fmt"
+	"net"
+	"strings"
+)
+
+// Plane names the three traffic planes a node presents. Each plane is a role
+// bound to a target, not a fixed NIC: the same three roles collapse onto one,
+// two or three interfaces depending on the hardware.
+type Plane string
+
+const (
+	// PlaneWAN carries public north-south traffic: ingress/egress, EIPs and
+	// SNAT/DNAT. It is the OVN external bridge-mapping.
+	PlaneWAN Plane = "wan"
+
+	// PlaneLAN carries internal cluster traffic: predastore replication, the
+	// NATS cluster mesh, OVN NB/SB control, Raft and S3DB.
+	PlaneLAN Plane = "lan"
+
+	// PlaneVPC carries the tenant overlay: OVN Geneve tunnels between nodes.
+	// It is the ovn-encap-ip endpoint.
+	PlaneVPC Plane = "vpc"
+)
+
+// Bridge returns the Linux bridge that carries this plane.
+func (p Plane) Bridge() string { return "br-" + string(p) }
+
+// NetworkRole binds one plane to a network target. An empty Interface means
+// the role is folded onto the next role up the chain (vpc <- lan <- wan) and
+// no bridge of its own is created.
+//
+// VLAN is what allows two roles to share one physical NIC while staying
+// separate planes: each gets its own NIC.<vlan> subinterface and bridge. Two
+// roles on the same Interface with no VLAN are the same plane, not two.
+type NetworkRole struct {
+	// Interface is the physical NIC this role binds to; empty means folded.
+	Interface string
+
+	// VLAN is the 802.1Q tag, or 0 for untagged.
+	VLAN int
+
+	// DHCPMode leaves addressing to the DHCP client; Address/Mask are then unset.
+	DHCPMode bool
+
+	Address string
+	Mask    string
+
+	// Gateway is only meaningful on the wan role — the other planes are
+	// link-local to the rack and must not install a default route.
+	Gateway string
+
+	DNS []string
+
+	// MTU is written as MTUBytes= on the link; 0 leaves the kernel default.
+	// East-west planes typically run 9000, wan stays at 1500.
+	MTU int
+
+	// WiFiSSID and WiFiPass are set when the selected NIC is wireless.
+	WiFiSSID string
+	WiFiPass string
+}
+
+// Folded reports whether this role has no interface of its own and inherits
+// the plane of the role above it.
+func (r NetworkRole) Folded() bool { return r.Interface == "" }
+
+// Link returns the name of the interface carrying this role's address: the
+// physical NIC, or its VLAN subinterface when tagged. Empty when folded.
+func (r NetworkRole) Link() string {
+	if r.Folded() {
+		return ""
+	}
+	if r.VLAN > 0 {
+		return fmt.Sprintf("%s.%d", r.Interface, r.VLAN)
+	}
+	return r.Interface
+}
+
 // Config holds all values collected by the installer UI.
 type Config struct {
-	// Disk is the block device path to install onto (e.g. /dev/sda).
-	Disk string
+	// Storage is the filesystem choice and the ordered set of disks it is
+	// built from. In ext4 mode it holds exactly one disk.
+	Storage DiskConfig
 
-	// WAN interface — always present. The management IP lives on br-wan (a Linux
-	// bridge over the NIC), not on the physical NIC itself. This allows OVN/OVS
-	// to safely attach to the bridge without disrupting host connectivity.
-	WANInterface string
-	WANDHCPMode  bool
-	WANAddress   string   // empty if WANDHCPMode
-	WANMask      string   // empty if WANDHCPMode
-	WANGateway   string   // empty if WANDHCPMode
-	WANDNS       []string // empty if WANDHCPMode; e.g. ["1.1.1.1", "8.8.8.8"]
-	WANWiFiSSID  string   // non-empty if a WiFi NIC was selected
-	WANWiFiPass  string   // non-empty if a WiFi NIC was selected
+	// WAN is always present and can never be folded — a node needs an uplink.
+	// The address lives on br-wan (a Linux bridge over the NIC), not on the
+	// physical NIC itself, so OVN/OVS can attach to the bridge without
+	// disrupting host connectivity.
+	WAN NetworkRole
 
-	// LAN interface — only set when 2+ NICs are present. The Geneve encap IP
-	// for OVN tunnels lives on br-lan. When empty, the WAN address is used.
-	LANInterface string
-	LANDHCPMode  bool
-	LANAddress   string
-	LANMask      string
-	LANDNS       []string
-	LANWiFiSSID  string
-	LANWiFiPass  string
+	// LAN carries internal cluster traffic. Folded onto WAN on single-NIC nodes.
+	LAN NetworkRole
+
+	// VPC carries Geneve overlay traffic and supplies ovn-encap-ip. Folded onto
+	// LAN unless a node dedicates an interface or VLAN to it.
+	VPC NetworkRole
 
 	// Node identity
 	Hostname string
 
-	// ClusterRole is "init" or "join".
-	ClusterRole string
-
-	// JoinAddr is the primary node address (host:port) when ClusterRole is "join".
-	JoinAddr string
-
 	// SkipFormation skips spx admin init/join in firstboot; the provisioning
 	// controller (e.g. bm-bootstrap.sh) owns cluster formation instead.
 	SkipFormation bool
@@ -60,3 +125,111 @@ type Config struct {
 	// Passes --gpu-passthrough to `spx admin init` during firstboot.
 	GPUPassthrough bool
 }
+
+// Resolve returns the role that actually carries the given plane after
+// collapsing folded roles up the chain vpc <- lan <- wan, along with the plane
+// it landed on. A folded vpc on a single-NIC node resolves to the wan role.
+func (c *Config) Resolve(p Plane) (NetworkRole, Plane) {
+	switch p {
+	case PlaneVPC:
+		if !c.VPC.Folded() {
+			return c.VPC, PlaneVPC
+		}
+		return c.Resolve(PlaneLAN)
+	case PlaneLAN:
+		if !c.LAN.Folded() {
+			return c.LAN, PlaneLAN
+		}
+		return c.WAN, PlaneWAN
+	default:
+		return c.WAN, PlaneWAN
+	}
+}
+
+// PlaneAddress returns the configured address for a plane after collapsing.
+// Empty when the resolved role uses DHCP, since no static address is known
+// until the lease is taken.
+func (c *Config) PlaneAddress(p Plane) string {
+	role, _ := c.Resolve(p)
+	if role.DHCPMode {
+		return ""
+	}
+	return role.Address
+}
+
+// PlaneBridge returns the bridge carrying a plane after collapsing, so callers
+// asking for the vpc bridge on a single-NIC node correctly get br-wan.
+func (c *Config) PlaneBridge(p Plane) string {
+	_, landed := c.Resolve(p)
+	return landed.Bridge()
+}
+
+// RoleBinding pairs a plane with the role that carries it.
+type RoleBinding struct {
+	Plane Plane
+	Role  NetworkRole
+}
+
+// Roles returns the bound roles in bridge-creation order, skipping folded ones.
+// wan comes first because br-lan and br-vpc are activated manually after it.
+func (c *Config) Roles() []RoleBinding {
+	out := []RoleBinding{{PlaneWAN, c.WAN}}
+	if !c.LAN.Folded() {
+		out = append(out, RoleBinding{PlaneLAN, c.LAN})
+	}
+	if !c.VPC.Folded() {
+		out = append(out, RoleBinding{PlaneVPC, c.VPC})
+	}
+	return out
+}
+
+// Validate enforces the sharing rules the UI and the headless path must both
+// obey: wan always needs a target, and two roles may only share an interface
+// when distinct VLAN ids keep them on separate broadcast domains.
+func (c *Config) Validate() error {
+	if c.WAN.Folded() {
+		return fmt.Errorf("wan role must be bound to an interface")
+	}
+
+	seen := map[string]Plane{}
+	for _, r := range c.Roles() {
+		link := r.Role.Link()
+		if prev, dup := seen[link]; dup {
+			return fmt.Errorf("%s and %s both bind %s: give each a distinct VLAN id or fold one onto the other",
+				prev, r.Plane, link)
+		}
+		seen[link] = r.Plane
+		if r.Role.VLAN < 0 || r.Role.VLAN > 4094 {
+			return fmt.Errorf("%s: VLAN id %d out of range (1-4094)", r.Plane, r.Role.VLAN)
+		}
+		if r.Role.MTU != 0 && (r.Role.MTU < 68 || r.Role.MTU > 9216) {
+			return fmt.Errorf("%s: MTU %d out of range (68-9216)", r.Plane, r.Role.MTU)
+		}
+		// Resolvers belong to the plane that holds the default route. Accepting
+		// one here and dropping it during generation would leave an unattended
+		// install believing it had set a resolver that was never written.
+		if r.Plane != PlaneWAN && len(r.Role.DNS) > 0 {
+			return fmt.Errorf("%s: DNS servers belong to the wan plane, which holds the default route", r.Plane)
+		}
+		// Netgen converts the mask with net.ParseIP, so a prefix length reaches
+		// it as garbage and fails the install after the disk is partitioned.
+		// Catch it here, where the operator can still fix it.
+		if m := strings.TrimSpace(r.Role.Mask); m != "" && !validDottedMask(m) {
+			return fmt.Errorf("%s: subnet mask %q must be dotted-decimal, e.g. 255.255.255.0", r.Plane, m)
+		}
+	}
+	return nil
+}
+
+// validDottedMask reports whether s is a contiguous dotted-decimal IPv4 subnet
+// mask. Prefix-length form ("24", "/24") is deliberately not accepted: netgen
+// parses the mask with net.ParseIP, so anything else fails the install itself.
+func validDottedMask(s string) bool {
+	ip := net.ParseIP(strings.TrimSpace(s))
+	if ip == nil || ip.To4() == nil {
+		return false
+	}
+	// Size reports zero bits for a non-contiguous mask, which netgen rejects too.
+	_, bits := net.IPMask(ip.To4()).Size()
+	return bits != 0
+}

cmd/installer/install/config_test.go

@@ -0,0 +1,463 @@
+package install
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// Node layouts drawn from the us-west-1 racks, which between them exercise
+// every collapse permutation the installer has to express.
+func threeNIC() *Config { // hydrogen / radon
+	return &Config{
+		WAN: NetworkRole{Interface: "eno6", Address: "216.218.163.99", Mask: "255.255.255.224", Gateway: "216.218.163.97"},
+		LAN: NetworkRole{Interface: "ens1f0np0", Address: "10.0.0.3", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "ens1f1np1", Address: "10.1.0.3", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func vlanSplit() *Config { // helium — one 100gbe carrying lan and vpc
+	return &Config{
+		WAN: NetworkRole{Interface: "enx6c1ff7c1475a", DHCPMode: true},
+		LAN: NetworkRole{Interface: "enp129s0np0", VLAN: 10, Address: "10.0.0.2", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "enp129s0np0", VLAN: 20, Address: "10.1.0.2", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func dualPortTagged() *Config { // us-west-1-az2 — wan and lan share port 1
+	return &Config{
+		WAN: NetworkRole{Interface: "enp1s0f0", VLAN: 30, Address: "216.218.163.10", Mask: "255.255.255.224", Gateway: "216.218.163.97"},
+		LAN: NetworkRole{Interface: "enp1s0f0", VLAN: 10, Address: "10.0.0.10", Mask: "255.255.0.0", MTU: 9000},
+		VPC: NetworkRole{Interface: "enp1s0f1", VLAN: 20, Address: "10.1.0.10", Mask: "255.255.0.0", MTU: 9000},
+	}
+}
+
+func TestResolveCollapsesUpTheChain(t *testing.T) {
+	singleNIC := &Config{WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"}}
+	twoNIC := &Config{
+		WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"},
+		LAN: NetworkRole{Interface: "eth1", Address: "10.0.0.5", Mask: "255.255.0.0"},
+	}
+
+	tests := []struct {
+		name      string
+		cfg       *Config
+		plane     Plane
+		wantAddr  string
+		wantPlane Plane
+	}{
+		{"1 NIC: lan folds onto wan", singleNIC, PlaneLAN, "192.168.1.10", PlaneWAN},
+		{"1 NIC: vpc folds all the way to wan", singleNIC, PlaneVPC, "192.168.1.10", PlaneWAN},
+		{"2 NIC: vpc folds onto lan", twoNIC, PlaneVPC, "10.0.0.5", PlaneLAN},
+		{"2 NIC: lan stays put", twoNIC, PlaneLAN, "10.0.0.5", PlaneLAN},
+		{"3 NIC: vpc is its own plane", threeNIC(), PlaneVPC, "10.1.0.3", PlaneVPC},
+		{"VLAN split: vpc is its own plane", vlanSplit(), PlaneVPC, "10.1.0.2", PlaneVPC},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.cfg.PlaneAddress(tt.plane); got != tt.wantAddr {
+				t.Errorf("PlaneAddress(%s) = %q, want %q", tt.plane, got, tt.wantAddr)
+			}
+			if _, landed := tt.cfg.Resolve(tt.plane); landed != tt.wantPlane {
+				t.Errorf("Resolve(%s) landed on %s, want %s", tt.plane, landed, tt.wantPlane)
+			}
+		})
+	}
+}
+
+// A DHCP plane has no address known at install time; setup-ovn.sh auto-detects
+// it from the default route at boot instead.
+func TestPlaneAddressEmptyUnderDHCP(t *testing.T) {
+	if got := vlanSplit().PlaneAddress(PlaneWAN); got != "" {
+		t.Errorf("PlaneAddress(wan) = %q, want empty for a DHCP role", got)
+	}
+}
+
+func TestValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     *Config
+		wantErr string
+	}{
+		{"three physical NICs", threeNIC(), ""},
+		{"VLAN split on one NIC", vlanSplit(), ""},
+		{"dual port, all tagged", dualPortTagged(), ""},
+		{
+			"wan cannot be folded",
+			&Config{LAN: NetworkRole{Interface: "eth1"}},
+			"wan role must be bound",
+		},
+		{
+			"two roles share a NIC untagged",
+			&Config{
+				WAN: NetworkRole{Interface: "eth0"},
+				LAN: NetworkRole{Interface: "eth0"},
+			},
+			"both bind eth0",
+		},
+		{
+			"duplicate VLAN id on one NIC",
+			&Config{
+				WAN: NetworkRole{Interface: "eth0", VLAN: 10},
+				LAN: NetworkRole{Interface: "eth0", VLAN: 10},
+			},
+			"both bind eth0.10",
+		},
+		{
+			"VLAN id out of range",
+			&Config{WAN: NetworkRole{Interface: "eth0", VLAN: 5000}},
+			"out of range",
+		},
+		{
+			"MTU out of range",
+			&Config{WAN: NetworkRole{Interface: "eth0", MTU: 70000}},
+			"MTU 70000 out of range",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("Validate() = %v, want nil", err)
+				}
+				return
+			}
+			if err == nil {
+				t.Fatalf("Validate() = nil, want error containing %q", tt.wantErr)
+			}
+			if !strings.Contains(err.Error(), tt.wantErr) {
+				t.Errorf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Two roles sharing one NIC must produce a single parent .network carrying both
+// VLAN declarations — not two files racing to configure the same interface.
+func TestWriteParentNICTaggedSharesOneFile(t *testing.T) {
+	dir := t.TempDir()
+	p := &parentNIC{vlans: []string{"enp129s0np0.10", "enp129s0np0.20"}, mtu: 9000}
+	if err := writeParentNIC(dir, "enp129s0np0", p); err != nil {
+		t.Fatal(err)
+	}
+
+	got := readFile(t, dir, "05-spinifex-nic-enp129s0np0.network")
+	for _, want := range []string{
+		"Name=enp129s0np0",
+		"MTUBytes=9000",
+		"VLAN=enp129s0np0.10",
+		"VLAN=enp129s0np0.20",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("parent .network missing %q:\n%s", want, got)
+		}
+	}
+	// A tagged parent is never itself enslaved — the VLAN devices are.
+	if strings.Contains(got, "Bridge=") {
+		t.Errorf("tagged parent must not be bridged directly:\n%s", got)
+	}
+}
+
+func TestWriteNetworkdBridgeVLAN(t *testing.T) {
+	dir := t.TempDir()
+	cfg := vlanSplit()
+	if err := writeNetworkdBridge(dir, PlaneVPC, cfg.VPC, true); err != nil {
+		t.Fatal(err)
+	}
+
+	netdev := readFile(t, dir, "10-spinifex-vpc-vlan.netdev")
+	if !strings.Contains(netdev, "Kind=vlan") || !strings.Contains(netdev, "Id=20") {
+		t.Errorf("VLAN netdev wrong:\n%s", netdev)
+	}
+	if !strings.Contains(netdev, "Name=enp129s0np0.20") {
+		t.Errorf("VLAN netdev name wrong:\n%s", netdev)
+	}
+
+	vlanNet := readFile(t, dir, "11-spinifex-vpc-vlan.network")
+	if !strings.Contains(vlanNet, "Bridge=br-vpc") {
+		t.Errorf("VLAN device must be enslaved to br-vpc:\n%s", vlanNet)
+	}
+
+	brNet := readFile(t, dir, "20-spinifex-vpc.network")
+	if !strings.Contains(brNet, "Address=10.1.0.2/16") {
+		t.Errorf("bridge address wrong:\n%s", brNet)
+	}
+	// Only wan installs a default route; an east-west plane must not.
+	if strings.Contains(brNet, "Gateway=") {
+		t.Errorf("vpc plane must not install a default route:\n%s", brNet)
+	}
+	if !strings.Contains(brNet, "ActivationPolicy=manual") {
+		t.Errorf("east-west bridge must not auto-activate:\n%s", brNet)
+	}
+}
+
+// The wan plane is the one that carries a default route and auto-activates.
+func TestWriteNetworkdBridgeWANUntagged(t *testing.T) {
+	dir := t.TempDir()
+	cfg := threeNIC()
+	if err := writeNetworkdBridge(dir, PlaneWAN, cfg.WAN, false); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := os.Stat(filepath.Join(dir, "10-spinifex-wan-vlan.netdev")); !os.IsNotExist(err) {
+		t.Error("untagged role must not produce a VLAN netdev")
+	}
+	brNet := readFile(t, dir, "20-spinifex-wan.network")
+	if !strings.Contains(brNet, "Gateway=216.218.163.97") {
+		t.Errorf("wan must install its default route:\n%s", brNet)
+	}
+	if strings.Contains(brNet, "ActivationPolicy=manual") {
+		t.Errorf("wan bridge must auto-activate:\n%s", brNet)
+	}
+}
+
+func TestToFirstbootConfigSourcesPlanes(t *testing.T) {
+	cfg := threeNIC()
+	cfg.Hostname = "hydrogen"
+	fb := cfg.toFirstbootConfig()
+
+	if fb.EncapIP != "10.1.0.3" {
+		t.Errorf("EncapIP = %q, want the vpc plane address", fb.EncapIP)
+	}
+	if fb.LANIP != "10.0.0.3" {
+		t.Errorf("LANIP = %q, want the lan plane address", fb.LANIP)
+	}
+	// The public address has to travel separately: spx echoes a concrete --bind
+	// back as the advertise address, so leaving this empty would publish the
+	// internal plane as the node's public dial target.
+	if fb.WANIP != "216.218.163.99" {
+		t.Errorf("WANIP = %q, want the wan plane address", fb.WANIP)
+	}
+
+	// On a single-NIC node all three collapse onto wan.
+	single := &Config{WAN: NetworkRole{Interface: "eth0", Address: "192.168.1.10", Mask: "255.255.255.0"}}
+	fb = single.toFirstbootConfig()
+	if fb.EncapIP != "192.168.1.10" || fb.LANIP != "192.168.1.10" || fb.WANIP != "192.168.1.10" {
+		t.Errorf("single-NIC collapse wrong: EncapIP=%q LANIP=%q WANIP=%q", fb.EncapIP, fb.LANIP, fb.WANIP)
+	}
+}
+
+func readFile(t *testing.T, dir, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(dir, name))
+	if err != nil {
+		t.Fatalf("reading %s: %v", name, err)
+	}
+	return string(b)
+}
+
+// Resolvers belong to the plane holding the default route. The east-west
+// bridges are link-local and activate manually, so a resolver pinned to one is
+// unreachable while resolved still tries it — a timeout in front of every
+// lookup. The TUI cannot express this; the kernel cmdline can, so it is caught.
+func TestValidateRejectsDNSOffWAN(t *testing.T) {
+	cfg := threeNIC()
+	cfg.LAN.DNS = []string{"10.0.0.1"}
+	if err := cfg.Validate(); err == nil {
+		t.Fatal("lan must not carry DNS servers")
+	}
+
+	cfg = threeNIC()
+	cfg.VPC.DNS = []string{"10.1.0.1"}
+	if err := cfg.Validate(); err == nil {
+		t.Fatal("vpc must not carry DNS servers")
+	}
+
+	cfg = threeNIC()
+	cfg.WAN.DNS = []string{"8.8.8.8", "1.1.1.1"}
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("wan carries the resolvers: %v", err)
+	}
+}
+
+// Even if a role somehow carries DNS, generation must not write it onto an
+// east-west bridge.
+func TestWriteNetworkdBridgeKeepsDNSOnWAN(t *testing.T) {
+	dir := t.TempDir()
+	lan := NetworkRole{Interface: "ens1f0np0", Address: "10.0.0.3", Mask: "255.255.0.0", DNS: []string{"10.0.0.1"}}
+	if err := writeNetworkdBridge(dir, PlaneLAN, lan, true); err != nil {
+		t.Fatalf("writeNetworkdBridge: %v", err)
+	}
+	if got := readFile(t, dir, "20-spinifex-lan.network"); strings.Contains(got, "DNS=") {
+		t.Errorf("lan bridge must carry no resolver:\n%s", got)
+	}
+
+	wan := NetworkRole{Interface: "eno6", Address: "216.218.163.99", Mask: "255.255.255.224", Gateway: "216.218.163.97", DNS: []string{"8.8.8.8"}}
+	if err := writeNetworkdBridge(dir, PlaneWAN, wan, false); err != nil {
+		t.Fatalf("writeNetworkdBridge: %v", err)
+	}
+	if got := readFile(t, dir, "20-spinifex-wan.network"); !strings.Contains(got, "DNS=8.8.8.8") {
+		t.Errorf("wan bridge must carry the resolver:\n%s", got)
+	}
+}
+
+func TestSwapSize(t *testing.T) {
+	dir := t.TempDir()
+
+	// Without an override, RAM drives the size and the disk caps it.
+	ram, err := totalRAM()
+	if err != nil {
+		t.Fatalf("totalRAM: %v", err)
+	}
+	got, err := swapSize(dir)
+	if err != nil {
+		t.Fatalf("swapSize: %v", err)
+	}
+	want := ram / 2
+	if ram >= swapRAMThreshold {
+		want = swapLargeBytes
+	}
+	if got > want {
+		t.Errorf("swapSize = %d, must not exceed what RAM implies (%d)", got, want)
+	}
+	if got != 0 && got < swapMinBytes {
+		t.Errorf("swapSize = %d, must be 0 rather than below the %d floor", got, int64(swapMinBytes))
+	}
+
+	// An explicit size wins, so a large node can be told to take the full 32G
+	// and a constrained one can opt out entirely.
+	for _, tt := range []struct {
+		env  string
+		want int64
+	}{
+		{"32G", 32 << 30},
+		{"512M", 512 << 20},
+		{"0", 0},
+		{"1073741824", 1 << 30},
+	} {
+		t.Setenv("SPINIFEX_SWAP_SIZE", tt.env)
+		got, err := swapSize(dir)
+		if err != nil {
+			t.Fatalf("swapSize(%s): %v", tt.env, err)
+		}
+		if got != tt.want {
+			t.Errorf("swapSize(%s) = %d, want %d", tt.env, got, tt.want)
+		}
+	}
+
+	t.Setenv("SPINIFEX_SWAP_SIZE", "banana")
+	if _, err := swapSize(dir); err == nil {
+		t.Error("an unparseable size must be reported, not silently ignored")
+	}
+}
+
+// The swap line must only appear when the file backing it exists — systemd
+// fails the boot's swap unit on a missing file rather than skipping it.
+func TestFstabSwapLineTracksTheFile(t *testing.T) {
+	if _, err := parseSize("8G"); err != nil {
+		t.Fatalf("parseSize: %v", err)
+	}
+	if n, _ := parseSize("2G"); n != 2<<30 {
+		t.Errorf("parseSize(2G) = %d, want %d", n, int64(2)<<30)
+	}
+	if _, err := parseSize("-5G"); err == nil {
+		t.Error("negative sizes must be rejected")
+	}
+}
+
+// Bridge ports and VLAN trunks never reach the "degraded" state
+// systemd-networkd-wait-online requires — the address sits on the bridge above
+// them. Leaving them required makes that unit fail on every boot and stall
+// until it times out, which is what a three-NIC node was doing.
+func TestSlaveLinksAreNotRequiredForOnline(t *testing.T) {
+	dir := t.TempDir()
+
+	// Untagged bridge port.
+	if err := writeParentNIC(dir, "ens4", &parentNIC{bridge: "br-lan", mtu: 9000}); err != nil {
+		t.Fatal(err)
+	}
+	got := readFile(t, dir, "05-spinifex-nic-ens4.network")
+	if !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("bridge port must not gate network-online:\n%s", got)
+	}
+	if !strings.Contains(got, "MTUBytes=9000") {
+		t.Errorf("MTU must survive alongside it:\n%s", got)
+	}
+
+	// VLAN trunk carrying tagged planes.
+	if err := writeParentNIC(dir, "enp129s0np0", &parentNIC{vlans: []string{"enp129s0np0.10"}}); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "05-spinifex-nic-enp129s0np0.network"); !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("VLAN trunk must not gate network-online:\n%s", got)
+	}
+
+	// The VLAN device itself is enslaved too.
+	cfg := vlanSplit()
+	if err := writeNetworkdBridge(dir, PlaneVPC, cfg.VPC, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "11-spinifex-vpc-vlan.network"); !strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("VLAN device must not gate network-online:\n%s", got)
+	}
+
+	// The wan bridge is what should gate it, and still does.
+	three := threeNIC()
+	if err := writeNetworkdBridge(dir, PlaneWAN, three.WAN, false); err != nil {
+		t.Fatal(err)
+	}
+	if got := readFile(t, dir, "20-spinifex-wan.network"); strings.Contains(got, "RequiredForOnline=no") {
+		t.Errorf("wan bridge must remain required for network-online:\n%s", got)
+	}
+}
+
+// A prefix length reaches netgen's net.ParseIP as garbage and fails the install
+// after the disk has been partitioned, so it has to be rejected up front. The
+// TUI advertised "255.255.255.0 or 24" while only the first ever worked.
+func TestValidateRejectsPrefixLengthMask(t *testing.T) {
+	for _, bad := range []string{"24", "/24", "16", "255.255.0", "255.0.255.0", "banana"} {
+		cfg := threeNIC()
+		cfg.LAN.Mask = bad
+		if err := cfg.Validate(); err == nil {
+			t.Errorf("mask %q must be rejected", bad)
+		} else if !strings.Contains(err.Error(), "dotted-decimal") {
+			t.Errorf("mask %q: error should name the expected form, got: %v", bad, err)
+		}
+	}
+
+	for _, good := range []string{"255.255.255.0", "255.255.0.0", "255.255.255.252", "255.255.255.255"} {
+		cfg := threeNIC()
+		cfg.LAN.Mask = good
+		if err := cfg.Validate(); err != nil {
+			t.Errorf("mask %q must be accepted: %v", good, err)
+		}
+	}
+
+	// A DHCP role carries no mask, and that must stay valid.
+	cfg := threeNIC()
+	cfg.LAN = NetworkRole{Interface: "ens1f0np0", DHCPMode: true}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("a DHCP role has no mask to check: %v", err)
+	}
+}
+
+// A bridged uplink must present the enslaved NIC's address. Left to its default
+// policy, systemd-networkd generates one for the bridge, so the node appears on
+// the wire as an address belonging to no NIC — DHCP requests carry a source that
+// disagrees with their own chaddr and the unicast reply is never delivered.
+func TestBridgeInheritsPortMAC(t *testing.T) {
+	dir := t.TempDir()
+	cfg := threeNIC()
+
+	for _, tc := range []struct {
+		plane  Plane
+		role   NetworkRole
+		manual bool
+	}{
+		{PlaneWAN, cfg.WAN, false},
+		{PlaneLAN, cfg.LAN, true},
+		{PlaneVPC, cfg.VPC, true},
+	} {
+		if err := writeNetworkdBridge(dir, tc.plane, tc.role, tc.manual); err != nil {
+			t.Fatal(err)
+		}
+		got := readFile(t, dir, "20-spinifex-"+string(tc.plane)+".netdev")
+		if !strings.Contains(got, "MACAddressPolicy=none") {
+			t.Errorf("%s bridge must inherit its port's MAC:\n%s", tc.plane, got)
+		}
+	}
+}

cmd/installer/install/disk.go

@@ -0,0 +1,158 @@
+package install
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"time"
+)
+
+// Partition numbers, identical on every disk the installer touches so that a
+// replacement drive can be cloned with `sgdisk -R` and nothing has to be
+// recomputed at 3am.
+const (
+	biosPartNum = 1
+	espPartNum  = 2
+	rootPartNum = 3
+)
+
+// GPT partition type GUIDs, by their sgdisk short code.
+const (
+	typeBIOSBoot = "EF02"
+	typeESP      = "EF00"
+	typeLinuxFS  = "8300"
+	typeZFS      = "BF00"
+)
+
+// Partition geometry. The ESP is sized as Proxmox does: 512MiB is enough for
+// two kernels, but a 1GiB ESP on a real disk costs nothing and keeps room for
+// the extra kernels an operator pins during an upgrade.
+const (
+	espSmallMiB     = 512
+	espLargeMiB     = 1024
+	espLargeDiskGiB = 100
+
+	// biosBootMiB is the BIOS Boot Partition GRUB's i386-pc core.img is embedded
+	// into. GPT leaves no post-MBR gap, so without it there is nowhere to put it.
+	biosBootMiB = 1
+
+	// tailReserveMiB is left unallocated at the end of every disk. A replacement
+	// drive of the "same" size is routinely a few thousand sectors smaller, and
+	// without slack `sgdisk -R` refuses to clone the table onto it — which is
+	// discovered while standing in front of a degraded pool.
+	tailReserveMiB = 1024
+)
+
+// espSizeMiB picks the ESP size for a disk.
+func espSizeMiB(d Disk) int {
+	if d.Bytes >= espLargeDiskGiB<<30 {
+		return espLargeMiB
+	}
+	return espSmallMiB
+}
+
+// partitionDisks lays down an identical GPT on every selected disk. Each disk
+// gets a BIOS boot partition and an ESP whether or not this machine booted in
+// EFI mode: the cost is 1GiB, and it means a board swap or a firmware change
+// from BIOS to UEFI does not leave an unbootable pool.
+func partitionDisks(cfg DiskConfig) error {
+	ptype := typeLinuxFS
+	if cfg.FS.IsZFS() {
+		ptype = typeZFS
+	}
+	for _, d := range cfg.Disks {
+		if err := partitionOne(d, ptype); err != nil {
+			return fmt.Errorf("partition %s: %w", d.Path, err)
+		}
+	}
+	return waitForPartitions(cfg)
+}
+
+// partitionOne writes the GPT for a single disk.
+func partitionOne(d Disk, rootType string) error {
+	// Stale ZFS labels and filesystem signatures survive a new partition table
+	// and make a retried install fail with errors that read like hardware faults.
+	if err := clearDisk(d); err != nil {
+		return fmt.Errorf("clear: %w", err)
+	}
+
+	esp := espSizeMiB(d)
+	if err := run("sgdisk",
+		"-n", fmt.Sprintf("%d:1M:+%dM", biosPartNum, biosBootMiB),
+		"-t", fmt.Sprintf("%d:%s", biosPartNum, typeBIOSBoot),
+		"-c", fmt.Sprintf("%d:bios_boot", biosPartNum),
+		"-n", fmt.Sprintf("%d:0:+%dM", espPartNum, esp),
+		"-t", fmt.Sprintf("%d:%s", espPartNum, typeESP),
+		"-c", fmt.Sprintf("%d:ESP", espPartNum),
+		"-n", fmt.Sprintf("%d:0:-%dM", rootPartNum, tailReserveMiB),
+		"-t", fmt.Sprintf("%d:%s", rootPartNum, rootType),
+		"-c", fmt.Sprintf("%d:root", rootPartNum),
+		d.Path,
+	); err != nil {
+		return err
+	}
+	// A fresh disk-level GUID: cloning a table with `sgdisk -R` copies it, and
+	// two disks sharing one leaves the pool ambiguous to udev.
+	if err := run("sgdisk", "-G", d.Path); err != nil {
+		slog.Warn("could not randomise GPT GUIDs", "disk", d.Path, "err", err)
+	}
+	return nil
+}
+
+// waitForPartitions ensures every partition device node the installer is about
+// to use exists. Trixie's udev is slow enough after a table rewrite that mkfs
+// races it and fails with ENOENT on a device the kernel has already accepted.
+func waitForPartitions(cfg DiskConfig) error {
+	for _, d := range cfg.Disks {
+		// Best-effort: partprobe failing is not fatal, since udev may still act
+		// on the BLKRRPART ioctl sgdisk issued itself.
+		if err := run("partprobe", d.Path); err != nil {
+			slog.Warn("partprobe failed, continuing", "disk", d.Path, "err", err)
+		}
+	}
+	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
+		slog.Warn("udevadm settle failed, continuing", "err", err)
+	}
+
+	deadline := time.Now().Add(15 * time.Second)
+	for _, d := range cfg.Disks {
+		wanted := []string{d.PartitionPath(espPartNum), d.PartitionPath(rootPartNum)}
+		// The pool is built from by-id paths, so their symlinks have to exist
+		// too — udev publishes them later than the kernel device node.
+		if d.ByID != "" {
+			wanted = append(wanted, d.StablePartitionPath(rootPartNum))
+		}
+		for _, part := range wanted {
+			if err := waitForPath(part, deadline); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func waitForPath(path string, deadline time.Time) error {
+	for {
+		if _, err := os.Stat(path); err == nil {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("device %s did not appear within timeout — the kernel or udev did not pick up the new partition table", path)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// formatESPs makes a FAT32 filesystem on every disk's ESP. Each one is
+// independent — there is no RAID here, they are kept in step by content.
+func formatESPs(cfg DiskConfig) error {
+	for _, d := range cfg.Disks {
+		esp := d.PartitionPath(espPartNum)
+		// A distinct label per member so `lsblk -f` on a degraded machine tells
+		// the operator which disk each ESP belongs to.
+		if err := run("mkfs.fat", "-F", "32", "-n", "EFI", esp); err != nil {
+			return fmt.Errorf("format ESP %s: %w", esp, err)
+		}
+	}
+	return nil
+}

cmd/installer/install/diskconfig.go

@@ -0,0 +1,318 @@
+package install
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"strings"
+)
+
+// Pool and dataset names. rpool matches Proxmox, which is what every ZFS
+// recovery guide and forum answer assumes — the name an operator will be
+// reading off a serial console at 3am.
+const (
+	ZFSPoolName    = "rpool"
+	ZFSRootDataset = ZFSPoolName + "/ROOT/spinifex"
+)
+
+// FSType selects the root filesystem and, for ZFS, the vdev topology.
+type FSType string
+
+const (
+	FSExt4      FSType = "ext4"
+	FSZFSRAID0  FSType = "zfs-raid0"
+	FSZFSRAID1  FSType = "zfs-raid1"
+	FSZFSRAID10 FSType = "zfs-raid10"
+	FSZFSRAIDZ1 FSType = "zfs-raidz1"
+	FSZFSRAIDZ2 FSType = "zfs-raidz2"
+	FSZFSRAIDZ3 FSType = "zfs-raidz3"
+)
+
+// AllFSTypes is the selector order presented by the UI. ext4 is first and is
+// the default, so ZFS — which holds the kernel at its shipped version and
+// reserves RAM for the ARC — is always an explicit choice.
+var AllFSTypes = []FSType{
+	FSExt4, FSZFSRAID0, FSZFSRAID1, FSZFSRAID10, FSZFSRAIDZ1, FSZFSRAIDZ2, FSZFSRAIDZ3,
+}
+
+// fsMeta carries the per-topology facts that drive display and validation.
+var fsMeta = map[FSType]struct {
+	label     string
+	minDisks  int
+	redundant bool   // subject to the same-size rule
+	vdev      string // zpool create keyword; empty means a plain stripe
+}{
+	FSExt4:      {"ext4", 1, false, ""},
+	FSZFSRAID0:  {"zfs (RAID0)", 1, false, ""},
+	FSZFSRAID1:  {"zfs (RAID1)", 2, true, "mirror"},
+	FSZFSRAID10: {"zfs (RAID10)", 4, true, "mirror"},
+	FSZFSRAIDZ1: {"zfs (RAIDZ-1)", 3, true, "raidz1"},
+	FSZFSRAIDZ2: {"zfs (RAIDZ-2)", 4, true, "raidz2"},
+	FSZFSRAIDZ3: {"zfs (RAIDZ-3)", 5, true, "raidz3"},
+}
+
+// IsZFS reports whether this mode creates a pool rather than a plain filesystem.
+func (f FSType) IsZFS() bool { return strings.HasPrefix(string(f), "zfs-") }
+
+// Label is the display name, matching proxinstall's wording so the screen is
+// recognisable to anyone who has installed Proxmox.
+func (f FSType) Label() string { return fsMeta[f].label }
+
+// MinDisks is the smallest member count the topology can be built from.
+func (f FSType) MinDisks() int { return fsMeta[f].minDisks }
+
+// Redundant reports whether the topology stores parity or mirrors, and so
+// requires members of matching size. A RAID0 stripe over unequal disks is
+// legitimate and is deliberately not covered.
+func (f FSType) Redundant() bool { return fsMeta[f].redundant }
+
+// vdevKeyword is the zpool create keyword introducing each vdev.
+func (f FSType) vdevKeyword() string { return fsMeta[f].vdev }
+
+// ParseFSType maps a SPINIFEX_FS value to an FSType.
+func ParseFSType(s string) (FSType, error) {
+	f := FSType(strings.ToLower(strings.TrimSpace(s)))
+	if _, ok := fsMeta[f]; !ok {
+		names := make([]string, 0, len(AllFSTypes))
+		for _, t := range AllFSTypes {
+			names = append(names, string(t))
+		}
+		return "", fmt.Errorf("unknown filesystem %q — must be one of: %s", s, strings.Join(names, ", "))
+	}
+	return f, nil
+}
+
+// ZFSOpts are the advanced tunables. A zero value means "use the computed
+// default", which is why Ashift and ARCMaxMiB are plain ints — 0 is not legal
+// for either, so it is unambiguous as a sentinel.
+type ZFSOpts struct {
+	Ashift    int
+	Compress  string
+	Checksum  string
+	Copies    int
+	ARCMaxMiB int
+}
+
+// DiskConfig is the storage half of the installer's configuration. Disks is
+// ordered and the order is significant: RAID10 pairs members two at a time.
+type DiskConfig struct {
+	FS    FSType
+	Disks []Disk
+	ZFS   ZFSOpts
+}
+
+// Primary is the first selected disk — the whole target in ext4 mode, and the
+// disk shown in single-disk summaries.
+func (d DiskConfig) Primary() Disk {
+	if len(d.Disks) == 0 {
+		return Disk{}
+	}
+	return d.Disks[0]
+}
+
+// Paths lists the selected device paths, for logs and confirmation screens.
+func (d DiskConfig) Paths() []string {
+	out := make([]string, len(d.Disks))
+	for i, disk := range d.Disks {
+		out[i] = disk.Path
+	}
+	return out
+}
+
+// Buildable reports whether the selection has enough members, of the right
+// count, to form the topology at all. Capacity and fault tolerance are
+// meaningless until it does — a 3-disk RAIDZ-3 has no geometry to describe.
+func (d DiskConfig) Buildable() bool {
+	if _, ok := fsMeta[d.FS]; !ok {
+		return false
+	}
+	if len(d.Disks) < d.FS.MinDisks() {
+		return false
+	}
+	return d.FS != FSZFSRAID10 || len(d.Disks)%2 == 0
+}
+
+// Requirement describes what the topology still needs, for the preview line
+// shown in place of a capacity that cannot be computed yet.
+func (d DiskConfig) Requirement() string {
+	req := fmt.Sprintf("needs %d disks", d.FS.MinDisks())
+	if d.FS == FSZFSRAID10 {
+		req = "needs an even number of disks, at least 4"
+	}
+	if d.FS.Redundant() {
+		req += " of the same size"
+	}
+	return req
+}
+
+// UsableBytes estimates post-parity capacity, for the geometry preview. It is
+// an operator-facing approximation and deliberately ignores metadata overhead.
+// Returns 0 for a selection that cannot form the topology, since the parity
+// arithmetic below goes to zero and then negative as members run out.
+func (d DiskConfig) UsableBytes() int64 {
+	n := int64(len(d.Disks))
+	if n == 0 || !d.Buildable() {
+		return 0
+	}
+	smallest := d.Disks[0].Bytes
+	for _, disk := range d.Disks {
+		smallest = min(smallest, disk.Bytes)
+	}
+	switch d.FS {
+	case FSExt4, FSZFSRAID0:
+		var total int64
+		for _, disk := range d.Disks {
+			total += disk.Bytes
+		}
+		return total
+	case FSZFSRAID1:
+		return smallest
+	case FSZFSRAID10:
+		return smallest * (n / 2)
+	case FSZFSRAIDZ1:
+		return smallest * (n - 1)
+	case FSZFSRAIDZ2:
+		return smallest * (n - 2)
+	case FSZFSRAIDZ3:
+		return smallest * (n - 3)
+	}
+	return 0
+}
+
+// Tolerated returns the number of simultaneous disk failures the topology
+// survives, for the geometry preview.
+func (d DiskConfig) Tolerated() int {
+	if !d.Buildable() {
+		return 0
+	}
+	switch d.FS {
+	case FSZFSRAID1:
+		return max(len(d.Disks)-1, 0)
+	case FSZFSRAID10:
+		return 1 // one per mirror pair; the honest worst case is one
+	case FSZFSRAIDZ1:
+		return 1
+	case FSZFSRAIDZ2:
+		return 2
+	case FSZFSRAIDZ3:
+		return 3
+	}
+	return 0
+}
+
+// sizeTolerance is the fraction by which pool members may differ in size.
+// Matches Proxmox's zfs_mirror_size_check: anything beyond 10% and the pool
+// silently truncates every member to the smallest.
+const sizeTolerance = 0.10
+
+// bootedEFI reports whether firmware booted this machine in EFI mode. GRUB's
+// i386-pc target cannot read a 4Kn drive, so legacy BIOS constrains selection.
+func bootedEFI() bool {
+	_, err := os.Stat("/sys/firmware/efi")
+	return err == nil
+}
+
+// Validate checks a disk selection against every rule that must hold before any
+// destructive command runs. The UI calls it on each keystroke so failures block
+// Continue, rather than aborting an install that has already repartitioned.
+func (d DiskConfig) Validate() error {
+	if _, ok := fsMeta[d.FS]; !ok {
+		return fmt.Errorf("no filesystem selected")
+	}
+	if len(d.Disks) == 0 {
+		return fmt.Errorf("%s: select at least one disk", d.FS.Label())
+	}
+	if !d.FS.IsZFS() && len(d.Disks) > 1 {
+		return fmt.Errorf("%s: supports a single disk only, %d selected — choose a zfs mode to use them all",
+			d.FS.Label(), len(d.Disks))
+	}
+	if n := len(d.Disks); n < d.FS.MinDisks() {
+		return fmt.Errorf("%s: needs at least %d disks, %d selected", d.FS.Label(), d.FS.MinDisks(), n)
+	}
+	if d.FS == FSZFSRAID10 && len(d.Disks)%2 != 0 {
+		return fmt.Errorf("%s: needs an even number of disks, %d selected", d.FS.Label(), len(d.Disks))
+	}
+
+	seen := map[string]bool{}
+	for _, disk := range d.Disks {
+		if seen[disk.Path] {
+			return fmt.Errorf("%s is selected more than once", disk.Path)
+		}
+		seen[disk.Path] = true
+
+		if disk.LiveMedia {
+			return fmt.Errorf("%s is the installer's own boot media and cannot be erased", disk.Path)
+		}
+		// GRUB's i386-pc target cannot read a 4Kn drive, so on legacy BIOS the
+		// installed system would have no reachable bootloader.
+		if !bootedEFI() && disk.LogicalBlockSize == 4096 {
+			return fmt.Errorf("%s is a 4Kn drive (4096-byte sectors) and this machine booted in legacy BIOS mode — booting from it is not supported; enable UEFI in firmware",
+				disk.Path)
+		}
+		if disk.Bytes < minRootBytes {
+			return fmt.Errorf("%s is too small (%s, need at least %dGiB)",
+				disk.Path, disk.SizeHuman(), minRootBytes>>30)
+		}
+	}
+
+	return d.validateSizes()
+}
+
+// minRootBytes matches Proxmox's hard floor for a root disk.
+const minRootBytes = 2 << 30
+
+// validateSizes enforces the same-size rule where the topology needs it:
+// across all members for mirrors and raidz, and per-pair for RAID10 — a
+// striped mirror only requires each pair to match, not every disk in the pool.
+func (d DiskConfig) validateSizes() error {
+	if !d.FS.Redundant() {
+		return nil
+	}
+	if d.FS == FSZFSRAID10 {
+		for i := 0; i+1 < len(d.Disks); i += 2 {
+			if err := sizesMatch(d.Disks[i], d.Disks[i+1]); err != nil {
+				return fmt.Errorf("%s mirror pair %d: %w", d.FS.Label(), i/2+1, err)
+			}
+		}
+		return nil
+	}
+	for _, disk := range d.Disks[1:] {
+		if err := sizesMatch(d.Disks[0], disk); err != nil {
+			return fmt.Errorf("%s: %w", d.FS.Label(), err)
+		}
+	}
+	return nil
+}
+
+// SizesWithinTolerance reports whether two disks could be members of the same
+// redundant pool. The UI uses it to decide whether suggesting ZFS is honest.
+func SizesWithinTolerance(a, b Disk) bool { return sizesMatch(a, b) == nil }
+
+// sizesMatch reports whether two members are within the tolerance, naming both
+// disks and their sizes — that message is the operator's only diagnostic.
+func sizesMatch(a, b Disk) error {
+	if math.Abs(float64(a.Bytes-b.Bytes)) > float64(a.Bytes)*sizeTolerance {
+		return fmt.Errorf("disks must be the same size within %.0f%%: %s is %s, %s is %s",
+			sizeTolerance*100, a.Path, a.SizeHuman(), b.Path, b.SizeHuman())
+	}
+	return nil
+}
+
+// Warnings returns non-blocking advice for a valid selection. A RAID0 stripe
+// over mismatched disks is legal but almost always a mistake, so it is
+// surfaced without preventing the install — which is what Proxmox does.
+func (d DiskConfig) Warnings() []string {
+	var out []string
+	if d.FS == FSZFSRAID0 && len(d.Disks) > 1 {
+		for _, disk := range d.Disks[1:] {
+			if sizesMatch(d.Disks[0], disk) != nil {
+				out = append(out, "striping disks of different sizes — capacity is usable but a failure of any disk loses the pool")
+				break
+			}
+		}
+	}
+	if d.FS.IsZFS() && d.FS != FSZFSRAID0 && d.Buildable() && d.Tolerated() == 0 {
+		out = append(out, "this topology stores no redundancy")
+	}
+	return out
+}

cmd/installer/install/diskconfig_test.go

@@ -0,0 +1,267 @@
+package install
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDiskConfigValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     DiskConfig
+		wantErr string
+	}{
+		{"ext4 on one disk", DiskConfig{FS: FSExt4, Disks: disks(1, 40)}, ""},
+		{"raidz1 on three matching disks", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)}, ""},
+		{"raid10 on four matching disks", DiskConfig{FS: FSZFSRAID10, Disks: disks(4, 40)}, ""},
+		// The single-drive ZFS case: no redundancy, but checksums, compression
+		// and snapshots still apply.
+		{"raid0 on one disk", DiskConfig{FS: FSZFSRAID0, Disks: disks(1, 40)}, ""},
+
+		{"no filesystem chosen", DiskConfig{Disks: disks(1, 40)}, "no filesystem selected"},
+		{"no disks", DiskConfig{FS: FSZFSRAIDZ1}, "select at least one disk"},
+		{"ext4 cannot span disks", DiskConfig{FS: FSExt4, Disks: disks(2, 40)}, "single disk only"},
+		{"raidz1 needs three", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(2, 40)}, "at least 3 disks"},
+		{"raidz2 needs four", DiskConfig{FS: FSZFSRAIDZ2, Disks: disks(3, 40)}, "at least 4 disks"},
+		{"raidz3 needs five", DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(4, 40)}, "at least 5 disks"},
+		{"raid1 needs two", DiskConfig{FS: FSZFSRAID1, Disks: disks(1, 40)}, "at least 2 disks"},
+		{"raid10 rejects an odd count", DiskConfig{FS: FSZFSRAID10, Disks: disks(5, 40)}, "even number"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			switch {
+			case tt.wantErr == "" && err != nil:
+				t.Fatalf("Validate() = %v, want nil", err)
+			case tt.wantErr != "" && err == nil:
+				t.Fatalf("Validate() = nil, want error containing %q", tt.wantErr)
+			case tt.wantErr != "" && !strings.Contains(err.Error(), tt.wantErr):
+				t.Fatalf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestValidateRejectsDuplicateDisk(t *testing.T) {
+	d := disk("a", 40)
+	err := DiskConfig{FS: FSZFSRAID1, Disks: []Disk{d, d}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "more than once") {
+		t.Fatalf("Validate() = %v, want a duplicate-selection error", err)
+	}
+}
+
+func TestValidateRejectsLiveMedia(t *testing.T) {
+	d := disk("a", 40)
+	d.LiveMedia = true
+	err := DiskConfig{FS: FSExt4, Disks: []Disk{d}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "boot media") {
+		t.Fatalf("Validate() = %v, want the installer's own media to be refused", err)
+	}
+}
+
+func TestValidateRejectsTooSmallDisk(t *testing.T) {
+	err := DiskConfig{FS: FSExt4, Disks: []Disk{disk("a", 1)}}.Validate()
+	if err == nil || !strings.Contains(err.Error(), "too small") {
+		t.Fatalf("Validate() = %v, want a minimum-size error", err)
+	}
+}
+
+// A pool truncates every member to the smallest, so mismatched disks in a
+// redundant topology silently throw capacity away.
+func TestValidateSizeTolerance(t *testing.T) {
+	tests := []struct {
+		name    string
+		fs      FSType
+		sizes   []int64
+		wantErr bool
+	}{
+		{"identical disks", FSZFSRAIDZ1, []int64{40, 40, 40}, false},
+		{"within 10%", FSZFSRAIDZ1, []int64{100, 95, 92}, false},
+		{"beyond 10%", FSZFSRAIDZ1, []int64{100, 100, 80}, true},
+		{"mirror with a mismatched member", FSZFSRAID1, []int64{100, 50}, true},
+		// A stripe has no parity to waste, so mismatched sizes are a warning
+		// rather than a refusal — matching Proxmox.
+		{"raid0 permits mismatched disks", FSZFSRAID0, []int64{100, 40}, false},
+		{"raid10 checks each pair, not the whole pool", FSZFSRAID10, []int64{40, 40, 100, 100}, false},
+		{"raid10 rejects a mismatched pair", FSZFSRAID10, []int64{40, 100, 100, 100}, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			members := make([]Disk, len(tt.sizes))
+			for i, gib := range tt.sizes {
+				members[i] = disk(string(rune('a'+i)), gib)
+			}
+			err := DiskConfig{FS: tt.fs, Disks: members}.Validate()
+			if tt.wantErr && err == nil {
+				t.Fatalf("Validate() = nil, want a size-mismatch error")
+			}
+			if !tt.wantErr && err != nil {
+				t.Fatalf("Validate() = %v, want nil", err)
+			}
+		})
+	}
+}
+
+func TestWarningsFlagMismatchedStripe(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAID0, Disks: []Disk{disk("a", 100), disk("b", 40)}}
+	warnings := cfg.Warnings()
+	if len(warnings) == 0 {
+		t.Fatal("a stripe over mismatched disks should warn")
+	}
+}
+
+func TestUsableBytes(t *testing.T) {
+	tests := []struct {
+		name    string
+		fs      FSType
+		n       int
+		wantGiB int64
+	}{
+		{"stripe adds every disk", FSZFSRAID0, 3, 120},
+		{"mirror gives one disk", FSZFSRAID1, 2, 40},
+		{"raid10 gives half", FSZFSRAID10, 4, 80},
+		{"raidz1 loses one to parity", FSZFSRAIDZ1, 3, 80},
+		{"raidz2 loses two", FSZFSRAIDZ2, 4, 80},
+		{"raidz3 loses three", FSZFSRAIDZ3, 5, 80},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if got := cfg.UsableBytes() >> 30; got != tt.wantGiB {
+				t.Errorf("UsableBytes() = %dG, want %dG", got, tt.wantGiB)
+			}
+		})
+	}
+}
+
+// The parity arithmetic goes to zero and then negative as members run out, and
+// a preview reading "-40GiB usable" is worse than no figure at all.
+func TestUsableBytesIsZeroWhenTopologyCannotBeBuilt(t *testing.T) {
+	tests := []struct {
+		name string
+		fs   FSType
+		n    int
+	}{
+		{"raidz3 with exactly its parity count", FSZFSRAIDZ3, 3},
+		{"raidz3 with fewer disks than parity", FSZFSRAIDZ3, 2},
+		{"raidz2 with two disks", FSZFSRAIDZ2, 2},
+		{"raidz1 with one disk", FSZFSRAIDZ1, 1},
+		{"mirror with one disk", FSZFSRAID1, 1},
+		{"raid10 with an odd count", FSZFSRAID10, 3},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if cfg.Buildable() {
+				t.Fatalf("Buildable() = true for %s across %d disks", tt.fs, tt.n)
+			}
+			if got := cfg.UsableBytes(); got != 0 {
+				t.Errorf("UsableBytes() = %d, want 0", got)
+			}
+			if got := cfg.Tolerated(); got != 0 {
+				t.Errorf("Tolerated() = %d, want 0", got)
+			}
+		})
+	}
+}
+
+// A selection that cannot be built must not also be described as unredundant:
+// the operator's problem is the disk count, not the topology.
+func TestWarningsStaySilentUntilBuildable(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(3, 40)}
+	if got := cfg.Warnings(); len(got) != 0 {
+		t.Errorf("Warnings() = %v, want none", got)
+	}
+}
+
+func TestRequirementNamesTheSameSizeRule(t *testing.T) {
+	if got := (DiskConfig{FS: FSZFSRAIDZ2}).Requirement(); !strings.Contains(got, "4 disks") ||
+		!strings.Contains(got, "same size") {
+		t.Errorf("Requirement() = %q, want the disk count and the same-size rule", got)
+	}
+	if got := (DiskConfig{FS: FSZFSRAID10}).Requirement(); !strings.Contains(got, "even") {
+		t.Errorf("Requirement() = %q, want the even-count rule", got)
+	}
+	// A stripe has no matching requirement, so claiming one would be wrong.
+	if got := (DiskConfig{FS: FSZFSRAID0}).Requirement(); strings.Contains(got, "same size") {
+		t.Errorf("Requirement() = %q, want no same-size rule for a stripe", got)
+	}
+}
+
+func TestTolerated(t *testing.T) {
+	tests := []struct {
+		fs   FSType
+		n    int
+		want int
+	}{
+		{FSExt4, 1, 0},
+		{FSZFSRAID0, 3, 0},
+		{FSZFSRAID1, 2, 1},
+		{FSZFSRAID1, 3, 2},
+		{FSZFSRAID10, 4, 1},
+		{FSZFSRAIDZ1, 3, 1},
+		{FSZFSRAIDZ2, 4, 2},
+		{FSZFSRAIDZ3, 5, 3},
+	}
+	for _, tt := range tests {
+		t.Run(string(tt.fs), func(t *testing.T) {
+			cfg := DiskConfig{FS: tt.fs, Disks: disks(tt.n, 40)}
+			if got := cfg.Tolerated(); got != tt.want {
+				t.Errorf("Tolerated() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseFSType(t *testing.T) {
+	for _, fs := range AllFSTypes {
+		got, err := ParseFSType(strings.ToUpper(string(fs)))
+		if err != nil {
+			t.Errorf("ParseFSType(%q): %v", fs, err)
+			continue
+		}
+		if got != fs {
+			t.Errorf("ParseFSType(%q) = %q", fs, got)
+		}
+	}
+	// The error has to list the valid values: it is read off a serial console
+	// by someone whose unattended install just stopped.
+	_, err := ParseFSType("btrfs")
+	if err == nil || !strings.Contains(err.Error(), "zfs-raidz1") {
+		t.Fatalf("ParseFSType(btrfs) = %v, want an error listing the accepted values", err)
+	}
+}
+
+func TestPartitionPathSeparator(t *testing.T) {
+	tests := []struct{ dev, want string }{
+		{"/dev/sda", "/dev/sda3"},
+		// A trailing digit needs a 'p' or the name is ambiguous.
+		{"/dev/nvme0n1", "/dev/nvme0n1p3"},
+		{"/dev/vda", "/dev/vda3"},
+	}
+	for _, tt := range tests {
+		if got := (Disk{Path: tt.dev}).PartitionPath(3); got != tt.want {
+			t.Errorf("PartitionPath(%s) = %s, want %s", tt.dev, got, tt.want)
+		}
+	}
+}
+
+// udev names partition links with a -partN suffix regardless of the kernel's
+// separator, so the by-id form must not reuse the 'p' rule.
+func TestStablePartitionPathUsesPartSuffix(t *testing.T) {
+	d := Disk{Path: "/dev/nvme0n1", ByID: "/dev/disk/by-id/nvme-SAMSUNG_1234"}
+	if got := d.StablePartitionPath(3); got != "/dev/disk/by-id/nvme-SAMSUNG_1234-part3" {
+		t.Errorf("StablePartitionPath() = %s", got)
+	}
+}
+
+func TestESPSizeMiB(t *testing.T) {
+	if got := espSizeMiB(disk("a", 40)); got != espSmallMiB {
+		t.Errorf("espSizeMiB(40G) = %d, want %d", got, espSmallMiB)
+	}
+	if got := espSizeMiB(disk("a", 500)); got != espLargeMiB {
+		t.Errorf("espSizeMiB(500G) = %d, want %d", got, espLargeMiB)
+	}
+}

cmd/installer/install/install.go

@@ -11,6 +11,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"slices"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -19,40 +20,55 @@ import (
 	"github.com/mulgadc/spinifex/cmd/installer/systemd"
 )
 
-const (
-	mountRoot = "/mnt/spinifex-install"
-	efiPart   = mountRoot + "/boot/efi"
-)
+// mountRoot is the install target. A variable, not a constant, so tests can
+// drive the file-writing steps against a temp tree rather than a real mount.
+var mountRoot = "/mnt/spinifex-install"
+
+// efiPart is the ESP mount point inside the target. Derived on each call so it
+// cannot go stale against mountRoot.
+func efiPart() string { return filepath.Join(mountRoot, "boot/efi") }
+
+// step is one named unit of work in the install sequence.
+type step struct {
+	name string
+	fn   func() error
+}
 
 // Run executes all installation steps in order. It is intentionally sequential
 // and explicit — each step is visible in logs so failures are easy to diagnose.
 func Run(cfg *Config) error {
+	// Re-checked here even though the UI and the headless path both validate:
+	// this is the last point before anything is erased, and it is the only one
+	// every caller must pass through.
+	if err := cfg.Storage.Validate(); err != nil {
+		return fmt.Errorf("storage: %w", err)
+	}
+
 	// The live environment may not have /sbin or /usr/sbin in PATH. Set it
 	// explicitly so exec.Command's LookPath finds system binaries like grub-install.
 	_ = os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
 
-	// Unmount unconditionally on exit so a failed step never leaves partitions
-	// mounted in the live environment, which would cause a retry to double-mount.
-	defer func() {
-		_ = run("umount", efiPart)
-		_ = run("umount", mountRoot)
-	}()
+	// Unmount unconditionally on exit so a failed step never leaves the target
+	// mounted in the live environment, which would make a retry double-mount or,
+	// on ZFS, fail with "pool is busy".
+	defer cleanupTarget(cfg.Storage)
 
-	steps := []struct {
-		name string
-		fn   func() error
-	}{
-		{"partition disk", func() error { return partitionDisk(cfg.Disk) }},
-		{"format partitions", func() error { return formatPartitions(cfg.Disk) }},
-		{"mount partitions", func() error { return mountPartitions(cfg.Disk) }},
-		{"copy rootfs", copyRootfs},
-		{"write fstab", func() error { return writeFstab(cfg.Disk) }},
-		{"install spinifex", func() error { return installSpinifex(cfg) }},
-		{"write network config", func() error { return writeNetworkConfig(cfg) }},
-		{"write firstboot service", func() error { return firstboot.Write(mountRoot, cfg.toFirstbootConfig()) }},
-		{"install bootloader", func() error { return installBootloader(cfg.Disk) }},
-		{"install CA cert", func() error { return installCACert(cfg) }},
+	steps := []step{{"partition disks", func() error { return partitionDisks(cfg.Storage) }}}
+	if cfg.Storage.FS.IsZFS() {
+		steps = append(steps, zfsRootSteps(cfg)...)
+	} else {
+		steps = append(steps, ext4RootSteps(cfg)...)
 	}
+	steps = append(steps,
+		step{"copy rootfs", func() error { return copyRootfs(cfg.Storage) }},
+		step{"create swap", func() error { return createSwap(cfg.Storage) }},
+		step{"write fstab", func() error { return writeFstab(cfg.Storage) }},
+		step{"install spinifex", func() error { return installSpinifex(cfg) }},
+		step{"write network config", func() error { return writeNetworkConfig(cfg) }},
+		step{"write firstboot service", func() error { return firstboot.Write(mountRoot, cfg.toFirstbootConfig()) }},
+		step{"install bootloader", func() error { return installBootloader(cfg.Storage) }},
+		step{"install CA cert", func() error { return installCACert(cfg) }},
+	)
 
 	for _, s := range steps {
 		slog.Info("installer", "step", s.name)
@@ -62,90 +78,83 @@ func Run(cfg *Config) error {
 	}
 
 	slog.Info("installation complete")
+	refreshBootToolNote(cfg.Storage)
 	fireInstallCallback()
 	promptRemoveUSB()
 	return reboot()
 }
 
-func partitionDisk(disk string) error {
-	// GPT table with three partitions:
-	//   p1: 1MiB BIOS Boot Partition — required for grub-install i386-pc on GPT
-	//   p2: 512MiB EFI System Partition
-	//   p3: remainder as root (ext4)
-	if err := run("parted", "--script", disk,
-		"mklabel", "gpt",
-		"mkpart", "bios_boot", "1MiB", "2MiB",
-		"set", "1", "bios_grub", "on",
-		"mkpart", "ESP", "fat32", "2MiB", "514MiB",
-		"set", "2", "esp", "on",
-		"mkpart", "root", "ext4", "514MiB", "100%",
-	); err != nil {
+// ext4RootSteps formats and mounts a single-disk ext4 root.
+func ext4RootSteps(cfg *Config) []step {
+	return []step{
+		{"format partitions", func() error { return formatPartitions(cfg.Storage) }},
+		{"mount partitions", func() error { return mountPartitions(cfg.Storage) }},
+	}
+}
+
+// zfsRootSteps builds the root pool. The ESPs are formatted here rather than
+// with the bootloader so that a failure to make a filesystem on one of them is
+// reported before the rootfs has been copied.
+func zfsRootSteps(cfg *Config) []step {
+	// Resolved once, before any step runs, so the values written to the pool,
+	// to modprobe.d and to the daemon's reserve are all the same numbers.
+	var opts ZFSOpts
+	return []step{
+		{"format ESPs", func() error { return formatESPs(cfg.Storage) }},
+		{"load zfs module", loadZFSModule},
+		{"create zfs pool", func() error {
+			opts = resolveZFSOpts(cfg.Storage)
+			slog.Info("zfs pool", "topology", cfg.Storage.FS, "disks", len(cfg.Storage.Disks),
+				"ashift", opts.Ashift, "compress", opts.Compress, "arc_max_mib", opts.ARCMaxMiB)
+			return createPool(cfg.Storage, opts)
+		}},
+		{"create zfs datasets", createDatasets},
+		{"configure zfs", func() error { return writeZFSSystemConfig(opts) }},
+	}
+}
+
+// cleanupTarget releases everything the install mounted, in the right order for
+// the filesystem in use.
+func cleanupTarget(cfg DiskConfig) {
+	unbindChrootMounts()
+	if cfg.FS.IsZFS() {
+		// Recursive: the layout is a dozen nested datasets, and leaving one
+		// mounted is enough to block the export.
+		_ = runQuiet("umount", "-R", mountRoot)
+		exportPool()
+		return
+	}
+	_ = runQuiet("umount", efiPart())
+	_ = runQuiet("umount", mountRoot)
+}
+
+func formatPartitions(cfg DiskConfig) error {
+	if err := formatESPs(cfg); err != nil {
 		return err
 	}
-	// Force the kernel to re-read the partition table and wait for udev to
-	// create the partition device nodes. Without this, mkfs.fat in the next
-	// step may race and fail with "No such file or directory" on /dev/sda2 —
-	// the kernel has accepted the new layout but /dev hasn't been populated
-	// yet. Trixie's udev seems slower at this than Bookworm's was.
-	return waitForPartitions(disk)
+	return run("mkfs.ext4", "-F", cfg.Primary().PartitionPath(rootPartNum))
 }
 
-// waitForPartitions ensures the EFI and root partition device nodes exist
-// after parted creates them. It runs partprobe (kernel re-read) and
-// udevadm settle (wait for queued events), then polls /dev for the files.
-func waitForPartitions(disk string) error {
-	// Best-effort: partprobe failure isn't fatal — udev may still pick up
-	// the change from the BLKRRPART ioctl that parted itself issued.
-	if err := run("partprobe", disk); err != nil {
-		slog.Warn("partprobe failed, continuing", "disk", disk, "err", err)
-	}
-	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
-		slog.Warn("udevadm settle failed, continuing", "err", err)
-	}
-	efi, root := partitionPaths(disk)
-	deadline := time.Now().Add(15 * time.Second)
-	for _, part := range []string{efi, root} {
-		for {
-			if _, err := os.Stat(part); err == nil {
-				break
-			}
-			if time.Now().After(deadline) {
-				return fmt.Errorf("partition device %s did not appear within timeout — kernel/udev did not pick up new partition table", part)
-			}
-			time.Sleep(200 * time.Millisecond)
-		}
-	}
-	return nil
-}
-
-func formatPartitions(disk string) error {
-	efi, root := partitionPaths(disk)
-	if err := run("mkfs.fat", "-F32", efi); err != nil {
-		return err
-	}
-	return run("mkfs.ext4", "-F", root)
-}
-
-func mountPartitions(disk string) error {
-	_, root := partitionPaths(disk)
+func mountPartitions(cfg DiskConfig) error {
+	d := cfg.Primary()
 	if err := os.MkdirAll(mountRoot, 0o755); err != nil {
 		return err
 	}
-	if err := run("mount", root, mountRoot); err != nil {
+	if err := run("mount", d.PartitionPath(rootPartNum), mountRoot); err != nil {
 		return err
 	}
-	if err := os.MkdirAll(efiPart, 0o755); err != nil {
+	if err := os.MkdirAll(efiPart(), 0o755); err != nil {
 		return err
 	}
-	efi, _ := partitionPaths(disk)
-	return run("mount", efi, efiPart)
+	return run("mount", d.PartitionPath(espPartNum), efiPart())
 }
 
 // copyRootfs copies the live squashfs environment onto the target disk using
 // rsync. This is the air-gapped alternative to debootstrap — all packages are
 // already embedded in the ISO so no network access is required.
-func copyRootfs() error {
-	if err := run("rsync", "-aHAX", "--delete", "--info=progress2",
+func copyRootfs(cfg DiskConfig) error {
+	args := []string{
+		"-aHAX", "--delete", "--info=progress2",
 		"--exclude=/proc",
 		"--exclude=/sys",
 		"--exclude=/dev",
@@ -161,8 +170,10 @@ func copyRootfs() error {
 		"--exclude=/etc/ssh/ssh_host_*",
 		"--exclude=/lost+found",
 		"--exclude=/boot/efi",
-		"/", mountRoot+"/",
-	); err != nil {
+	}
+	args = append(args, datasetProtectFilters(cfg)...)
+	args = append(args, "/", mountRoot+"/")
+	if err := run("rsync", args...); err != nil {
 		return err
 	}
 
@@ -278,7 +289,7 @@ func installSpinifex(cfg *Config) error {
 	// MANAGEMENT_IFACE is the bridge (br-wan), not the physical NIC.
 	// MANAGEMENT_IP is empty for DHCP — banner's --boot-check fills it in at boot.
 	nodeConf := fmt.Sprintf("MANAGEMENT_IP=%s\nMANAGEMENT_IFACE=br-wan\nNODE_HOSTNAME=%s\n",
-		cfg.WANAddress, cfg.Hostname)
+		cfg.WAN.Address, cfg.Hostname)
 	confDir := filepath.Join(mountRoot, "etc/spinifex")
 	if err := os.MkdirAll(confDir, 0o755); err != nil {
 		return err
@@ -309,23 +320,47 @@ func writeNetworkConfig(cfg *Config) error {
 		return err
 	}
 
-	if err := writeNetworkdBridge(netdDir, cfg.WANInterface, "br-wan", false,
-		cfg.WANDHCPMode, cfg.WANAddress, cfg.WANMask, cfg.WANGateway, cfg.WANDNS,
-		cfg.WANWiFiSSID, cfg.WANWiFiPass); err != nil {
-		return err
+	if err := cfg.Validate(); err != nil {
+		return fmt.Errorf("network roles: %w", err)
 	}
 
-	if cfg.LANInterface != "" {
-		if err := writeNetworkdBridge(netdDir, cfg.LANInterface, "br-lan", true,
-			cfg.LANDHCPMode, cfg.LANAddress, cfg.LANMask, "", cfg.LANDNS,
-			cfg.LANWiFiSSID, cfg.LANWiFiPass); err != nil {
+	// A physical NIC is configured once even when two roles share it via
+	// VLANs, so parent-interface files are keyed by NIC rather than by plane.
+	parents := map[string]*parentNIC{}
+	for _, rb := range cfg.Roles() {
+		p, ok := parents[rb.Role.Interface]
+		if !ok {
+			p = &parentNIC{}
+			parents[rb.Role.Interface] = p
+		}
+		if rb.Role.VLAN > 0 {
+			p.vlans = append(p.vlans, rb.Role.Link())
+		} else {
+			p.bridge = rb.Plane.Bridge()
+		}
+		// The parent must carry at least the largest MTU of any VLAN riding it.
+		if rb.Role.MTU > p.mtu {
+			p.mtu = rb.Role.MTU
+		}
+	}
+	for iface, p := range parents {
+		if err := writeParentNIC(netdDir, iface, p); err != nil {
 			return err
 		}
-		// br-lan timing is controlled by spinifex-lan-bridge.service, not
-		// networkd auto-activation — ActivationPolicy=manual in the .network
-		// file ensures networkd creates the bridge but does not bring it up.
-		if err := systemd.WriteLANBridgeUnit(mountRoot); err != nil {
-			return fmt.Errorf("lan bridge unit: %w", err)
+	}
+
+	for _, rb := range cfg.Roles() {
+		// Only br-wan auto-activates; the east-west bridges are brought up by
+		// their own unit after network-online.target so a missing cable or a
+		// DHCP timeout on them can never stall the management path.
+		manual := rb.Plane != PlaneWAN
+		if err := writeNetworkdBridge(netdDir, rb.Plane, rb.Role, manual); err != nil {
+			return err
+		}
+		if manual {
+			if err := systemd.WriteBridgeUnit(mountRoot, rb.Plane.Bridge()); err != nil {
+				return fmt.Errorf("%s bridge unit: %w", rb.Plane, err)
+			}
 		}
 	}
 
@@ -353,9 +388,9 @@ func writeNetworkConfig(cfg *Config) error {
 
 	// Disable IPv6 via sysctl — belt-and-suspenders alongside IPv6AcceptRA=no
 	// in the networkd .network files.
-	bridges := []string{"br-wan"}
-	if cfg.LANInterface != "" {
-		bridges = append(bridges, "br-lan")
+	var bridges []string
+	for _, rb := range cfg.Roles() {
+		bridges = append(bridges, rb.Plane.Bridge())
 	}
 	var sysctl strings.Builder
 	sysctl.WriteString("# Generated by Spinifex installer — IPv6 disabled on management bridges\n")
@@ -377,10 +412,14 @@ func writeNetworkConfig(cfg *Config) error {
 		return err
 	}
 	var udevRules strings.Builder
-	for _, iface := range []string{cfg.WANInterface, cfg.LANInterface} {
-		if iface == "" {
+	pinned := map[string]bool{}
+	for _, rb := range cfg.Roles() {
+		iface := rb.Role.Interface
+		// Two roles can share one NIC via VLANs; pin the physical name once.
+		if iface == "" || pinned[iface] {
 			continue
 		}
+		pinned[iface] = true
 		mac, err := os.ReadFile("/sys/class/net/" + iface + "/address")
 		if err != nil {
 			slog.Warn("writeNetworkConfig: could not read NIC MAC, skipping udev pin", "iface", iface, "err", err)
@@ -395,35 +434,90 @@ func writeNetworkConfig(cfg *Config) error {
 	return nil
 }
 
-// writeNetworkdBridge writes the three systemd-networkd files that configure a
-// physical NIC enslaved to a Linux bridge:
+// parentNIC accumulates how one physical interface is used across roles. A NIC
+// is either enslaved directly to a bridge (untagged) or carries VLAN
+// subinterfaces — never both for the same plane.
+type parentNIC struct {
+	bridge string
+	vlans  []string
+	mtu    int
+}
+
+// writeParentNIC writes the .network file for a physical interface. An
+// untagged NIC is enslaved straight to its bridge; a tagged one declares its
+// VLAN subinterfaces and stays unbridged, since the VLAN devices are what get
+// enslaved.
+func writeParentNIC(dir, iface string, p *parentNIC) error {
+	var b strings.Builder
+	fmt.Fprintf(&b, "[Match]\nName=%s\n\n", iface)
+	// A physical NIC here is either a bridge port or a VLAN trunk, and neither
+	// carries the address — the bridge above it does. Such a link never reaches
+	// the "degraded" state systemd-networkd-wait-online requires, so leaving it
+	// required makes that unit fail on every boot and stall until it times out.
+	b.WriteString("[Link]\nRequiredForOnline=no\n")
+	if p.mtu > 0 {
+		fmt.Fprintf(&b, "MTUBytes=%d\n", p.mtu)
+	}
+	b.WriteString("\n[Network]\n")
+	if p.bridge != "" {
+		fmt.Fprintf(&b, "Bridge=%s\n", p.bridge)
+	}
+	for _, v := range p.vlans {
+		fmt.Fprintf(&b, "VLAN=%s\n", v)
+	}
+	b.WriteString("IPv6AcceptRA=no\n")
+	return os.WriteFile(filepath.Join(dir, fmt.Sprintf("05-spinifex-nic-%s.network", iface)), []byte(b.String()), 0o644)
+}
+
+// writeNetworkdBridge writes the systemd-networkd files that put a plane's
+// address on its own Linux bridge:
 //
-//   - {prio}-spinifex-{suffix}-nic.network  enslaves the NIC to the bridge
-//   - {prio+1}-spinifex-{suffix}.netdev     declares the bridge device
-//   - {prio+1}-spinifex-{suffix}.network    configures IP on the bridge
+//   - 1{n}-spinifex-{plane}-vlan.netdev   VLAN subinterface (tagged roles only)
+//   - 1{n}-spinifex-{plane}-vlan.network  enslaves the VLAN device to the bridge
+//   - 2{n}-spinifex-{plane}.netdev        declares the bridge device
+//   - 2{n}-spinifex-{plane}.network       configures IP on the bridge
+//
+// The physical NIC itself is written separately by writeParentNIC, because two
+// roles may share one NIC and it must be configured exactly once.
 //
 // manual=true sets ActivationPolicy=manual so networkd creates the bridge but
-// does not auto-activate it — used for br-lan, which spinifex-lan-bridge.service
-// activates after network-online.target.
-func writeNetworkdBridge(dir, nicIface, bridgeName string, manual, dhcp bool,
-	addr, mask, gw string, dns []string, wifiSSID, wifiPass string) error {
-	suffix := strings.TrimPrefix(bridgeName, "br-")
-	nicPrio, brPrio := 10, 11
-	if manual {
-		nicPrio, brPrio = 12, 13
+// does not auto-activate it — used for br-lan and br-vpc, which their own
+// spinifex-{plane}-bridge.service activates after network-online.target.
+func writeNetworkdBridge(dir string, plane Plane, role NetworkRole, manual bool) error {
+	name := string(plane)
+	bridgeName := plane.Bridge()
+
+	if role.VLAN > 0 {
+		link := role.Link()
+		vlanNetdev := fmt.Sprintf("[NetDev]\nName=%s\nKind=vlan\n\n[VLAN]\nId=%d\n", link, role.VLAN)
+		if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("10-spinifex-%s-vlan.netdev", name)), []byte(vlanNetdev), 0o644); err != nil {
+			return err
+		}
+		var v strings.Builder
+		fmt.Fprintf(&v, "[Match]\nName=%s\n\n", link)
+		// Enslaved to the bridge below, so it is not what "online" means here.
+		v.WriteString("[Link]\nRequiredForOnline=no\n")
+		if role.MTU > 0 {
+			fmt.Fprintf(&v, "MTUBytes=%d\n", role.MTU)
+		}
+		fmt.Fprintf(&v, "\n[Network]\nBridge=%s\nIPv6AcceptRA=no\n", bridgeName)
+		if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("11-spinifex-%s-vlan.network", name)), []byte(v.String()), 0o644); err != nil {
+			return err
+		}
 	}
 
-	// NIC .network — enslaves the physical NIC to the bridge.
-	nicNet := fmt.Sprintf("[Match]\nName=%s\n\n[Network]\nBridge=%s\n", nicIface, bridgeName)
-	nicFile := fmt.Sprintf("%02d-spinifex-%s-nic.network", nicPrio, suffix)
-	if err := os.WriteFile(filepath.Join(dir, nicFile), []byte(nicNet), 0o644); err != nil {
-		return err
+	// Without a policy systemd-networkd invents a MAC for the bridge, so the node
+	// appears on the wire as an address that belongs to no NIC — the DHCP server
+	// sees a request whose source does not match its own chaddr and the unicast
+	// reply goes nowhere. "none" leaves the kernel to inherit the enslaved port's
+	// address, which is what a bridged uplink is supposed to present, and keeps
+	// it stable across boots for switch MAC tables and DHCP reservations.
+	brNetdev := fmt.Sprintf("[NetDev]\nName=%s\nKind=bridge\nMACAddressPolicy=none\n", bridgeName)
+	if role.MTU > 0 {
+		brNetdev += fmt.Sprintf("MTUBytes=%d\n", role.MTU)
 	}
-
-	// Bridge .netdev — declares the bridge device.
-	brNetdev := fmt.Sprintf("[NetDev]\nName=%s\nKind=bridge\n\n[Bridge]\nSTP=no\n", bridgeName)
-	brNetdevFile := fmt.Sprintf("%02d-spinifex-%s.netdev", brPrio, suffix)
-	if err := os.WriteFile(filepath.Join(dir, brNetdevFile), []byte(brNetdev), 0o644); err != nil {
+	brNetdev += "\n[Bridge]\nSTP=no\n"
+	if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("20-spinifex-%s.netdev", name)), []byte(brNetdev), 0o644); err != nil {
 		return err
 	}
 
@@ -434,36 +528,43 @@ func writeNetworkdBridge(dir, nicIface, bridgeName string, manual, dhcp bool,
 		b.WriteString("[Link]\nActivationPolicy=manual\nRequiredForOnline=no\n\n")
 	}
 	b.WriteString("[Network]\n")
-	if dhcp {
+	if role.DHCPMode {
 		b.WriteString("DHCP=ipv4\n")
 	} else {
-		cidr, err := addrCIDR(addr, mask)
+		cidr, err := addrCIDR(role.Address, role.Mask)
 		if err != nil {
 			return fmt.Errorf("bridge %s: %w", bridgeName, err)
 		}
 		fmt.Fprintf(&b, "Address=%s\n", cidr)
-		if gw != "" {
-			fmt.Fprintf(&b, "Gateway=%s\n", gw)
+		// Only the wan plane installs a default route; the east-west planes
+		// are link-local to the rack.
+		if role.Gateway != "" && plane == PlaneWAN {
+			fmt.Fprintf(&b, "Gateway=%s\n", role.Gateway)
 		}
-		for _, ns := range dns {
-			if ns = strings.TrimSpace(ns); ns != "" {
-				fmt.Fprintf(&b, "DNS=%s\n", ns)
+		// Resolvers follow the default route for the same reason. A resolver
+		// pinned to an east-west bridge is unreachable — those bridges are
+		// link-local and activate manually — but resolved would still try it,
+		// putting a timeout in front of every lookup on the node.
+		if plane == PlaneWAN {
+			for _, ns := range role.DNS {
+				if ns = strings.TrimSpace(ns); ns != "" {
+					fmt.Fprintf(&b, "DNS=%s\n", ns)
+				}
 			}
 		}
 	}
 	b.WriteString("IPv6AcceptRA=no\nConfigureWithoutCarrier=yes\n")
-	if manual && dhcp {
-		// br-lan is non-critical; fail fast on DHCP so a missing LAN cable
-		// does not stall spinifex-lan-bridge.service indefinitely.
+	if manual && role.DHCPMode {
+		// The east-west bridges are non-critical; fail fast on DHCP so a
+		// missing cable does not stall their activation unit indefinitely.
 		b.WriteString("\n[DHCP]\nTimeout=10\n")
 	}
-	brNetFile := fmt.Sprintf("%02d-spinifex-%s.network", brPrio, suffix)
-	if err := os.WriteFile(filepath.Join(dir, brNetFile), []byte(b.String()), 0o644); err != nil {
+	if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("20-spinifex-%s.network", name)), []byte(b.String()), 0o644); err != nil {
 		return err
 	}
 
-	if wifiSSID != "" {
-		if err := writeWPASupplicant(nicIface, wifiSSID, wifiPass); err != nil {
+	if role.WiFiSSID != "" {
+		if err := writeWPASupplicant(role.Interface, role.WiFiSSID, role.WiFiPass); err != nil {
 			return err
 		}
 	}
@@ -513,7 +614,59 @@ func writeWPASupplicant(nicIface, ssid, psk string) error {
 	return os.Symlink("/lib/systemd/system/wpa_supplicant@.service", link)
 }
 
-func installBootloader(disk string) error {
+func installBootloader(cfg DiskConfig) error {
+	// Branding assets have to be in place before either path regenerates
+	// grub.cfg — the ZFS path copies them out of /boot onto each ESP.
+	copySplashImage(mountRoot)
+	copyGrubFont(mountRoot)
+
+	if cfg.FS.IsZFS() {
+		// The initramfs must be able to import the pool before the ESPs are
+		// synced, since refresh copies whatever initrd exists at that moment.
+		if err := buildZFSInitramfs(); err != nil {
+			return err
+		}
+		return installBootToolZFS(cfg)
+	}
+	return installBootloaderExt4(cfg.Primary())
+}
+
+// buildZFSInitramfs regenerates the target's initramfs so it carries the ZFS
+// module, the pool cache and the hostid. The image copied from the live ISO was
+// built for a machine with an ext4 root and cannot import a pool.
+func buildZFSInitramfs() error {
+	// ZFS refuses to import a pool last touched by a different host without
+	// -f, and the hostid the initramfs sees must match the one recorded on the
+	// pool — otherwise every boot stops in the initramfs shell.
+	if err := run("cp", "-f", "/etc/hostid", filepath.Join(mountRoot, "etc/hostid")); err != nil {
+		slog.Warn("could not copy /etc/hostid, generating one in the target", "err", err)
+		if err := run("zgenhostid", "-f", "-o", filepath.Join(mountRoot, "etc/hostid")); err != nil {
+			return fmt.Errorf("zgenhostid: %w", err)
+		}
+	}
+
+	// The pool was created with cachefile=none so the live environment would
+	// not claim it. The installed system needs the cache to import at boot
+	// without scanning every block device.
+	cacheDir := filepath.Join(mountRoot, "etc/zfs")
+	if err := os.MkdirAll(cacheDir, 0o755); err != nil {
+		return err
+	}
+	if err := run("zpool", "set", "cachefile="+filepath.Join(cacheDir, "zpool.cache"), ZFSPoolName); err != nil {
+		return fmt.Errorf("set zpool cachefile: %w", err)
+	}
+
+	if err := bindChrootMounts(); err != nil {
+		return err
+	}
+	defer unbindChrootMounts()
+	if err := run("chroot", mountRoot, "update-initramfs", "-u", "-k", "all"); err != nil {
+		return fmt.Errorf("update-initramfs: %w", err)
+	}
+	return nil
+}
+
+func installBootloaderExt4(disk Disk) error {
 	// grub-install runs in the live environment (not chroot) using the
 	// grub-pc-bin and grub-efi-amd64-bin packages already present on the ISO.
 	// --boot-directory points at the installed system's /boot.
@@ -535,7 +688,7 @@ func installBootloader(disk string) error {
 		"--target=i386-pc",
 		"--boot-directory="+bootDir,
 		"--recheck",
-		disk,
+		disk.Path,
 	); biosErr != nil {
 		if efiErr != nil {
 			// Both targets failed — the system will not boot.
@@ -543,56 +696,9 @@ func installBootloader(disk string) error {
 		}
 		return biosErr
 	}
-	// Copy splash image and unicode font from the ISO (mounted at /cdrom) so the
-	// installed GRUB shows the same branded background as the installer GRUB.
-	// The font must be at /boot/grub/fonts/unicode.pf2 so update-grub finds it
-	// there and emits the same loadfont path as the ISO's grub.cfg — GRUB 2.12
-	// (trixie) needs the font in the boot partition, not just /usr/share/grub/.
-	copySplashImage(mountRoot)
-	copyGrubFont(mountRoot)
 
-	// Both consoles listed: tty0 stays primary for local install, ttyS0 mirrors
-	// kernel output to serial so headless installs (CI, racks with serial-only
-	// IPMI, remote-dev SSH-to-QEMU) see boot messages. The last `console=` on
-	// the cmdline becomes the system console, so ttyS0 wins on serial-only
-	// hardware while tty0 still receives output for local display.
-	grubDefault := `GRUB_DEFAULT=0
-GRUB_TIMEOUT=5
-GRUB_DISTRIBUTOR=Spinifex
-GRUB_CMDLINE_LINUX_DEFAULT=""
-GRUB_CMDLINE_LINUX="console=tty0 console=ttyS0,115200 systemd.show_status=1"
-`
-
-	if err := os.WriteFile(filepath.Join(mountRoot, "etc/default/grub"), []byte(grubDefault), 0o644); err != nil {
-		return fmt.Errorf("write /etc/default/grub: %w", err)
-	}
-
-	// Mirror the ISO grub.cfg graphical block exactly so the installed GRUB menu
-	// looks identical to the installer menu. gfxterm MUST be activated before
-	// serial is appended — background_image silently does nothing in text mode.
-	// Using exec tail so update-grub includes everything from line 3 as raw GRUB config.
-	grubTheme := `#!/bin/sh
-exec tail -n +3 $0
-insmod all_video
-insmod font
-if loadfont /boot/grub/fonts/unicode.pf2; then
-  set gfxmode=auto
-  insmod gfxterm
-  terminal_output gfxterm
-fi
-insmod serial
-if serial --unit=0 --speed=115200 --timeout=1; then
-  terminal_input  --append serial
-  terminal_output --append serial
-fi
-
-# --- Branding ---
-insmod png
-set theme=/boot/grub/theme.txt
-export theme
-`
-	if err := os.WriteFile(filepath.Join(mountRoot, "etc/grub.d/06_spinifex"), []byte(grubTheme), 0o755); err != nil {
-		return fmt.Errorf("write /etc/grub.d/06_spinifex: %w", err)
+	if err := writeGrubDefaults(DiskConfig{FS: FSExt4}); err != nil {
+		return err
 	}
 
 	if err := bindChrootMounts(); err != nil {
@@ -667,42 +773,247 @@ func reboot() error {
 
 // toFirstbootConfig maps installer Config to the firstboot package's Config.
 func (c *Config) toFirstbootConfig() firstboot.Config {
-	// Geneve encap IP: use LAN bridge IP when a dedicated LAN NIC is present,
-	// otherwise fall back to WAN bridge IP. Empty for DHCP — setup-ovn.sh
+	// Both addresses come from their plane after collapsing, so a single-NIC
+	// node resolves them to the wan address and a three-plane rack keeps
+	// Geneve on vpc and cluster traffic on lan. Empty for DHCP — setup-ovn.sh
 	// auto-detects the IP from the default route at boot in that case.
-	encapIP := c.WANAddress
-	if c.LANInterface != "" && c.LANAddress != "" {
-		encapIP = c.LANAddress
-	}
 	return firstboot.Config{
 		Hostname:        c.Hostname,
-		EncapIP:         encapIP,
-		ClusterRole:     c.ClusterRole,
-		JoinAddr:        c.JoinAddr,
+		EncapIP:         c.PlaneAddress(PlaneVPC),
+		LANIP:           c.PlaneAddress(PlaneLAN),
+		WANIP:           c.PlaneAddress(PlaneWAN),
 		Email:           c.Email,
 		InstallCallback: strings.TrimSpace(os.Getenv("SPINIFEX_INSTALL_CALLBACK")),
 		SkipFormation:   c.SkipFormation,
 	}
 }
 
-// writeFstab writes /etc/fstab on the installed system using partition UUIDs so
-// the root filesystem is mounted read-write at boot and the EFI partition is
-// mounted at /boot/efi.
-func writeFstab(disk string) error {
-	efi, root := partitionPaths(disk)
-	rootUUID, err := partUUID(root)
-	if err != nil {
-		return fmt.Errorf("get root UUID: %w", err)
+// writeFstab writes /etc/fstab on the installed system.
+//
+// On ZFS the root and every dataset are mounted by ZFS itself from properties
+// stored on the pool, and the ESPs are mounted on demand by spinifex-boot-tool,
+// so the file holds nothing but swap.
+func writeFstab(cfg DiskConfig) error {
+	fstab := "# /etc/fstab — generated by Spinifex installer\n"
+
+	if !cfg.FS.IsZFS() {
+		d := cfg.Primary()
+		rootUUID, err := partUUID(d.PartitionPath(rootPartNum))
+		if err != nil {
+			return fmt.Errorf("get root UUID: %w", err)
+		}
+		efiUUID, err := partUUID(d.PartitionPath(espPartNum))
+		if err != nil {
+			return fmt.Errorf("get EFI UUID: %w", err)
+		}
+		fstab += fmt.Sprintf("UUID=%s / ext4 errors=remount-ro 0 1\nUUID=%s /boot/efi vfat umask=0077 0 1\n",
+			rootUUID, efiUUID)
 	}
-	efiUUID, err := partUUID(efi)
-	if err != nil {
-		return fmt.Errorf("get EFI UUID: %w", err)
+
+	// Only claim the swap the previous step actually produced; systemd fails the
+	// boot on a missing swap unit rather than ignoring the line.
+	switch {
+	case cfg.FS.IsZFS():
+		if _, err := os.Stat("/dev/zvol/" + swapZvolName); err == nil {
+			fstab += fmt.Sprintf("/dev/zvol/%s none swap discard 0 0\n", swapZvolName)
+		}
+	default:
+		if _, err := os.Stat(filepath.Join(mountRoot, swapFileName)); err == nil {
+			fstab += fmt.Sprintf("/%s none swap sw 0 0\n", swapFileName)
+		}
 	}
-	fstab := fmt.Sprintf("# /etc/fstab — generated by Spinifex installer\nUUID=%s / ext4 errors=remount-ro 0 1\nUUID=%s /boot/efi vfat umask=0077 0 1\n",
-		rootUUID, efiUUID)
 	return os.WriteFile(filepath.Join(mountRoot, "etc/fstab"), []byte(fstab), 0o644)
 }
 
+const (
+	swapFileName = "swapfile"
+	// Nodes below this get half their RAM, which keeps a 4 GiB test VM to a
+	// 2 GiB file instead of eating its disk. At or above it the machine is a
+	// real host with a real disk, so swap is worth sizing for large imports.
+	swapRAMThreshold = 32 << 30
+	swapLargeBytes   = 64 << 30
+	// Below this there is not enough to be worth the space it costs.
+	swapMinBytes = 512 << 20
+	// Hard ceiling as a share of free disk, whatever RAM suggests: a 64 GiB file
+	// must never fill a small root filesystem.
+	swapDiskShare = 4
+)
+
+// createSwapFile lays down /swapfile on the installed root, sized to the disk.
+//
+// Importing an image streams the decompressed disk through viperblock into
+// predastore, which a 2-4 GiB node cannot hold in RAM — the kernel OOM-kills the
+// import mid-flush. Swap trades speed for completing the operation at all. It is
+// a stopgap for the memory profile of that path, not a fix for it.
+//
+// Failure is logged and not fatal: a node that boots without swap is worse at
+// large imports but is otherwise fine, and losing a whole install over it would
+// be the greater harm.
+// swapZvolName is the swap volume on a ZFS root. ZFS cannot host a swap *file*
+// at all, so a zvol is the only option — the same one Proxmox uses.
+const swapZvolName = ZFSPoolName + "/swap"
+
+// createSwap lays down swap in whichever form the root filesystem supports.
+func createSwap(cfg DiskConfig) error {
+	if cfg.FS.IsZFS() {
+		return createSwapZvol()
+	}
+	return createSwapFile()
+}
+
+// createSwapZvol creates the swap volume on a ZFS root.
+//
+// The properties are not tuning preferences. A zvol swapping under memory
+// pressure can deadlock against the ARC, and these are the settings that make
+// it survivable: page-sized blocks so a swap-out is one record, no data
+// caching so swap pages are never copied into the ARC, and sync writes so a
+// page is on disk before the kernel frees it.
+func createSwapZvol() error {
+	size, err := swapSize(mountRoot)
+	if err != nil {
+		slog.Warn("swap: cannot size swap volume, continuing without one", "err", err)
+		return nil
+	}
+	if size == 0 {
+		slog.Warn("swap: pool too small for a useful swap volume, continuing without one")
+		return nil
+	}
+
+	if err := run("zfs", "create",
+		"-V", strconv.FormatInt(size, 10),
+		"-b", strconv.Itoa(os.Getpagesize()),
+		"-o", "logbias=throughput",
+		"-o", "sync=always",
+		"-o", "primarycache=metadata",
+		"-o", "secondarycache=none",
+		"-o", "compression=zle",
+		"-o", "com.sun:auto-snapshot=false",
+		swapZvolName,
+	); err != nil {
+		slog.Warn("swap: could not create swap volume, continuing without swap", "err", err)
+		return nil
+	}
+
+	dev := "/dev/zvol/" + swapZvolName
+	if err := waitForPath(dev, time.Now().Add(10*time.Second)); err != nil {
+		slog.Warn("swap: zvol device node never appeared, continuing without swap", "err", err)
+		return nil
+	}
+	if err := run("mkswap", "-f", dev); err != nil {
+		slog.Warn("swap: mkswap failed, continuing without swap", "err", err)
+		return nil
+	}
+	slog.Info("swap: created", "zvol", swapZvolName, "bytes", size)
+	return nil
+}
+
+func createSwapFile() error {
+	size, err := swapSize(mountRoot)
+	if err != nil {
+		slog.Warn("swap: cannot size swap file, continuing without one", "err", err)
+		return nil
+	}
+	if size == 0 {
+		slog.Warn("swap: disk too small for a useful swap file, continuing without one",
+			"min_bytes", swapMinBytes)
+		return nil
+	}
+
+	path := filepath.Join(mountRoot, swapFileName)
+	// fallocate rather than dd: on ext4 it reserves real extents, so mkswap and
+	// swapon accept it, and a 32 GiB file costs no write time during install.
+	if err := run("fallocate", "-l", strconv.FormatInt(size, 10), path); err != nil {
+		slog.Warn("swap: fallocate failed, continuing without swap", "err", err)
+		_ = os.Remove(path)
+		return nil
+	}
+	// mkswap refuses a world-readable file, and its contents are process memory.
+	if err := os.Chmod(path, 0o600); err != nil {
+		slog.Warn("swap: chmod failed, continuing without swap", "err", err)
+		_ = os.Remove(path)
+		return nil
+	}
+	if err := run("mkswap", path); err != nil {
+		slog.Warn("swap: mkswap failed, continuing without swap", "err", err)
+		_ = os.Remove(path)
+		return nil
+	}
+	slog.Info("swap: created", "path", "/"+swapFileName, "bytes", size)
+	return nil
+}
+
+// swapSize returns the swap file size for the filesystem mounted at root, or 0
+// when the disk cannot spare a useful amount. SPINIFEX_SWAP_SIZE overrides the
+// calculation; 0 disables swap entirely.
+//
+// RAM drives the size and the disk only caps it, so a small test VM stays cheap
+// on disk while a real host gets enough to carry a large import.
+func swapSize(root string) (int64, error) {
+	if v := strings.TrimSpace(os.Getenv("SPINIFEX_SWAP_SIZE")); v != "" {
+		size, err := parseSize(v)
+		if err != nil {
+			return 0, fmt.Errorf("SPINIFEX_SWAP_SIZE: %w", err)
+		}
+		return size, nil
+	}
+
+	ram, err := totalRAM()
+	if err != nil {
+		return 0, err
+	}
+	size := ram / 2
+	if ram >= swapRAMThreshold {
+		size = swapLargeBytes
+	}
+
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(root, &st); err != nil {
+		return 0, fmt.Errorf("statfs %s: %w", root, err)
+	}
+	// Free space, not total: the rootfs is already copied in, so this is what is
+	// genuinely left to give away.
+	free := int64(st.Bavail) * st.Bsize
+	if diskCap := free / swapDiskShare; size > diskCap {
+		size = diskCap
+	}
+	if size < swapMinBytes {
+		return 0, nil
+	}
+	return size, nil
+}
+
+// totalRAM reports installed memory in bytes.
+func totalRAM() (int64, error) {
+	var si syscall.Sysinfo_t
+	if err := syscall.Sysinfo(&si); err != nil {
+		return 0, fmt.Errorf("sysinfo: %w", err)
+	}
+	unit := int64(si.Unit)
+	if unit == 0 {
+		unit = 1
+	}
+	return int64(si.Totalram) * unit, nil
+}
+
+// parseSize reads a plain byte count or a G/M suffixed size ("32G", "512M").
+func parseSize(s string) (int64, error) {
+	mult := int64(1)
+	switch {
+	case strings.HasSuffix(s, "G"), strings.HasSuffix(s, "g"):
+		mult, s = 1<<30, s[:len(s)-1]
+	case strings.HasSuffix(s, "M"), strings.HasSuffix(s, "m"):
+		mult, s = 1<<20, s[:len(s)-1]
+	}
+	n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("invalid size %q", s)
+	}
+	if n < 0 {
+		return 0, fmt.Errorf("negative size %q", s)
+	}
+	return n * mult, nil
+}
+
 func partUUID(dev string) (string, error) {
 	out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", dev).Output()
 	if err != nil {
@@ -715,18 +1026,6 @@ func partUUID(dev string) (string, error) {
 	return uuid, nil
 }
 
-// partitionPaths returns the EFI and root partition device paths for a given
-// disk. p1 is the BIOS Boot Partition (no filesystem), p2 is EFI, p3 is root.
-// Handles both /dev/sdX (→ /dev/sdX2, /dev/sdX3) and /dev/nvmeXnY
-// (→ /dev/nvmeXnYp2, /dev/nvmeXnYp3).
-func partitionPaths(disk string) (efi, root string) {
-	// NVMe devices use a 'p' separator before the partition number.
-	if len(disk) > 0 && disk[len(disk)-1] >= '0' && disk[len(disk)-1] <= '9' {
-		return disk + "p2", disk + "p3"
-	}
-	return disk + "2", disk + "3"
-}
-
 // copyGrubFont copies the unicode.pf2 GRUB font into the installed system's
 // /boot/grub/fonts/ directory. This ensures update-grub finds the font at
 // /boot/grub/fonts/unicode.pf2 — the same path the ISO's grub.cfg uses —
@@ -836,13 +1135,31 @@ func bindChrootMounts() error {
 // Errors are logged but not returned — this is best-effort cleanup.
 func unbindChrootMounts() {
 	for _, v := range slices.Backward(chrootMountPaths) {
-		_ = run("umount", filepath.Join(mountRoot, v))
+		// Quiet: this also runs from the deferred cleanup, where nothing is
+		// mounted and three "no mount point specified" lines above a real error
+		// message are pure distraction.
+		_ = runQuiet("umount", filepath.Join(mountRoot, v))
 	}
 }
 
-func run(name string, args ...string) error {
+var run = func(name string, args ...string) error {
+	return runEnv(nil, name, args...)
+}
+
+// runQuiet is run with output discarded, for best-effort probes whose failure
+// is expected and whose stderr would otherwise look like an install fault.
+var runQuiet = func(name string, args ...string) error {
+	cmd := exec.Command(name, args...)
+	return cmd.Run()
+}
+
+// runEnv is run with extra environment variables appended to the installer's own.
+var runEnv = func(env []string, name string, args ...string) error {
 	cmd := exec.Command(name, args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
+	if len(env) > 0 {
+		cmd.Env = append(os.Environ(), env...)
+	}
 	return cmd.Run()
 }

cmd/installer/install/install_test.go

@@ -9,7 +9,7 @@ func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
 	const url = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
 	t.Setenv("SPINIFEX_INSTALL_CALLBACK", url)
 
-	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	cfg := &Config{Hostname: "node1"}
 	fb := cfg.toFirstbootConfig()
 	if fb.InstallCallback != url {
 		t.Errorf("InstallCallback = %q, want %q", fb.InstallCallback, url)
@@ -19,7 +19,7 @@ func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
 func TestToFirstbootConfigEmptyCallbackWhenEnvUnset(t *testing.T) {
 	os.Unsetenv("SPINIFEX_INSTALL_CALLBACK")
 
-	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	cfg := &Config{Hostname: "node1"}
 	fb := cfg.toFirstbootConfig()
 	if fb.InstallCallback != "" {
 		t.Errorf("InstallCallback = %q, want empty", fb.InstallCallback)

cmd/installer/install/steps_test.go

@@ -0,0 +1,821 @@
+package install
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// fakeExec records every command the install steps shell out to and lets a
+// test fail a chosen one. Without it these steps can only be exercised by
+// running a real install against real hardware.
+type fakeExec struct {
+	calls []string
+	fail  func(name string, args []string) error
+}
+
+func fakeCommands(t *testing.T) *fakeExec {
+	t.Helper()
+	f := &fakeExec{}
+	prevRun, prevQuiet, prevEnv := run, runQuiet, runEnv
+
+	record := func(name string, args ...string) error {
+		f.calls = append(f.calls, strings.TrimSpace(name+" "+strings.Join(args, " ")))
+		if f.fail != nil {
+			return f.fail(name, args)
+		}
+		return nil
+	}
+	run = record
+	runQuiet = record
+	runEnv = func(env []string, name string, args ...string) error {
+		return record(name, append(env, args...)...)
+	}
+
+	t.Cleanup(func() { run, runQuiet, runEnv = prevRun, prevQuiet, prevEnv })
+	return f
+}
+
+// ran reports whether any recorded command contains every given fragment.
+func (f *fakeExec) ran(fragments ...string) bool {
+	for _, c := range f.calls {
+		all := true
+		for _, frag := range fragments {
+			if !strings.Contains(c, frag) {
+				all = false
+				break
+			}
+		}
+		if all {
+			return true
+		}
+	}
+	return false
+}
+
+func (f *fakeExec) mustRun(t *testing.T, fragments ...string) {
+	t.Helper()
+	if !f.ran(fragments...) {
+		t.Errorf("no command matched %v; recorded:\n  %s", fragments, strings.Join(f.calls, "\n  "))
+	}
+}
+
+func (f *fakeExec) mustNotRun(t *testing.T, fragments ...string) {
+	t.Helper()
+	if f.ran(fragments...) {
+		t.Errorf("unexpected command matching %v; recorded:\n  %s", fragments, strings.Join(f.calls, "\n  "))
+	}
+}
+
+// indexOf returns the position of the first command matching frag, or -1.
+func (f *fakeExec) indexOf(frag string) int {
+	for i, c := range f.calls {
+		if strings.Contains(c, frag) {
+			return i
+		}
+	}
+	return -1
+}
+
+// indexOfExact is indexOf for a whole command, for cases where one command
+// line is a prefix of another.
+func (f *fakeExec) indexOfExact(cmd string) int {
+	for i, c := range f.calls {
+		if c == cmd {
+			return i
+		}
+	}
+	return -1
+}
+
+// withMountRoot points the install target at a temp tree.
+func withMountRoot(t *testing.T) string {
+	t.Helper()
+	prev := mountRoot
+	mountRoot = t.TempDir()
+	t.Cleanup(func() { mountRoot = prev })
+	return mountRoot
+}
+
+// targetSkeleton creates the directories rsync would already have copied in by
+// the time these steps run.
+func targetSkeleton(t *testing.T, root string) {
+	t.Helper()
+	for _, d := range []string{"etc/default", "etc/grub.d", "etc/systemd/system", "etc"} {
+		if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// tempDisks builds disks whose device nodes are real files, so the steps that
+// wait for partition nodes to appear can complete.
+func tempDisks(t *testing.T, n int, gib int64) []Disk {
+	t.Helper()
+	dir := t.TempDir()
+	out := make([]Disk, n)
+	for i := range out {
+		d := Disk{
+			Path:              filepath.Join(dir, fmt.Sprintf("sd%c", 'a'+i)),
+			Bytes:             gib << 30,
+			LogicalBlockSize:  512,
+			PhysicalBlockSize: 512,
+		}
+		for _, part := range []int{biosPartNum, espPartNum, rootPartNum} {
+			if err := os.WriteFile(d.PartitionPath(part), nil, 0o644); err != nil {
+				t.Fatalf("create partition node: %v", err)
+			}
+		}
+		out[i] = d
+	}
+	return out
+}
+
+func TestPartitionDisksWritesTheSameLayoutOnEveryDisk(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 200)}
+
+	if err := partitionDisks(cfg); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+
+	for _, d := range cfg.Disks {
+		// Stale signatures survive a new table, so the wipe comes first.
+		f.mustRun(t, "wipefs -a "+d.Path)
+		f.mustRun(t, "sgdisk -Z "+d.Path)
+		f.mustRun(t,
+			"sgdisk", d.Path,
+			"-n 1:1M:+1M", "-t 1:EF02", "-c 1:bios_boot",
+			// A 200GiB disk is a real host, so it gets the large ESP.
+			"-n 2:0:+1024M", "-t 2:EF00", "-c 2:ESP",
+			"-n 3:0:-1024M", "-t 3:BF00", "-c 3:root",
+		)
+		f.mustRun(t, "sgdisk -G "+d.Path)
+		f.mustRun(t, "partprobe "+d.Path)
+
+		if wipe, part := f.indexOf("wipefs -a "+d.Path), f.indexOf("-c 3:root "+d.Path); wipe > part {
+			t.Errorf("wipefs ran after the table was written (%d > %d)", wipe, part)
+		}
+	}
+	f.mustRun(t, "udevadm settle --timeout=10")
+}
+
+func TestPartitionDisksTypesTheRootPartitionForTheFilesystem(t *testing.T) {
+	f := fakeCommands(t)
+	if err := partitionDisks(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+	f.mustRun(t, "-t 3:8300")
+	f.mustNotRun(t, "-t 3:BF00")
+	// Under 100GiB the ESP stays at the smaller size.
+	f.mustRun(t, "-n 2:0:+512M")
+}
+
+func TestPartitionDisksNamesTheDiskThatFailed(t *testing.T) {
+	f := fakeCommands(t)
+	disks := tempDisks(t, 2, 40)
+	f.fail = func(name string, args []string) error {
+		if name == "sgdisk" && strings.Contains(strings.Join(args, " "), "bios_boot") {
+			return errors.New("device busy")
+		}
+		return nil
+	}
+
+	err := partitionDisks(DiskConfig{FS: FSExt4, Disks: disks})
+	if err == nil || !strings.Contains(err.Error(), disks[0].Path) {
+		t.Fatalf("err = %v, want it to name %s", err, disks[0].Path)
+	}
+}
+
+func TestPartitionDisksSurvivesBestEffortFailures(t *testing.T) {
+	f := fakeCommands(t)
+	// partprobe, udevadm and the GUID randomisation are all advisory: sgdisk
+	// has already issued BLKRRPART and the nodes exist.
+	f.fail = func(name string, args []string) error {
+		switch {
+		case name == "partprobe", name == "udevadm":
+			return errors.New("not available")
+		case name == "sgdisk" && len(args) > 0 && args[0] == "-G":
+			return errors.New("no")
+		}
+		return nil
+	}
+	if err := partitionDisks(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+}
+
+func TestWaitForPathFailsWhenTheNodeNeverAppears(t *testing.T) {
+	missing := filepath.Join(t.TempDir(), "sda3")
+
+	err := waitForPath(missing, time.Now().Add(-time.Second))
+	if err == nil || !strings.Contains(err.Error(), missing) {
+		t.Fatalf("err = %v, want a timeout naming the missing node", err)
+	}
+}
+
+func TestWaitForPathReturnsOnceTheNodeExists(t *testing.T) {
+	f, err := os.CreateTemp(t.TempDir(), "node")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := waitForPath(f.Name(), time.Now().Add(-time.Second)); err != nil {
+		t.Errorf("waitForPath on an existing node: %v", err)
+	}
+}
+
+func TestEspSizeMiB(t *testing.T) {
+	if got := espSizeMiB(Disk{Bytes: 99 << 30}); got != espSmallMiB {
+		t.Errorf("99GiB -> %d, want %d", got, espSmallMiB)
+	}
+	if got := espSizeMiB(Disk{Bytes: 100 << 30}); got != espLargeMiB {
+		t.Errorf("100GiB -> %d, want %d", got, espLargeMiB)
+	}
+}
+
+func TestClearDiskOnlyLabelclearsAnExistingPartition(t *testing.T) {
+	f := fakeCommands(t)
+	disks := tempDisks(t, 1, 40)
+	if err := clearDisk(disks[0]); err != nil {
+		t.Fatalf("clearDisk: %v", err)
+	}
+	f.mustRun(t, "zpool labelclear -f "+disks[0].PartitionPath(rootPartNum))
+
+	f2 := fakeCommands(t)
+	blank := Disk{Path: filepath.Join(t.TempDir(), "sdb")}
+	if err := clearDisk(blank); err != nil {
+		t.Fatalf("clearDisk: %v", err)
+	}
+	f2.mustNotRun(t, "labelclear")
+	f2.mustRun(t, "wipefs -a "+blank.Path)
+}
+
+func TestFormatESPsLabelsEveryMember(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 40)}
+	if err := formatESPs(cfg); err != nil {
+		t.Fatalf("formatESPs: %v", err)
+	}
+	for _, d := range cfg.Disks {
+		f.mustRun(t, "mkfs.fat -F 32 -n EFI "+d.PartitionPath(espPartNum))
+	}
+}
+
+func TestFormatESPsNamesTheFailingESP(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: tempDisks(t, 2, 40)}
+	f.fail = func(name string, _ []string) error {
+		if name == "mkfs.fat" {
+			return errors.New("no such device")
+		}
+		return nil
+	}
+	err := formatESPs(cfg)
+	if err == nil || !strings.Contains(err.Error(), cfg.Disks[0].PartitionPath(espPartNum)) {
+		t.Fatalf("err = %v, want it to name the ESP", err)
+	}
+}
+
+func TestFormatPartitionsMakesTheExt4Root(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}
+	if err := formatPartitions(cfg); err != nil {
+		t.Fatalf("formatPartitions: %v", err)
+	}
+	f.mustRun(t, "mkfs.ext4 -F "+cfg.Disks[0].PartitionPath(rootPartNum))
+}
+
+func TestMountPartitionsMountsRootThenESP(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}
+
+	if err := mountPartitions(cfg); err != nil {
+		t.Fatalf("mountPartitions: %v", err)
+	}
+	d := cfg.Disks[0]
+	f.mustRun(t, "mount "+d.PartitionPath(rootPartNum)+" "+root)
+	f.mustRun(t, "mount "+d.PartitionPath(espPartNum)+" "+efiPart())
+	if _, err := os.Stat(efiPart()); err != nil {
+		t.Errorf("ESP mountpoint was not created: %v", err)
+	}
+	if f.indexOf("mount "+d.PartitionPath(rootPartNum)) > f.indexOf("mount "+d.PartitionPath(espPartNum)) {
+		t.Error("the ESP was mounted before the root it lives under")
+	}
+}
+
+func TestMountPartitionsStopsWhenTheRootWillNotMount(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "mount" {
+			return errors.New("wrong fs type")
+		}
+		return nil
+	}
+	if err := mountPartitions(DiskConfig{FS: FSExt4, Disks: tempDisks(t, 1, 40)}); err == nil {
+		t.Fatal("expected an error when the root mount fails")
+	}
+	f.mustNotRun(t, "boot/efi")
+}
+
+func TestCleanupTargetExportsThePoolOnZFS(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	cleanupTarget(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
+
+	// Recursive: one dataset left mounted is enough to block the export.
+	f.mustRun(t, "umount -R "+root)
+	f.mustRun(t, "zpool export "+ZFSPoolName)
+	if f.indexOf("umount -R") > f.indexOf("zpool export") {
+		t.Error("the pool was exported before its datasets were unmounted")
+	}
+}
+
+func TestCleanupTargetUnmountsESPBeforeRootOnExt4(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	cleanupTarget(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
+
+	f.mustNotRun(t, "zpool export")
+	esp, rootIdx := f.indexOfExact("umount "+efiPart()), f.indexOfExact("umount "+root)
+	if esp < 0 || rootIdx < 0 {
+		t.Fatalf("missing umounts; recorded:\n  %s", strings.Join(f.calls, "\n  "))
+	}
+	if esp > rootIdx {
+		t.Error("the root was unmounted before the ESP nested inside it")
+	}
+}
+
+func TestCreatePoolBuildsTheZpoolCommand(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	opts := ZFSOpts{Ashift: 12, Compress: "lz4", Checksum: "blake3", Copies: 1, ARCMaxMiB: 4096}
+
+	if err := createPool(cfg, opts); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f.mustRun(t, "zpool create -f",
+		"cachefile=none", "ashift=12", "compression=lz4", "checksum=blake3", "copies=1",
+		// altroot keeps the datasets under the target rather than over the live system.
+		"-R "+root, ZFSPoolName, "mirror",
+		cfg.Disks[0].StablePartitionPath(rootPartNum),
+		cfg.Disks[1].StablePartitionPath(rootPartNum),
+	)
+}
+
+func TestCreatePoolEnablesAutotrimOnFlashOnly(t *testing.T) {
+	withMountRoot(t)
+	flash := disks(2, 40)
+	f := fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: flash}, ZFSOpts{Ashift: 12, Copies: 1}); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f.mustRun(t, "zpool set autotrim=on "+ZFSPoolName)
+
+	spinning := disks(2, 40)
+	spinning[1].Rotational = true
+	f2 := fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: spinning}, ZFSOpts{Ashift: 12, Copies: 1}); err != nil {
+		t.Fatalf("createPool: %v", err)
+	}
+	f2.mustNotRun(t, "autotrim")
+}
+
+func TestCreatePoolWrapsTheZpoolFailure(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, args []string) error {
+		if name == "zpool" && len(args) > 0 && args[0] == "create" {
+			return errors.New("device is in use")
+		}
+		return nil
+	}
+	err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}, ZFSOpts{Ashift: 12, Copies: 1})
+	if err == nil || !strings.Contains(err.Error(), "zpool create") {
+		t.Fatalf("err = %v, want a zpool create failure", err)
+	}
+}
+
+func TestCreatePoolRejectsAnUnbuildableTopology(t *testing.T) {
+	withMountRoot(t)
+	fakeCommands(t)
+	if err := createPool(DiskConfig{FS: FSZFSRAID1, Disks: disks(1, 40)}, ZFSOpts{Ashift: 12}); err == nil {
+		t.Fatal("expected a vdev error for a one-disk mirror")
+	}
+}
+
+func TestCreateDatasetsCreatesEveryDataset(t *testing.T) {
+	f := fakeCommands(t)
+	if err := createDatasets(); err != nil {
+		t.Fatalf("createDatasets: %v", err)
+	}
+	for _, ds := range datasets {
+		f.mustRun(t, "zfs create", "mountpoint="+ds.mountpoint, ds.name)
+	}
+}
+
+func TestCreateDatasetsNamesTheDatasetThatFailed(t *testing.T) {
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "zfs" {
+			return errors.New("out of space")
+		}
+		return nil
+	}
+	err := createDatasets()
+	if err == nil || !strings.Contains(err.Error(), datasets[0].name) {
+		t.Fatalf("err = %v, want it to name %s", err, datasets[0].name)
+	}
+}
+
+func TestExportPoolToleratesABusyPool(t *testing.T) {
+	f := fakeCommands(t)
+	f.fail = func(string, []string) error { return errors.New("pool is busy") }
+	exportPool()
+	f.mustRun(t, "zpool export "+ZFSPoolName)
+}
+
+func TestWriteZFSSystemConfigDeclaresTheARCCeilingToBothConsumers(t *testing.T) {
+	root := withMountRoot(t)
+
+	if err := writeZFSSystemConfig(ZFSOpts{ARCMaxMiB: 4096}); err != nil {
+		t.Fatalf("writeZFSSystemConfig: %v", err)
+	}
+
+	modprobe := readFile(t, root, "etc/modprobe.d/zfs.conf")
+	if !strings.Contains(modprobe, fmt.Sprintf("zfs_arc_max=%d", int64(4096)<<20)) {
+		t.Errorf("zfs.conf does not cap the ARC in bytes:\n%s", modprobe)
+	}
+
+	// The daemon cannot see the ARC, so the reserve has to account for it or
+	// guests get admitted into memory the ARC already owns.
+	env := readFile(t, root, "etc/spinifex/host.env")
+	want := fmt.Sprintf("SPINIFEX_RESERVED_MEM_GB=%.1f", defaultHostReserveGB+4.0)
+	if !strings.Contains(env, want) {
+		t.Errorf("host.env = %q, want it to contain %q", env, want)
+	}
+}
+
+func TestWriteTargetFileAppliesTheModeOnRewrite(t *testing.T) {
+	root := withMountRoot(t)
+
+	if err := writeTargetFile("/usr/sbin/tool", []byte("#!/bin/sh\n"), 0o600); err != nil {
+		t.Fatalf("writeTargetFile: %v", err)
+	}
+	// A retried install rewrites an existing file, and WriteFile would leave
+	// the original mode in place.
+	if err := writeTargetFile("/usr/sbin/tool", []byte("#!/bin/sh\ntrue\n"), 0o755); err != nil {
+		t.Fatalf("writeTargetFile rewrite: %v", err)
+	}
+
+	st, err := os.Stat(filepath.Join(root, "usr/sbin/tool"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st.Mode().Perm() != 0o755 {
+		t.Errorf("mode = %o, want 755", st.Mode().Perm())
+	}
+	if got := readFile(t, root, "usr/sbin/tool"); !strings.Contains(got, "true") {
+		t.Errorf("content not rewritten: %q", got)
+	}
+}
+
+func TestWriteGrubDefaultsPinsTheZFSRoot(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeGrubDefaults(DiskConfig{FS: FSZFSRAID1}); err != nil {
+		t.Fatalf("writeGrubDefaults: %v", err)
+	}
+	got := readFile(t, root, "etc/default/grub")
+	if !strings.Contains(got, "root=ZFS="+ZFSRootDataset) || !strings.Contains(got, "boot=zfs") {
+		t.Errorf("grub defaults do not name the ZFS root:\n%s", got)
+	}
+	// Serial is listed last so it wins as the system console on headless racks.
+	if !strings.Contains(got, "console=tty0 console=ttyS0,115200") {
+		t.Errorf("grub defaults do not set both consoles:\n%s", got)
+	}
+	if theme := readFile(t, root, "etc/grub.d/06_spinifex"); !strings.Contains(theme, "terminal_output gfxterm") {
+		t.Errorf("branding snippet missing gfxterm:\n%s", theme)
+	}
+}
+
+func TestWriteGrubDefaultsLeavesExt4RootToGrubProbe(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	if err := writeGrubDefaults(DiskConfig{FS: FSExt4}); err != nil {
+		t.Fatalf("writeGrubDefaults: %v", err)
+	}
+	if got := readFile(t, root, "etc/default/grub"); strings.Contains(got, "root=ZFS=") {
+		t.Errorf("ext4 install should not pin a ZFS root:\n%s", got)
+	}
+}
+
+func TestDivertGrubInstallReplacesTheBinary(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	if err := divertGrubInstall(); err != nil {
+		t.Fatalf("divertGrubInstall: %v", err)
+	}
+	// The diversion is what makes the replacement survive a grub-pc upgrade.
+	f.mustRun(t, "dpkg-divert --divert "+grubInstallReal, "--rename --add "+grubInstallPath)
+	if _, err := os.Stat(filepath.Join(root, strings.TrimPrefix(grubInstallPath, "/"))); err != nil {
+		t.Errorf("wrapper not written: %v", err)
+	}
+}
+
+func TestDivertGrubInstallFailsLoudly(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(string, []string) error { return errors.New("dpkg is locked") }
+
+	if err := divertGrubInstall(); err == nil || !strings.Contains(err.Error(), "divert grub-install") {
+		t.Fatalf("err = %v, want a diversion failure", err)
+	}
+}
+
+// espFixture writes the state a successful boot-tool run leaves behind: the
+// registered ESP list, and a grub.cfg on the ESP that verifyBootConfig mounts.
+func espFixture(t *testing.T, root string, uuids string, grubCfg string) {
+	t.Helper()
+	list := filepath.Join(root, strings.TrimPrefix(bootToolUUIDList, "/"))
+	if err := os.MkdirAll(filepath.Dir(list), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(list, []byte(uuids), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if grubCfg == "" {
+		return
+	}
+	dir := filepath.Join(root, "var/tmp/verify-esp/grub")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "grub.cfg"), []byte(grubCfg), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestVerifyBootConfig(t *testing.T) {
+	goodCfg := "linux /vmlinuz-6.12.0 root=ZFS=" + ZFSRootDataset + " boot=zfs\n"
+
+	tests := []struct {
+		name    string
+		uuids   string
+		grubCfg string
+		wantErr string
+	}{
+		{name: "every ESP registered and pointing at the pool", uuids: "AAAA-1111\nBBBB-2222\n", grubCfg: goodCfg},
+		{
+			name:    "a disk got no bootloader",
+			uuids:   "AAAA-1111\n",
+			grubCfg: goodCfg,
+			wantErr: "expected 2 registered ESPs, found 1",
+		},
+		{
+			// grub-mkconfig degrades quietly when grub-probe cannot read the
+			// pool, and the failure only shows up as a panic at boot.
+			name:    "grub.cfg names no root",
+			uuids:   "AAAA-1111\nBBBB-2222\n",
+			grubCfg: "linux /vmlinuz-6.12.0 root=/dev/sda3\n",
+			wantErr: "would not find its root filesystem",
+		},
+		{
+			name:    "grub.cfg names no kernel",
+			uuids:   "AAAA-1111\nBBBB-2222\n",
+			grubCfg: "set root=ZFS=" + ZFSRootDataset + "\n",
+			wantErr: "references no kernel",
+		},
+		{name: "no ESP list at all", wantErr: "read ESP list"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			root := withMountRoot(t)
+			fakeCommands(t)
+			cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+			if tt.uuids != "" {
+				espFixture(t, root, tt.uuids, tt.grubCfg)
+			}
+
+			err := verifyBootConfig(cfg)
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("verifyBootConfig: %v", err)
+				}
+				return
+			}
+			if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+				t.Fatalf("err = %v, want %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestInstallBootToolZFSSetsUpEveryESP(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	targetSkeleton(t, root)
+	espFixture(t, root, "AAAA-1111\nBBBB-2222\n", "linux /vmlinuz-6.12.0 root=ZFS="+ZFSRootDataset+"\n")
+
+	if err := installBootToolZFS(cfg); err != nil {
+		t.Fatalf("installBootToolZFS: %v", err)
+	}
+
+	if _, err := os.Stat(filepath.Join(root, strings.TrimPrefix(bootToolPath, "/"))); err != nil {
+		t.Errorf("boot tool not installed: %v", err)
+	}
+	// A partially-updated set of ESPs boots today and fails on the next kernel
+	// upgrade, so every disk is initialised under strict mode.
+	for _, d := range cfg.Disks {
+		f.mustRun(t, "SPINIFEX_BOOT_TOOL_STRICT=1", "init", d.PartitionPath(espPartNum))
+	}
+	// NVRAM is written exactly once, by the initial refresh.
+	f.mustRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1", "refresh")
+	f.mustRun(t, "dpkg-divert")
+	for _, m := range chrootMountPaths {
+		f.mustRun(t, "mount --bind /"+m)
+	}
+}
+
+func TestInstallBootToolZFSStopsWhenADiskFails(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	targetSkeleton(t, root)
+	espFixture(t, root, "AAAA-1111\nBBBB-2222\n", "linux /vmlinuz root=ZFS="+ZFSRootDataset+"\n")
+	f.fail = func(name string, args []string) error {
+		if strings.Contains(strings.Join(args, " "), "init") {
+			return errors.New("no ESP")
+		}
+		return nil
+	}
+
+	err := installBootToolZFS(cfg)
+	if err == nil || !strings.Contains(err.Error(), "spinifex-boot-tool init") {
+		t.Fatalf("err = %v, want the init failure", err)
+	}
+	f.mustNotRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1")
+}
+
+func TestRefreshBootToolNoteOnlyAppliesToZFS(t *testing.T) {
+	// Log-only, but it is the operator's only record of the disk-replacement
+	// command, so both branches are walked.
+	refreshBootToolNote(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
+	refreshBootToolNote(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
+}
+
+func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
+	root := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(root, "etc/systemd/system"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	for range 2 {
+		if err := maskSystemdUnit(root, "getty@tty1.service"); err != nil {
+			t.Fatalf("maskSystemdUnit: %v", err)
+		}
+	}
+	target, err := os.Readlink(filepath.Join(root, "etc/systemd/system/getty@tty1.service"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if target != "/dev/null" {
+		t.Errorf("mask target = %q, want /dev/null", target)
+	}
+}
+
+func TestCopyAssetsAreBestEffort(t *testing.T) {
+	root := t.TempDir()
+	// Neither asset exists outside the live ISO; a missing one must not fail
+	// the install, only the branding.
+	copyGrubFont(root)
+	copySplashImage(root)
+}
+
+func TestWriteFstabOnZFSHoldsNothingButSwap(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeFstab(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	got := readFile(t, root, "etc/fstab")
+	// Root and every dataset are mounted from pool properties, and the ESPs on
+	// demand by the boot tool.
+	if strings.Contains(got, " / ") || strings.Contains(got, "/boot/efi") {
+		t.Errorf("ZFS fstab should carry no root or ESP line:\n%s", got)
+	}
+}
+
+func TestWriteNetworkConfigWritesTheWholeNetworkdTree(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+
+	if err := writeNetworkConfig(threeNIC()); err != nil {
+		t.Fatalf("writeNetworkConfig: %v", err)
+	}
+
+	netd := filepath.Join(root, "etc/systemd/network")
+	for _, plane := range []Plane{PlaneWAN, PlaneLAN, PlaneVPC} {
+		if got := readFile(t, netd, fmt.Sprintf("20-spinifex-%s.netdev", plane)); !strings.Contains(got, "Kind=bridge") {
+			t.Errorf("%s netdev is not a bridge:\n%s", plane, got)
+		}
+	}
+	// Only br-wan may block network-online.target; the east-west bridges are
+	// brought up by their own unit afterwards.
+	if got := readFile(t, netd, "20-spinifex-wan.network"); strings.Contains(got, "ActivationPolicy=manual") {
+		t.Errorf("br-wan must auto-activate:\n%s", got)
+	}
+	if got := readFile(t, netd, "20-spinifex-lan.network"); !strings.Contains(got, "ActivationPolicy=manual") {
+		t.Errorf("br-lan must not auto-activate:\n%s", got)
+	}
+
+	// The live ISO masks wait-online; the mask is copied in by rsync and has to
+	// be lifted, then scoped to br-wan so br-lan never blocks the boot.
+	dropin := readFile(t, root, "etc/systemd/system/systemd-networkd-wait-online.service.d/spinifex-wan-only.conf")
+	if !strings.Contains(dropin, "--interface=br-wan") || !strings.Contains(dropin, "--timeout=60") {
+		t.Errorf("wait-online drop-in is not scoped to br-wan:\n%s", dropin)
+	}
+
+	sysctl := readFile(t, root, "etc/sysctl.d/99-spinifex-network.conf")
+	for _, br := range []string{"br-wan", "br-lan", "br-vpc"} {
+		if !strings.Contains(sysctl, "net.ipv6.conf."+br+".disable_ipv6=1") {
+			t.Errorf("IPv6 not disabled on %s:\n%s", br, sysctl)
+		}
+	}
+}
+
+func TestWriteNetworkConfigRejectsAnInvalidRoleSet(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	cfg := threeNIC()
+	cfg.LAN.Interface = cfg.WAN.Interface
+
+	if err := writeNetworkConfig(cfg); err == nil || !strings.Contains(err.Error(), "network roles") {
+		t.Fatalf("err = %v, want the role validation to reject two planes on one link", err)
+	}
+}
+
+func TestWriteNetworkConfigLiftsTheWaitOnlineMask(t *testing.T) {
+	root := withMountRoot(t)
+	mask := filepath.Join(root, "etc/systemd/system/systemd-networkd-wait-online.service")
+	if err := os.MkdirAll(filepath.Dir(mask), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink("/dev/null", mask); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := writeNetworkConfig(threeNIC()); err != nil {
+		t.Fatalf("writeNetworkConfig: %v", err)
+	}
+	if _, err := os.Lstat(mask); err == nil {
+		t.Error("the wait-online mask survived; firstboot would run before br-wan has a lease")
+	}
+}
+
+func TestBindChrootMountsCreatesEveryMountpoint(t *testing.T) {
+	root := withMountRoot(t)
+	f := fakeCommands(t)
+
+	if err := bindChrootMounts(); err != nil {
+		t.Fatalf("bindChrootMounts: %v", err)
+	}
+	for _, m := range chrootMountPaths {
+		if _, err := os.Stat(filepath.Join(root, m)); err != nil {
+			t.Errorf("mountpoint /%s not created: %v", m, err)
+		}
+		f.mustRun(t, "mount --bind /"+m+" "+filepath.Join(root, m))
+	}
+
+	unbindChrootMounts()
+	// Reverse order: /sys and /proc come out before /dev they were nested under.
+	if f.indexOf("umount "+filepath.Join(root, "sys")) > f.indexOf("umount "+filepath.Join(root, "dev")) {
+		t.Error("chroot mounts were unbound in bind order, not reverse")
+	}
+}
+
+func TestBindChrootMountsReportsAFailedBind(t *testing.T) {
+	withMountRoot(t)
+	f := fakeCommands(t)
+	f.fail = func(name string, _ []string) error {
+		if name == "mount" {
+			return errors.New("permission denied")
+		}
+		return nil
+	}
+	if err := bindChrootMounts(); err == nil || !strings.Contains(err.Error(), "bind-mount") {
+		t.Fatalf("err = %v, want a bind-mount failure", err)
+	}
+}

cmd/installer/install/zfs.go

@@ -0,0 +1,362 @@
+package install
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math"
+	"math/bits"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// ARC bounds. Spinifex RAM is guest RAM: Proxmox defaults non-PVE products to
+// half of system memory, which on a 256GB compute node would remove 128GB of
+// schedulable instance capacity. 10% capped at 16GiB is their PVE behaviour and
+// the right trade for a hypervisor.
+const (
+	arcMinMiB     = 64
+	arcMaxCapMiB  = 16 * 1024
+	arcFraction   = 0.10
+	arcHeadroomMi = 1024 // always leave the OS a GiB above the ARC ceiling
+)
+
+// zfsDataset describes one dataset created at install time. Order matters:
+// parents must precede children so mountpoints nest correctly.
+type zfsDataset struct {
+	name       string
+	mountpoint string
+	props      []string
+}
+
+// datasets is the layout described in the plan. Recordsizes are matched to each
+// service's access pattern; 16K on the predastore index keeps RAIDZ-1 parity
+// overhead at 33% rather than the ~50% a 4-8K record would cost.
+var datasets = []zfsDataset{
+	{ZFSPoolName + "/ROOT", "none", []string{"canmount=off"}},
+	{ZFSRootDataset, "/", []string{"acltype=posix"}},
+	{ZFSPoolName + "/data", "/var/lib/spinifex", []string{"canmount=off"}},
+	{ZFSPoolName + "/data/viperblock", "/var/lib/spinifex/viperblock",
+		[]string{"recordsize=128K", "logbias=throughput"}},
+	{ZFSPoolName + "/data/predastore", "/var/lib/spinifex/predastore",
+		[]string{"recordsize=1M"}},
+	// Broken out so the later move of object data onto dedicated drives is a
+	// zfs send | zfs recv rather than an rsync with downtime.
+	{ZFSPoolName + "/data/predastore-nodes", "/var/lib/spinifex/predastore/distributed/nodes", nil},
+	// BadgerDB index plus the Raft BoltDB: small random, fsync-heavy.
+	{ZFSPoolName + "/data/predastore-db", "/var/lib/spinifex/predastore/distributed/db",
+		[]string{"recordsize=16K", "logbias=latency"}},
+	{ZFSPoolName + "/data/nats", "/var/lib/spinifex/nats", []string{"recordsize=16K"}},
+	{ZFSPoolName + "/log", "/var/log/spinifex", []string{"compression=zstd"}},
+}
+
+// buildVdev renders the vdev specification for zpool create. Pure: it is the
+// function that decides pool shape, so it is exhaustively unit-tested rather
+// than discovered to be wrong on hardware.
+func buildVdev(cfg DiskConfig) (string, error) {
+	if err := cfg.Validate(); err != nil {
+		return "", err
+	}
+	if !cfg.FS.IsZFS() {
+		return "", fmt.Errorf("buildVdev: %s is not a zfs topology", cfg.FS)
+	}
+
+	// Members are named by their partition-3 by-id path: a pool built on
+	// /dev/sdX breaks the first time a controller enumerates in a new order.
+	members := make([]string, len(cfg.Disks))
+	for i, d := range cfg.Disks {
+		members[i] = d.StablePartitionPath(rootPartNum)
+	}
+
+	keyword := cfg.FS.vdevKeyword()
+	if keyword == "" {
+		return strings.Join(members, " "), nil // plain stripe
+	}
+	// RAID10 is a stripe of two-disk mirrors, paired in selection order — which
+	// is why the UI numbers selected disks instead of just ticking them.
+	if cfg.FS == FSZFSRAID10 {
+		var parts []string
+		for i := 0; i+1 < len(members); i += 2 {
+			parts = append(parts, fmt.Sprintf("mirror %s %s", members[i], members[i+1]))
+		}
+		return strings.Join(parts, " "), nil
+	}
+	return keyword + " " + strings.Join(members, " "), nil
+}
+
+// detectAshift derives the pool sector size from the widest sector any member
+// reports. ashift cannot be changed after creation, and one that is too small
+// costs read-modify-write on every block for the life of the pool, so the floor
+// is 12 (4K) even when every disk claims 512.
+func detectAshift(disks []Disk) int {
+	widest := 4096
+	for _, d := range disks {
+		widest = max(widest, d.PhysicalBlockSize, d.LogicalBlockSize)
+	}
+	// ashift is the base-2 exponent of the sector size.
+	return bits.Len(uint(widest)) - 1
+}
+
+// defaultARCMaxMiB computes the ARC ceiling from total system memory.
+func defaultARCMaxMiB(totalMiB int) int {
+	want := int(math.Round(float64(totalMiB) * arcFraction))
+	want = min(want, arcMaxCapMiB)
+	// Never let the ARC crowd the OS out of its last GiB on a small node.
+	if ceiling := totalMiB - arcHeadroomMi; ceiling > 0 {
+		want = min(want, ceiling)
+	}
+	return max(want, arcMinMiB)
+}
+
+// totalMemoryMiB reads MemTotal from /proc/meminfo.
+func totalMemoryMiB() int {
+	data, err := os.ReadFile("/proc/meminfo")
+	if err != nil {
+		return 0
+	}
+	for line := range strings.SplitSeq(string(data), "\n") {
+		if !strings.HasPrefix(line, "MemTotal:") {
+			continue
+		}
+		var kb int
+		if _, err := fmt.Sscanf(line, "MemTotal: %d kB", &kb); err == nil {
+			return kb / 1024
+		}
+	}
+	return 0
+}
+
+// resolveZFSOpts fills unset advanced options with their computed defaults, so
+// every value written to the pool is explicit and auditable afterwards.
+func resolveZFSOpts(cfg DiskConfig) ZFSOpts {
+	o := cfg.ZFS
+	if o.Ashift == 0 {
+		o.Ashift = detectAshift(cfg.Disks)
+	}
+	if o.Compress == "" {
+		// Named explicitly rather than "on" so a future OpenZFS default change
+		// cannot silently alter what this pool does.
+		o.Compress = "lz4"
+	}
+	if o.Checksum == "" {
+		o.Checksum = "on"
+	}
+	if o.Copies == 0 {
+		o.Copies = 1
+	}
+	if o.ARCMaxMiB == 0 {
+		o.ARCMaxMiB = defaultARCMaxMiB(totalMemoryMiB())
+	}
+	return o
+}
+
+// loadZFSModule brings up the ZFS kernel module in the live environment and
+// waits for /dev/zfs. The module is prebuilt against the shipped kernel at ISO
+// build time, so failure here means the ISO is broken, not the hardware.
+func loadZFSModule() error {
+	if err := run("modprobe", "zfs"); err != nil {
+		slog.Warn("modprobe zfs failed, checking for /dev/zfs anyway", "err", err)
+	}
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		if _, err := os.Stat("/dev/zfs"); err == nil {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("/dev/zfs did not appear after modprobe zfs — the ZFS module is missing or does not match the running kernel")
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// ImportablePools returns the names of pools visible to `zpool import`. There
+// is still no machine-readable output in OpenZFS 2.3, so the human format is
+// parsed — the same approach Proxmox::Sys::ZFS takes.
+func ImportablePools() ([]string, error) {
+	out, err := exec.Command("zpool", "import").Output()
+	if err != nil {
+		// zpool exits non-zero when there is nothing to import, which is the
+		// normal case. Anything else — no binary, no /dev/zfs — must surface, or
+		// a machine with an existing pool looks empty to the caller.
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("zpool import: %w", err)
+	}
+	return parseImportablePools(string(out)), nil
+}
+
+// parseImportablePools extracts pool names from `zpool import` output. Split
+// out from the command call so it can be tested against captured fixtures.
+func parseImportablePools(out string) []string {
+	var pools []string
+	scanner := bufio.NewScanner(strings.NewReader(out))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		// Only the "pool:" key introduces a pool; "id:", "state:" and the
+		// config tree that follows are attributes of the one before it.
+		if name, ok := strings.CutPrefix(line, "pool:"); ok {
+			if n := strings.TrimSpace(name); n != "" {
+				pools = append(pools, n)
+			}
+		}
+	}
+	return pools
+}
+
+// createPool builds the root pool across the selected disks. altroot keeps
+// every dataset mounted under the install target instead of over the live
+// filesystem, and cachefile=none stops the live environment from claiming it.
+func createPool(cfg DiskConfig, opts ZFSOpts) error {
+	vdev, err := buildVdev(cfg)
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"create", "-f",
+		"-o", "cachefile=none",
+		"-o", fmt.Sprintf("ashift=%d", opts.Ashift),
+		"-O", "atime=on", "-O", "relatime=on",
+		"-O", "xattr=sa",
+		"-O", fmt.Sprintf("compression=%s", opts.Compress),
+		"-O", fmt.Sprintf("checksum=%s", opts.Checksum),
+		"-O", fmt.Sprintf("copies=%d", opts.Copies),
+		"-O", "mountpoint=none",
+		"-R", mountRoot,
+		ZFSPoolName,
+	}
+	args = append(args, strings.Fields(vdev)...)
+	if err := run("zpool", args...); err != nil {
+		return fmt.Errorf("zpool create: %w", err)
+	}
+
+	// TRIM matters on a pool that churns EBS volumes: without it the drives'
+	// garbage collection degrades write latency over months, and diagnosing
+	// that after the fact is miserable. Detected, never prompted.
+	if allFlash(cfg.Disks) {
+		if err := run("zpool", "set", "autotrim=on", ZFSPoolName); err != nil {
+			slog.Warn("could not enable autotrim", "err", err)
+		}
+	}
+	return nil
+}
+
+// allFlash reports whether every member is non-rotational.
+func allFlash(disks []Disk) bool {
+	for _, d := range disks {
+		if d.Rotational {
+			return false
+		}
+	}
+	return len(disks) > 0
+}
+
+// createDatasets lays out the service datasets. Mountpoints are set relative to
+// the pool's altroot, so they land under the install target now and at their
+// real paths once the installed system imports the pool.
+func createDatasets() error {
+	for _, ds := range datasets {
+		args := []string{"create", "-o", "mountpoint=" + ds.mountpoint}
+		for _, p := range ds.props {
+			args = append(args, "-o", p)
+		}
+		args = append(args, ds.name)
+		if err := run("zfs", args...); err != nil {
+			return fmt.Errorf("create dataset %s: %w", ds.name, err)
+		}
+	}
+	return nil
+}
+
+// datasetProtectFilters keeps rsync --delete away from the dataset mountpoints.
+//
+// The datasets are mounted before the copy so files land on the right one, but
+// the live source has nothing at those paths, so --delete tries to rmdir a busy
+// mountpoint and fails the whole transfer. Protect rules exempt them.
+func datasetProtectFilters(cfg DiskConfig) []string {
+	if !cfg.FS.IsZFS() {
+		return nil
+	}
+	var out []string
+	for _, ds := range datasets {
+		if ds.mountpoint == "/" || ds.mountpoint == "none" {
+			continue
+		}
+		out = append(out, "--filter=P "+ds.mountpoint)
+	}
+	return out
+}
+
+// writeZFSSystemConfig writes the target's ZFS tunables: the ARC ceiling, and
+// the matching host reserve so the daemon's admission gate stops handing out
+// memory the ARC is holding.
+func writeZFSSystemConfig(opts ZFSOpts) error {
+	modprobeDir := filepath.Join(mountRoot, "etc/modprobe.d")
+	if err := os.MkdirAll(modprobeDir, 0o755); err != nil {
+		return err
+	}
+	arcBytes := int64(opts.ARCMaxMiB) << 20
+	conf := fmt.Sprintf("# Generated by the Spinifex installer.\n"+
+		"# Guest RAM is the scarce resource on a hypervisor, so the ARC is\n"+
+		"# capped well below the ZFS default of half of system memory.\n"+
+		"options zfs zfs_arc_max=%d\n", arcBytes)
+	if err := os.WriteFile(filepath.Join(modprobeDir, "zfs.conf"), []byte(conf), 0o644); err != nil {
+		return fmt.Errorf("write zfs.conf: %w", err)
+	}
+
+	// The daemon computes schedulable memory as host total minus this reserve
+	// (defaultHostReserve in daemon/resource.go). ARC is invisible to it, so
+	// the ceiling has to be declared or instances get admitted into memory the
+	// ARC already owns.
+	reserve := defaultHostReserveGB + float64(opts.ARCMaxMiB)/1024.0
+	env := fmt.Sprintf("# Generated by the Spinifex installer.\n"+
+		"# %.1fGB base reserve + %.1fGB ZFS ARC ceiling.\n"+
+		"SPINIFEX_RESERVED_MEM_GB=%.1f\n",
+		defaultHostReserveGB, float64(opts.ARCMaxMiB)/1024.0, reserve)
+	confDir := filepath.Join(mountRoot, "etc/spinifex")
+	if err := os.MkdirAll(confDir, 0o755); err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(confDir, "host.env"), []byte(env), 0o644)
+}
+
+// defaultHostReserveGB mirrors defaultHostReserve.memGB in
+// spinifex/daemon/resource.go. Changing one without the other silently
+// misreports how much memory a ZFS node can schedule.
+const defaultHostReserveGB = 2.0
+
+// exportPool releases the pool so a retry after a failed install is not blocked
+// by "pool is busy". The live environment runs spinifex-init as PID 1, so
+// nothing else will ever clean this up.
+func exportPool() {
+	if err := run("zpool", "export", ZFSPoolName); err != nil {
+		slog.Warn("zpool export failed", "pool", ZFSPoolName, "err", err)
+	}
+}
+
+// clearDisk removes any prior filesystem, partition table and ZFS label from a
+// disk so a retry after a partial install is not rejected for stale metadata —
+// a failure that otherwise reads as mysterious rather than as leftover debris.
+//
+// The probes are quiet because a blank disk makes all of them fail, and their
+// stderr on the install console reads like the install is going wrong.
+func clearDisk(d Disk) error {
+	// ZFS labels live at both ends of the device and wipefs does not know about
+	// them, so an old pool member has to be cleared before its partition goes.
+	if old := d.PartitionPath(rootPartNum); exists(old) {
+		_ = runQuiet("zpool", "labelclear", "-f", old)
+	}
+	_ = runQuiet("wipefs", "-a", d.Path)
+	return runQuiet("sgdisk", "-Z", d.Path)
+}
+
+func exists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}

cmd/installer/install/zfs_test.go

@@ -0,0 +1,311 @@
+package install
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+// disk builds a test disk with a by-id path, since that is what pool members
+// are actually named by.
+func disk(name string, gib int64) Disk {
+	return Disk{
+		Path:              "/dev/" + name,
+		ByID:              "/dev/disk/by-id/virtio-" + name,
+		Bytes:             gib << 30,
+		LogicalBlockSize:  512,
+		PhysicalBlockSize: 512,
+	}
+}
+
+func disks(n int, gib int64) []Disk {
+	out := make([]Disk, n)
+	for i := range out {
+		out[i] = disk(string(rune('a'+i)), gib)
+	}
+	return out
+}
+
+func TestBuildVdev(t *testing.T) {
+	tests := []struct {
+		name  string
+		fs    FSType
+		disks []Disk
+		want  string
+	}{
+		{
+			"raid0 is a bare stripe with no keyword",
+			FSZFSRAID0, disks(2, 40),
+			"/dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3",
+		},
+		{
+			"single-disk raid0 is the one-drive case",
+			FSZFSRAID0, disks(1, 40),
+			"/dev/disk/by-id/virtio-a-part3",
+		},
+		{
+			"raid1 is one mirror vdev",
+			FSZFSRAID1, disks(2, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3",
+		},
+		{
+			"raid1 across three disks is a three-way mirror, not a stripe",
+			FSZFSRAID1, disks(3, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 /dev/disk/by-id/virtio-c-part3",
+		},
+		{
+			"raid10 pairs members in selection order",
+			FSZFSRAID10, disks(4, 40),
+			"mirror /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 " +
+				"mirror /dev/disk/by-id/virtio-c-part3 /dev/disk/by-id/virtio-d-part3",
+		},
+		{
+			"raidz1 across three disks",
+			FSZFSRAIDZ1, disks(3, 40),
+			"raidz1 /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 /dev/disk/by-id/virtio-c-part3",
+		},
+		{
+			"raidz2 needs four",
+			FSZFSRAIDZ2, disks(4, 40),
+			"raidz2 /dev/disk/by-id/virtio-a-part3 /dev/disk/by-id/virtio-b-part3 " +
+				"/dev/disk/by-id/virtio-c-part3 /dev/disk/by-id/virtio-d-part3",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := buildVdev(DiskConfig{FS: tt.fs, Disks: tt.disks})
+			if err != nil {
+				t.Fatalf("buildVdev: %v", err)
+			}
+			if got != tt.want {
+				t.Errorf("buildVdev()\n got: %s\nwant: %s", got, tt.want)
+			}
+		})
+	}
+}
+
+// A pool built on /dev/sdX silently breaks when the controller enumerates in a
+// different order, so the fallback only applies when there is no by-id path.
+func TestBuildVdevFallsBackToKernelNameWithoutByID(t *testing.T) {
+	d := disk("a", 40)
+	d.ByID = ""
+	got, err := buildVdev(DiskConfig{FS: FSZFSRAID0, Disks: []Disk{d}})
+	if err != nil {
+		t.Fatalf("buildVdev: %v", err)
+	}
+	if got != "/dev/a3" {
+		t.Errorf("buildVdev() = %q, want /dev/a3", got)
+	}
+}
+
+func TestBuildVdevRejectsExt4(t *testing.T) {
+	if _, err := buildVdev(DiskConfig{FS: FSExt4, Disks: disks(1, 40)}); err == nil {
+		t.Fatal("buildVdev accepted ext4")
+	}
+}
+
+func TestBuildVdevValidatesFirst(t *testing.T) {
+	// Three disks cannot form striped mirrors, and producing a two-disk mirror
+	// plus a silently dropped disk would be worse than an error.
+	if _, err := buildVdev(DiskConfig{FS: FSZFSRAID10, Disks: disks(3, 40)}); err == nil {
+		t.Fatal("buildVdev accepted an odd disk count for RAID10")
+	}
+}
+
+func TestDetectAshift(t *testing.T) {
+	tests := []struct {
+		name     string
+		logical  int
+		physical int
+		want     int
+	}{
+		// ashift cannot be changed after creation and one that is too small
+		// costs a read-modify-write per block forever, so 512e drives are
+		// deliberately treated as 4K.
+		{"512n drive is floored at 4K", 512, 512, 12},
+		{"512e drive reports 4K physical", 512, 4096, 12},
+		{"4Kn drive", 4096, 4096, 12},
+		{"8K native drive", 8192, 8192, 13},
+		{"16K native drive", 16384, 16384, 14},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			d := Disk{LogicalBlockSize: tt.logical, PhysicalBlockSize: tt.physical}
+			if got := detectAshift([]Disk{d}); got != tt.want {
+				t.Errorf("detectAshift() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+// A pool cannot have two sector sizes, so the widest member wins — the reverse
+// would make every write to the large-sector disk a read-modify-write.
+func TestDetectAshiftTakesWidestMember(t *testing.T) {
+	got := detectAshift([]Disk{
+		{LogicalBlockSize: 512, PhysicalBlockSize: 512},
+		{LogicalBlockSize: 512, PhysicalBlockSize: 8192},
+	})
+	if got != 13 {
+		t.Errorf("detectAshift() = %d, want 13", got)
+	}
+}
+
+func TestDefaultARCMaxMiB(t *testing.T) {
+	tests := []struct {
+		name    string
+		totalMi int
+		want    int
+	}{
+		{"tiny VM gets 10% of very little", 2048, 205},
+		{"8GB node gets 10%", 8192, 819},
+		{"64GB node gets 10%", 65536, 6554},
+		// The point of the cap: guest RAM is what this machine sells, and half
+		// of a 256GB node is 128GB of instance capacity the ARC would swallow.
+		{"256GB node is capped at 16GiB", 262144, 16384},
+		{"1TB node is still capped at 16GiB", 1048576, 16384},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := defaultARCMaxMiB(tt.totalMi); got != tt.want {
+				t.Errorf("defaultARCMaxMiB(%d) = %d, want %d", tt.totalMi, got, tt.want)
+			}
+		})
+	}
+}
+
+// An unreadable /proc/meminfo must not produce a zero or negative ceiling,
+// which ZFS would either reject or read as "no limit".
+func TestDefaultARCMaxMiBNeverGoesBelowFloor(t *testing.T) {
+	for _, total := range []int{0, 128, 512} {
+		if got := defaultARCMaxMiB(total); got < arcMinMiB {
+			t.Errorf("defaultARCMaxMiB(%d) = %d, below the floor of %d", total, got, arcMinMiB)
+		}
+	}
+}
+
+func TestResolveZFSOptsFillsDefaults(t *testing.T) {
+	got := resolveZFSOpts(DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)})
+	if got.Ashift != 12 {
+		t.Errorf("Ashift = %d, want 12", got.Ashift)
+	}
+	if got.Compress != "lz4" {
+		t.Errorf("Compress = %q, want lz4", got.Compress)
+	}
+	if got.Checksum != "on" {
+		t.Errorf("Checksum = %q, want on", got.Checksum)
+	}
+	if got.Copies != 1 {
+		t.Errorf("Copies = %d, want 1", got.Copies)
+	}
+	if got.ARCMaxMiB <= 0 {
+		t.Errorf("ARCMaxMiB = %d, want a positive ceiling", got.ARCMaxMiB)
+	}
+}
+
+func TestResolveZFSOptsKeepsOperatorChoices(t *testing.T) {
+	in := DiskConfig{
+		FS:    FSZFSRAIDZ1,
+		Disks: disks(3, 40),
+		ZFS:   ZFSOpts{Ashift: 13, Compress: "zstd", Checksum: "blake3", Copies: 2, ARCMaxMiB: 2048},
+	}
+	got := resolveZFSOpts(in)
+	if got != in.ZFS {
+		t.Errorf("resolveZFSOpts overrode explicit options: got %+v, want %+v", got, in.ZFS)
+	}
+}
+
+func TestParseImportablePools(t *testing.T) {
+	// Captured from `zpool import` on a machine with a pool from a previous
+	// install: the config tree below each header must not be mistaken for one.
+	const out = `   pool: rpool
+     id: 8543219876543210987
+  state: ONLINE
+ action: The pool can be imported using its name or numeric identifier.
+ config:
+
+	rpool                     ONLINE
+	  raidz1-0                ONLINE
+	    virtio-spinifex-disk1-part3  ONLINE
+	    virtio-spinifex-disk2-part3  ONLINE
+
+   pool: tank
+     id: 1111111111111111111
+  state: ONLINE
+ config:
+
+	tank                      ONLINE
+`
+	got := parseImportablePools(out)
+	want := []string{"rpool", "tank"}
+	if len(got) != len(want) {
+		t.Fatalf("parseImportablePools() = %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("pool %d = %q, want %q", i, got[i], want[i])
+		}
+	}
+}
+
+func TestParseImportablePoolsEmpty(t *testing.T) {
+	if got := parseImportablePools("no pools available to import\n"); len(got) != 0 {
+		t.Errorf("parseImportablePools() = %v, want none", got)
+	}
+}
+
+// Every dataset must be creatable in order: a child cannot be created before
+// its parent, and getting this wrong only shows up on real hardware.
+func TestDatasetsAreOrderedParentFirst(t *testing.T) {
+	seen := map[string]bool{ZFSPoolName: true}
+	for _, ds := range datasets {
+		parent := ds.name[:strings.LastIndex(ds.name, "/")]
+		if !seen[parent] {
+			t.Errorf("dataset %s is created before its parent %s", ds.name, parent)
+		}
+		seen[ds.name] = true
+	}
+}
+
+// The root dataset has to be the one the bootloader names, or the machine
+// mounts an empty filesystem and panics.
+func TestRootDatasetIsCreated(t *testing.T) {
+	for _, ds := range datasets {
+		if ds.name == ZFSRootDataset {
+			if ds.mountpoint != "/" {
+				t.Fatalf("%s mountpoint = %q, want /", ZFSRootDataset, ds.mountpoint)
+			}
+			return
+		}
+	}
+	t.Fatalf("%s is never created", ZFSRootDataset)
+}
+
+// rsync --delete tries to rmdir a dataset mountpoint the live source does not
+// have, which fails on a busy mount and aborts the whole copy. Every dataset
+// with a real mountpoint must be protected.
+func TestDatasetProtectFilters(t *testing.T) {
+	got := datasetProtectFilters(DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(3, 40)})
+	for _, ds := range datasets {
+		if ds.mountpoint == "/" || ds.mountpoint == "none" {
+			continue
+		}
+		want := "--filter=P " + ds.mountpoint
+		if !slices.Contains(got, want) {
+			t.Errorf("missing protect rule for %s (%s)", ds.name, ds.mountpoint)
+		}
+	}
+	// The root dataset is the destination itself, and "none" datasets are never
+	// mounted, so protecting either would be meaningless.
+	for _, f := range got {
+		if f == "--filter=P /" || f == "--filter=P none" {
+			t.Errorf("unexpected protect rule %q", f)
+		}
+	}
+}
+
+func TestDatasetProtectFiltersSkipsExt4(t *testing.T) {
+	if got := datasetProtectFilters(DiskConfig{FS: FSExt4, Disks: disks(1, 40)}); got != nil {
+		t.Errorf("datasetProtectFilters() = %v on ext4, want none", got)
+	}
+}

cmd/installer/netprobe/netprobe.go

@@ -0,0 +1,426 @@
+// Package netprobe identifies physical network interfaces by hardware rather
+// than kernel name, so an operator choosing between eno1, ens1f0np0 and
+// ens1f1np1 can tell the 1gbe Broadcom from the 25gbe Mellanox.
+package netprobe
+
+import (
+	"cmp"
+	"fmt"
+	"log/slog"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// sysClassNet is the kernel's per-interface attribute tree. A variable, not a
+// constant, so tests can point the probe at a fixture instead of the running
+// machine's hardware.
+var sysClassNet = "/sys/class/net"
+
+// The external helpers the probe leans on, indirected so tests can exercise it
+// without the host's real tooling.
+var (
+	listInterfaces = net.Interfaces
+	commandOutput  = func(name string, args ...string) ([]byte, error) {
+		return exec.Command(name, args...).Output()
+	}
+)
+
+// NIC describes one physical interface as presented in the installer's
+// selection table.
+type NIC struct {
+	Name string
+
+	// Vendor and Model come from the udev hardware database, e.g. "Mellanox
+	// Technologies" and "MT27710 Family [ConnectX-4 Lx]". Empty when the
+	// database carries no entry for the card.
+	Vendor string
+	Model  string
+
+	// Driver is the kernel module that claimed the port, e.g. "mlx5_core". It
+	// comes straight from sysfs, so it still identifies a card the hardware
+	// database misses.
+	Driver string
+
+	// DeviceID is the PCI or USB "vendor:device" pair, the last-resort identity
+	// for hardware neither the database nor a driver name pins down.
+	DeviceID string
+
+	// AltName is the first alternative name systemd assigns, which is what
+	// operators often see in switch and cabling documentation.
+	AltName string
+
+	// Speed is the negotiated link speed rendered for display ("25 Gbps") and
+	// SpeedMbps the raw figure behind it. Both are zero on a port with no
+	// carrier, which is why Probe brings links up before reading.
+	Speed     string
+	SpeedMbps int
+
+	// Carrier reports a cable with link. It orders the picker so the port the
+	// operator has just plugged in is the one offered first.
+	Carrier bool
+
+	// State is the link summary shown in the table: "online", "no cable" or
+	// "down".
+	State string
+
+	MTU    int
+	IsWiFi bool
+}
+
+// Label renders the NIC's identity line: kernel name, negotiated speed and link
+// state. The hardware detail is deliberately absent — it goes on its own line
+// via Hardware, where there is room to read it.
+func (n NIC) Label() string {
+	speed := n.Speed
+	if speed == "" {
+		speed = "—"
+	}
+	state := n.State
+	if state == "" {
+		state = "unknown"
+	}
+	return fmt.Sprintf("%s  %s  %s", n.Name, speed, state)
+}
+
+// Hardware renders the card's identity for the detail line, degrading from the
+// database name to the driver and device id rather than to "unknown". An
+// operator needs something to match against the card in their hand.
+func (n NIC) Hardware() string {
+	if hw := strings.TrimSpace(n.Vendor + " " + n.Model); hw != "" {
+		return hw
+	}
+	switch {
+	case n.Driver != "" && n.DeviceID != "":
+		return n.Driver + " [" + n.DeviceID + "]"
+	case n.Driver != "":
+		return n.Driver
+	case n.DeviceID != "":
+		return n.DeviceID
+	}
+	return "no hardware detail"
+}
+
+// linkUpTimeout bounds how long Probe waits after bringing links up for speed
+// and carrier to settle. A 25gbe port negotiating against a cold switch can
+// take a second or two to report a speed.
+var linkUpTimeout = 2 * time.Second
+
+// Probe enumerates non-loopback physical interfaces and enriches each from
+// sysfs and the udev hardware database, ordered so the ports worth assigning
+// come first. Links are brought up first: a down port reports no speed and no
+// carrier, which would leave every fast port unidentified at exactly the moment
+// the operator needs to pick one.
+//
+// Nothing the probe depends on needs a running systemd. The live installer
+// boots straight into spinifex-init with only udevd started, so networkctl —
+// which has to reach systemd-networkd — enriches gaps and is never the source.
+func Probe() ([]NIC, error) {
+	ifaces, err := listInterfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	var names, physical []string
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagLoopback != 0 {
+			continue
+		}
+		if len(iface.HardwareAddr) == 0 {
+			continue
+		}
+		names = append(names, iface.Name)
+		if hasDevice(iface.Name) {
+			physical = append(physical, iface.Name)
+		}
+	}
+
+	// Bridges, bonds and veth pairs carry a MAC like any port but have no
+	// device behind them, and offering them would let a plane be bound to
+	// something that does not exist until after the install. Keep them only if
+	// the filter found nothing, so an unusual platform still gets a list.
+	if len(physical) > 0 {
+		names = physical
+	}
+
+	brought := false
+	for _, name := range names {
+		if _, err := commandOutput("ip", "link", "set", "dev", name, "up"); err != nil {
+			slog.Debug("netprobe: could not bring link up", "iface", name, "err", err)
+			continue
+		}
+		brought = true
+	}
+	if brought {
+		time.Sleep(linkUpTimeout)
+	}
+
+	nics := make([]NIC, 0, len(names))
+	for _, name := range names {
+		nics = append(nics, describe(name))
+	}
+	slices.SortFunc(nics, byUsefulness)
+	return nics, nil
+}
+
+// describe reads everything about one interface that needs no daemon, then lets
+// networkctl fill the gaps if it happens to be answering.
+func describe(name string) NIC {
+	nic := NIC{
+		Name:     name,
+		Driver:   sysLinkBase(name, "device/driver"),
+		DeviceID: deviceID(name),
+		MTU:      sysInt(name, "mtu"),
+		IsWiFi:   isWiFi(name),
+	}
+
+	// The kernel reports -1 for a port that cannot state its speed; treat that
+	// as unknown rather than letting it sort below every real figure.
+	nic.SpeedMbps = max(sysInt(name, "speed"), 0)
+	nic.Speed = formatSpeed(nic.SpeedMbps)
+	nic.Carrier = sysAttr(name, "carrier") == "1"
+	nic.State = linkState(name, nic.Carrier)
+
+	props := udevProperties(name)
+	nic.Vendor = props["ID_VENDOR_FROM_DATABASE"]
+	nic.Model = props["ID_MODEL_FROM_DATABASE"]
+
+	// udev's predictable names are the ones that appear in cabling notes; the
+	// path-based name is the one networkctl lists first.
+	for _, key := range []string{"ID_NET_NAME_PATH", "ID_NET_NAME_ONBOARD", "ID_NET_NAME_SLOT"} {
+		if alt := props[key]; alt != "" && alt != name {
+			nic.AltName = alt
+			break
+		}
+	}
+
+	if out, err := commandOutput("networkctl", "status", "--no-pager", name); err == nil {
+		fill(&nic, ParseStatus(string(out)))
+	} else {
+		slog.Debug("netprobe: networkctl unavailable, using sysfs only", "iface", name, "err", err)
+	}
+	return nic
+}
+
+// fill copies parsed hardware detail onto a NIC without overwriting anything
+// sysfs already established. State is never taken: the carrier bit is a truer
+// answer to "is this cable live" than networkd's configuration status.
+func fill(dst *NIC, src NIC) {
+	if dst.Vendor == "" {
+		dst.Vendor = src.Vendor
+	}
+	if dst.Model == "" {
+		dst.Model = src.Model
+	}
+	if dst.AltName == "" {
+		dst.AltName = src.AltName
+	}
+	if dst.Speed == "" {
+		dst.Speed = src.Speed
+	}
+	if dst.MTU == 0 {
+		dst.MTU = src.MTU
+	}
+}
+
+// byUsefulness orders the picker so the interfaces an operator would actually
+// assign come first: a live cable, then wired over wireless, then the fastest
+// port. The rest still appear, just below the plausible choices.
+func byUsefulness(a, b NIC) int {
+	if a.Carrier != b.Carrier {
+		return boolOrder(a.Carrier)
+	}
+	if a.IsWiFi != b.IsWiFi {
+		return boolOrder(b.IsWiFi)
+	}
+	if a.SpeedMbps != b.SpeedMbps {
+		return cmp.Compare(b.SpeedMbps, a.SpeedMbps)
+	}
+	return cmp.Compare(a.Name, b.Name)
+}
+
+// boolOrder sorts true ahead of false.
+func boolOrder(first bool) int {
+	if first {
+		return -1
+	}
+	return 1
+}
+
+// formatSpeed renders the kernel's Mbit/s figure the way a datasheet does.
+func formatSpeed(mbps int) string {
+	switch {
+	case mbps <= 0:
+		return ""
+	case mbps%1000 == 0:
+		return fmt.Sprintf("%d Gbps", mbps/1000)
+	default:
+		return fmt.Sprintf("%d Mbps", mbps)
+	}
+}
+
+// linkState collapses the kernel's flags into the three outcomes an operator
+// acts on: the port is live, the port is enabled but unplugged, or the port
+// never came up at all. operstate reports "down" for the latter two alike.
+func linkState(name string, carrier bool) string {
+	switch {
+	case carrier:
+		return "online"
+	case adminUp(name):
+		return "no cable"
+	default:
+		return "down"
+	}
+}
+
+// adminUp reads the IFF_UP bit out of the interface flags.
+func adminUp(name string) bool {
+	flags, err := strconv.ParseUint(strings.TrimPrefix(sysAttr(name, "flags"), "0x"), 16, 64)
+	return err == nil && flags&0x1 != 0
+}
+
+// deviceID reads the PCI or USB "vendor:device" pair. A NIC's device link
+// points at the PCI function directly; on USB it points at the interface, whose
+// parent carries the ids.
+func deviceID(name string) string {
+	dev := filepath.Join(sysClassNet, name, "device")
+	if id := idPair(dev, "vendor", "device"); id != "" {
+		return id
+	}
+	return idPair(filepath.Join(dev, ".."), "idVendor", "idProduct")
+}
+
+func idPair(dir, vendorAttr, productAttr string) string {
+	vendor := readTrimmed(filepath.Join(dir, vendorAttr))
+	product := readTrimmed(filepath.Join(dir, productAttr))
+	if vendor == "" || product == "" {
+		return ""
+	}
+	return strings.TrimPrefix(vendor, "0x") + ":" + strings.TrimPrefix(product, "0x")
+}
+
+// udevProperties reads the interface's udev database entry, which is where the
+// hardware-database vendor and model names live. udevd populates it during the
+// installer's device enumeration, so this works with no systemd running.
+func udevProperties(name string) map[string]string {
+	out, err := commandOutput("udevadm", "info", "--query=property", "--path="+filepath.Join(sysClassNet, name))
+	if err != nil {
+		slog.Debug("netprobe: udevadm info failed", "iface", name, "err", err)
+		return nil
+	}
+
+	props := make(map[string]string)
+	for line := range strings.SplitSeq(string(out), "\n") {
+		if key, value, ok := strings.Cut(strings.TrimSpace(line), "="); ok {
+			props[key] = value
+		}
+	}
+	return props
+}
+
+// ParseStatus extracts the hardware fields from `networkctl status <iface>`
+// output. The format is a set of right-aligned "Key: value" lines, with
+// Alternative Names continuing across subsequent indented lines.
+func ParseStatus(out string) NIC {
+	var nic NIC
+	var inAltNames bool
+
+	for line := range strings.SplitSeq(out, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" {
+			inAltNames = false
+			continue
+		}
+
+		key, value, found := strings.Cut(trimmed, ": ")
+		if !found {
+			// A continuation line under "Alternative Names:" — take the first
+			// only, which is the one operators see in cabling notes.
+			if inAltNames && nic.AltName == "" {
+				nic.AltName = trimmed
+			}
+			continue
+		}
+		inAltNames = false
+
+		key = strings.TrimSpace(key)
+		value = strings.TrimSpace(value)
+		switch key {
+		case "Vendor":
+			nic.Vendor = value
+		case "Model":
+			nic.Model = value
+		case "Speed":
+			nic.Speed = value
+		case "Online state":
+			// Rendered as "online" or "online (configured)".
+			if f := strings.Fields(value); len(f) > 0 {
+				nic.State = f[0]
+			}
+		case "MTU":
+			// Rendered as "9000 (min: 68, max: 9978)".
+			if f := strings.Fields(value); len(f) > 0 {
+				if mtu, err := strconv.Atoi(f[0]); err == nil {
+					nic.MTU = mtu
+				}
+			}
+		case "Alternative Names":
+			inAltNames = true
+			if f := strings.Fields(value); len(f) > 0 && nic.AltName == "" {
+				nic.AltName = f[0]
+			}
+		}
+	}
+	return nic
+}
+
+// hasDevice reports whether the interface is backed by real hardware, which
+// sysfs shows as a device link to the PCI, USB or platform node behind it.
+func hasDevice(name string) bool {
+	_, err := os.Stat(filepath.Join(sysClassNet, name, "device"))
+	return err == nil
+}
+
+// isWiFi reports whether the interface has a wireless subdirectory in sysfs.
+func isWiFi(name string) bool {
+	_, err := os.Stat(filepath.Join(sysClassNet, name, "wireless"))
+	return err == nil
+}
+
+// sysAttr reads one sysfs attribute. A missing or unreadable attribute is not
+// an error worth surfacing: speed and carrier both fail on a port with no link,
+// which the state column already conveys.
+func sysAttr(name, attr string) string {
+	return readTrimmed(filepath.Join(sysClassNet, name, attr))
+}
+
+func sysInt(name, attr string) int {
+	n, err := strconv.Atoi(sysAttr(name, attr))
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+// sysLinkBase resolves a sysfs symlink and returns its final element, which is
+// how the driver name is exposed.
+func sysLinkBase(name, attr string) string {
+	target, err := os.Readlink(filepath.Join(sysClassNet, name, attr))
+	if err != nil {
+		return ""
+	}
+	return filepath.Base(target)
+}
+
+func readTrimmed(path string) string {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(b))
+}

cmd/installer/netprobe/netprobe_test.go

@@ -0,0 +1,292 @@
+package netprobe
+
+import (
+	"errors"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Captured from hydrogen (HP DL325 Gen10, dual 25gbe Mellanox) in us-west-1-az1.
+const mellanox25g = `● 6: ens1f0np0
+                   Link File: /usr/lib/systemd/network/99-default.link
+                Network File: /etc/systemd/network/12-spinifex-lan-nic.network
+                       State: enslaved (configured)
+                Online state: online
+                        Type: ether
+                        Path: pci-0000:84:00.0
+                      Driver: mlx5_core
+                      Vendor: Mellanox Technologies
+                       Model: MT27710 Family [ConnectX-4 Lx]
+           Alternative Names: enp132s0f0np0
+                              enx040973e5ce60
+            Hardware Address: 04:09:73:e5:ce:60 (Hewlett Packard Enterprise)
+                         MTU: 9000 (min: 68, max: 9978)
+                       QDisc: mq
+                      Master: br-lan
+IPv6 Address Generation Mode: none
+    Number of Queues (Tx/Rx): 384/48
+            Auto negotiation: yes
+                       Speed: 25Gbps
+                      Duplex: full
+           Activation Policy: up
+         Required For Online: yes
+                Connected To: MikroTik (MikroTik RouterOS 7.11.3 (stable) Sep/27/2023 13:09:44 CRS504-4XQ) on port bridge/qsfp28-3-3
+`
+
+// A link with no carrier reports neither Speed nor an online state, which is
+// the case Probe brings links up to avoid.
+const linkDown = `● 3: eno6
+                   Link File: /usr/lib/systemd/network/99-default.link
+                       State: off
+                Online state: off
+                        Type: ether
+                      Driver: bnxt_en
+                      Vendor: Broadcom Inc. and subsidiaries
+                       Model: NetXtreme BCM5720 Gigabit Ethernet PCIe
+            Hardware Address: 04:09:73:e5:ce:5f
+                         MTU: 1500 (min: 60, max: 9000)
+`
+
+func TestParseStatusMellanox(t *testing.T) {
+	nic := ParseStatus(mellanox25g)
+
+	if nic.Vendor != "Mellanox Technologies" {
+		t.Errorf("Vendor = %q, want %q", nic.Vendor, "Mellanox Technologies")
+	}
+	if nic.Model != "MT27710 Family [ConnectX-4 Lx]" {
+		t.Errorf("Model = %q, want %q", nic.Model, "MT27710 Family [ConnectX-4 Lx]")
+	}
+	if nic.Speed != "25Gbps" {
+		t.Errorf("Speed = %q, want %q", nic.Speed, "25Gbps")
+	}
+	if nic.State != "online" {
+		t.Errorf("State = %q, want %q", nic.State, "online")
+	}
+	if nic.MTU != 9000 {
+		t.Errorf("MTU = %d, want 9000", nic.MTU)
+	}
+	// Only the first alternative name is taken; enx040973e5ce60 is ignored.
+	if nic.AltName != "enp132s0f0np0" {
+		t.Errorf("AltName = %q, want %q", nic.AltName, "enp132s0f0np0")
+	}
+}
+
+func TestParseStatusLinkDown(t *testing.T) {
+	nic := ParseStatus(linkDown)
+
+	if nic.Speed != "" {
+		t.Errorf("Speed = %q, want empty for a down link", nic.Speed)
+	}
+	if nic.State != "off" {
+		t.Errorf("State = %q, want %q", nic.State, "off")
+	}
+	if nic.Model != "NetXtreme BCM5720 Gigabit Ethernet PCIe" {
+		t.Errorf("Model = %q, want the Broadcom model", nic.Model)
+	}
+	if nic.MTU != 1500 {
+		t.Errorf("MTU = %d, want 1500", nic.MTU)
+	}
+}
+
+func TestParseStatusEmpty(t *testing.T) {
+	// A truncated or unparseable status must not panic or invent values.
+	nic := ParseStatus("● 2: eth0\n     Online state:\n     Alternative Names:\n")
+	if nic.State != "" || nic.AltName != "" {
+		t.Errorf("expected empty fields from valueless keys, got State=%q AltName=%q", nic.State, nic.AltName)
+	}
+}
+
+func TestLabelFallsBackWhenSpeedUnknown(t *testing.T) {
+	got := NIC{Name: "eth0", State: "no cable"}.Label()
+	want := "eth0  —  no cable"
+	if got != want {
+		t.Errorf("Label() = %q, want %q", got, want)
+	}
+}
+
+// Hardware must never render "unknown": an operator standing at the console
+// needs something to match against the card in their hand, and sysfs always has
+// at least a driver or a device id.
+func TestHardwareDegradesRatherThanUnknown(t *testing.T) {
+	tests := []struct {
+		name string
+		nic  NIC
+		want string
+	}{
+		{
+			name: "database name wins",
+			nic:  NIC{Vendor: "Mellanox Technologies", Model: "MT27710", Driver: "mlx5_core", DeviceID: "15b3:1015"},
+			want: "Mellanox Technologies MT27710",
+		},
+		{"vendor only", NIC{Vendor: "Intel Corporation"}, "Intel Corporation"},
+		{"driver and id", NIC{Driver: "igb", DeviceID: "8086:1533"}, "igb [8086:1533]"},
+		{"driver only", NIC{Driver: "virtio_net"}, "virtio_net"},
+		{"id only", NIC{DeviceID: "0bda:8153"}, "0bda:8153"},
+		{"nothing at all", NIC{Name: "eth0"}, "no hardware detail"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.nic.Hardware(); got != tt.want {
+				t.Errorf("Hardware() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestFormatSpeed(t *testing.T) {
+	tests := []struct {
+		mbps int
+		want string
+	}{
+		{25000, "25 Gbps"},
+		{1000, "1 Gbps"},
+		{100, "100 Mbps"},
+		{2500, "2500 Mbps"},
+		{0, ""},
+		{-1, ""},
+	}
+
+	for _, tt := range tests {
+		if got := formatSpeed(tt.mbps); got != tt.want {
+			t.Errorf("formatSpeed(%d) = %q, want %q", tt.mbps, got, tt.want)
+		}
+	}
+}
+
+// A live cable outranks everything, then wired over wireless, then speed. This
+// is what puts the ports an operator would actually assign at indices 0..2,
+// where newModel pre-fills wan, lan and vpc from.
+func TestByUsefulnessOrdersPluggedPortsFirst(t *testing.T) {
+	nics := []NIC{
+		{Name: "eno1", SpeedMbps: 1000},
+		{Name: "wlp2s0", Carrier: true, IsWiFi: true, SpeedMbps: 866},
+		{Name: "ens1f1np1", SpeedMbps: 25000},
+		{Name: "ens1f0np0", Carrier: true, SpeedMbps: 25000},
+		{Name: "eno2", Carrier: true, SpeedMbps: 1000},
+	}
+	slices.SortFunc(nics, byUsefulness)
+
+	want := []string{"ens1f0np0", "eno2", "wlp2s0", "ens1f1np1", "eno1"}
+	for i, name := range want {
+		if nics[i].Name != name {
+			t.Fatalf("order = %v, want %v", names(nics), want)
+		}
+	}
+}
+
+func names(nics []NIC) []string {
+	out := make([]string, len(nics))
+	for i, n := range nics {
+		out[i] = n.Name
+	}
+	return out
+}
+
+func TestProbeReadsSysfsWithoutSystemd(t *testing.T) {
+	root := t.TempDir()
+	sysClassNet = root
+	linkUpTimeout = 0
+	t.Cleanup(func() {
+		sysClassNet = "/sys/class/net"
+		linkUpTimeout = 2 * time.Second
+	})
+
+	// ens1f0np0: a 25gbe Mellanox with a live cable, identified by the hardware
+	// database. eno1: a 1gbe port that is up but unplugged, and absent from the
+	// database, so only its driver and PCI id identify it.
+	writeIface(t, root, "ens1f0np0", map[string]string{
+		"speed": "25000", "carrier": "1", "mtu": "9000", "flags": "0x1003",
+		"device/vendor": "0x15b3", "device/device": "0x1015",
+	}, "mlx5_core")
+	writeIface(t, root, "eno1", map[string]string{
+		"speed": "1000", "carrier": "0", "mtu": "1500", "flags": "0x1003",
+		"device/vendor": "0x8086", "device/device": "0x1533",
+	}, "igb")
+
+	listInterfaces = func() ([]net.Interface, error) {
+		return []net.Interface{
+			{Name: "lo", Flags: net.FlagLoopback},
+			{Name: "eno1", HardwareAddr: net.HardwareAddr{4, 9, 0x73, 0xe5, 0xce, 0x5f}},
+			{Name: "ens1f0np0", HardwareAddr: net.HardwareAddr{4, 9, 0x73, 0xe5, 0xce, 0x60}},
+			{Name: "veth0"}, // no hardware address, so not a physical port
+			{Name: "br-lan", HardwareAddr: net.HardwareAddr{2, 0, 0, 0, 0, 1}}, // a bridge, so no device behind it
+		}, nil
+	}
+	t.Cleanup(func() { listInterfaces = net.Interfaces })
+
+	// Only udevadm answers. networkctl fails the way it does in the live
+	// installer, where there is no systemd-networkd for it to reach.
+	commandOutput = func(name string, args ...string) ([]byte, error) {
+		switch {
+		case name == "ip":
+			return nil, nil
+		case name == "udevadm" && strings.HasSuffix(args[len(args)-1], "ens1f0np0"):
+			return []byte("ID_VENDOR_FROM_DATABASE=Mellanox Technologies\n" +
+				"ID_MODEL_FROM_DATABASE=MT27710 Family [ConnectX-4 Lx]\n" +
+				"ID_NET_NAME_PATH=enp132s0f0np0\n"), nil
+		case name == "udevadm":
+			return []byte("ID_NET_NAME_PATH=enp1s0f0\n"), nil
+		default:
+			return nil, errors.New("Failed to connect to bus: No such file or directory")
+		}
+	}
+	t.Cleanup(func() {
+		commandOutput = func(name string, args ...string) ([]byte, error) {
+			return exec.Command(name, args...).Output()
+		}
+	})
+
+	nics, err := Probe()
+	if err != nil {
+		t.Fatalf("Probe: %v", err)
+	}
+	if got := names(nics); !slices.Equal(got, []string{"ens1f0np0", "eno1"}) {
+		t.Fatalf("probed %v, want the two physical ports with the plugged one first", got)
+	}
+
+	mlx := nics[0]
+	if mlx.Label() != "ens1f0np0  25 Gbps  online" {
+		t.Errorf("Label() = %q", mlx.Label())
+	}
+	if want := "Mellanox Technologies MT27710 Family [ConnectX-4 Lx]"; mlx.Hardware() != want {
+		t.Errorf("Hardware() = %q, want %q", mlx.Hardware(), want)
+	}
+	if mlx.AltName != "enp132s0f0np0" || mlx.MTU != 9000 || !mlx.Carrier {
+		t.Errorf("unexpected detail: %+v", mlx)
+	}
+
+	// Up but unplugged, and unknown to the database: the driver and PCI id are
+	// all the operator gets, which is still enough to identify the card.
+	igb := nics[1]
+	if igb.Label() != "eno1  1 Gbps  no cable" {
+		t.Errorf("Label() = %q", igb.Label())
+	}
+	if igb.Hardware() != "igb [8086:1533]" {
+		t.Errorf("Hardware() = %q", igb.Hardware())
+	}
+}
+
+// writeIface lays out the sysfs attributes Probe reads for one interface,
+// including the driver symlink it resolves the module name from.
+func writeIface(t *testing.T, root, name string, attrs map[string]string, driver string) {
+	t.Helper()
+	for attr, value := range attrs {
+		path := filepath.Join(root, name, attr)
+		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(path, []byte(value+"\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := os.Symlink("../../../bus/pci/drivers/"+driver, filepath.Join(root, name, "device/driver")); err != nil {
+		t.Fatal(err)
+	}
+}

cmd/installer/systemd/systemd.go

@@ -6,44 +6,38 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
-// WriteLANBridgeUnit installs a non-critical oneshot service that activates
-// br-lan via networkctl *after* network-online.target. br-lan has
-// ActivationPolicy=manual in its networkd .network file, so networkd creates
-// the bridge at boot but does not bring it up automatically. This service
-// triggers the activation once the WAN path is confirmed online, ensuring a
-// missing LAN cable or DHCP timeout on the secondary bridge never stalls
+// WriteBridgeUnit installs a non-critical oneshot service that activates an
+// east-west bridge (br-lan, br-vpc) via networkctl *after* network-online.target.
+// Those bridges carry ActivationPolicy=manual in their networkd .network file,
+// so networkd creates them at boot but does not bring them up automatically.
+// This service triggers the activation once the WAN path is confirmed online,
+// ensuring a missing cable or DHCP timeout on a secondary bridge never stalls
 // the management interface or firstboot.
-func WriteLANBridgeUnit(root string) error {
-	unit := `[Unit]
-Description=Spinifex LAN bridge (non-critical)
+func WriteBridgeUnit(root, bridge string) error {
+	name := "spinifex-" + strings.TrimPrefix(bridge, "br-") + "-bridge.service"
+	unit := fmt.Sprintf(`[Unit]
+Description=Spinifex %s bridge (non-critical)
 After=network-online.target
 Wants=network-online.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/networkctl up br-lan
+ExecStart=/usr/bin/networkctl up %s
 RemainAfterExit=yes
 # Failure is non-critical — cable unplugged or switch not ready at boot.
 SuccessExitStatus=0 1
 
 [Install]
 WantedBy=multi-user.target
-`
-	unitPath := filepath.Join(root, "etc/systemd/system/spinifex-lan-bridge.service")
+`, bridge, bridge)
+	unitPath := filepath.Join(root, "etc/systemd/system", name)
 	if err := os.WriteFile(unitPath, []byte(unit), 0o644); err != nil {
 		return err
 	}
-	wantsDir := filepath.Join(root, "etc/systemd/system/multi-user.target.wants")
-	if err := os.MkdirAll(wantsDir, 0o755); err != nil {
-		return err
-	}
-	link := filepath.Join(wantsDir, "spinifex-lan-bridge.service")
-	if err := os.Remove(link); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("remove stale symlink %s: %w", link, err)
-	}
-	return os.Symlink("/etc/systemd/system/spinifex-lan-bridge.service", link)
+	return EnableUnit(root, name)
 }
 
 // WriteFirstbootUnit writes the spinifex-firstboot.service oneshot unit that

cmd/installer/ui/network.go

@@ -0,0 +1,696 @@
+package ui
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	"github.com/charmbracelet/bubbles/textinput"
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/charmbracelet/lipgloss"
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+	"github.com/mulgadc/spinifex/cmd/installer/netprobe"
+)
+
+// foldedNIC marks a role with no interface of its own, which collapses onto
+// the plane above it (vpc <- lan <- wan).
+const foldedNIC = -1
+
+// roleField identifies one editable element on the per-role screen. Which of
+// these are actually shown depends on the plane, the addressing mode, whether
+// the NIC is wireless, and whether Advanced is on.
+type roleField int
+
+const (
+	roleFieldNIC roleField = iota
+	roleFieldMethod
+	roleFieldIP
+	roleFieldMask
+	roleFieldGateway
+	roleFieldDNS
+	roleFieldVLAN
+	roleFieldMTU
+	roleFieldSSID
+	roleFieldWiFiPass
+)
+
+// roleForm is the editable state for one plane. The three forms are the whole
+// of the installer's network configuration.
+type roleForm struct {
+	plane install.Plane
+
+	// nic indexes into model.nics, or foldedNIC when the role is folded.
+	nic   int
+	dhcp  bool
+	focus roleField
+
+	ip       textinput.Model
+	mask     textinput.Model
+	gateway  textinput.Model
+	dns      textinput.Model
+	vlan     textinput.Model
+	mtu      textinput.Model
+	ssid     textinput.Model
+	wifiPass textinput.Model
+}
+
+func newRoleForm(plane install.Plane, nic int) roleForm {
+	text := func(placeholder string, limit int) textinput.Model {
+		in := textinput.New()
+		in.Prompt = ""
+		in.Placeholder = placeholder
+		if limit > 0 {
+			in.CharLimit = limit
+		}
+		return in
+	}
+
+	f := roleForm{
+		plane:    plane,
+		nic:      nic,
+		dhcp:     plane == install.PlaneWAN,
+		ip:       text(defaultIPPlaceholder(plane), 0),
+		mask:     text("255.255.255.0", 0),
+		gateway:  text("192.168.1.1", 0),
+		dns:      text("1.1.1.1, 8.8.8.8", 0),
+		vlan:     text("untagged", 4),
+		mtu:      text("1500", 5),
+		ssid:     text("Network SSID", 64),
+		wifiPass: text("WiFi password", 128),
+	}
+	f.wifiPass.EchoMode = textinput.EchoPassword
+	return f
+}
+
+// defaultIPPlaceholder hints the addressing convention for each plane so an
+// operator filling in a rack does not have to remember it.
+func defaultIPPlaceholder(plane install.Plane) string {
+	switch plane {
+	case install.PlaneLAN:
+		return "10.0.0.3"
+	case install.PlaneVPC:
+		return "10.1.0.3"
+	default:
+		return "192.168.1.10"
+	}
+}
+
+func (f *roleForm) folded() bool { return f.nic == foldedNIC }
+
+// canFold reports whether this role may be left unbound. wan always needs an
+// uplink, so only lan and vpc can fold.
+func (f *roleForm) canFold() bool { return f.plane != install.PlaneWAN }
+
+// visibleFields returns the fields shown for this form, in tab order. Hiding
+// rather than disabling keeps the default screen narrow: a DHCP role shows two
+// lines, and VLAN/MTU only appear once Advanced is on.
+func (f *roleForm) visibleFields(isWiFi, advanced bool) []roleField {
+	if f.folded() {
+		return []roleField{roleFieldNIC}
+	}
+	fields := []roleField{roleFieldNIC, roleFieldMethod}
+	if !f.dhcp {
+		fields = append(fields, roleFieldIP, roleFieldMask)
+		// Default route and resolvers are properties of the node, not of each
+		// plane, and wan is the only one that carries either: it holds the
+		// default route, so it is the only plane that can reach an upstream
+		// resolver. On DHCP the lease supplies both.
+		if f.plane == install.PlaneWAN {
+			fields = append(fields, roleFieldGateway, roleFieldDNS)
+		}
+	}
+	if advanced {
+		fields = append(fields, roleFieldVLAN, roleFieldMTU)
+	}
+	if isWiFi {
+		fields = append(fields, roleFieldSSID, roleFieldWiFiPass)
+	}
+	return fields
+}
+
+// input returns the textinput backing a field, or nil for the non-text fields.
+func (f *roleForm) input(field roleField) *textinput.Model {
+	switch field {
+	case roleFieldIP:
+		return &f.ip
+	case roleFieldMask:
+		return &f.mask
+	case roleFieldGateway:
+		return &f.gateway
+	case roleFieldDNS:
+		return &f.dns
+	case roleFieldVLAN:
+		return &f.vlan
+	case roleFieldMTU:
+		return &f.mtu
+	case roleFieldSSID:
+		return &f.ssid
+	case roleFieldWiFiPass:
+		return &f.wifiPass
+	default:
+		return nil
+	}
+}
+
+// blurAll drops focus from every input so only the focused field has a cursor.
+func (f *roleForm) blurAll() {
+	for _, field := range []roleField{
+		roleFieldIP, roleFieldMask, roleFieldGateway, roleFieldDNS,
+		roleFieldVLAN, roleFieldMTU, roleFieldSSID, roleFieldWiFiPass,
+	} {
+		f.input(field).Blur()
+	}
+}
+
+// focusCurrent gives the cursor to the focused field, if it is a text field.
+func (f *roleForm) focusCurrent() {
+	f.blurAll()
+	if in := f.input(f.focus); in != nil {
+		in.Focus()
+	}
+}
+
+// toRole converts the form into the install package's role model.
+func (f *roleForm) toRole(nics []netprobe.NIC) install.NetworkRole {
+	if f.folded() || f.nic >= len(nics) {
+		return install.NetworkRole{}
+	}
+	role := install.NetworkRole{
+		Interface: nics[f.nic].Name,
+		DHCPMode:  f.dhcp,
+		VLAN:      atoiOr(f.vlan.Value(), 0),
+		MTU:       atoiOr(f.mtu.Value(), 0),
+	}
+	if !f.dhcp {
+		role.Address = strings.TrimSpace(f.ip.Value())
+		role.Mask = strings.TrimSpace(f.mask.Value())
+		role.DNS = parseDNS(f.dns.Value())
+		if f.plane == install.PlaneWAN {
+			role.Gateway = strings.TrimSpace(f.gateway.Value())
+		}
+	}
+	if nics[f.nic].IsWiFi {
+		role.WiFiSSID = strings.TrimSpace(f.ssid.Value())
+		role.WiFiPass = f.wifiPass.Value()
+	}
+	return role
+}
+
+func atoiOr(s string, fallback int) int {
+	n, err := strconv.Atoi(strings.TrimSpace(s))
+	if err != nil {
+		return fallback
+	}
+	return n
+}
+
+// ── Roles overview screen ─────────────────────────────────────────────────────
+
+// continueRow is the cursor position below the three role rows.
+const continueRow = 3
+
+func (m model) handleRolesKey(key string) (tea.Model, tea.Cmd) {
+	switch key {
+	case "up", "k":
+		if m.roleCursor > 0 {
+			m.roleCursor--
+		}
+	case "down", "j":
+		if m.roleCursor < continueRow {
+			m.roleCursor++
+		}
+	case "a":
+		// Advanced reveals VLAN and MTU on every role at once, rather than
+		// per-role, so the operator flips it on for the whole node.
+		m.advanced = !m.advanced
+		m.validationErr = ""
+	case "enter":
+		if m.roleCursor == continueRow {
+			if err := m.buildConfig().Validate(); err != nil {
+				m.validationErr = capitalise(err.Error())
+				return m, nil
+			}
+			m.validationErr = ""
+			m.screen = screenIdentity
+			m.hostnameInput.Focus()
+			return m, nil
+		}
+		m.validationErr = ""
+		m.screen = screenNetworkRole
+		m.roles[m.roleCursor].focus = roleFieldNIC
+		m.roles[m.roleCursor].blurAll()
+	case "esc":
+		m.screen = screenDiskConfirm
+		m.eraseInput.Focus()
+	}
+	return m, nil
+}
+
+func (m model) viewNetworkRoles(w int) string {
+	title := styleTitle.Render("Network Planes")
+	subtitle := styleMuted.Render("A role left unbound folds onto the plane above it.")
+
+	adv := styleMuted.Render("Advanced: off")
+	if m.advanced {
+		adv = styleSuccess.Render("Advanced: on")
+	}
+
+	header := fmt.Sprintf("  %-*s%-*s", colRole, "ROLE", colIface, "INTERFACE")
+	if m.advanced {
+		header += fmt.Sprintf("%-*s%-*s", colVLAN, "VLAN", colMTU, "MTU")
+	}
+	header += "ADDRESS"
+
+	lines := []string{title, subtitle, "", adv, "", styleMuted.Render(header)}
+
+	for i := range m.roles {
+		row := m.renderRoleRow(i)
+		if i == m.roleCursor {
+			lines = append(lines, styleSelected.Render("> "+row))
+		} else {
+			lines = append(lines, "  "+row)
+		}
+		if detail := m.roleHardware(i, bodyWidth(w)); detail != "" {
+			lines = append(lines, detail)
+		}
+	}
+
+	lines = append(lines, "")
+	cont := "  Continue"
+	if m.roleCursor == continueRow {
+		cont = styleSelected.Render("> Continue")
+	}
+	lines = append(lines, cont)
+
+	if m.validationErr != "" {
+		lines = append(lines, "", styleError.Render(m.validationErr))
+	}
+	lines = append(lines, styleHelp.Render("↑/↓ move • Enter edit • a advanced • Esc back"))
+
+	body := lipgloss.JoinVertical(lipgloss.Left, lines...)
+	return lipgloss.Place(w, m.height, lipgloss.Center, lipgloss.Center,
+		styleBox.Width(min(w-4, 78)).Render(body),
+	)
+}
+
+// renderRoleRow renders one line of the overview table. A folded role names
+// the plane it collapsed onto rather than showing a blank, so the operator can
+// always see where the traffic will actually go.
+func (m model) renderRoleRow(i int) string {
+	f := &m.roles[i]
+	name := fmt.Sprintf("%-5s", f.plane)
+
+	if f.folded() {
+		_, landed := m.buildConfig().Resolve(f.plane)
+		return fmt.Sprintf("%-*s%s", colRole, name, styleMuted.Render("folds onto "+string(landed)))
+	}
+
+	iface := styleMuted.Render("—")
+	if f.nic < len(m.nics) {
+		iface = nicSummary(m.nics[f.nic])
+	}
+
+	addr := "DHCP"
+	if !f.dhcp {
+		addr = formatCIDR(f.ip.Value(), f.mask.Value())
+		if addr == "" {
+			addr = styleWarning.Render("not set")
+		}
+	}
+
+	row := padCell(name, colRole) + padCell(iface, colIface)
+	if m.advanced {
+		row += padCell(valueOr(f.vlan.Value(), "—"), colVLAN) + padCell(valueOr(f.mtu.Value(), "—"), colMTU)
+	}
+	return row + addr
+}
+
+// roleHardware is the vendor/model line beneath a role row, indented under the
+// interface column. It is what tells an operator that wan really is the 25gbe
+// Mellanox and not the onboard 1gbe.
+func (m model) roleHardware(i, width int) string {
+	f := &m.roles[i]
+	if f.folded() || f.nic >= len(m.nics) {
+		return ""
+	}
+	indent := 2 + colRole
+	return strings.Repeat(" ", indent) + styleMuted.Render(nicHardware(m.nics[f.nic], width-indent))
+}
+
+// nicSummary renders name, speed and link state, colouring the state so a port
+// with no cable stands out rather than having to be read for.
+func nicSummary(n netprobe.NIC) string {
+	state := styleWarning.Render(n.State)
+	if n.Carrier {
+		state = styleSuccess.Render(n.State)
+	}
+	speed := n.Speed
+	if speed == "" {
+		speed = "—"
+	}
+	return fmt.Sprintf("%s  %s  %s", n.Name, speed, state)
+}
+
+// nicHardware is the detail line: what the card is, plus the predictable name
+// it also answers to, which is the one written on cabling notes. The alt name
+// is dropped rather than truncated when the two will not fit together.
+func nicHardware(n netprobe.NIC, width int) string {
+	hw := n.Hardware()
+	const separator = "  ·  "
+	if n.AltName != "" && len([]rune(hw))+len(separator)+len([]rune(n.AltName)) <= width {
+		return hw + separator + n.AltName
+	}
+	return truncate(hw, width)
+}
+
+// bodyWidth is the text width inside the framed body. Detail lines are trimmed
+// to it rather than left to wrap, which would push the table out of alignment.
+func bodyWidth(w int) int {
+	return min(w-4, 78) - 4
+}
+
+// padCell pads a cell to width by display width, which the %-*s verb cannot do
+// once a cell carries the ANSI escapes of a style.
+func padCell(s string, width int) string {
+	return lipgloss.PlaceHorizontal(width, lipgloss.Left, s)
+}
+
+// Column widths for the overview table, shared by the header and the rows.
+const (
+	colRole  = 6
+	colIface = 32
+	colVLAN  = 7
+	colMTU   = 7
+)
+
+// focusMark gives the focused text field a visible caret without every field
+// rendering its own prompt.
+func focusMark(focused bool) string {
+	if focused {
+		return styleLabel.Render("> ")
+	}
+	return "  "
+}
+
+// formatCIDR renders an address and mask the way an operator writes them,
+// falling back to the raw mask when it is not dotted-decimal.
+func formatCIDR(addr, mask string) string {
+	addr = strings.TrimSpace(addr)
+	mask = strings.TrimSpace(mask)
+	if addr == "" {
+		return ""
+	}
+	if mask == "" {
+		return addr
+	}
+	if ip := net.ParseIP(mask); ip != nil && ip.To4() != nil {
+		if ones, bits := net.IPMask(ip.To4()).Size(); bits != 0 {
+			return fmt.Sprintf("%s/%d", addr, ones)
+		}
+	}
+	return addr + "/" + strings.TrimPrefix(mask, "/")
+}
+
+func valueOr(v, fallback string) string {
+	if v = strings.TrimSpace(v); v != "" {
+		return v
+	}
+	return fallback
+}
+
+// ── Per-role editor screen ────────────────────────────────────────────────────
+
+func (m model) handleRoleKey(key string, msg tea.KeyMsg) (tea.Model, tea.Cmd) {
+	f := &m.roles[m.roleCursor]
+	fields := f.visibleFields(m.nicIsWiFi(f.nic), m.advanced)
+	pos := fieldIndex(fields, f.focus)
+
+	switch key {
+	case "esc":
+		f.blurAll()
+		m.screen = screenNetworkRoles
+		m.validationErr = ""
+		return m, nil
+
+	case "enter":
+		if err := m.validateRole(m.roleCursor); err != "" {
+			m.validationErr = err
+			return m, nil
+		}
+		f.blurAll()
+		m.screen = screenNetworkRoles
+		m.validationErr = ""
+		return m, nil
+
+	case "tab", "down":
+		pos = (pos + 1) % len(fields)
+		f.focus = fields[pos]
+		f.focusCurrent()
+		return m, nil
+
+	case "shift+tab", "up":
+		pos = (pos - 1 + len(fields)) % len(fields)
+		f.focus = fields[pos]
+		f.focusCurrent()
+		return m, nil
+
+	case "left", "right":
+		// Only the two selector fields consume arrows; text fields need them
+		// for cursor movement.
+		switch f.focus {
+		case roleFieldNIC:
+			return m.cycleNIC(m.roleCursor, key == "right"), nil
+		case roleFieldMethod:
+			f.dhcp = key == "left"
+			f.focusCurrent()
+			return m, nil
+		}
+	}
+
+	if in := f.input(f.focus); in != nil {
+		var cmd tea.Cmd
+		*in, cmd = in.Update(msg)
+		return m, cmd
+	}
+	return m, nil
+}
+
+// cycleNIC steps the interface selection, including the fold option for lan
+// and vpc. Wrapping keeps a single key usable on a node with one NIC.
+func (m model) cycleNIC(idx int, forward bool) model {
+	f := &m.roles[idx]
+	low := 0
+	if f.canFold() {
+		low = foldedNIC
+	}
+	high := len(m.nics) - 1
+
+	if forward {
+		if f.nic >= high {
+			f.nic = low
+		} else {
+			f.nic++
+		}
+		return m
+	}
+	if f.nic <= low {
+		f.nic = high
+	} else {
+		f.nic--
+	}
+	return m
+}
+
+func (m model) nicIsWiFi(idx int) bool {
+	return idx >= 0 && idx < len(m.nics) && m.nics[idx].IsWiFi
+}
+
+func (m model) viewNetworkRole(w int) string {
+	f := &m.roles[m.roleCursor]
+	title := styleTitle.Render(fmt.Sprintf("Configure %s plane", strings.ToUpper(string(f.plane))))
+	subtitle := styleMuted.Render(planeDescription(f.plane))
+
+	lines := []string{title, subtitle, ""}
+	for _, field := range f.visibleFields(m.nicIsWiFi(f.nic), m.advanced) {
+		lines = append(lines, m.renderRoleField(f, field, bodyWidth(w)), "")
+	}
+
+	if m.validationErr != "" {
+		lines = append(lines, styleError.Render(m.validationErr), "")
+	}
+	lines = append(lines, styleHelp.Render("Tab/↑↓ move • ←/→ change • Enter done • Esc cancel"))
+
+	body := lipgloss.JoinVertical(lipgloss.Left, lines...)
+	return lipgloss.Place(w, m.height, lipgloss.Center, lipgloss.Center,
+		styleBox.Width(min(w-4, 78)).Render(body),
+	)
+}
+
+// planeDescription explains what rides each plane, so the choice is not just a
+// three-letter name.
+func planeDescription(p install.Plane) string {
+	switch p {
+	case install.PlaneWAN:
+		return "Public ingress and egress, EIPs and NAT (br-wan)."
+	case install.PlaneLAN:
+		return "Storage replication, NATS mesh and OVN control (br-lan)."
+	default:
+		return "Geneve tunnels carrying EC2 traffic between nodes (br-vpc)."
+	}
+}
+
+// roleLabelWidth is the field-label column on the editor screen, shared so a
+// wrapped value can indent to line up underneath it.
+const roleLabelWidth = 14
+
+func (m model) renderRoleField(f *roleForm, field roleField, width int) string {
+	focused := f.focus == field
+	label := styleLabel.Render(fmt.Sprintf("%-*s", roleLabelWidth, roleFieldLabel(field)))
+
+	switch field {
+	case roleFieldNIC:
+		return label + m.renderNICChoice(f, focused, width)
+
+	case roleFieldMethod:
+		return label + renderToggle([]string{"DHCP (automatic)", "Static"}, boolToIdx(!f.dhcp), focused)
+
+	default:
+		if in := f.input(field); in != nil {
+			return label + focusMark(focused) + in.View()
+		}
+		return label
+	}
+}
+
+func roleFieldLabel(field roleField) string {
+	switch field {
+	case roleFieldNIC:
+		return "Interface"
+	case roleFieldMethod:
+		return "IP method"
+	case roleFieldIP:
+		return "IP address"
+	case roleFieldMask:
+		return "Subnet mask"
+	case roleFieldGateway:
+		return "Gateway"
+	case roleFieldDNS:
+		return "DNS"
+	case roleFieldVLAN:
+		return "VLAN id"
+	case roleFieldMTU:
+		return "MTU"
+	case roleFieldSSID:
+		return "WiFi SSID"
+	default:
+		return "WiFi password"
+	}
+}
+
+// renderNICChoice shows the selected interface on the field line and its
+// hardware identity on a line of its own beneath. Vendor and model are what
+// make a port identifiable, and they do not fit alongside the selector.
+func (m model) renderNICChoice(f *roleForm, focused bool, width int) string {
+	var text, detail string
+	switch {
+	case f.folded():
+		_, landed := m.buildConfig().Resolve(f.plane)
+		text = fmt.Sprintf("(fold onto %s)", landed)
+	case f.nic < len(m.nics):
+		text = nicSummary(m.nics[f.nic])
+		detail = nicHardware(m.nics[f.nic], width-roleLabelWidth-2)
+	default:
+		text = "(none detected)"
+	}
+
+	if focused {
+		text = styleSelected.Render(" ← " + text + " → ")
+	} else {
+		text = styleLabel.Render("[ " + text + " ]")
+	}
+	if detail == "" {
+		return text
+	}
+	return text + "\n" + strings.Repeat(" ", roleLabelWidth+2) + styleMuted.Render(detail)
+}
+
+func renderToggle(options []string, selected int, focused bool) string {
+	var parts []string
+	for i, opt := range options {
+		switch {
+		case i == selected && focused:
+			parts = append(parts, styleSelected.Render(" "+opt+" "))
+		case i == selected:
+			parts = append(parts, styleLabel.Render("["+opt+"]"))
+		default:
+			parts = append(parts, styleMuted.Render(opt))
+		}
+	}
+	return strings.Join(parts, "  ")
+}
+
+func boolToIdx(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+func fieldIndex(fields []roleField, want roleField) int {
+	for i, f := range fields {
+		if f == want {
+			return i
+		}
+	}
+	return 0
+}
+
+// validateRole checks one plane's entries. Cross-role rules (a shared NIC
+// without distinct VLAN ids, an unbound wan) belong to install.Config.Validate
+// and run when the operator continues.
+func (m model) validateRole(idx int) string {
+	f := &m.roles[idx]
+	if f.folded() {
+		return ""
+	}
+	if len(m.nics) == 0 {
+		return "No network interfaces detected"
+	}
+
+	if !f.dhcp {
+		if net.ParseIP(strings.TrimSpace(f.ip.Value())) == nil {
+			return "Enter a valid IP address"
+		}
+		if !validSubnetMask(f.mask.Value()) {
+			return "Enter a dotted-decimal subnet mask, e.g. 255.255.255.0 or 255.255.0.0"
+		}
+		if f.plane == install.PlaneWAN && net.ParseIP(strings.TrimSpace(f.gateway.Value())) == nil {
+			return "Enter a valid gateway address"
+		}
+	}
+
+	if v := strings.TrimSpace(f.vlan.Value()); v != "" {
+		id, err := strconv.Atoi(v)
+		if err != nil || id < 1 || id > 4094 {
+			return "VLAN id must be between 1 and 4094"
+		}
+	}
+	if v := strings.TrimSpace(f.mtu.Value()); v != "" {
+		mtu, err := strconv.Atoi(v)
+		if err != nil || mtu < 68 || mtu > 9216 {
+			return "MTU must be between 68 and 9216"
+		}
+	}
+	if m.nicIsWiFi(f.nic) && strings.TrimSpace(f.ssid.Value()) == "" {
+		return "Enter the WiFi network SSID"
+	}
+	return ""
+}
+
+func capitalise(s string) string {
+	if s == "" {
+		return s
+	}
+	return strings.ToUpper(s[:1]) + s[1:]
+}

cmd/installer/ui/network_test.go

@@ -0,0 +1,357 @@
+package ui
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+	"github.com/mulgadc/spinifex/cmd/installer/netprobe"
+)
+
+func nics(n int) []netprobe.NIC {
+	out := make([]netprobe.NIC, 0, n)
+	names := []string{"eno1", "ens1f0np0", "ens1f1np1", "ens2f0np0"}
+	for i := range n {
+		out = append(out, netprobe.NIC{Name: names[i], Speed: "25Gbps", Vendor: "Mellanox", State: "online"})
+	}
+	return out
+}
+
+// The pre-fill is the whole of the simple path: an operator with a
+// conventional node should be able to press Continue without editing anything.
+func TestNewModelPrefillsRolesByNICCount(t *testing.T) {
+	tests := []struct {
+		name             string
+		count            int
+		wantLAN, wantVPC int
+	}{
+		{"one NIC folds everything onto wan", 1, foldedNIC, foldedNIC},
+		{"two NICs dedicate the second to lan, vpc folds", 2, 1, foldedNIC},
+		{"three NICs give each plane its own", 3, 1, 2},
+		{"four NICs still only bind three", 4, 1, 2},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(tt.count))
+
+			if got := m.roles[0].nic; got != 0 {
+				t.Errorf("wan nic = %d, want 0", got)
+			}
+			if got := m.roles[1].nic; got != tt.wantLAN {
+				t.Errorf("lan nic = %d, want %d", got, tt.wantLAN)
+			}
+			if got := m.roles[2].nic; got != tt.wantVPC {
+				t.Errorf("vpc nic = %d, want %d", got, tt.wantVPC)
+			}
+		})
+	}
+}
+
+// Hiding rather than disabling is what keeps the default screen narrow.
+func TestVisibleFields(t *testing.T) {
+	tests := []struct {
+		name     string
+		mutate   func(*roleForm)
+		isWiFi   bool
+		advanced bool
+		want     []roleField
+	}{
+		{
+			name:   "wan on DHCP asks two questions",
+			mutate: func(f *roleForm) { f.dhcp = true },
+			want:   []roleField{roleFieldNIC, roleFieldMethod},
+		},
+		{
+			name:   "wan static adds a gateway",
+			mutate: func(f *roleForm) { f.dhcp = false },
+			want:   []roleField{roleFieldNIC, roleFieldMethod, roleFieldIP, roleFieldMask, roleFieldGateway, roleFieldDNS},
+		},
+		{
+			name:     "advanced reveals VLAN and MTU",
+			mutate:   func(f *roleForm) { f.dhcp = true },
+			advanced: true,
+			want:     []roleField{roleFieldNIC, roleFieldMethod, roleFieldVLAN, roleFieldMTU},
+		},
+		{
+			name:   "wireless adds credentials",
+			mutate: func(f *roleForm) { f.dhcp = true },
+			isWiFi: true,
+			want:   []roleField{roleFieldNIC, roleFieldMethod, roleFieldSSID, roleFieldWiFiPass},
+		},
+		{
+			name:   "a folded role has nothing to configure",
+			mutate: func(f *roleForm) { f.nic = foldedNIC },
+			want:   []roleField{roleFieldNIC},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newRoleForm(install.PlaneWAN, 0)
+			tt.mutate(&f)
+			got := f.visibleFields(tt.isWiFi, tt.advanced)
+
+			if len(got) != len(tt.want) {
+				t.Fatalf("visibleFields() = %v, want %v", got, tt.want)
+			}
+			for i := range got {
+				if got[i] != tt.want[i] {
+					t.Fatalf("visibleFields() = %v, want %v", got, tt.want)
+				}
+			}
+		})
+	}
+}
+
+// A non-wan role must be able to reach the folded state, and wan must not.
+func TestCycleNICFoldReachableOnlyOffWAN(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(2))
+
+	// lan: 1 -> 0 -> folded -> wraps to the last NIC.
+	m.roles[1].nic = 1
+	for _, want := range []int{0, foldedNIC, 1} {
+		m = m.cycleNIC(1, false)
+		if m.roles[1].nic != want {
+			t.Fatalf("lan nic = %d, want %d", m.roles[1].nic, want)
+		}
+	}
+
+	// wan cycles only through real interfaces.
+	m.roles[0].nic = 0
+	for range 4 {
+		m = m.cycleNIC(0, false)
+		if m.roles[0].nic == foldedNIC {
+			t.Fatal("wan reached the folded state, but it always needs an uplink")
+		}
+	}
+}
+
+func TestToRoleReadsVLANAndMTU(t *testing.T) {
+	f := newRoleForm(install.PlaneLAN, 0)
+	f.dhcp = false
+	f.ip.SetValue("10.0.0.2")
+	f.mask.SetValue("255.255.0.0")
+	f.vlan.SetValue("10")
+	f.mtu.SetValue("9000")
+
+	role := f.toRole(nics(1))
+
+	if role.VLAN != 10 || role.MTU != 9000 {
+		t.Errorf("VLAN/MTU = %d/%d, want 10/9000", role.VLAN, role.MTU)
+	}
+	if role.Link() != "eno1.10" {
+		t.Errorf("Link() = %q, want eno1.10", role.Link())
+	}
+	// Only wan carries a default route, even if a gateway was typed earlier.
+	f.gateway.SetValue("10.0.0.1")
+	if got := f.toRole(nics(1)).Gateway; got != "" {
+		t.Errorf("lan Gateway = %q, want empty", got)
+	}
+}
+
+func TestToRoleFoldedIsEmpty(t *testing.T) {
+	f := newRoleForm(install.PlaneVPC, foldedNIC)
+	if !f.toRole(nics(2)).Folded() {
+		t.Error("a folded form must produce a folded role")
+	}
+}
+
+func TestValidateRole(t *testing.T) {
+	tests := []struct {
+		name    string
+		plane   install.Plane
+		mutate  func(*roleForm)
+		wantErr string
+	}{
+		{"valid static lan", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("10.0.0.2")
+			f.mask.SetValue("255.255.0.0")
+		}, ""},
+		{"bad ip", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("not-an-ip")
+			f.mask.SetValue("255.255.0.0")
+		}, "valid IP"},
+		{"wan needs a gateway", install.PlaneWAN, func(f *roleForm) {
+			f.dhcp = false
+			f.ip.SetValue("216.218.163.99")
+			f.mask.SetValue("255.255.255.224")
+		}, "gateway"},
+		{"VLAN out of range", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = true
+			f.vlan.SetValue("9000")
+		}, "VLAN id must be"},
+		{"MTU out of range", install.PlaneLAN, func(f *roleForm) {
+			f.dhcp = true
+			f.mtu.SetValue("70000")
+		}, "MTU must be"},
+		{"folded roles skip validation", install.PlaneVPC, func(f *roleForm) {
+			f.nic = foldedNIC
+			f.ip.SetValue("nonsense")
+		}, ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(3))
+			m.roles[0] = newRoleForm(tt.plane, 0)
+			tt.mutate(&m.roles[0])
+
+			got := m.validateRole(0)
+			if tt.wantErr == "" {
+				if got != "" {
+					t.Fatalf("validateRole() = %q, want no error", got)
+				}
+				return
+			}
+			if !strings.Contains(got, tt.wantErr) {
+				t.Fatalf("validateRole() = %q, want it to mention %q", got, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Continue must refuse a layout install.Config.Validate would reject, rather
+// than failing later during the install itself.
+func TestContinueBlocksOnSharedNICWithoutVLANs(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(2))
+	m.screen = screenNetworkRoles
+	m.roleCursor = continueRow
+	m.roles[1].nic = 0 // lan onto the wan NIC, both untagged
+
+	next, _ := m.handleRolesKey("enter")
+	got := next.(model)
+
+	if got.screen != screenNetworkRoles {
+		t.Error("expected to stay on the roles screen when the layout is invalid")
+	}
+	if !strings.Contains(got.validationErr, "both bind eno1") {
+		t.Errorf("validationErr = %q, want it to name the shared interface", got.validationErr)
+	}
+
+	// Distinct VLAN ids make the same pairing legal.
+	got.roles[0].vlan.SetValue("30")
+	got.roles[1].vlan.SetValue("10")
+	next, _ = got.handleRolesKey("enter")
+	if next.(model).screen != screenIdentity {
+		t.Error("distinct VLAN ids should allow two roles to share one NIC")
+	}
+}
+
+func TestAdvancedToggle(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(3))
+	m.screen = screenNetworkRoles
+
+	next, _ := m.handleRolesKey("a")
+	if !next.(model).advanced {
+		t.Fatal("'a' should turn Advanced on")
+	}
+	next, _ = next.(model).handleRolesKey("a")
+	if next.(model).advanced {
+		t.Fatal("'a' should turn Advanced back off")
+	}
+}
+
+// The default route and the resolvers are node-level, and wan is the only plane
+// that carries either, so the east-west editors must not ask for them. Their
+// bridges are link-local and activate manually — a resolver pinned there is
+// unreachable, and a gateway would fight the real default route.
+func TestVisibleFieldsOmitsRoutingOffWAN(t *testing.T) {
+	for _, plane := range []install.Plane{install.PlaneLAN, install.PlaneVPC} {
+		t.Run(string(plane), func(t *testing.T) {
+			f := newRoleForm(plane, 1)
+			f.dhcp = false
+			got := f.visibleFields(false, false)
+
+			want := []roleField{roleFieldNIC, roleFieldMethod, roleFieldIP, roleFieldMask}
+			if len(got) != len(want) {
+				t.Fatalf("visibleFields() = %v, want %v", got, want)
+			}
+			for i := range got {
+				if got[i] != want[i] {
+					t.Fatalf("visibleFields() = %v, want %v", got, want)
+				}
+			}
+		})
+	}
+}
+
+// The mask field must accept only what netgen can consume. It previously
+// advertised and accepted a prefix length, which passed the TUI and then failed
+// the install itself with "invalid mask: 24".
+func TestValidSubnetMaskRejectsPrefixLength(t *testing.T) {
+	for _, bad := range []string{"24", "/24", "8", "", "255.255.0", "255.0.255.0", "::1"} {
+		if validSubnetMask(bad) {
+			t.Errorf("validSubnetMask(%q) = true, want false", bad)
+		}
+	}
+	for _, good := range []string{"255.255.255.0", "255.255.0.0", "255.0.0.0", "255.255.255.252"} {
+		if !validSubnetMask(good) {
+			t.Errorf("validSubnetMask(%q) = false, want true", good)
+		}
+	}
+}
+
+// The hardware detail is the point of the NIC table: an operator has to be able
+// to tell which physical port each plane landed on. It gets a line of its own,
+// under the interface column, rather than being squeezed into the row.
+func TestRoleHardwareRendersOnItsOwnLine(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, []netprobe.NIC{{
+		Name: "ens1f0np0", Vendor: "Mellanox Technologies", Model: "MT27710",
+		AltName: "enp132s0f0np0", Speed: "25 Gbps", Carrier: true, State: "online",
+	}})
+
+	view := m.viewNetworkRoles(100)
+	if !strings.Contains(view, "ens1f0np0  25 Gbps  online") {
+		t.Errorf("expected the identity line in:\n%s", view)
+	}
+	detail := m.roleHardware(0, bodyWidth(100))
+	if !strings.Contains(detail, "Mellanox Technologies MT27710") {
+		t.Errorf("roleHardware = %q, want the vendor and model", detail)
+	}
+	if !strings.Contains(detail, "enp132s0f0np0") {
+		t.Errorf("roleHardware = %q, want the alternative name", detail)
+	}
+	// Every line has to fit the box, or lipgloss wraps it and the columns shear.
+	for line := range strings.SplitSeq(view, "\n") {
+		if len([]rune(line)) > 100 {
+			t.Errorf("line exceeds the terminal width: %q", line)
+		}
+	}
+}
+
+// A narrow box drops the alternative name before it truncates the model, since
+// half a model name identifies nothing.
+func TestNICHardwareDropsAltNameBeforeTruncating(t *testing.T) {
+	n := netprobe.NIC{Vendor: "Broadcom Inc. and subsidiaries", Model: "NetXtreme BCM5720", AltName: "enp1s0f1"}
+
+	if got := nicHardware(n, 60); got != "Broadcom Inc. and subsidiaries NetXtreme BCM5720" {
+		t.Errorf("nicHardware(60) = %q, want the name without the alt name", got)
+	}
+	if got := nicHardware(n, 20); got != "Broadcom Inc. and s…" {
+		t.Errorf("nicHardware(20) = %q, want a marked truncation", got)
+	}
+}
+
+// A role bound to a port with no hardware database entry still gets something
+// to identify the card by, never the bare word "unknown".
+func TestRoleHardwareFallsBackToDriver(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, []netprobe.NIC{{
+		Name: "eno1", Driver: "bnxt_en", DeviceID: "14e4:165f", State: "no cable",
+	}})
+
+	if got := m.roleHardware(0, bodyWidth(100)); !strings.Contains(got, "bnxt_en [14e4:165f]") {
+		t.Errorf("roleHardware = %q, want the driver and device id", got)
+	}
+}
+
+// A folded role has no interface, so it must not render an orphan detail line
+// under the plane it collapsed onto.
+func TestRoleHardwareEmptyWhenFolded(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(1))
+	if got := m.roleHardware(1, bodyWidth(100)); got != "" {
+		t.Errorf("roleHardware for a folded role = %q, want empty", got)
+	}
+}

cmd/spinifex/cmd/admin.go

@@ -359,6 +359,8 @@ func init() {
 	adminJoinCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)")
 	adminJoinCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)")
 	adminJoinCmd.Flags().String("token", "", "Join token from the init node (required)")
+	adminJoinCmd.Flags().Bool("force", false, "Join even though this node is already initialized, discarding its own CA and master key")
+	adminJoinCmd.Flags().Duration("join-timeout", 20*time.Minute, "How long to keep retrying while the formation server is unreachable")
 	adminJoinCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all)")
 	adminJoinCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during join (default: enabled)")
 	adminJoinCmd.Flags().String("email", "", "Operator email address (used for update and security notifications)")
@@ -1945,6 +1947,54 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	fmt.Println()
 }
 
+// joinRetryInterval paces retries against an unreachable formation server. The
+// primary may still be booting, so this is measured in "how long until the other
+// machine finishes starting", not in fractions of a second.
+const joinRetryInterval = 5 * time.Second
+
+// joinRetryable separates a formation server that is not up yet from one that
+// has answered and said no. Bare-metal nodes cannot be sequenced reliably, so a
+// joiner racing ahead of the primary must wait rather than fail. A rejected
+// token or a duplicate node name answers the same way on every attempt, so
+// those are reported at once instead of after the whole timeout.
+func joinRetryable(err error, statusCode int) bool {
+	if err != nil {
+		// Connection refused, DNS failure, TLS handshake, timeout: the primary
+		// is not listening yet.
+		return true
+	}
+	return statusCode >= 500
+}
+
+// checkJoinPreconditions rejects a join that would silently destroy this node's
+// existing cluster identity. Joining adopts the primary's CA and master key,
+// overwriting whatever is here: correct on a freshly installed node, and on one
+// that has been in service it orphans every fragment and volume sealed under the
+// old key. runAdminInit guards the same way before re-initializing.
+func checkJoinPreconditions(configDir string, force bool) error {
+	if force {
+		return nil
+	}
+	tomlPath := filepath.Join(configDir, "spinifex.toml")
+	if !admin.FileExists(tomlPath) {
+		return nil
+	}
+	return fmt.Errorf("this node is already initialized: %s", tomlPath)
+}
+
+// joinDiscardsIdentityMsg spells out what a forced join throws away. Kept out of
+// the error string so that stays short enough to wrap in another.
+const joinDiscardsIdentityMsg = `Joining will discard this node's own cluster identity:
+  - CA certificate and key
+  - master key, and any data sealed under it
+  - viperblock key, and any volumes encrypted under it
+
+That is safe on a freshly installed node — an ISO install initializes a
+single-node cluster at first boot, and nothing has been sealed under these keys
+yet. On a node that has been in service it is unrecoverable data loss.
+
+To proceed: spx admin join --force ...`
+
 func runAdminJoin(cmd *cobra.Command, args []string) {
 	if os.Getuid() != 0 {
 		fmt.Fprintln(os.Stderr, "⚠️  Warning: 'spx admin join' is not running as root.")
@@ -1965,6 +2015,8 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	clusterBind, _ := cmd.Flags().GetString("cluster-bind")
 	services, _ := cmd.Flags().GetStringSlice("services")
 	compactionInterval, _ := cmd.Flags().GetInt("predastore-compaction-interval")
+	force, _ := cmd.Flags().GetBool("force")
+	joinTimeout, _ := cmd.Flags().GetDuration("join-timeout")
 
 	email, _ := cmd.Flags().GetString("email")
 	email = strings.TrimSpace(email)
@@ -2013,6 +2065,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	// Checked before any network call so a node that will not join says so
+	// immediately. Unlike init this exits non-zero: a node that did not join
+	// must not look like success to a provisioning script.
+	if err := checkJoinPreconditions(configDir, force); err != nil {
+		fmt.Fprintf(os.Stderr, "⚠️  %v\n\n%s\n", err, joinDiscardsIdentityMsg)
+		os.Exit(1)
+	}
+
 	// Default cluster-bind to bind IP if not specified
 	if clusterBind == "" {
 		clusterBind = bindIP
@@ -2059,19 +2119,47 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}
 
 	joinURL := fmt.Sprintf("https://%s/formation/join", leaderHost)
-	req, err := http.NewRequest(http.MethodPost, joinURL, bytes.NewBuffer(reqBody))
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error creating join request: %v\n", err)
-		os.Exit(1)
-	}
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Authorization", "Bearer "+joinToken)
+	deadline := time.Now().Add(joinTimeout)
 
-	resp, err := client.Do(req)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error connecting to formation server: %v\n", err)
-		fmt.Fprintf(os.Stderr, "Make sure the leader node has run 'spx admin init' and is accessible at %s\n", leaderHost)
-		os.Exit(1)
+	// A fresh body per attempt: bytes.Buffer is consumed by the first send.
+	var resp *http.Response
+	for attempt := 1; ; attempt++ {
+		req, reqErr := http.NewRequest(http.MethodPost, joinURL, bytes.NewBuffer(reqBody))
+		if reqErr != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error creating join request: %v\n", reqErr)
+			os.Exit(1)
+		}
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Authorization", "Bearer "+joinToken)
+
+		var doErr error
+		resp, doErr = client.Do(req)
+		statusCode := 0
+		if doErr == nil {
+			statusCode = resp.StatusCode
+		}
+		if !joinRetryable(doErr, statusCode) {
+			break
+		}
+		if doErr == nil {
+			resp.Body.Close()
+		}
+
+		if time.Now().After(deadline) {
+			if doErr != nil {
+				fmt.Fprintf(os.Stderr, "❌ Error connecting to formation server: %v\n", doErr)
+			} else {
+				fmt.Fprintf(os.Stderr, "❌ Formation server returned status %d\n", statusCode)
+			}
+			fmt.Fprintf(os.Stderr, "Gave up after %s (%d attempts).\n", joinTimeout, attempt)
+			fmt.Fprintf(os.Stderr, "Make sure the leader node has run 'spx admin init' and is accessible at %s\n", leaderHost)
+			os.Exit(1)
+		}
+		if attempt == 1 {
+			fmt.Printf("⏳ Formation server at %s not ready, retrying every %s (up to %s)...\n",
+				leaderHost, joinRetryInterval, joinTimeout)
+		}
+		time.Sleep(joinRetryInterval)
 	}
 
 	body, err := io.ReadAll(resp.Body)

cmd/spinifex/cmd/admin_test.go

@@ -148,6 +148,48 @@ func TestSpinifexTomlTemplate_AdvertiseField(t *testing.T) {
 	assert.Contains(t, content, `advertise = "192.168.1.21"`, "off-host dial target")
 }
 
+// The gateway is the public AWS-API surface, so its listener must stay on the
+// wildcard even when --bind names a single internal plane; pinning it to BindIP
+// makes a multi-NIC node unreachable from the wan side. The internal services
+// around it still follow BindIP.
+func TestSpinifexTomlTemplate_GatewayListensOnAllPlanes(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:        "hydrogen",
+		Az:          "us-west-1a",
+		Port:        "4432",
+		Region:      "us-west-1",
+		BindIP:      "10.0.0.3",
+		AdvertiseIP: "216.218.163.99",
+		AccessKey:   "AKIATEST",
+		SecretKey:   "SECRET",
+		AccountID:   "123456789012",
+		NatsToken:   "token",
+		ConfigDir:   dir,
+		OVNNBAddr:   "tcp:10.0.0.3:6641",
+		OVNSBAddr:   "tcp:10.0.0.3:6642",
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `host = "0.0.0.0:9999"`, "gateway must answer on every plane")
+	assert.NotContains(t, content, `host = "10.0.0.3:9999"`, "gateway must not be pinned to the internal plane")
+	assert.Contains(t, content, `host = "10.0.0.3:4222"`, "nats stays internal")
+
+	// predastore-start.sh scrapes this into SPINIFEX_PREDASTORE_HOST, which
+	// outranks predastore.toml, so pinning it here makes S3 unreachable from the
+	// wan plane and breaks the UI, which proxies to localhost:8443.
+	assert.Contains(t, content, `host = "0.0.0.0:8443"`, "S3 must answer on every plane")
+	assert.NotContains(t, content, `host = "10.0.0.3:8443"`, "S3 must not be pinned to the internal plane")
+
+	// The daemon is cluster-internal and peers dial it by advertise address,
+	// so it stays on the bind address rather than following the public ones.
+	assert.Contains(t, content, `host = "10.0.0.3:4432"`, "daemon stays internal")
+}
+
 // Empty AdvertiseIP (e.g. loading an existing cluster pre-siv-8) must NOT
 // render an empty advertise = "" line — downstream fallback to Host kicks in.
 func TestSpinifexTomlTemplate_AdvertiseOmittedWhenEmpty(t *testing.T) {
@@ -1249,3 +1291,77 @@ func TestAMIVolumeSizeGiB(t *testing.T) {
 		})
 	}
 }
+
+func TestCheckJoinPreconditions(t *testing.T) {
+	writeToml := func(t *testing.T) string {
+		t.Helper()
+		dir := t.TempDir()
+		if err := os.WriteFile(filepath.Join(dir, "spinifex.toml"), []byte("# cluster\n"), 0o644); err != nil {
+			t.Fatalf("seed config: %v", err)
+		}
+		return dir
+	}
+
+	t.Run("uninitialized node joins", func(t *testing.T) {
+		if err := checkJoinPreconditions(t.TempDir(), false); err != nil {
+			t.Fatalf("want nil, got %v", err)
+		}
+	})
+
+	t.Run("initialized node is refused", func(t *testing.T) {
+		err := checkJoinPreconditions(writeToml(t), false)
+		if err == nil {
+			t.Fatal("want refusal, got nil")
+		}
+		if !strings.Contains(err.Error(), "already initialized") {
+			t.Errorf("unexpected message: %v", err)
+		}
+	})
+
+	// The guidance is the whole point of the guard: an operator must be able to
+	// tell what a forced join would destroy without reading the source.
+	t.Run("guidance names what is discarded", func(t *testing.T) {
+		for _, want := range []string{"CA certificate", "master key", "viperblock key", "--force"} {
+			if !strings.Contains(joinDiscardsIdentityMsg, want) {
+				t.Errorf("guidance missing %q", want)
+			}
+		}
+	})
+
+	t.Run("force overrides", func(t *testing.T) {
+		if err := checkJoinPreconditions(writeToml(t), true); err != nil {
+			t.Fatalf("want nil with force, got %v", err)
+		}
+	})
+}
+
+func TestJoinRetryable(t *testing.T) {
+	tests := []struct {
+		name   string
+		err    error
+		status int
+		want   bool
+	}{
+		// The primary may still be booting — bare-metal nodes cannot be
+		// sequenced, so a joiner that arrives first has to wait.
+		{"connection refused", errors.New("connection refused"), 0, true},
+		{"tls handshake failure", errors.New("tls: handshake failure"), 0, true},
+		{"timeout", errors.New("context deadline exceeded"), 0, true},
+		{"server error", nil, 500, true},
+		{"bad gateway", nil, 502, true},
+
+		// Answered and said no: retrying changes nothing, and a mistyped token
+		// must not hang a console for the whole timeout.
+		{"bad token", nil, 401, false},
+		{"duplicate node name", nil, 409, false},
+		{"bad request", nil, 400, false},
+		{"success", nil, 200, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := joinRetryable(tt.err, tt.status); got != tt.want {
+				t.Errorf("joinRetryable(%v, %d) = %v, want %v", tt.err, tt.status, got, tt.want)
+			}
+		})
+	}
+}

cmd/spinifex/cmd/templates/spinifex.toml

@@ -89,7 +89,11 @@ gpu_passthrough = true
 {{- end}}
 
 [nodes.{{.Node}}.awsgw]
-host = "{{.BindIP}}:9999"
+# Wildcard rather than the bind IP: the gateway is the public AWS-API surface and
+# has to answer on the wan plane as well as internally, while --bind names only
+# the internal plane. TLS + SigV4 gate it, and the ECR registry host falls back
+# to the advertise address when the listener is unspecified.
+host = "0.0.0.0:9999"
 tlskey = "config/server.key"
 tlscert = "config/server.pem"
 debug = false
@@ -122,7 +126,16 @@ bridge_mode = "{{.BridgeMode}}"
 {{- end}}
 
 [nodes.{{.Node}}.predastore]
-host = "{{.BindIP}}:8443"
+# Wildcard rather than the bind IP: S3 is a public service and has to answer on
+# the wan plane, and predastore-start.sh scrapes this line into
+# SPINIFEX_PREDASTORE_HOST, which outranks the host set in predastore.toml. Only
+# the S3 API moves — the [[db]] and [[nodes]] peer addresses in
+# predastore-multinode.toml stay on the bind address.
+#
+# This doubles as the local dial target, and 0.0.0.0 is not one: config loading
+# rewrites it to 127.0.0.1 for this node, and admin.DialTarget does the same for
+# callers that read it directly.
+host = "0.0.0.0:8443"
 region = "{{.Region}}"
 bucket = "predastore"
 accesskey = "{{.AccessKey}}"

docs/install/install-multi-node/README.md

@@ -6,7 +6,11 @@ tags:
   - install
   - multi node
   - cluster
+  - iso
+  - bare-metal
 resources:
+  - title: "Bootable USB Install"
+    url: "/docs/install-usb"
   - title: "Spinifex Repository"
     url: "https://github.com/mulgadc/spinifex"
   - title: "Predastore (S3)"
@@ -24,6 +28,7 @@ resources:
 - [Overview](#overview)
 - [Prerequisites](#prerequisites)
 - [Instructions](#instructions)
+- [Converting ISO-Installed Nodes](#converting-iso-installed-nodes)
 - [Troubleshooting](#troubleshooting)
 
 ---
@@ -32,11 +37,22 @@ resources:
 
 A Spinifex cluster distributes services across multiple servers for high availability, data durability, and fault tolerance. Cluster formation is automatic — the init node waits for peers to join, then distributes credentials, CA certificates, and configuration.
 
+**Installing on bare metal?** Step 1 below can be done either with the binary installer or by booting each server from the Spinifex ISO — see [Bootable USB Install](/docs/install-usb). The ISO installs the operating system, disks and network configuration as well as Spinifex itself, which makes it the better option for servers with no existing OS. Either way, install every server first, then return here and continue from Step 2.
+
+**Cluster Sizing:**
+
+**Three servers is the minimum recommended for a multi-server deployment.** At three or more, the OVN control-plane databases run clustered across three nodes and the cluster tolerates the loss of any one of them. Below three, OVN runs standalone on the first node, which becomes a single point of failure for control-plane operations.
+
+A standalone OVN outage is less severe than it sounds. `ovn-controller` has already programmed the forwarding rules into each host, so **running instances keep full networking** — east-west, north-south, NAT and security groups all continue. What stops is change: creating VPCs, launching instances, and updating security groups all require the control plane.
+
+Servers beyond the third join as compute nodes and do not run a database, which keeps write latency stable as the cluster grows.
+
 **Network Requirements:**
 
 - Minimum 1 NIC per server (2 recommended for production)
 - UDP port 6081 open between hosts (Geneve tunnels)
 - TCP ports 4222, 4248, 6641, 6642 open between hosts (NATS, OVN)
+- TCP ports 6643, 6644 open between database servers (OVN database clustering)
 
 ## Prerequisites
 
@@ -58,10 +74,33 @@ A Spinifex cluster distributes services across multiple servers for high availab
 
 ## Step 1. Install Spinifex on Each Server
 
+Choose one of the two methods below and apply it to **every** server in the cluster.
+
+**Option A — existing OS.** On a server already running Ubuntu 26.04 or Debian 13:
+
 ```bash
 curl -fsSL https://install.mulgadc.com | bash
 ```
 
+**Option B — bare metal, from the ISO.** Boot each server from the Spinifex ISO and follow [Bootable USB Install](/docs/install-usb). This installs the operating system, partitions the disks, and configures the hostname and network interfaces alongside Spinifex. The ISO installer does not form a cluster — that is what the remaining steps do.
+
+Complete this step on all servers before continuing. The nodes must be installed and reachable from one another before the cluster is formed, because Step 4 requires every node to be available at the same time.
+
+### Shortcut: form the cluster with one command
+
+Steps 2 to 6 are a fixed sequence, and `scripts/install-node.sh` in the Spinifex repository runs it for you over SSH — from a workstation that can reach every server, not from the servers themselves:
+
+```bash
+scripts/install-node.sh \
+  --external-pool 10.0.1.100-10.0.1.150 \
+  --external-gateway 10.0.1.1 --external-prefix-len 24 \
+  server1 server2 server3
+```
+
+The first host initializes and the rest join. It resolves each node's plane addresses, rebuilds the OVN databases in clustered form, forms the cluster, restarts services and verifies the result — and `--dry-run` prints every command it would run without touching anything. It needs passwordless `sudo` on each host and requires all of them to be on the same Spinifex version.
+
+The remaining steps document what it does, and are the path to follow when installing by hand or when something needs to be adjusted mid-way.
+
 ## Step 2. Set Node IP Variables
 
 On **each server**, export the management IPs for all nodes:
@@ -74,15 +113,38 @@ export AWS_REGION=us-east-1
 export AWS_AZ=us-east-1a
 ```
 
+## Converting ISO-Installed Nodes
+
+Skip this section if you installed with the binary installer (Option A).
+
+The ISO installs each server as a **running standalone single-node cluster** — it initializes Spinifex, starts a standalone OVN database, and brings up `spinifex.target` at first boot. That is the right behaviour for a single server, and it means forming a cluster is a conversion rather than a fresh setup. Two extra things are needed:
+
+**Before Step 3**, stop services on every server:
+
+```bash
+sudo systemctl stop spinifex.target
+```
+
+**In Step 4**, pass `--force` to `spx admin init` and `spx admin join`. Servers 2 and 3 each arrived with their own CA and master key from the single-node install, and joining replaces them with server 1's. `--force` is how you confirm that.
+
+That discard is safe here — nothing has been sealed under those keys on a freshly installed node — but it is unrecoverable on a server that has been in service. `spx admin join` refuses without `--force` for exactly this reason.
+
 ## Step 3. Setup OVN Networking
 
-Server 1 runs OVN central and must be set up first. If your WAN interface is already a bridge, setup-ovn.sh auto-detects it. Otherwise use `--wan-bridge=br-wan --wan-iface=eth1` (dedicated WAN NIC).
+All three servers run a clustered OVN database, so the control plane survives the loss of any one of them. Server 1 creates the cluster and must be set up first; servers 2 and 3 join it.
 
-**Server 1:**
+If your WAN interface is already a bridge, setup-ovn.sh auto-detects it. Otherwise use `--wan-bridge=br-wan --wan-iface=eth1` (dedicated WAN NIC).
+
+> [!IMPORTANT]
+> **`--recreate-db` destroys all logical network state** — every logical switch, router, port and ACL, and so every VPC on that node. It is required here because the `ovn-central` package starts a standalone database on install, and a clustered database can only be created from scratch. That is safe on a freshly installed server and is **not** safe on one already running workloads. If you are converting a server that has been in service, back up its NB database first.
+
+**Server 1 — create the cluster:**
 
 ```bash
 sudo /usr/local/share/spinifex/setup-ovn.sh \
   --management \
+  --db-cluster-local-addr=$SPINIFEX_NODE1 \
+  --recreate-db \
   --encap-ip=$SPINIFEX_NODE1
 ```
 
@@ -90,7 +152,10 @@ sudo /usr/local/share/spinifex/setup-ovn.sh \
 
 ```bash
 sudo /usr/local/share/spinifex/setup-ovn.sh \
-  --ovn-remote=tcp:$SPINIFEX_NODE1:6642 \
+  --management \
+  --db-cluster-local-addr=$SPINIFEX_NODE2 \
+  --db-cluster-remote-addr=$SPINIFEX_NODE1 \
+  --recreate-db \
   --encap-ip=$SPINIFEX_NODE2
 ```
 
@@ -98,20 +163,36 @@ sudo /usr/local/share/spinifex/setup-ovn.sh \
 
 ```bash
 sudo /usr/local/share/spinifex/setup-ovn.sh \
-  --ovn-remote=tcp:$SPINIFEX_NODE1:6642 \
+  --management \
+  --db-cluster-local-addr=$SPINIFEX_NODE3 \
+  --db-cluster-remote-addr=$SPINIFEX_NODE1 \
+  --recreate-db \
   --encap-ip=$SPINIFEX_NODE3
 ```
 
-Verify all 3 chassis registered:
+**Servers 4 and beyond** are compute nodes and run no database. Point them at all three database servers so they survive any one of them failing:
 
 ```bash
+sudo /usr/local/share/spinifex/setup-ovn.sh \
+  --ovn-remote=tcp:$SPINIFEX_NODE1:6642,tcp:$SPINIFEX_NODE2:6642,tcp:$SPINIFEX_NODE3:6642 \
+  --encap-ip=$SPINIFEX_NODE4
+```
+
+Verify the database cluster formed, then that all chassis registered:
+
+```bash
+sudo ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
 sudo ovn-sbctl show
 ```
 
+`cluster/status` should list three servers with one leader. If it reports a standalone database, the cluster did not form — re-check that `--db-cluster-local-addr` was passed on every database server.
+
 ## Step 4. Form the Cluster
 
 Run init and join concurrently — init blocks until all nodes join.
 
+If you installed from the ISO, add `--force` to every command below (see [Converting ISO-Installed Nodes](#converting-iso-installed-nodes)).
+
 **Server 1 — Initialize:**
 
 ```bash
@@ -185,6 +266,20 @@ The init command must still be running when join executes. If init exited, re-ru
 curl -sk https://$SPINIFEX_NODE1:4432/health
 ```
 
+### Join Refuses: "this node is already initialized"
+
+The node has its own cluster configuration — normal for anything installed from the ISO, which initializes a single-node cluster at first boot. Joining replaces that node's CA and master key with the primary's, so it must be confirmed with `--force`.
+
+Safe on a freshly installed node. On one that has been in service it orphans every volume and fragment sealed under the old key, so check before forcing.
+
+### OVN Database Cluster Not Forming
+
+```bash
+sudo ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
+```
+
+If this reports a standalone database rather than three servers, the database was created before the cluster flags were supplied. A clustered database can only be created from scratch — re-run Step 3 with `--recreate-db`, remembering that it discards all logical network state.
+
 ### OVN Chassis Not Registering
 
 ```bash

docs/install/install-usb/README.md

@@ -13,6 +13,8 @@ tags:
 resources:
   - title: "Spinifex ISO"
     url: "https://iso.mulgadc.com/spinifex.iso"
+  - title: "Multi-Node Install"
+    url: "/docs/install-multi-node"
   - title: "Balena Etcher"
     url: "https://etcher.balena.io"
   - title: "Spinifex Repository"
@@ -85,7 +87,9 @@ For network configuration, it is recommended to use automatic IP (DHCP), but thi
 
 Network interfaces will be automatically detected. In the event that none are detected, the user can manually input the name of the network interface. In the event that multiple network interfaces are detected, the installer will prompt for WAN selection first, followed by LAN.
 
-A hostname (eg `node1`) and admin password must be set, followed by the "cluster role" — whether the node is the first in a new cluster, or joining an already existing one.
+A hostname (eg `node1`) and admin password must be set.
+
+The installer does not ask about clustering. It installs and configures a standalone node — operating system, disks, hostname, network interfaces and plane addressing — and leaves Spinifex service configuration to a post-install step. This applies equally to single-node and multi-node deployments; see [Configure Spinifex](#configure-spinifex) below.
 
 Once configuration is complete, a summary of the configuration will be shown.
 
@@ -97,20 +101,38 @@ Once the USB drive is removed, press enter or wait for the auto-reboot.
 
 ### Log In
 
-The device will reboot and spend ~2-5 minutes configuring the Spinifex node. The user will then be prompted to log in. Use the following credentials for login:
+The device will reboot and briefly finalise the install — setting the hostname and bringing up the configured network interfaces — then prompt for login. Use the following credentials:
 
 - Login: `spinifex`
 - Password: Set by user during installation
 
-Both before and after login, a banner will be printed specifying important information, such as details for SSH into the node and accessing the web dashboard for further configuration of the node.
+Both before and after login, a banner will be printed specifying important information, such as details for SSH into the node and the commands needed to configure Spinifex. The web dashboard becomes available once the node has been configured and `spinifex.target` has been started.
 
 <img src="../../../.github/assets/images/banner1.png" alt="banner">
 
+### Configure Spinifex
+
+**Spinifex is now installed on this node, but not yet configured.**
+
+The ISO performs the same job as **Step 1** of the install guides — it puts Spinifex and its dependencies on the machine. The remaining steps configure OVN networking and form the cluster, and they run after installation because cluster membership determines how OVN's clustered database is brought up. That set cannot be known while the nodes are still being installed.
+
+Continue from **Step 2** of whichever guide applies:
+
+**Single node** — follow [Single-Node Install](/docs/install) from Step 2. In brief:
+
+```bash
+sudo /usr/local/share/spinifex/setup-ovn.sh --management
+sudo spx admin init --node node1 --nodes 1
+sudo systemctl start spinifex.target
+```
+
+**Multi-node cluster** — install Spinifex from the ISO on **every** server first, following this guide on each one. Once all servers are installed and reachable, follow [Multi-Node Install](/docs/install-multi-node) from Step 2. Do not configure any node until all of them are installed: the first node's `spx admin init` waits for the others to join, and the join must happen while it is waiting.
+
 ### Setup Complete
 
 **Congratulations! Spinifex is installed.**
 
-Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create a VPC, and launch your first instance.
+Once configured and started, continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create a VPC, and launch your first instance.
 
 ## Troubleshooting
 

scripts/README.md

@@ -0,0 +1,100 @@
+# Spinifex Scripts
+
+Node installation, dev-environment lifecycle, guest image builds, and verification
+tooling for Spinifex.
+
+Repo-level tooling (releases, CI infrastructure, ISO builds, beads) lives in
+[`mulga/scripts/`](../../scripts/README.md).
+
+## Install and node setup
+
+| Script | What it does |
+|---|---|
+| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, files, systemd units, logrotate and udev rules. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
+| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. `--management` also starts the OVN central services. |
+| `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
+| `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |
+| `clone-deps.sh` | Clones or updates the viperblock and predastore checkouts alongside spinifex for cross-repo development. |
+
+## Dev environment
+
+| Script | What it does |
+|---|---|
+| `reset-dev-env.sh` | Single-node reset: stops services, wipes `/etc/`, `/var/lib/`, `/var/log/` and `/run/spinifex`, rebuilds, reinstalls via `dev-install.sh`, and relaunches a smoke-test instance. Preserves the existing node/network topology from `spinifex.toml`. |
+| `dev-env/*.conf` | Topology profiles for `reset-dev-env.sh --profile=NAME`, declaring which bridge carries the wan, lan and vpc planes: `single-nic` (all collapsed onto wan), `two-nic` (vpc folds onto lan), `three-plane` (a NIC each). |
+
+## Verification and benchmarks
+
+| Script | What it does |
+|---|---|
+| `smoke-test.sh` | End-to-end sanity check on a running node: imports an SSH keypair and the Ubuntu AMI, then launches an instance. `--create-vpc` builds the VPC, gateway, subnet and routes first and tears them down after; `--nodes N` launches through a spread placement group so each instance lands on a different physical server. `TEST_GPU=1` adds GPU passthrough checks. |
+| `verify-network.sh` | Asserts a node's datapath matches its declared planes — EIP reachability, which plane carries Geneve, and which interfaces services bind. Catches what `smoke-test.sh` cannot, since none of its traffic leaves the box. |
+| `test-ddil-phase1.sh` | systemd-level smoke test for daemon-local-autonomy tier 1. Destructive on the local node: stops and starts `spinifex-nats` and restarts `spinifex-daemon`, restoring both via a cleanup trap. |
+| `check-coverage.sh` | Fails the build if total Go coverage falls below the minimum threshold, excluding packages exempt from the requirement. |
+| `diff-coverage.sh` | Coverage check scoped to changed lines only, against an auto-detected base ref (`HEAD~1` on main, `origin/main` on dev, otherwise `origin/dev`). |
+| `run-bench.sh` | Launches a benchmark against running Ubuntu instances, auto-detecting the AMI for the host architecture. Tracks nbdkit and perf alongside the run. |
+| `disk-performance.sh` | Runs an in-guest `fio` random 70/30 read-write benchmark. Installs fio and sysstat first. |
+| `workload-performance.sh` | In-guest memory, disk, CPU and network workload test. Installs Go and runs the Badger test suite with jemalloc disabled and reduced parallelism. |
+
+## Guest images
+
+| Script | What it does |
+|---|---|
+| `build-system-image.sh` | Builds a minimal system AMI (Alpine or Ubuntu) from a manifest. All customisation happens inside the libguestfs appliance, so the build touches no host block device, no host mount, and needs no sudo. `--import` also registers it as an AMI. |
+| `publish-system-image.sh` | Converts a built raw image to compressed qcow2, writes the `.sha256` sidecar the catalog verifier expects, and uploads both to the R2 bucket behind `iso.mulgadc.com/system-ami/`. |
+| `build-microvm-image.sh` | Builds the minimal Alpine kernel + initramfs used by the QEMU microvm machine type and the lb-agent, carrying haproxy (ALB) and nginx-stream (NLB) data planes. |
+| `mkosi-build.sh` | Runs an mkosi system-image build inside the pinned builder container, so a runner needs only Docker and the toolchain cannot drift. `--image` names an output and expands to the right profile list. |
+| `mkosi-publish.sh` | Publishes an mkosi-built image to R2 from inside the same pinned container — qcow2 conversion, `.sha256` sidecar, and upload. |
+| `mkosi-common.sh` | Shared plumbing sourced by both mkosi scripts. Builds the toolchain image once, with the builder account matched to the invoking user so bind-mounted output is not written back as root. |
+| `mkosi-builder.Dockerfile` | The pinned image-build toolchain. Debian-hosted regardless of target distro, since mkosi bootstraps the target from its own archive. |
+
+### `images/` — per-image build inputs
+
+Each subdirectory holds a `manifest.conf`, init scripts, and a `setup.sh` that
+`build-system-image.sh` runs inside the libguestfs appliance after packages and files are
+placed. `*_test.sh` files are shell unit tests for the helper of the same name.
+
+| Path | What it builds |
+|---|---|
+| `ecs-agent/` | The Alpine `spinifex-ecs-node` container-instance AMI (containerd + ecs-agent). `setup.sh` sets OpenRC exec bits, wires the CNI bin path, and applies the serial-console tweaks. |
+| `ecs-agent-gpu/` | The Ubuntu ECS GPU container-instance AMI. Same CNI and containerd wiring adjusted for Ubuntu's package layout, plus the headless NVIDIA driver and nvidia-container-toolkit. |
+| `eks-node/` | The unified Alpine EKS node AMI (K3s server *or* agent, selected at first boot). `setup.sh` installs the pinned, SHA-verified K3s binary and the role init scripts. |
+| `eks-node-gpu/` | The Ubuntu EKS GPU worker AMI. Agent role only — GPU nodes never run the control plane — plus the NVIDIA GPU stack. |
+| `rds-postgres/` | The `spinifex-rds-postgres` AMI. `setup.sh` sets exec bits on the OpenRC services and creates the root-only directory cloud-init drops the agent env file and gateway CA into. |
+| `ubuntu-gpu-{amd,nvidia}-setup.sh` | Chroot setup for GPU-capable Ubuntu guest images: ROCm CLI tools and AMD firmware, or the DKMS-built headless NVIDIA driver. Guest-side only — host VFIO setup is `spx admin gpu setup`. |
+
+### `images/eks-node/` — in-guest helpers
+
+Installed into the EKS node AMI and run at boot or on a timer. All host communication
+goes through the AWS gateway over SigV4; the VM never speaks NATS directly.
+
+| Script | What it does |
+|---|---|
+| `eks-node-role.sh` | First-boot role selector. Reads `SPINIFEX_K3S_ROLE` from the cloud-init env file and enables the matching services — nothing is enabled at bake time. |
+| `k3s-prestart.sh` | Server prestart: refuses to start on a restore-block marker, derives `config.yaml` from its skeleton, stages the konnectivity-agent and CoreDNS manifests, and waits for the token-webhook kubeconfig. |
+| `k3s-agent-prestart.sh` | Agent prestart shared by the OpenRC and systemd units. Requires `K3S_URL` and `K3S_TOKEN`, then resolves the kubelet provider-id from IMDSv2 for EKS parity. |
+| `k3s-first-boot.sh` | Runs once after the server is healthy: reads the node-token and admin kubeconfig, rewrites the kubeconfig server address to the cluster NLB endpoint, and publishes both to the host. |
+| `konnectivity-server-prestart.sh` | Waits for the K3s CA and admin kubeconfig, mints the konnectivity serving cert, and writes the apiserver replica count for the caller to source. |
+| `mulga-eks-addon-sync.sh` | Pulls staged managed-addon descriptors through the gateway, renders each bundle into the K3s auto-deploy dir, and reports per-addon delivery status back. |
+| `mulga-eks-etcd-snapshot.sh` | Snapshots K3s embedded etcd to predastore. Installed into two crond periodic dirs and picks its tier — `frequent` or `daily` — from the dir it was invoked from. |
+| `mulga-eks-k3s-recovery.sh` | Applies a control-plane etcd-quorum recovery directive before k3s starts. Pull-based and a no-op unless the reconciler escalated a wedged control plane. |
+| `mulga-eks-state-report.sh` | Publishes server health (`healthz`, node count) to the cluster reconciler. The apiserver binds the VPC node-ip, which the host daemon cannot probe directly. |
+| `mulga-eks-provider-id.sh` | Writes the K3s config drop-in supplying the metadata the EBS CSI driver needs from IMDS: provider-id plus the region, zone and instance-type labels. |
+| `mulga-ebs-byid.sh` | Mints the EBS-style `/dev/disk/by-id` symlink the upstream EBS CSI node plugin resolves, from the virtio-blk serial. Alpine's busybox mdev has no eudev to do it. |
+| `mulga-mgmt-net.sh` | Boot oneshot bringing a multi-NIC system VM's interfaces up by MAC from the QEMU fw_cfg netcfg blob, before cloud-init's network stage. cloud-init cannot reliably pick the right NIC out of two. |
+| `mulga-vpc-mtu.sh` | Boot oneshot pinning the VPC data NIC MTU so flannel sizes its VXLAN overlay to fit the OVN geneve underlay. Otherwise large frames are silently dropped while handshakes pass. |
+
+## Launch helpers
+
+| Script | What it does |
+|---|---|
+| `launch-gpu.sh` | Launches a GPU instance from the AMD or NVIDIA GPU AMI, selecting the AMI from the instance type's GPU brand. `--amd` / `--nvidia` override the detection. |
+| `ecs-node-bringup.sh` | Launches an ECS container-instance VM. Renders the cloud-init user-data seeding the agent's control-plane config, IAM credentials and gateway CA — there is no ECS-specific launch API, by design. |
+
+## `demo/` — GPU demo
+
+| Script | What it does |
+|---|---|
+| `deploy.sh` | Deploys and starts the four-instance GPU demo (dashboard, YOLO, Ollama, qwen3-vl) from the `*_IP` and `VIDEO_PATH` env vars. |
+| `dashboard_server.py` | Serves the demo web UI and proxies the streams from the GPU instances. |
+| `yolo_stream.py` | Runs YOLO detection over a video file, serving an MJPEG stream of annotated frames and an SSE stream of qwen3-vl scene descriptions. |

scripts/dev-env/single-nic.conf

@@ -0,0 +1,21 @@
+# single-nic.conf — every plane collapsed onto the WAN bridge.
+#
+# The eval / laptop layout, and what an ISO install produces on a one-NIC box.
+# Leaving lan and vpc unset is what triggers the collapse: both resolve up the
+# chain to wan, so bind, advertise and encap all land on the one address.
+#
+# Usable on a multi-NIC box to test the collapsed path without touching netplan
+# — the other bridges keep their addresses, nothing is bound to them.
+
+DEV_WAN_BRIDGE=br-wan
+DEV_LAN_BRIDGE=
+DEV_VPC_BRIDGE=
+
+DEV_MGMT_BRIDGE=br-mgmt
+DEV_MGMT_CIDR=10.15.8.1/24
+
+DEV_UPLINK=bridged
+
+# Binding to the lan plane is meaningless when lan is wan: it would pin every
+# internal service to the public address rather than leaving it on the wildcard.
+DEV_BIND_PLANES=0

scripts/dev-env/three-plane.conf

@@ -0,0 +1,17 @@
+# three-plane.conf — wan, lan and vpc each on their own NIC.
+#
+# The colo layout (radon, hydrogen) and the local dev box. Internal cluster
+# traffic and Geneve are both off the public interface, so this is the profile
+# that actually exercises the plane model.
+
+DEV_WAN_BRIDGE=br-wan
+DEV_LAN_BRIDGE=br-lan
+DEV_VPC_BRIDGE=br-vpc
+
+# Guest→host bridge for system instances. Orthogonal to the three planes and
+# never folded into lan — keep it off the plane bridge names.
+DEV_MGMT_BRIDGE=br-mgmt
+DEV_MGMT_CIDR=10.15.8.1/24
+
+DEV_UPLINK=bridged
+DEV_BIND_PLANES=1

scripts/dev-env/two-nic.conf

@@ -0,0 +1,15 @@
+# two-nic.conf — dedicated WAN, with lan and vpc sharing the second NIC.
+#
+# The typical small-production layout. vpc is left unset so it folds onto lan:
+# Geneve and internal cluster traffic share a bridge, but both stay off the
+# public interface.
+
+DEV_WAN_BRIDGE=br-wan
+DEV_LAN_BRIDGE=br-lan
+DEV_VPC_BRIDGE=
+
+DEV_MGMT_BRIDGE=br-mgmt
+DEV_MGMT_CIDR=10.15.8.1/24
+
+DEV_UPLINK=bridged
+DEV_BIND_PLANES=1

scripts/diff-coverage.sh

@@ -68,7 +68,9 @@ trap 'rm -rf "$TMPDIR"' EXIT
 # Only non-test .go files that were added or modified.
 # -M enables rename detection so file splits/moves only count truly changed lines.
 # -C enables copy detection for the same reason (file split = partial copy).
-git diff "${BASE_REF}" HEAD --unified=0 -M -C --diff-filter=AM -- '*.go' ':!*_test.go' ':!spinifex/migrate/*' | awk '
+# cmd/installer/ui is excluded: interactive TUI rendering has no meaningful unit
+# test, and the installer is validated by the ISO hardware run instead.
+git diff "${BASE_REF}" HEAD --unified=0 -M -C --diff-filter=AM -- '*.go' ':!*_test.go' ':!spinifex/migrate/*' ':!cmd/installer/ui/*' | awk '
     /^\+\+\+ / {
         file = substr($2, 3)
         next

scripts/disk-performance.sh

@@ -7,15 +7,15 @@ sudo apt install -y fio sysstat jq util-linux
 # make sure not written to tmp disk, etc
 mkdir $HOME/bench
 
+# Require larger size (4G) since smaller tests will not correctly stress nbdkit/viperblock/predastore stack
 fio --name=randrw_70_30 \
     --directory=$HOME/bench \
     --rw=randrw \
     --rwmixread=70 \
     --bs=4k \
-    --size=128M \
+    --size=4G \
     --numjobs=4 \
     --iodepth=32 \
     --ioengine=libaio \
     --direct=1 \
     --group_reporting
-

scripts/install-node.sh

@@ -0,0 +1,610 @@
+#!/bin/bash
+# install-node.sh — form a multi-node Spinifex cluster across running nodes.
+#
+# Every node installed from the ISO comes up as its own single-node cluster.
+# This script converts a set of them into one cluster: it rebuilds the OVN
+# database in clustered form, runs `spx admin init` on the first host and
+# `spx admin join` on the rest, restarts services and verifies the result.
+#
+# It is the scripted form of docs/install/install-multi-node. It installs no
+# software — every host must already be running the version you want. Use
+# mulga's scripts/update-nodes.sh, or the release installer, for that first.
+#
+# Usage:
+#   scripts/install-node.sh --external-pool 10.0.1.100-10.0.1.150 \
+#       --external-gateway 10.0.1.1 --external-prefix-len 24 \
+#       node1.example.com node2.example.com node3.example.com
+#
+# The first host is the init node; the rest join it.
+#
+# Options:
+#   --external-pool A-B      Public IP range for instances (required)
+#   --external-gateway IP    Gateway for that range (required)
+#   --external-prefix-len N  Prefix length of the range's subnet (required)
+#   --external-iface NAME    WAN NIC for br-external (default: auto-detected)
+#   --node-names A,B,C       Cluster node names (default: node1..nodeN)
+#   --lan-bridge NAME        Bridge carrying internal cluster traffic
+#   --vpc-bridge NAME        Bridge carrying Geneve overlay traffic
+#   --region R               Region (default: ap-southeast-2)
+#   --az AZ                  Availability zone (default: <region>a)
+#   --email ADDR             Operator email for update and security notices
+#   --user U                 SSH user (default: spinifex)
+#   --identity FILE          SSH private key
+#   --hosts-file FILE        One host per line; blank lines and # comments ignored
+#   --token-ttl D            Join token validity (default: 30m)
+#   --wipe                   Reset every host to its pre-install state first.
+#                            Destroys all data on ALL hosts, including the first.
+#                            Confirmed separately from --yes; unattended runs
+#                            must set SPX_WIPE_CONFIRM=wipe.
+#   --smoke                  Run smoke-test.sh --create-vpc on the first host
+#   --yes                    Skip the confirmation prompt
+#   --dry-run                Print what would happen and touch nothing
+#
+# WHAT THIS DESTROYS
+#
+# Forming a cluster is not additive. --recreate-db discards all OVN logical
+# network state on every database node, and `spx admin join --force` replaces
+# each joining node's CA and master key with the first host's. Every VPC,
+# subnet and instance on the joining nodes is lost. That is safe on freshly
+# installed servers and unrecoverable on ones that have been in service.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+SSH_USER="${SSH_USER:-spinifex}"
+IDENTITY=""
+HOSTS=()
+NODE_NAMES=()
+EXT_POOL=""
+EXT_GATEWAY=""
+EXT_PREFIX=""
+EXT_IFACE=""
+LAN_BRIDGE=""
+VPC_BRIDGE=""
+LAN_BRIDGE_SET=false
+VPC_BRIDGE_SET=false
+REGION="ap-southeast-2"
+AZ=""
+EMAIL=""
+PORT=4432
+TOKEN_TTL="30m"
+RUN_SMOKE=false
+WIPE=false
+ASSUME_YES=false
+DRY_RUN=false
+
+# How long to wait for the join token to appear, and for formation to complete
+# once the last node has joined.
+TOKEN_TIMEOUT=180
+FORMATION_TIMEOUT=600
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --external-pool)       EXT_POOL="$2"; shift 2 ;;
+        --external-gateway)    EXT_GATEWAY="$2"; shift 2 ;;
+        --external-prefix-len) EXT_PREFIX="$2"; shift 2 ;;
+        --external-iface)      EXT_IFACE="$2"; shift 2 ;;
+        --node-names)          IFS=',' read -ra NODE_NAMES <<<"$2"; shift 2 ;;
+        --lan-bridge)          LAN_BRIDGE="$2"; LAN_BRIDGE_SET=true; shift 2 ;;
+        --vpc-bridge)          VPC_BRIDGE="$2"; VPC_BRIDGE_SET=true; shift 2 ;;
+        --region)              REGION="$2"; shift 2 ;;
+        --az)                  AZ="$2"; shift 2 ;;
+        --email)               EMAIL="$2"; shift 2 ;;
+        --user)                SSH_USER="$2"; shift 2 ;;
+        --identity)            IDENTITY="$2"; shift 2 ;;
+        --token-ttl)           TOKEN_TTL="$2"; shift 2 ;;
+        --hosts-file)
+            while IFS= read -r line; do
+                line="${line%%#*}"
+                line="$(echo "$line" | xargs)"
+                [ -n "$line" ] && HOSTS+=("$line")
+            done <"$2"
+            shift 2
+            ;;
+        --smoke)    RUN_SMOKE=true; shift ;;
+        --wipe)     WIPE=true; shift ;;
+        --yes|-y)   ASSUME_YES=true; shift ;;
+        --dry-run)  DRY_RUN=true; shift ;;
+        -h|--help)  sed -n '2,49p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
+        -*)         echo "ERROR: unknown option: $1" >&2; exit 2 ;;
+        *)          HOSTS+=("$1"); shift ;;
+    esac
+done
+
+log()  { echo "[install-node] $*"; }
+fail() { echo "[install-node] ERROR: $*" >&2; exit 1; }
+
+N=${#HOSTS[@]}
+
+if [ "$N" -lt 2 ]; then
+    echo "ERROR: give at least two hosts — the first initializes, the rest join." >&2
+    echo "Usage: $(basename "$0") [options] host1 host2 [host3...]" >&2
+    exit 2
+fi
+
+# Two-node RAFT is strictly worse than standalone: quorum is a majority, so
+# 2-of-2 tolerates zero failures and either node dying loses the database.
+# setup-ovn.sh documents the same rule; this is where the operator finds out.
+if [ "$N" -eq 2 ]; then
+    log "WARNING: two nodes run a standalone OVN database on ${HOSTS[0]}."
+    log "         Losing that host takes the control plane down. Three is the minimum"
+    log "         for a fault-tolerant deployment."
+fi
+
+for required in "--external-pool:$EXT_POOL" "--external-gateway:$EXT_GATEWAY" "--external-prefix-len:$EXT_PREFIX"; do
+    [ -n "${required#*:}" ] || fail "${required%%:*} is required.
+       Instances get their public addresses from this range, and there is no safe
+       default to guess. Example:
+         --external-pool 216.218.163.101-216.218.163.110 \\
+         --external-gateway 216.218.163.97 --external-prefix-len 27"
+done
+
+[[ "$EXT_POOL" =~ ^[0-9.]+-[0-9.]+$ ]] || fail "--external-pool must be START-END, got: $EXT_POOL"
+[[ "$EXT_PREFIX" =~ ^[0-9]+$ ]] && [ "$EXT_PREFIX" -ge 1 ] && [ "$EXT_PREFIX" -le 32 ] ||
+    fail "--external-prefix-len must be 1-32, got: $EXT_PREFIX"
+
+AZ="${AZ:-${REGION}a}"
+
+if [ ${#NODE_NAMES[@]} -eq 0 ]; then
+    for i in $(seq 1 "$N"); do NODE_NAMES+=("node$i"); done
+elif [ ${#NODE_NAMES[@]} -ne "$N" ]; then
+    fail "--node-names has ${#NODE_NAMES[@]} entries for $N hosts"
+fi
+
+# The node name is the cluster's identity for a host: it keys the [nodes.X]
+# config block, the NATS server name and the IPsec peer certificate. Two nodes
+# sharing one is not a cosmetic clash.
+if [ "$(printf '%s\n' "${NODE_NAMES[@]}" | sort -u | wc -l)" -ne "$N" ]; then
+    fail "--node-names contains duplicates: ${NODE_NAMES[*]}"
+fi
+
+SSH_OPTS=(-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes)
+[ -n "$IDENTITY" ] && SSH_OPTS+=(-i "$IDENTITY")
+
+# on runs a command on a host. Under --dry-run it prints instead, so a rehearsal
+# shows the exact remote command line rather than a summary of it.
+on() {
+    local host="$1"
+    shift
+    if $DRY_RUN; then
+        echo "  ssh $SSH_USER@$host $*"
+        return 0
+    fi
+    ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "$@"
+}
+
+# out runs a command and returns its output even under --dry-run, for the reads
+# that later phases depend on. Nothing it runs changes host state.
+out() {
+    local host="$1"
+    shift
+    ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "$@" 2>/dev/null || true
+}
+
+SETUP_OVN="/usr/local/share/spinifex/setup-ovn.sh"
+
+# --- Preflight -------------------------------------------------------------
+#
+# Every host is checked, and every address resolved, before any host is
+# touched: a typo in the third address should not leave the first two stopped
+# with a half-formed cluster.
+
+declare -a WAN_IPS LAN_IPS VPC_IPS
+
+# plane_ip <host> <bridge> — first IPv4 address on a link, empty if absent.
+plane_ip() {
+    out "$1" "ip -4 -o addr show $2 2>/dev/null | awk 'NR==1 {split(\$4, a, \"/\"); print a[1]}'"
+}
+
+# resolve_plane <host> <requested bridge> <explicitly set?> <wan ip> — the
+# address a plane lands on. An unset bridge that does not exist folds onto the
+# wan plane (the canonical vpc <- lan <- wan collapse); an explicitly named one
+# that has no address is an error rather than a silent fold, because a plane
+# the operator asked for is one they expect to be used.
+resolve_plane() {
+    local host="$1" bridge="$2" explicit="$3" wan_ip="$4" ip=""
+    [ -n "$bridge" ] && ip=$(plane_ip "$host" "$bridge")
+    ip="${ip//[$'\r\n']/}"
+    if [ -n "$ip" ]; then
+        echo "$ip"
+        return 0
+    fi
+    if [ "$explicit" = true ]; then
+        return 1
+    fi
+    echo "$wan_ip"
+}
+
+log "preflight on $N host(s)"
+
+versions=""
+for i in $(seq 0 $((N - 1))); do
+    host="${HOSTS[$i]}"
+
+    ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" true 2>/dev/null ||
+        fail "cannot reach $SSH_USER@$host over SSH"
+    ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "sudo -n true" 2>/dev/null ||
+        fail "$host: $SSH_USER does not have passwordless sudo"
+
+    ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "test -x /usr/local/bin/spx" 2>/dev/null ||
+        fail "$host: no spx installed. This script forms a cluster; it does not install
+       Spinifex. Install every host first, then re-run."
+    ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "test -x $SETUP_OVN" 2>/dev/null ||
+        fail "$host: $SETUP_OVN is missing — this node predates the clustered-OVN
+       installer. Update it before forming a cluster."
+
+    version=$(out "$host" "spx version 2>/dev/null | head -1")
+    version="${version//[$'\r\n']/}"
+    versions+="${version}"$'\n'
+
+    # A node that is already in a multi-node cluster is one whose keys and
+    # logical network state are load-bearing. Refusing here is deliberate:
+    # forming a cluster must never be the command that destroys a live one.
+    # --wipe is the deliberate teardown this asks for, and carries its own
+    # typed confirmation, so it is the one way past.
+    node_count=$(out "$host" \
+        "sudo grep -oP '^\[nodes\.\K[^.\]]+' /etc/spinifex/spinifex.toml 2>/dev/null | sort -u | wc -l")
+    node_count="${node_count//[$'\r\n']/}"
+    [[ "$node_count" =~ ^[0-9]+$ ]] || node_count=0
+    if [ "$node_count" -gt 1 ] && ! $WIPE; then
+        fail "$host is already part of a $node_count-node cluster.
+       Re-forming it would discard its CA and master key, and any data sealed
+       under them. Tear it down deliberately before re-running."
+    fi
+
+    wan_ip=$(out "$host" "ip -4 route show default | awk 'NR==1 {print \$5}'")
+    wan_ip="${wan_ip//[$'\r\n']/}"
+    wan_ip=$(plane_ip "$host" "$wan_ip")
+    wan_ip="${wan_ip//[$'\r\n']/}"
+    [ -n "$wan_ip" ] || fail "$host: no IPv4 address on the default-route interface"
+
+    lan_ip=$(resolve_plane "$host" "${LAN_BRIDGE:-br-lan}" "$LAN_BRIDGE_SET" "$wan_ip") ||
+        fail "$host: --lan-bridge $LAN_BRIDGE has no IPv4 address"
+    vpc_ip=$(resolve_plane "$host" "${VPC_BRIDGE:-br-vpc}" "$VPC_BRIDGE_SET" "$lan_ip") ||
+        fail "$host: --vpc-bridge $VPC_BRIDGE has no IPv4 address"
+
+    WAN_IPS[i]="$wan_ip"
+    LAN_IPS[i]="$lan_ip"
+    VPC_IPS[i]="$vpc_ip"
+
+    # Node names do not have to match hostnames, but when they disagree it is
+    # almost always because the operator meant a different host. Cheap to say.
+    hostname=$(out "$host" "hostname")
+    hostname="${hostname//[$'\r\n']/}"
+    if [ "$hostname" != "${NODE_NAMES[$i]}" ]; then
+        log "  NOTE: $host is '$hostname' but will be named '${NODE_NAMES[$i]}' in the cluster"
+    fi
+
+    log "  $host — ${version:-unknown} wan=$wan_ip lan=$lan_ip vpc=$vpc_ip"
+done
+
+# Mixed versions form a cluster whose NATS mesh, predastore and OVN schema
+# disagree with each other. That failure surfaces hours later as unexplained
+# errors, so it is worth catching here.
+if [ "$(sort -u <<<"$versions" | grep -c .)" -gt 1 ]; then
+    fail "hosts are running different spx versions:
+$(sort -u <<<"$versions" | sed 's/^/         /')
+       Bring them all to the same build before forming a cluster."
+fi
+
+# --- Confirm ---------------------------------------------------------------
+
+echo ""
+echo "  Cluster: $N nodes, region $REGION, az $AZ"
+printf '  %-4s %-22s %-10s %-16s %-16s %s\n' "" HOST NAME WAN LAN VPC
+for i in $(seq 0 $((N - 1))); do
+    role=$([ "$i" -eq 0 ] && echo "init" || echo "join")
+    printf '  %-4s %-22s %-10s %-16s %-16s %s\n' \
+        "$role" "${HOSTS[$i]}" "${NODE_NAMES[$i]}" "${WAN_IPS[$i]}" "${LAN_IPS[$i]}" "${VPC_IPS[$i]}"
+done
+echo ""
+echo "  Instance pool: $EXT_POOL via $EXT_GATEWAY/$EXT_PREFIX"
+echo ""
+if $WIPE; then
+    echo "  --wipe: this DESTROYS, on ALL $N hosts INCLUDING ${HOSTS[0]}:"
+    echo "    - every instance and every volume backing them"
+    echo "    - every S3 object held on these nodes"
+    echo "    - each node's CA and master key — data sealed under them is unreadable"
+    echo "      afterwards even if the bytes are restored from elsewhere"
+    echo "    - all OVN logical network state"
+else
+    echo "  This DESTROYS, on every host except ${HOSTS[0]}:"
+    echo "    - all OVN logical network state: every VPC, subnet, router and port"
+    echo "    - every instance depending on it"
+    echo "    - the node's own CA and master key, replaced by ${HOSTS[0]}'s"
+fi
+echo ""
+
+if ! $ASSUME_YES && ! $DRY_RUN; then
+    read -r -p "  Type 'yes' to continue: " reply
+    [ "$reply" = "yes" ] || fail "aborted"
+fi
+
+# --yes deliberately does not cover --wipe. Unattended reruns are the whole
+# point of --yes, and a flag that silently also means "destroy every node's
+# data" is one stray shell-history recall away from an incident. Automation
+# opts in separately through the environment, so a lab rerun stays possible
+# without making the destructive path reachable by one extra flag.
+if $WIPE && ! $DRY_RUN; then
+    if [ -t 0 ]; then
+        read -r -p "  --wipe destroys all data on $N hosts. Type 'wipe' to confirm: " reply
+        [ "$reply" = "wipe" ] || fail "aborted"
+    elif [ "${SPX_WIPE_CONFIRM:-}" != "wipe" ]; then
+        fail "--wipe needs confirmation, and there is no terminal to ask on.
+       Re-run with SPX_WIPE_CONFIRM=wipe to confirm destroying all data on $N hosts."
+    fi
+fi
+
+# --- Stop ------------------------------------------------------------------
+#
+# Each of these is its own single-node cluster, so there is no coordinated
+# shutdown to run: spinifex.target's own ExecStop drains that node's guests
+# before any storage service stops.
+
+log "stopping spinifex.target on every host"
+for host in "${HOSTS[@]}"; do
+    on "$host" "sudo systemctl stop spinifex.target" || fail "$host: could not stop spinifex.target"
+done
+
+# --- Wipe ------------------------------------------------------------------
+#
+# node-reset.sh is not part of the release tarball, so it is pushed from this
+# checkout the same way smoke-test.sh is. After this a node has no CA, master
+# key, OVN database or chassis identity left for the formation phase to collide
+# with — which is the whole reason --wipe exists.
+if $WIPE; then
+    log "wiping every host back to its pre-install state"
+    for host in "${HOSTS[@]}"; do
+        if ! $DRY_RUN; then
+            scp "${SSH_OPTS[@]}" -q "$SCRIPT_DIR/node-reset.sh" "$SSH_USER@$host:/tmp/spx-node-reset.sh" ||
+                fail "$host: could not copy node-reset.sh"
+        fi
+        on "$host" "chmod 0755 /tmp/spx-node-reset.sh && sudo /tmp/spx-node-reset.sh --yes" ||
+            fail "$host: node-reset.sh failed"
+        on "$host" "rm -f /tmp/spx-node-reset.sh"
+        log "  $host wiped"
+    done
+fi
+
+# --- OVN database cluster --------------------------------------------------
+#
+# ovn-ctl consults the cluster flags only when it creates the database, and
+# the ovn-central package starts a standalone one on install — so converting
+# to RAFT means recreating the DB. Three or more nodes run RAFT across the
+# first three; the rest are compute nodes pointing at all three, so they
+# survive any one of them failing.
+
+DB_NODES=$((N >= 3 ? 3 : 1))
+
+# --node-name pins the OVS system-id, which is the OVN chassis-id, which
+# ovs-monitor-ipsec uses as the IKEv2 `@<name>` peer identity. `spx admin init`
+# enables IPsec by default and bakes the cluster node name into each peer
+# certificate as CN and dnsName, so a chassis left on its package-generated
+# UUID authenticates as @<uuid> against a cert naming the node and every
+# Geneve tunnel fails with AUTHENTICATION_FAILED. Safe to pin here because
+# --recreate-db means no stale chassis row survives to be orphaned by it.
+log "building the OVN database ($([ "$DB_NODES" -eq 3 ] && echo "RAFT across 3" || echo standalone))"
+
+if [ "$DB_NODES" -eq 3 ]; then
+    on "${HOSTS[0]}" "sudo $SETUP_OVN --management \
+        --node-name=${NODE_NAMES[0]} \
+        --db-cluster-local-addr=${LAN_IPS[0]} \
+        --recreate-db \
+        --encap-ip=${VPC_IPS[0]}" || fail "${HOSTS[0]}: could not create the OVN database cluster"
+    log "  ${HOSTS[0]} created the database cluster"
+
+    for i in 1 2; do
+        on "${HOSTS[$i]}" "sudo $SETUP_OVN --management \
+            --node-name=${NODE_NAMES[$i]} \
+            --db-cluster-local-addr=${LAN_IPS[$i]} \
+            --db-cluster-remote-addr=${LAN_IPS[0]} \
+            --recreate-db \
+            --encap-ip=${VPC_IPS[$i]}" || fail "${HOSTS[$i]}: could not join the OVN database cluster"
+        log "  ${HOSTS[$i]} joined the database cluster"
+    done
+
+    OVN_REMOTE="tcp:${LAN_IPS[0]}:6642,tcp:${LAN_IPS[1]}:6642,tcp:${LAN_IPS[2]}:6642"
+else
+    on "${HOSTS[0]}" "sudo $SETUP_OVN --management \
+        --node-name=${NODE_NAMES[0]} \
+        --encap-ip=${VPC_IPS[0]}" || fail "${HOSTS[0]}: setup-ovn.sh failed"
+    log "  ${HOSTS[0]} is running a standalone database"
+
+    OVN_REMOTE="tcp:${LAN_IPS[0]}:6642"
+fi
+
+# Compute nodes: no database of their own, pointed at every database node.
+for i in $(seq "$DB_NODES" $((N - 1))); do
+    on "${HOSTS[$i]}" "sudo $SETUP_OVN \
+        --node-name=${NODE_NAMES[$i]} \
+        --ovn-remote=$OVN_REMOTE \
+        --encap-ip=${VPC_IPS[$i]}" || fail "${HOSTS[$i]}: setup-ovn.sh failed"
+    log "  ${HOSTS[$i]} attached as a compute node"
+done
+
+# --- Form the cluster ------------------------------------------------------
+
+# --advertise must be explicit whenever --bind is pinned. resolveAdvertiseIP
+# echoes a non-wildcard --bind straight back, so binding services to the lan
+# plane without this publishes the internal address as the node's public dial
+# target.
+init_args=(
+    --force
+    --node "${NODE_NAMES[0]}"
+    --nodes "$N"
+    --bind "${LAN_IPS[0]}"
+    --cluster-bind "${LAN_IPS[0]}"
+    --advertise "${WAN_IPS[0]}"
+    --port "$PORT"
+    --region "$REGION"
+    --az "$AZ"
+    --token-ttl "$TOKEN_TTL"
+    --external-mode=pool
+    --external-source=static
+    --external-pool="$EXT_POOL"
+    --external-gateway="$EXT_GATEWAY"
+    --external-prefix-len="$EXT_PREFIX"
+)
+[ -n "$EXT_IFACE" ] && init_args+=(--external-iface="$EXT_IFACE")
+[ -n "$EMAIL" ] && init_args+=(--email="$EMAIL")
+
+INIT_LOG=/tmp/spx-init.log
+
+log "initializing ${HOSTS[0]} as ${NODE_NAMES[0]} (waiting for $((N - 1)) node(s) to join)"
+
+# init blocks in a formation server until every node has joined, so it runs
+# detached and the token is read back out of its output.
+on "${HOSTS[0]}" "sudo rm -f $INIT_LOG; \
+    sudo setsid sh -c 'spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
+    exit 0" || fail "${HOSTS[0]}: could not start spx admin init"
+
+TOKEN=""
+if $DRY_RUN; then
+    TOKEN="spx_join_<scraped-from-$INIT_LOG>"
+else
+    elapsed=0
+    while [ "$elapsed" -lt "$TOKEN_TIMEOUT" ]; do
+        TOKEN=$(out "${HOSTS[0]}" "sudo grep -o 'spx_join_[A-Za-z0-9_-]*' $INIT_LOG | head -1")
+        TOKEN="${TOKEN//[$'\r\n']/}"
+        [ -n "$TOKEN" ] && break
+        sleep 5
+        elapsed=$((elapsed + 5))
+    done
+    if [ -z "$TOKEN" ]; then
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
+        fail "${HOSTS[0]}: spx admin init produced no join token within ${TOKEN_TIMEOUT}s"
+    fi
+    log "  join token issued"
+fi
+
+# --force: each joining node arrived from the ISO already initialized, with its
+# own CA and master key. Joining replaces them with the init node's, and
+# `spx admin join` refuses to do that silently.
+#
+# Every join must run concurrently. `spx admin join` registers and then blocks
+# until the formation server has all N nodes, so joining one at a time deadlocks
+# the moment there is more than one joiner: node 2 waits for node 3, which is
+# waiting on a shell that node 2 is still holding.
+JOIN_PIDS=()
+for i in $(seq 1 $((N - 1))); do
+    on "${HOSTS[$i]}" "sudo spx admin join --force \
+        --node ${NODE_NAMES[$i]} \
+        --bind ${LAN_IPS[$i]} --cluster-bind ${LAN_IPS[$i]} --advertise ${WAN_IPS[$i]} \
+        --host ${LAN_IPS[0]}:$PORT --token $TOKEN \
+        --region $REGION --az $AZ" &
+    JOIN_PIDS+=("$!")
+done
+
+# Collect every join before failing on any of them, so a partial formation
+# reports which node broke it rather than whichever was reaped first.
+JOIN_FAILED=()
+for i in $(seq 1 $((N - 1))); do
+    if wait "${JOIN_PIDS[$((i - 1))]}"; then
+        log "  ${NODE_NAMES[$i]} joined"
+    else
+        JOIN_FAILED+=("${HOSTS[$i]}")
+    fi
+done
+[ ${#JOIN_FAILED[@]} -eq 0 ] || fail "spx admin join failed on: ${JOIN_FAILED[*]}"
+
+# init exits once the last node registers. A formation server still running
+# means a node registered but formation never completed.
+if ! $DRY_RUN; then
+    elapsed=0
+    while [ "$elapsed" -lt "$FORMATION_TIMEOUT" ]; do
+        # [s]px, not spx: pgrep -f scans full command lines, and the shell ssh
+        # spawns to run it carries the pattern itself. Unbracketed, this always
+        # matches and formation never looks finished.
+        ssh "${SSH_OPTS[@]}" "$SSH_USER@${HOSTS[0]}" "pgrep -f '[s]px admin init' >/dev/null" >/dev/null 2>&1 || break
+        sleep 5
+        elapsed=$((elapsed + 5))
+    done
+    if [ "$elapsed" -ge "$FORMATION_TIMEOUT" ]; then
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
+        fail "${HOSTS[0]}: formation did not complete within ${FORMATION_TIMEOUT}s"
+    fi
+    log "  formation complete"
+fi
+
+# --- Start -----------------------------------------------------------------
+
+log "starting spinifex.target on every host"
+for host in "${HOSTS[@]}"; do
+    on "$host" "sudo systemctl start spinifex.target" || fail "$host: could not start spinifex.target"
+done
+
+if $DRY_RUN; then
+    echo ""
+    log "dry run complete — nothing was changed"
+    exit 0
+fi
+
+# --- Verify ----------------------------------------------------------------
+
+log "verifying"
+
+# Services take a moment to register on NATS after start, so the node list is
+# worth a few retries before it means anything. spx colourises unconditionally,
+# including down a pipe, so the escapes come off before anything parses fields.
+nodes=""
+for _ in $(seq 1 12); do
+    nodes=$(out "${HOSTS[0]}" "sudo spx get nodes" | sed 's/\x1b\[[0-9;]*m//g')
+    [ -n "$nodes" ] && ! grep -q NotReady <<<"$nodes" && break
+    sleep 10
+done
+echo ""
+sed 's/^/  /' <<<"$nodes"
+echo ""
+
+# Match Ready as a whole field, so NotReady is not counted as Ready.
+ready=$(awk '{for (i = 1; i <= NF; i++) if ($i == "Ready") { c++; break }} END {print c + 0}' <<<"$nodes")
+[ "$ready" -eq "$N" ] || fail "expected $N Ready nodes, got $ready — check journalctl -u 'spinifex-*'"
+
+if [ "$DB_NODES" -eq 3 ]; then
+    status=$(out "${HOSTS[0]}" "sudo ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound")
+    members=$(grep -c ' at tcp:' <<<"$status" || true)
+    [ "$members" -eq 3 ] ||
+        fail "OVN Northbound reports $members cluster members, expected 3 — a standalone
+       database here means --recreate-db did not take on every database node"
+    log "  OVN Northbound: 3 members, $(grep -oiE '^Role: .*' <<<"$status" | head -1)"
+fi
+
+sb=$(out "${HOSTS[0]}" "sudo ovn-sbctl show 2>/dev/null")
+chassis=$(grep -c '^Chassis' <<<"$sb" || true)
+log "  OVN Southbound: $chassis chassis registered (expected $N)"
+
+# Each chassis must be named for its node, not left on a UUID: that name is the
+# IPsec peer identity, so a mismatch here is a cluster whose Geneve tunnels will
+# not authenticate.
+for name in "${NODE_NAMES[@]}"; do
+    grep -q "^Chassis \"\?$name\"\?" <<<"$sb" ||
+        log "  WARNING: no chassis named '$name' in the Southbound DB. ovn-controller may
+       still be registering — if it persists, IPsec will fail to authenticate.
+       Check 'sudo ovs-vsctl get Open_vSwitch . external_ids:system-id' on that host."
+done
+
+# The pool is the one thing the operator supplied by hand and the one thing
+# nothing else validates, so print what actually landed. dns_servers in
+# particular is detected rather than passed, and is wrong often enough to be
+# worth seeing.
+echo ""
+log "external pool as written:"
+out "${HOSTS[0]}" "sudo sed -n '/^\[\[network.external_pools\]\]/,/^$/p' /etc/spinifex/spinifex.toml" |
+    sed 's/^/  /'
+
+# --- Smoke test ------------------------------------------------------------
+
+if $RUN_SMOKE; then
+    echo ""
+    log "running the smoke test on ${HOSTS[0]}"
+    # smoke-test.sh is not part of the release tarball, so it is pushed from
+    # this checkout — which also guarantees the test and this script are the
+    # same version.
+    scp "${SSH_OPTS[@]}" -q "$SCRIPT_DIR/smoke-test.sh" "$SSH_USER@${HOSTS[0]}:/tmp/spx-smoke-test.sh"
+    on "${HOSTS[0]}" "chmod 0755 /tmp/spx-smoke-test.sh && /tmp/spx-smoke-test.sh --create-vpc --nodes $N" ||
+        fail "smoke test failed on ${HOSTS[0]}"
+    on "${HOSTS[0]}" "rm -f /tmp/spx-smoke-test.sh"
+fi
+
+echo ""
+log "cluster formed: $N nodes, pool $EXT_POOL"

scripts/node-reset.sh

@@ -0,0 +1,227 @@
+#!/bin/bash
+# node-reset.sh — return one Spinifex node to a freshly-installed state.
+#
+# Teardown only: it stops services, removes state, and exits. It installs
+# nothing, starts nothing and decides nothing. Callers do the rebuilding —
+# reset-dev-env.sh locally, install-node.sh --wipe over SSH.
+#
+# Usage:
+#   sudo scripts/node-reset.sh [--keep-data] [--yes] [--dry-run]
+#
+# Options:
+#   --keep-data   Leave /var/lib/spinifex alone: volumes, S3 objects and
+#                 JetStream survive. For when only the control plane is suspect.
+#   --yes         Skip the confirmation prompt
+#   --dry-run     Print what would be removed and touch nothing
+#
+# WHAT THIS DESTROYS
+#
+# Every instance on this node and every volume backing them, the node's CA and
+# master key, all OVN logical network state, and the S3 objects held here. None
+# of it is recoverable. Data sealed under the master key cannot be read back
+# even if the bytes are restored from elsewhere.
+set -euo pipefail
+
+ETC_DIR=/etc/spinifex
+DATA_DIR=/var/lib/spinifex
+LOG_DIR=/var/log/spinifex
+RUN_DIR=/run/spinifex
+
+KEEP_DATA=false
+ASSUME_YES=false
+DRY_RUN=false
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --keep-data) KEEP_DATA=true ;;
+        --yes | -y)  ASSUME_YES=true ;;
+        --dry-run)   DRY_RUN=true ;;
+        -h | --help)
+            sed -n '2,23p' "$0" | sed 's/^# \?//'
+            exit 0
+            ;;
+        *)
+            echo "ERROR: unknown option: $1" >&2
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+log() { echo "[node-reset] $*"; }
+run() {
+    if $DRY_RUN; then
+        echo "  would run: $*"
+        return 0
+    fi
+    "$@"
+}
+
+WIPE_DIRS=("$ETC_DIR" "$LOG_DIR" "$RUN_DIR")
+$KEEP_DATA || WIPE_DIRS+=("$DATA_DIR")
+
+# Directories and symlinks are structure, not state: setup.sh builds both and
+# nothing in a cluster is stored as either. Two of those symlinks are load
+# bearing and easy to miss — /var/lib/spinifex/config and awsgw/config both
+# point at /etc/spinifex, the second nested a level down, and awsgw reads its
+# IAM master key through it. Keeping every symlink covers them without this
+# script having to know where setup.sh put them.
+#
+# keep_args adds the regular files worth keeping: the service helper scripts
+# and the env files the units load, at the top level of $1 only. A *.sh deeper
+# in the tree is cluster state, not an installed helper.
+KEEP_TOP=()
+keep_args() {
+    KEEP_TOP=(-path "$1/*" ! -path "$1/*/*" '(' -name '*.sh' -o -name '*.env' ')')
+}
+
+# Report what is at stake in figures rather than adjectives. An operator who
+# reads "3 instances, 12 volumes, 840G" makes a better decision than one who
+# reads a warning banner.
+# `|| true` on each: under pipefail a find over a directory that was never
+# created fails the whole pipeline, and a node with nothing to report is the
+# most ordinary case there is.
+if [ -d "$DATA_DIR" ]; then
+    instances=$(sudo find "$DATA_DIR/instances" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l || true)
+    volumes=$(sudo find "$DATA_DIR/volumes" -maxdepth 1 -mindepth 1 2>/dev/null | wc -l || true)
+    size=$(sudo du -sh "$DATA_DIR" 2>/dev/null | cut -f1 || true)
+    log "on $(hostname): $instances instance(s), $volumes volume(s), ${size:-unknown} under $DATA_DIR"
+    $KEEP_DATA && log "  --keep-data: $DATA_DIR will be preserved"
+fi
+log "removing: ${WIPE_DIRS[*]}"
+
+if ! $ASSUME_YES && ! $DRY_RUN; then
+    read -r -p "Destroy this node's state? Type 'destroy' to continue: " reply
+    [ "$reply" = "destroy" ] || { log "aborted"; exit 1; }
+fi
+
+log "stopping services"
+run sudo systemctl stop spinifex.target 2>/dev/null || true
+run sudo systemctl reset-failed 'spinifex-*' 2>/dev/null || true
+run sudo pkill -x qemu-system-x86_64 2>/dev/null || true
+run sudo pkill -x qemu-system-aarch64 2>/dev/null || true
+
+# Viperblock state must not be torn out from under a live guest, so wait for
+# QEMU to actually exit rather than assuming the signal was enough.
+if ! $DRY_RUN; then
+    elapsed=0
+    while pgrep -x 'qemu-system-x86_64|qemu-system-aarch64' >/dev/null 2>&1; do
+        if [ "$elapsed" -ge 30 ]; then
+            echo "ERROR: QEMU still running after 30s:" >&2
+            pgrep -af 'qemu-system-' >&2 || true
+            echo "  Kill them manually and re-run." >&2
+            exit 1
+        fi
+        sleep 1
+        elapsed=$((elapsed + 1))
+    done
+fi
+
+# Clearing external_ids drops system-id along with it. That is the point: a
+# node keeping its old system-id across a reset re-registers under the old
+# chassis name and collides with the new one over the encap IP.
+log "removing OVS bridges and identity"
+if command -v ovs-vsctl >/dev/null 2>&1; then
+    run sudo systemctl start openvswitch-switch 2>/dev/null || true
+    $DRY_RUN || sleep 1
+    # Listed even under --dry-run: "which bridges" is the question an operator
+    # actually has here, and br-wan is a Linux bridge so it is never in this set.
+    for br in $(sudo ovs-vsctl list-br 2>/dev/null || true); do
+        log "  deleting bridge: $br"
+        run sudo ovs-vsctl --if-exists del-br "$br"
+    done
+    run sudo ovs-vsctl --if-exists clear Open_vSwitch . external_ids 2>/dev/null || true
+    run sudo systemctl stop openvswitch-switch 2>/dev/null || true
+fi
+run sudo rm -f /etc/openvswitch/system-id.conf
+
+# Delete the OVN DBs outright — a caller re-running setup-ovn.sh gets fresh
+# empty ones. This is what clears stale chassis rows and port bindings, which
+# otherwise accumulate across resets and wedge ovn-controller in a commit loop.
+log "removing OVN databases"
+run sudo systemctl stop ovn-central 2>/dev/null || true
+run sudo systemctl stop ovn-controller 2>/dev/null || true
+run sudo rm -f /var/lib/ovn/ovnnb_db.db /var/lib/ovn/ovnsb_db.db
+
+if ip link show veth-wan-br >/dev/null 2>&1; then
+    log "  deleting veth pair: veth-wan-br <-> veth-wan-ovs"
+    run sudo ip link del veth-wan-br 2>/dev/null || true
+fi
+
+# Without this, systemd-networkd recreates the veth on the next reboot even
+# after a full reset.
+if [ -e /etc/systemd/network/15-spinifex-veth-wan.netdev ] ||
+    [ -e /etc/systemd/network/15-spinifex-veth-wan.network ] ||
+    [ -e /etc/systemd/network/16-spinifex-veth-wan-ovs.network ]; then
+    log "  deleting veth persistence units"
+    run sudo rm -f /etc/systemd/network/15-spinifex-veth-wan.netdev \
+        /etc/systemd/network/15-spinifex-veth-wan.network \
+        /etc/systemd/network/16-spinifex-veth-wan-ovs.network
+    run sudo networkctl reload 2>/dev/null || true
+fi
+
+# `rm -rf` is wrong here on two counts. On an ISO-installed node several of
+# these are separate ZFS datasets — rpool/log and rpool/data/{nats,viperblock,
+# predastore,predastore-db,predastore-nodes} — and a mountpoint cannot be
+# removed at all. And the directory tree is itself install state worth keeping:
+# it carries the ownership setup.sh assigned per service.
+log "wiping ${WIPE_DIRS[*]}"
+for dir in "${WIPE_DIRS[@]}"; do
+    [ -d "$dir" ] || continue
+    if $DRY_RUN; then
+        echo "  would empty: $dir"
+        continue
+    fi
+    keep_args "$dir"
+    # Regular files only, so directories and symlinks survive along with the
+    # per-service ownership setup.sh assigned. Nothing then has to be
+    # reinstalled afterwards, which is what lets this script install nothing
+    # and leave a node on the exact build it was already running.
+    sudo find "$dir" -mindepth 1 \( -type f -o -type s -o -type p \) \
+        ! \( "${KEEP_TOP[@]}" \) -delete 2>/dev/null || true
+done
+
+# Directories may survive — a mountpoint cannot be removed, and neither can the
+# path leading down to one. Files may not: a surviving file is state that would
+# carry into the new cluster, which is the exact failure this script exists to
+# prevent, so refuse rather than hand back a node that looks clean.
+if ! $DRY_RUN; then
+    for dir in "${WIPE_DIRS[@]}"; do
+        [ -d "$dir" ] || continue
+        keep_args "$dir"
+        left=$(sudo find "$dir" -mindepth 1 ! -type d ! -type l \
+            ! \( "${KEEP_TOP[@]}" \) 2>/dev/null | head -5)
+        if [ -n "$left" ]; then
+            echo "ERROR: $dir still holds files after the wipe:" >&2
+            echo "$left" | sed 's/^/  /' >&2
+            exit 1
+        fi
+    done
+fi
+
+if [ ! -d /etc/spinifex ]; then
+    echo "ERROR: /etc/spinifex is missing and could not be restored." >&2
+    echo "  spx would treat this node as a dev install and build it under ~/spinifex," >&2
+    echo "  forming a cluster whose services can never start. Re-run the installer." >&2
+    $DRY_RUN || exit 1
+fi
+
+# A node that has already been through that fallback carries a stray dev-layout
+# tree holding its own keys and configs. It is not user data — it is wreckage
+# from a misdetected layout — and leaving it invites a later dev-mode run to
+# adopt it.
+DEV_ROOT="$(getent passwd spinifex | cut -d: -f6)/spinifex"
+if [ -n "${DEV_ROOT#/spinifex}" ] && [ -d "$DEV_ROOT" ]; then
+    log "removing the dev-layout fallback tree at $DEV_ROOT"
+    run sudo rm -rf "$DEV_ROOT"
+fi
+
+# The next init writes a fresh CA. Leaving the old one trusted means the host
+# trusts a CA nobody holds the key for.
+if [ -f /usr/local/share/ca-certificates/spinifex-ca.crt ]; then
+    log "removing the stale CA from the trust store"
+    run sudo rm -f /usr/local/share/ca-certificates/spinifex-ca.crt
+    run sudo update-ca-certificates
+fi
+
+log "done — node is installed but uninitialized, ready to init or join"

scripts/reset-dev-env.sh

@@ -10,6 +10,28 @@
 # pool range) rather than applying defaults. On a fresh box with no config,
 # falls back to ap-southeast-2 + pool mode derived from the WAN subnet.
 #
+# Usage:
+#   ./scripts/reset-dev-env.sh [options]
+#
+# Options:
+#   --profile=NAME   Topology profile from scripts/dev-env/NAME.conf. Declares
+#                    which bridge carries the wan, lan and vpc planes. Omit to
+#                    auto-detect from the host, which is the historical
+#                    behaviour: wan off the default route, lan and vpc unset.
+#                    Also settable as SPINIFEX_DEV_PROFILE.
+#   --dry-run        Resolve the planes and print the setup-ovn.sh and
+#                    'spx admin init' command lines, then exit without
+#                    touching any host state.
+#
+# Every profile variable is overridable from the environment, e.g.
+#   DEV_VPC_BRIDGE=br-vpc2 ./scripts/reset-dev-env.sh --profile=three-plane
+#
+# Planes follow the canonical collapse vpc <- lan <- wan, so a profile that
+# leaves vpc unset folds it onto lan, and one that leaves both unset is a
+# single-NIC node. --encap-ip is always passed explicitly from the resolved
+# vpc plane rather than relying on setup-ovn.sh sniffing for a bridge named
+# br-vpc, which silently falls back to the WAN address when it misses.
+#
 # WARNING: setup-ovn.sh converts the WAN NIC into an OVS bridge. On hosts
 # where the WAN NIC is also the SSH NIC, SSH will drop mid-run. Run from
 # the console or via a separate management NIC.
@@ -37,6 +59,66 @@ if [ -z "$INVOKING_HOME" ]; then
     exit 1
 fi
 
+# --- Arguments ---
+DEV_PROFILE="${SPINIFEX_DEV_PROFILE:-}"
+DRY_RUN=false
+for arg in "$@"; do
+    case "$arg" in
+        --profile=*) DEV_PROFILE="${arg#*=}" ;;
+        --dry-run)   DRY_RUN=true ;;
+        --help|-h)
+            sed -n '2,/^set -euo/{/^set -euo/!p}' "$0"
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $arg (try --help)"
+            exit 1
+            ;;
+    esac
+done
+
+# --- Topology profile ---
+# Profiles only set DEV_* variables, and every one is honoured from the
+# environment first, so an operator can override a single plane without
+# copying a profile.
+if [ -n "$DEV_PROFILE" ]; then
+    PROFILE_FILE="$SCRIPT_DIR/dev-env/${DEV_PROFILE}.conf"
+    if [ ! -f "$PROFILE_FILE" ]; then
+        echo "❌ No such profile: $DEV_PROFILE ($PROFILE_FILE)"
+        echo "   Available:"
+        for f in "$SCRIPT_DIR"/dev-env/*.conf; do
+            [ -e "$f" ] || continue
+            echo "     $(basename "$f" .conf)"
+        done
+        exit 1
+    fi
+    # Environment wins: stash anything already set, source, then restore.
+    for v in DEV_WAN_BRIDGE DEV_LAN_BRIDGE DEV_VPC_BRIDGE DEV_MGMT_BRIDGE \
+             DEV_MGMT_CIDR DEV_UPLINK DEV_BIND_PLANES; do
+        if [ -n "${!v+x}" ]; then
+            eval "__override_$v=\${$v}"
+        fi
+    done
+    # shellcheck disable=SC1090  # profile path is resolved at runtime
+    . "$PROFILE_FILE"
+    for v in DEV_WAN_BRIDGE DEV_LAN_BRIDGE DEV_VPC_BRIDGE DEV_MGMT_BRIDGE \
+             DEV_MGMT_CIDR DEV_UPLINK DEV_BIND_PLANES; do
+        ov="__override_$v"
+        if [ -n "${!ov+x}" ]; then
+            eval "$v=\${$ov}"
+        fi
+    done
+    echo "==> Profile: $DEV_PROFILE ($PROFILE_FILE)"
+fi
+
+DEV_WAN_BRIDGE="${DEV_WAN_BRIDGE:-}"
+DEV_LAN_BRIDGE="${DEV_LAN_BRIDGE:-}"
+DEV_VPC_BRIDGE="${DEV_VPC_BRIDGE:-}"
+DEV_MGMT_BRIDGE="${DEV_MGMT_BRIDGE:-br-mgmt}"
+DEV_MGMT_CIDR="${DEV_MGMT_CIDR:-10.15.8.1/24}"
+DEV_UPLINK="${DEV_UPLINK:-bridged}"
+DEV_BIND_PLANES="${DEV_BIND_PLANES:-0}"
+
 # --- Guard: refuse to run on multi-node clusters ---
 if sudo test -f "$CONFIG_FILE"; then
     NODE_COUNT=$(sudo grep -cE '^\[nodes\.[^.]+\]' "$CONFIG_FILE" 2>/dev/null || echo 0)
@@ -110,6 +192,162 @@ if [ -n "$OPERATOR_EMAIL" ]; then
     echo "  operator email: $OPERATOR_EMAIL"
 fi
 
+# --- Resolve network planes ---
+# Everything below runs before the wipe so a bad topology fails while the
+# environment is still intact, and so --dry-run can print the real commands.
+
+# iface_ip <link> — first IPv4 address, empty when the link has none or does
+# not exist. The trailing `|| true` matters: under `set -o pipefail` a missing
+# link makes ip(8) fail the whole pipeline, which would abort the script here
+# instead of letting preflight report the plane properly.
+iface_ip() {
+    ip -4 -o addr show "$1" 2>/dev/null | awk 'NR==1 {split($4, a, "/"); print a[1]}' || true
+}
+
+# iface_prefix <link> — prefix length of the first IPv4 address.
+iface_prefix() {
+    ip -4 -o addr show "$1" 2>/dev/null | awk 'NR==1 {split($4, a, "/"); print a[2]}' || true
+}
+
+# Auto-detect the wan plane from the default route when no profile named it.
+WAN_IFACE=$(ip -4 route show default | awk '{print $5}' | head -1)
+WAN_GW=$(ip -4 route show default | awk '{print $3}' | head -1)
+WAN_BRIDGE="${DEV_WAN_BRIDGE:-$WAN_IFACE}"
+WAN_IP=$(iface_ip "$WAN_BRIDGE")
+WAN_PREFIX_HOST=$(iface_prefix "$WAN_BRIDGE")
+WAN_PREFIX_HOST="${WAN_PREFIX_HOST:-24}"
+
+# Collapse vpc <- lan <- wan. An unset role inherits the plane above it, which
+# is what makes one profile express one, two and three NIC layouts.
+LAN_BRIDGE="${DEV_LAN_BRIDGE:-$WAN_BRIDGE}"
+VPC_BRIDGE="${DEV_VPC_BRIDGE:-$LAN_BRIDGE}"
+LAN_IP=$(iface_ip "$LAN_BRIDGE")
+VPC_IP=$(iface_ip "$VPC_BRIDGE")
+
+printf '==> Planes\n'
+printf '      %-6s %-10s %s\n' wan "$WAN_BRIDGE" "${WAN_IP:-no address}"
+printf '      %-6s %-10s %s%s\n' lan "$LAN_BRIDGE" "${LAN_IP:-no address}" \
+    "$([ "$LAN_BRIDGE" = "$WAN_BRIDGE" ] && echo '  (folds onto wan)')"
+printf '      %-6s %-10s %s%s\n' vpc "$VPC_BRIDGE" "${VPC_IP:-no address}" \
+    "$([ "$VPC_BRIDGE" = "$LAN_BRIDGE" ] && echo '  (folds onto lan)')"
+printf '      %-6s %-10s %s\n' mgmt "$DEV_MGMT_BRIDGE" "$DEV_MGMT_CIDR"
+
+# --- Preflight ---
+PREFLIGHT_FAIL=0
+fail() { echo "❌ $*"; PREFLIGHT_FAIL=1; }
+
+if [ "$DEV_UPLINK" != "bridged" ] && [ "$DEV_UPLINK" != "nat" ]; then
+    fail "DEV_UPLINK must be 'bridged' or 'nat', got: $DEV_UPLINK"
+fi
+
+if [ "$DEV_UPLINK" = "bridged" ]; then
+    if [ -z "$WAN_BRIDGE" ]; then
+        fail "No wan plane: no default route and no DEV_WAN_BRIDGE."
+    elif ! ip link show "$WAN_BRIDGE" >/dev/null 2>&1; then
+        fail "wan bridge '$WAN_BRIDGE' does not exist."
+    fi
+fi
+
+# An addressless vpc plane is the silent failure this script exists to stop:
+# setup-ovn.sh would fall back to the default-route source IP and put Geneve
+# on WAN while reporting a successful auto-detection.
+if [ -z "$VPC_IP" ]; then
+    fail "vpc plane '$VPC_BRIDGE' has no IPv4 address — Geneve encap would silently fall back to WAN."
+fi
+
+# br-mgmt is the guest→host bridge and must be an OVS bridge. A plane bridge
+# sharing the name leaves setup-ovn.sh with a half-built bridge record.
+for plane_pair in "wan:$WAN_BRIDGE" "lan:$LAN_BRIDGE" "vpc:$VPC_BRIDGE"; do
+    if [ "${plane_pair#*:}" = "$DEV_MGMT_BRIDGE" ]; then
+        fail "${plane_pair%%:*} plane uses '$DEV_MGMT_BRIDGE', which is reserved for the guest→host bridge. Rename the plane bridge (br-wan/br-lan/br-vpc) or set DEV_MGMT_BRIDGE."
+    fi
+done
+
+if [ "$DEV_BIND_PLANES" = "1" ]; then
+    if [ -z "$LAN_IP" ]; then
+        fail "DEV_BIND_PLANES=1 but the lan plane '$LAN_BRIDGE' has no IPv4 address."
+    fi
+    if [ -z "$WAN_IP" ]; then
+        fail "DEV_BIND_PLANES=1 but the wan plane '$WAN_BRIDGE' has no IPv4 address to advertise."
+    fi
+    # Pinning --bind moves every service rendered as {{.BindIP}} onto the lan
+    # plane. awsgw and predastore must be on the wildcard first, or the public
+    # AWS API and S3 endpoint leave the wan plane with them.
+    TEMPLATE="$PROJECT_ROOT/cmd/spinifex/cmd/templates/spinifex.toml"
+    if [ -f "$TEMPLATE" ] && ! grep -q '^host = "0.0.0.0:9999"' "$TEMPLATE"; then
+        fail "DEV_BIND_PLANES=1 but $TEMPLATE still renders awsgw on {{.BindIP}}. This branch predates the wildcard public listeners — rebase, or set DEV_BIND_PLANES=0."
+    fi
+fi
+
+if [ "$PREFLIGHT_FAIL" -ne 0 ]; then
+    echo "   Preflight failed — no host state was touched."
+    exit 1
+fi
+
+# --- Assemble the commands both --dry-run and the real run use ---
+SETUP_OVN_FLAGS="--encap-ip=$VPC_IP --mgmt-bridge=$DEV_MGMT_BRIDGE --mgmt-cidr=$DEV_MGMT_CIDR"
+if [ "$DEV_UPLINK" = "nat" ]; then
+    # Routed NAT bridges no WAN NIC, and setup-ovn.sh rejects the WAN flags.
+    SETUP_OVN_FLAGS="--nat-uplink $SETUP_OVN_FLAGS"
+elif ip -d link show "$WAN_BRIDGE" 2>/dev/null | grep -q "bridge"; then
+    echo "==> WAN is a bridge: $WAN_BRIDGE (setup-ovn.sh links it via veth)"
+else
+    SETUP_OVN_FLAGS="--wan-bridge=br-wan --wan-iface=$WAN_BRIDGE $SETUP_OVN_FLAGS"
+    echo "==> WAN is physical: $WAN_BRIDGE (direct bridge br-wan)"
+fi
+
+# Mirror the ISO installer: internal services on lan, public dial target on
+# wan. --advertise must be explicit, because resolveAdvertiseIP echoes a
+# non-wildcard --bind straight back and would publish the internal address.
+BIND_INIT_ARGS=""
+if [ "$DEV_BIND_PLANES" = "1" ]; then
+    BIND_INIT_ARGS="--bind=$LAN_IP --cluster-bind=$LAN_IP --advertise=$WAN_IP"
+fi
+
+# --- Assemble external-networking args for spx admin init ---
+EXTERNAL_INIT_ARGS=""
+case "$EXT_MODE" in
+    nat)
+        # NAT gateway IP isn't captured from config (not stored in a
+        # predictable location); derive from WAN subnet — same rule as
+        # a fresh install. Operators who hand-edited the gateway IP will
+        # need to re-apply it after reset.
+        if [ -n "$WAN_GW" ]; then
+            IFS='.' read -r o1 o2 o3 _ <<< "$WAN_GW"
+            GATEWAY_IP="${o1}.${o2}.${o3}.200"
+            EXTERNAL_INIT_ARGS="--external-mode=nat --gateway-ip=$GATEWAY_IP --external-gateway=$WAN_GW --external-prefix-len=${EXT_PREFIX:-$WAN_PREFIX_HOST}"
+            echo "==> External mode: nat (gateway IP $GATEWAY_IP)"
+        fi
+        ;;
+    pool|*)
+        if [ -n "$POOL_START" ] && [ -n "$POOL_END" ]; then
+            EXTERNAL_INIT_ARGS="--external-mode=pool --external-pool=${POOL_START}-${POOL_END} --external-gateway=${EXT_GATEWAY:-$WAN_GW} --external-prefix-len=${EXT_PREFIX:-$WAN_PREFIX_HOST}"
+            echo "==> External mode: pool (preserved $POOL_START-$POOL_END)"
+        elif [ -n "$WAN_GW" ]; then
+            # Fresh box: derive pool from WAN subnet.
+            IFS='.' read -r o1 o2 o3 _ <<< "$WAN_GW"
+            POOL_START="${o1}.${o2}.${o3}.200"
+            POOL_END="${o1}.${o2}.${o3}.250"
+            EXTERNAL_INIT_ARGS="--external-mode=pool --external-pool=${POOL_START}-${POOL_END} --external-gateway=$WAN_GW --external-prefix-len=$WAN_PREFIX_HOST"
+            echo "==> External mode: pool (default $POOL_START-$POOL_END)"
+        fi
+        ;;
+esac
+
+EMAIL_INIT_ARG=""
+if [ -n "$OPERATOR_EMAIL" ]; then
+    EMAIL_INIT_ARG="--email=$OPERATOR_EMAIL"
+fi
+
+echo "==> Planned commands"
+echo "      setup-ovn.sh --management $SETUP_OVN_FLAGS"
+echo "      spx admin init --force --region $REGION --az $AZ --node node1 --nodes 1 $EXTERNAL_INIT_ARGS $BIND_INIT_ARGS $EMAIL_INIT_ARG"
+
+if [ "$DRY_RUN" = true ]; then
+    echo "==> Dry run — no host state touched."
+    exit 0
+fi
+
 # --- Shutdown services ---
 echo "==> Stopping services"
 sudo systemctl stop spinifex.target 2>/dev/null || true
@@ -172,6 +410,44 @@ if [ -e /etc/systemd/network/15-spinifex-veth-wan.netdev ] || \
     sudo networkctl reload 2>/dev/null || true
 fi
 
+# The mgmt address and the OVS-unmanaged marker are re-created by setup-ovn.sh
+# with whatever bridge/CIDR this run resolves. Leaving the old units behind
+# re-applies the previous run's address on the next boot.
+if [ -e /etc/systemd/network/10-spinifex-mgmt.network ] || \
+   [ -e /etc/systemd/network/05-spinifex-ovs-internal.network ]; then
+    echo "  Deleting mgmt + OVS-internal networkd units"
+    sudo rm -f /etc/systemd/network/10-spinifex-mgmt.network \
+               /etc/systemd/network/05-spinifex-ovs-internal.network
+    sudo networkctl reload 2>/dev/null || true
+fi
+
+# Routed-NAT artifacts from a previous --nat-uplink run. Without this, flipping
+# DEV_UPLINK back to bridged leaves a transit veth and masquerade rules that
+# keep NATting behind the reset's back.
+if ip link show spx-nat-host >/dev/null 2>&1; then
+    echo "  Deleting NAT transit veth: spx-nat-host"
+    sudo ip link del spx-nat-host 2>/dev/null || true
+fi
+if [ -e /etc/systemd/network/17-spinifex-nat.netdev ]; then
+    echo "  Deleting NAT transit persistence units"
+    sudo rm -f /etc/systemd/network/17-spinifex-nat.netdev \
+               /etc/systemd/network/17-spinifex-nat.network \
+               /etc/systemd/network/18-spinifex-nat-ovs.network
+    sudo networkctl reload 2>/dev/null || true
+fi
+
+# Masquerade + forward rules carry the spinifex-nat-egress comment, so the
+# rule spec can be read back out of iptables-save and deleted verbatim.
+for tbl in nat filter; do
+    while read -r spec; do
+        [ -n "$spec" ] || continue
+        echo "  Deleting iptables -t $tbl rule: $spec"
+        # shellcheck disable=SC2086  # spec is a rule argument list
+        sudo iptables -t "$tbl" -D $spec 2>/dev/null || true
+    done < <(sudo iptables-save -t "$tbl" 2>/dev/null |
+             grep -- 'spinifex-nat-egress' | sed -e 's/^-A //' -e 's/"//g')
+done
+
 # Legacy: clean up macvlan interfaces left over from pre-veth setup-ovn.sh
 # runs. Safe to remove once no operator system retains these.
 for iface in $(ip -o link show type macvlan 2>/dev/null | awk -F': ' '{print $2}' | grep '^spx-ext-'); do
@@ -195,69 +471,26 @@ fi
 echo "==> Rebuilding and reinstalling (dev-install.sh, setup-only mode)"
 DEV_INSTALL_SKIP_INIT=1 "$SCRIPT_DIR/dev-install.sh"
 
-# --- Detect WAN topology (bridge only — no macvlan) ---
-WAN_IFACE=$(ip -4 route show default | awk '{print $5}' | head -1)
-WAN_GW=$(ip -4 route show default | awk '{print $3}' | head -1)
-WAN_PREFIX_HOST=""
-if [ -n "$WAN_IFACE" ]; then
-    WAN_PREFIX_HOST=$(ip -4 -o addr show "$WAN_IFACE" 2>/dev/null | awk '{print $4}' | cut -d/ -f2 | head -1)
-fi
-WAN_PREFIX_HOST="${WAN_PREFIX_HOST:-24}"
-
-SETUP_OVN_FLAGS=""
-if [ -n "$WAN_IFACE" ]; then
-    if ip -d link show "$WAN_IFACE" 2>/dev/null | grep -q "bridge"; then
-        echo "==> WAN is a bridge: $WAN_IFACE (setup-ovn.sh auto-detects)"
-    else
-        SETUP_OVN_FLAGS="--wan-bridge=br-wan --wan-iface=$WAN_IFACE"
-        echo "==> WAN is physical: $WAN_IFACE (direct bridge br-wan)"
-    fi
-fi
-
 echo "==> Running setup-ovn.sh"
 # shellcheck disable=SC2086  # intentional word-splitting for flag list
 sudo /usr/local/share/spinifex/setup-ovn.sh --management $SETUP_OVN_FLAGS
 
-# --- Assemble external-networking args for spx admin init ---
-EXTERNAL_INIT_ARGS=""
-case "$EXT_MODE" in
-    nat)
-        # NAT gateway IP isn't captured from config (not stored in a
-        # predictable location); derive from WAN subnet — same rule as
-        # a fresh install. Operators who hand-edited the gateway IP will
-        # need to re-apply it after reset.
-        if [ -n "$WAN_GW" ]; then
-            IFS='.' read -r o1 o2 o3 _ <<< "$WAN_GW"
-            GATEWAY_IP="${o1}.${o2}.${o3}.200"
-            EXTERNAL_INIT_ARGS="--external-mode=nat --gateway-ip=$GATEWAY_IP --external-gateway=$WAN_GW --external-prefix-len=${EXT_PREFIX:-$WAN_PREFIX_HOST}"
-            echo "==> External mode: nat (gateway IP $GATEWAY_IP)"
-        fi
-        ;;
-    pool|*)
-        if [ -n "$POOL_START" ] && [ -n "$POOL_END" ]; then
-            EXTERNAL_INIT_ARGS="--external-mode=pool --external-pool=${POOL_START}-${POOL_END} --external-gateway=${EXT_GATEWAY:-$WAN_GW} --external-prefix-len=${EXT_PREFIX:-$WAN_PREFIX_HOST}"
-            echo "==> External mode: pool (preserved $POOL_START-$POOL_END)"
-        elif [ -n "$WAN_GW" ]; then
-            # Fresh box: derive pool from WAN subnet.
-            IFS='.' read -r o1 o2 o3 _ <<< "$WAN_GW"
-            POOL_START="${o1}.${o2}.${o3}.200"
-            POOL_END="${o1}.${o2}.${o3}.250"
-            EXTERNAL_INIT_ARGS="--external-mode=pool --external-pool=${POOL_START}-${POOL_END} --external-gateway=$WAN_GW --external-prefix-len=$WAN_PREFIX_HOST"
-            echo "==> External mode: pool (default $POOL_START-$POOL_END)"
-        fi
-        ;;
-esac
-
 # --- Initialize platform ---
 echo "==> Initializing platform (region=$REGION az=$AZ)"
-EMAIL_INIT_ARG=""
-if [ -n "$OPERATOR_EMAIL" ]; then
-    EMAIL_INIT_ARG="--email=$OPERATOR_EMAIL"
-fi
 # shellcheck disable=SC2086  # intentional word-splitting for arg list
 sudo /usr/local/bin/spx admin init --force \
     --region "$REGION" --az "$AZ" --node node1 --nodes 1 \
-    $EXTERNAL_INIT_ARGS $EMAIL_INIT_ARG
+    $EXTERNAL_INIT_ARGS $BIND_INIT_ARGS $EMAIL_INIT_ARG
+
+# --- Pin the mgmt bridge in the daemon config ---
+# spx admin init has no flag for it, and the daemon defaults to br-mgmt. A
+# profile that moved the guest→host bridge has to say so here, or system
+# instance TAPs attach to a bridge the host never addressed.
+if [ "$DEV_MGMT_BRIDGE" != "br-mgmt" ]; then
+    echo "==> Pinning daemon mgmt_bridge = $DEV_MGMT_BRIDGE"
+    sudo sed -i "/^\[nodes\..*\.daemon\]/,/^\[/ s|^dev_networking = .*|&\nmgmt_bridge = \"$DEV_MGMT_BRIDGE\"|" \
+        "$CONFIG_FILE"
+fi
 
 # --- Install CA cert into system trust store ---
 echo "==> Installing CA certificate"
@@ -312,4 +545,16 @@ aws_as_user ec2 authorize-security-group-ingress \
 echo "  Authorized SSH on $DEFAULT_SG_ID ($DEFAULT_VPC_ID)"
 
 # --- Smoke test ---
+# Exports the instance it launched so verify-network.sh can assert egress from
+# inside the guest rather than launching a second one.
 sudo -u "$INVOKING_USER" "$SCRIPT_DIR/smoke-test.sh"
+
+# --- Verify the datapath ---
+# The smoke test proves an instance booted and answered SSH from the host,
+# which never leaves the box. This asserts the planes actually carry what they
+# were asked to.
+DEV_WAN_BRIDGE="$WAN_BRIDGE" DEV_LAN_BRIDGE="$LAN_BRIDGE" \
+DEV_VPC_BRIDGE="$VPC_BRIDGE" DEV_MGMT_BRIDGE="$DEV_MGMT_BRIDGE" \
+DEV_MGMT_CIDR="$DEV_MGMT_CIDR" DEV_BIND_PLANES="$DEV_BIND_PLANES" \
+DEV_UPLINK="$DEV_UPLINK" \
+    sudo -E -u "$INVOKING_USER" "$SCRIPT_DIR/verify-network.sh"

scripts/setup-ovn.sh

@@ -37,6 +37,35 @@
 #                        the cluster; set ⇒ join it.
 #   --db-cluster-remote-addr=IP  An existing cluster DB node's IP to join. Omit
 #                        on the first (init) DB node that forms the cluster.
+#   --recreate-db        Destroy and recreate the NB/SB DBs so they can be
+#                        created in clustered format. Required when converting
+#                        a node that has already run a standalone ovn-central
+#                        (which is every node — the ovn-central package starts
+#                        one on install). DISCARDS ALL LOGICAL NETWORK STATE:
+#                        safe on a fresh node, destroys every VPC on a live one.
+#
+# Cluster Sizing — which DB topology to run:
+#
+#   Nodes  DB topology                              Tolerates
+#   1-2    standalone ovn-central on node 1         nothing
+#   3      RAFT across all 3                        1 node
+#   4+     RAFT across 3, remainder compute-only    1 node
+#
+#   Three nodes is the minimum recommended multi-server deployment.
+#
+#   NEVER run RAFT with 2 members. Quorum is a majority, so 2-of-2 tolerates
+#   zero failures — and it is strictly worse than standalone, because either
+#   node failing loses quorum rather than only node 1. Two-node deployments
+#   run standalone and accept the single point of failure.
+#
+#   Do not scale DB members past 3. RAFT write latency is bounded by the
+#   slowest member of the majority, and a 4th member buys no extra fault
+#   tolerance. Nodes beyond the third are compute-only, pointing at all three
+#   SB endpoints via the comma-separated --ovn-remote form.
+#
+#   Losing ovn-central does not stop the data plane: ovn-controller has already
+#   programmed flows into br-int, so running instances keep networking. Only
+#   control-plane change stops — new VPCs, instance launches, port bindings.
 #
 # WAN Bridge Auto-Detection:
 #   When no --wan-bridge is given, the script checks the default route interface:
@@ -90,6 +119,9 @@ ENCAP_IP=""
 # DBs run clustered; REMOTE_ADDR empty ⇒ create the cluster, set ⇒ join it.
 DB_CLUSTER_LOCAL_ADDR=""
 DB_CLUSTER_REMOTE_ADDR=""
+# Recreating the NB/SB DBs discards all logical network state, so it is opt-in.
+RECREATE_DB=false
+OVN_DBDIR="${OVN_DBDIR:-/var/lib/ovn}"
 # NODE_NAME is left empty by default. The chassis-id pin block at Step 4
 # only runs when --node-name=NAME is explicitly given. Passing nothing
 # preserves whatever system-id already lives in OVS (gold-image UUID,
@@ -115,6 +147,7 @@ for arg in "$@"; do
         --encap-ip=*)       ENCAP_IP="${arg#*=}" ;;
         --db-cluster-local-addr=*)  DB_CLUSTER_LOCAL_ADDR="${arg#*=}" ;;
         --db-cluster-remote-addr=*) DB_CLUSTER_REMOTE_ADDR="${arg#*=}" ;;
+        --recreate-db)      RECREATE_DB=true ;;
         --node-name=*)      NODE_NAME="${arg#*=}" ;;
         --help|-h)
             sed -n '3,/^set -e/{/^set -e/!p}' "$0"
@@ -138,6 +171,11 @@ if [ -n "$DB_CLUSTER_REMOTE_ADDR" ] && [ -z "$DB_CLUSTER_LOCAL_ADDR" ]; then
     echo "ERROR: --db-cluster-remote-addr requires --db-cluster-local-addr"
     exit 1
 fi
+if [ "$RECREATE_DB" = true ] && [ -z "$DB_CLUSTER_LOCAL_ADDR" ]; then
+    echo "ERROR: --recreate-db requires --db-cluster-local-addr (it exists to allow"
+    echo "       clustered DB creation over an existing standalone DB)"
+    exit 1
+fi
 
 # --- WAN bridge auto-detection ---
 # Determine the WAN bridge name and how to set it up.
@@ -307,6 +345,46 @@ if [ -d /etc/apparmor.d/local ]; then
 fi
 
 # --- Step 2: Enable services ---
+# ovn-ctl consults the RAFT flags only when it creates a database. The
+# ovn-central package starts a standalone ovsdb-server on install, so a
+# standalone-format DB is always already present by the time this script first
+# runs: without this check ovn-ctl serves that DB, silently ignores the cluster
+# flags, and the script reports success on a cluster that was never formed.
+ensure_clustered_db_storage() {
+    local db
+    local standalone=()
+    for db in "$OVN_DBDIR/ovnnb_db.db" "$OVN_DBDIR/ovnsb_db.db"; do
+        if [ -f "$db" ] && ! sudo ovsdb-tool db-is-clustered "$db" 2>/dev/null; then
+            standalone+=("$db")
+        fi
+    done
+    if [ ${#standalone[@]} -eq 0 ]; then
+        return 0
+    fi
+
+    if [ "$RECREATE_DB" != true ]; then
+        echo "ERROR: clustered DBs requested, but these are in standalone format:" >&2
+        printf '         %s\n' "${standalone[@]}" >&2
+        echo "" >&2
+        echo "  A clustered DB can only be created from scratch, so these must be" >&2
+        echo "  removed first. That DISCARDS ALL LOGICAL NETWORK STATE — every logical" >&2
+        echo "  switch, router, port and ACL, and so every VPC on this node." >&2
+        echo "" >&2
+        echo "  A freshly installed node has nothing to lose: re-run with --recreate-db." >&2
+        echo "  A node running workloads does: do not." >&2
+        exit 1
+    fi
+
+    echo "  --recreate-db: removing standalone DBs so ovn-ctl can create clustered ones"
+
+    # ovn-controller must go down with them. Left running, it reconnects to the
+    # fresh SB and re-registers under whatever system-id it started with — which
+    # on a renamed node is the old one, and that row then holds this node's encap
+    # IP so the correctly-named chassis can never commit. Step 5 restarts it.
+    sudo systemctl stop ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-central ovn-controller 2>/dev/null || true
+    sudo rm -f "${standalone[@]}"
+}
+
 echo ""
 echo "Step 2: Enabling services..."
 
@@ -318,6 +396,8 @@ if [ "$MANAGEMENT" = true ]; then
     sudo systemctl enable ovn-central
 
     if [ -n "$DB_CLUSTER_LOCAL_ADDR" ]; then
+        ensure_clustered_db_storage
+
         # Clustered NB/SB via native OVSDB RAFT. Both per-DB units source one
         # shared OVN_CTL_OPTS from /etc/default/ovn-central; each run_*_ovsdb
         # consumes only its own --db-{nb,sb}-* flags. RAFT ports default to NB
@@ -379,6 +459,20 @@ EOF
         sleep 1
     done
 
+    # Verify rather than assume: a DB that came up standalone despite the RAFT
+    # flags is the exact failure this guards against, and it stays invisible
+    # until a node goes down and the cluster turns out not to exist.
+    if [ -n "$DB_CLUSTER_LOCAL_ADDR" ]; then
+        for db in "$OVN_DBDIR/ovnnb_db.db" "$OVN_DBDIR/ovnsb_db.db"; do
+            if ! sudo ovsdb-tool db-is-clustered "$db" 2>/dev/null; then
+                echo "ERROR: $db is not in clustered format after startup." >&2
+                echo "       The RAFT configuration did not take effect." >&2
+                exit 1
+            fi
+        done
+        echo "  DB storage:       clustered (NB + SB verified)"
+    fi
+
     # Wait for the Southbound DB to be serving before ovn-controller (Step 5)
     # dials it. On a single node ovn-controller races a fresh SB RAFT election
     # with no join step to absorb the window — attaching before the leader is
@@ -764,6 +858,22 @@ if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
     echo ""
     echo "Step 3d: Configuring management bridge ($MGMT_BRIDGE)..."
 
+    # --may-exist is idempotent against OVS, not against the kernel. When a
+    # Linux bridge already holds the name, OVS creates the bridge record but
+    # cannot create the internal netdev, leaving a half-built bridge whose
+    # only symptom is an error buried in `ovs-vsctl show`.
+    if ip link show "$MGMT_BRIDGE" >/dev/null 2>&1 && \
+       ! sudo ovs-vsctl br-exists "$MGMT_BRIDGE" 2>/dev/null; then
+        echo "  ERROR: '$MGMT_BRIDGE' already exists as a non-OVS link"
+        ip -d link show "$MGMT_BRIDGE" | head -2
+        echo ""
+        echo "  The management bridge must be an OVS bridge. Either:"
+        echo "    1. Point this script elsewhere:  --mgmt-bridge=<name>"
+        echo "    2. Rename the existing link (a plane bridge belongs on br-wan/br-lan/br-vpc)"
+        echo "    3. Skip mgmt provisioning entirely:  --no-mgmt-bridge"
+        exit 1
+    fi
+
     sudo ovs-vsctl --may-exist add-br "$MGMT_BRIDGE"
     sudo ovs-vsctl set Bridge "$MGMT_BRIDGE" \
         fail-mode=standalone \
@@ -846,9 +956,21 @@ fi
 # refused to start. Preserving the on-disk value is always safe — any
 # caller that needs IPsec cert identity matching pins it themselves.
 if [ -n "$NODE_NAME" ]; then
+    OLD_ID=$(sudo ovs-vsctl get Open_vSwitch . external_ids:system-id 2>/dev/null | tr -d '"')
     echo "$NODE_NAME" | sudo tee /etc/openvswitch/system-id.conf >/dev/null
     sudo ovs-vsctl set Open_vSwitch . external_ids:system-id="$NODE_NAME"
     echo "  system-id:      $NODE_NAME (pinned via --node-name)"
+
+    # A renamed chassis cannot register while the row it left behind still
+    # holds this node's encap IP: ovn-controller loops on "OVNSB commit
+    # failed" and never appears in `ovn-sbctl show`. The stale row owns no
+    # state worth keeping — ovn-controller rebuilds everything on register —
+    # but it must go before Step 5 restarts the controller under the new name.
+    if [ -n "$OLD_ID" ] && [ "$OLD_ID" != "$NODE_NAME" ]; then
+        if sudo ovn-sbctl --db="$OVN_REMOTE" --timeout=10 chassis-del "$OLD_ID" 2>/dev/null; then
+            echo "  stale chassis:  removed '$OLD_ID' (renamed to $NODE_NAME)"
+        fi
+    fi
 else
     CURRENT_ID=$(sudo ovs-vsctl get Open_vSwitch . external_ids:system-id 2>/dev/null | tr -d '"')
     echo "  system-id:      ${CURRENT_ID:-<unset>} (preserved; no --node-name given)"

scripts/setup.sh

@@ -320,6 +320,14 @@ install_apt_deps() {
             > /dev/null
 
         info "System dependencies installed"
+
+        # Only on a machine that already boots from ZFS. Pulling zfsutils-linux
+        # in unconditionally would drag zfs-dkms onto every dev box and rebuild
+        # the module on each kernel upgrade for no benefit.
+        if [ "$(findmnt -no FSTYPE /)" = "zfs" ] && ! command -v zpool >/dev/null 2>&1; then
+            info "ZFS root detected, installing zfsutils-linux..."
+            DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq zfsutils-linux > /dev/null
+        fi
     fi
 
     # Mask the standalone dhcpcd.service auto-enabled on Debian Trixie when
@@ -467,14 +475,16 @@ create_directories() {
     $SUDO chmod 0750 /var/lib/spinifex
     $SUDO chown "root:$SPINIFEX_GROUP" /var/lib/spinifex
 
-    # Symlink so services that expect BaseDir/config/ can find /etc/spinifex/
-    if [ ! -e /var/lib/spinifex/config ]; then
-        $SUDO ln -s /etc/spinifex /var/lib/spinifex/config
+    # Symlink so services that expect BaseDir/config/ can find /etc/spinifex/.
+    # -n so a re-run replaces the link rather than creating one inside the
+    # directory it already points at.
+    if $SUDO test ! -e /var/lib/spinifex/config; then
+        $SUDO ln -sfn /etc/spinifex /var/lib/spinifex/config
     fi
 
     # Symlink so services that write logs to BaseDir/logs/ use /var/log/spinifex/
-    if [ ! -e /var/lib/spinifex/logs ]; then
-        $SUDO ln -s /var/log/spinifex /var/lib/spinifex/logs
+    if $SUDO test ! -e /var/lib/spinifex/logs; then
+        $SUDO ln -sfn /var/log/spinifex /var/lib/spinifex/logs
     fi
 
     $SUDO mkdir -p /var/log/spinifex
@@ -559,9 +569,14 @@ TMPEOF
     $SUDO chown "spinifex-ui:$SPINIFEX_GROUP" /var/lib/spinifex/spinifex-ui
     $SUDO chmod 0700 /var/lib/spinifex/spinifex-ui
 
-    # Symlink so awsgw's {BaseDir}/config/ paths resolve to /etc/spinifex/
-    if [ ! -e /var/lib/spinifex/awsgw/config ]; then
-        $SUDO ln -s /etc/spinifex /var/lib/spinifex/awsgw/config
+    # Symlink so awsgw's {BaseDir}/config/ paths resolve to /etc/spinifex/.
+    # The test runs under $SUDO because the parent is 0700 spinifex-gw: an
+    # unprivileged -e cannot stat through it and always reports "missing", so
+    # every re-run reached the ln below. That ln then resolved through the
+    # existing link and tried to create /etc/spinifex/spinifex, which
+    # CreateServiceDirectories already owns — failing the whole install.
+    if $SUDO test ! -e /var/lib/spinifex/awsgw/config; then
+        $SUDO ln -sfn /etc/spinifex /var/lib/spinifex/awsgw/config
     fi
 
     # Service helper scripts (root-owned, group-executable by all service users)

scripts/smoke-test.sh

@@ -7,6 +7,18 @@
 # Assumes services are running and AWS_PROFILE=spinifex is configured
 # by a prior 'spx admin init'.
 #
+# Options:
+#   --create-vpc   Build the VPC, internet gateway, public subnet, route table
+#                  and security group first, following the sequence in
+#                  docs/admin/setting-up-your-cluster, and tear them down at the
+#                  end. Without this the test launches into whatever public
+#                  subnet already exists, which only works where a usable
+#                  default VPC is present.
+#   --nodes N      Cluster size. Above 1, launch N instances into a spread
+#                  placement group so each one lands on a different physical
+#                  server, and verify that it did.
+#   --keep         Leave the created VPC, placement group and instances in place.
+#
 # GPU passthrough test (opt-in):
 #   TEST_GPU=1 scripts/smoke-test.sh
 #   TEST_GPU=1 GPU_VENDOR_ID=10de GPU_DEVICE_ID=2236 scripts/smoke-test.sh
@@ -23,6 +35,25 @@ GPU_VENDOR_ID="${GPU_VENDOR_ID:-}"
 GPU_DEVICE_ID="${GPU_DEVICE_ID:-}"
 GPU_FAMILY="${GPU_FAMILY:-g5}"
 
+CREATE_VPC=0
+NODES=1
+KEEP=0
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --create-vpc) CREATE_VPC=1; shift ;;
+        --nodes)      NODES="$2"; shift 2 ;;
+        --keep)       KEEP=1; shift ;;
+        -h|--help)    sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
+        *)            echo "❌ Unknown option: $1 (try --help)" >&2; exit 2 ;;
+    esac
+done
+
+[[ "$NODES" =~ ^[0-9]+$ ]] && [ "$NODES" -ge 1 ] || { echo "❌ --nodes must be >= 1" >&2; exit 2; }
+
+VPC_CIDR="${VPC_CIDR:-10.200.0.0/16}"
+SUBNET_CIDR="${SUBNET_CIDR:-10.200.1.0/24}"
+
 export AWS_PROFILE=spinifex
 
 # AWS commands run as the invoking user so their ~/.aws/config (spinifex profile
@@ -133,6 +164,93 @@ else
     fi
 fi
 
+# --- Build the VPC topology (--create-vpc) ---
+# Follows docs/admin/setting-up-your-cluster step for step, so a pass here means
+# the documented sequence works and not merely that some pre-existing subnet does.
+CREATED_VPC_ID=""
+CREATED_IGW_ID=""
+CREATED_SUBNET_ID=""
+CREATED_RTB_ID=""
+CREATED_SG_ID=""
+CREATED_PG_NAME=""
+LAUNCHED_IDS=""
+
+# Torn down in reverse creation order, on failure as well as success: resources
+# left behind by a failed run are ones the next run trips over. The placement
+# group only deletes once its instances are gone, hence the wait.
+cleanup_resources() {
+    [ "$KEEP" = "1" ] && { echo "==> --keep: leaving the created resources in place"; return 0; }
+    echo "==> Cleaning up"
+    if [ -n "$LAUNCHED_IDS" ]; then
+        # shellcheck disable=SC2086  # deliberate word-splitting of the id list
+        aws_as_user ec2 terminate-instances --instance-ids $LAUNCHED_IDS >/dev/null 2>&1 || true
+        for _i in $(seq 1 60); do
+            # shellcheck disable=SC2086
+            _left=$(aws_as_user ec2 describe-instances --instance-ids $LAUNCHED_IDS \
+                --query 'Reservations[].Instances[?State.Name!=`terminated`].InstanceId' \
+                --output text 2>/dev/null || true)
+            [ -z "$_left" ] && break
+            sleep 2
+        done
+    fi
+    [ -n "$CREATED_PG_NAME" ] && aws_as_user ec2 delete-placement-group --group-name "$CREATED_PG_NAME" >/dev/null 2>&1 || true
+    [ -n "$CREATED_RTB_ID" ] && aws_as_user ec2 delete-route-table --route-table-id "$CREATED_RTB_ID" >/dev/null 2>&1 || true
+    [ -n "$CREATED_SUBNET_ID" ] && aws_as_user ec2 delete-subnet --subnet-id "$CREATED_SUBNET_ID" >/dev/null 2>&1 || true
+    if [ -n "$CREATED_IGW_ID" ]; then
+        aws_as_user ec2 detach-internet-gateway --internet-gateway-id "$CREATED_IGW_ID" --vpc-id "$CREATED_VPC_ID" >/dev/null 2>&1 || true
+        aws_as_user ec2 delete-internet-gateway --internet-gateway-id "$CREATED_IGW_ID" >/dev/null 2>&1 || true
+    fi
+    [ -n "$CREATED_VPC_ID" ] && aws_as_user ec2 delete-vpc --vpc-id "$CREATED_VPC_ID" >/dev/null 2>&1 || true
+    echo "  Removed"
+}
+
+if [ "$CREATE_VPC" = "1" ] || [ "$NODES" -gt 1 ]; then
+    trap cleanup_resources EXIT
+fi
+
+if [ "$CREATE_VPC" = "1" ]; then
+    echo "==> Creating VPC $VPC_CIDR"
+    CREATED_VPC_ID=$(aws_as_user ec2 create-vpc --cidr-block "$VPC_CIDR" \
+        --query 'Vpc.VpcId' --output text)
+    [ -n "$CREATED_VPC_ID" ] && [ "$CREATED_VPC_ID" != "None" ] || { echo "❌ create-vpc failed"; exit 1; }
+    echo "  VPC: $CREATED_VPC_ID"
+
+    echo "==> Creating and attaching internet gateway"
+    CREATED_IGW_ID=$(aws_as_user ec2 create-internet-gateway \
+        --query 'InternetGateway.InternetGatewayId' --output text)
+    aws_as_user ec2 attach-internet-gateway \
+        --internet-gateway-id "$CREATED_IGW_ID" --vpc-id "$CREATED_VPC_ID" >/dev/null
+    echo "  IGW: $CREATED_IGW_ID"
+
+    echo "==> Creating public subnet $SUBNET_CIDR"
+    CREATED_SUBNET_ID=$(aws_as_user ec2 create-subnet \
+        --vpc-id "$CREATED_VPC_ID" --cidr-block "$SUBNET_CIDR" \
+        --query 'Subnet.SubnetId' --output text)
+    CREATED_RTB_ID=$(aws_as_user ec2 create-route-table --vpc-id "$CREATED_VPC_ID" \
+        --query 'RouteTable.RouteTableId' --output text)
+    aws_as_user ec2 create-route --route-table-id "$CREATED_RTB_ID" \
+        --destination-cidr-block 0.0.0.0/0 --gateway-id "$CREATED_IGW_ID" >/dev/null
+    aws_as_user ec2 associate-route-table \
+        --route-table-id "$CREATED_RTB_ID" --subnet-id "$CREATED_SUBNET_ID" >/dev/null
+    aws_as_user ec2 modify-subnet-attribute \
+        --subnet-id "$CREATED_SUBNET_ID" --map-public-ip-on-launch >/dev/null
+    echo "  Subnet: $CREATED_SUBNET_ID  route table: $CREATED_RTB_ID"
+
+    # The default SG denies all inbound, so without this the SSH probe below
+    # times out on an instance that is otherwise perfectly healthy.
+    echo "==> Authorizing SSH and ICMP on the default security group"
+    CREATED_SG_ID=$(aws_as_user ec2 describe-security-groups \
+        --filters "Name=vpc-id,Values=$CREATED_VPC_ID" "Name=group-name,Values=default" \
+        --query 'SecurityGroups[0].GroupId' --output text)
+    [ -n "$CREATED_SG_ID" ] && [ "$CREATED_SG_ID" != "None" ] ||
+        { echo "❌ No default security group in $CREATED_VPC_ID"; exit 1; }
+    aws_as_user ec2 authorize-security-group-ingress \
+        --group-id "$CREATED_SG_ID" --protocol tcp --port 22 --cidr 0.0.0.0/0 >/dev/null 2>&1 || true
+    aws_as_user ec2 authorize-security-group-ingress \
+        --group-id "$CREATED_SG_ID" --protocol icmp --port -1 --cidr 0.0.0.0/0 >/dev/null 2>&1 || true
+    echo "  Security group: $CREATED_SG_ID"
+fi
+
 # --- Launch smoke-test instance ---
 echo "==> Launching smoke-test instance"
 if [[ "$TEST_GPU" == "1" ]]; then
@@ -152,22 +270,50 @@ if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
     exit 1
 fi
 
-SUBNET_ID=$(aws_as_user ec2 describe-subnets --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId | [0]' --output text)
+if [ -n "$CREATED_SUBNET_ID" ]; then
+    SUBNET_ID="$CREATED_SUBNET_ID"
+else
+    SUBNET_ID=$(aws_as_user ec2 describe-subnets --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId | [0]' --output text)
+fi
 if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
     echo "❌ No subnet found"
     exit 1
 fi
 
-echo "  AMI: $AMI_ID  type: $INSTANCE_TYPE  subnet: $SUBNET_ID"
+# On a cluster, launch through a spread placement group. Spread is strict
+# 1-per-node, so the API itself refuses the launch with
+# InsufficientInstanceCapacity unless it can put every instance on a different
+# physical server — which makes the launch succeeding the assertion, rather
+# than something to be checked afterwards and hoped for.
+PLACEMENT_ARGS=()
+if [ "$NODES" -gt 1 ]; then
+    CREATED_PG_NAME="spinifex-smoke-spread-$$"
+    echo "==> Creating spread placement group $CREATED_PG_NAME"
+    aws_as_user ec2 create-placement-group \
+        --group-name "$CREATED_PG_NAME" --strategy spread >/dev/null
+    PLACEMENT_ARGS=(--placement "GroupName=$CREATED_PG_NAME")
+fi
 
-INSTANCE_ID=$(aws_as_user ec2 run-instances \
+echo "  AMI: $AMI_ID  type: $INSTANCE_TYPE  subnet: $SUBNET_ID  count: $NODES"
+
+if ! ALL_IDS=$(aws_as_user ec2 run-instances \
     --image-id "$AMI_ID" \
     --instance-type "$INSTANCE_TYPE" \
     --key-name spinifex-key \
     --subnet-id "$SUBNET_ID" \
-    --count 1 \
-    --query 'Instances[0].InstanceId' --output text)
+    --count "$NODES" \
+    "${PLACEMENT_ARGS[@]}" \
+    --query 'Instances[].InstanceId' --output text); then
+    if [ "$NODES" -gt 1 ]; then
+        echo "❌ run-instances failed. On a spread placement group this usually means"
+        echo "   fewer than $NODES nodes had capacity, so the cluster could not place one"
+        echo "   instance per physical server. Check 'sudo spx get nodes'."
+    fi
+    exit 1
+fi
+LAUNCHED_IDS="$ALL_IDS"
 
+INSTANCE_ID=$(awk '{print $1}' <<<"$ALL_IDS")
 if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" = "None" ] || [ "$INSTANCE_ID" = "null" ]; then
     echo "❌ run-instances returned no InstanceId"
     exit 1
@@ -277,3 +423,34 @@ if [[ "$TEST_GPU" == "1" ]]; then
 else
     echo "✅ Smoke test passed — instance $INSTANCE_ID launched, running, and SSH-verified"
 fi
+
+# --- Multi-node placement ---
+# The spread group already refused to launch unless one instance could go on
+# each node, so this confirms the reservation actually became distinct physical
+# placement rather than re-testing the scheduler.
+if [ "$NODES" -gt 1 ]; then
+    echo "==> Confirming placement across $NODES nodes"
+    # spx colourises unconditionally, including down a pipe. Field 1 is the
+    # instance id and field 8 the node, in the ' | ' separated table.
+    PLACED=""
+    for _i in $(seq 1 30); do
+        PLACEMENT=$(sudo /usr/local/bin/spx get vms 2>/dev/null |
+            sed 's/\x1b\[[0-9;]*m//g' |
+            awk -F'|' 'NR > 1 {gsub(/ /, "", $1); gsub(/ /, "", $8); if ($1 != "") print $1, $8}')
+        PLACED=$(tr '\t' '\n' <<<"$ALL_IDS" | while read -r id; do
+            [ -n "$id" ] && grep "^$id " <<<"$PLACEMENT" | awk '{print $2}'
+        done)
+        [ "$(grep -c . <<<"$PLACED")" -eq "$NODES" ] && break
+        sleep 2
+    done
+
+    sort <<<"$PLACED" | uniq -c | awk '{printf "  %s: %s instance(s)\n", $2, $1}'
+    DISTINCT=$(sort -u <<<"$PLACED" | grep -c . || true)
+    if [ "$DISTINCT" -ne "$NODES" ]; then
+        echo "❌ $NODES instances landed on $DISTINCT node(s), expected one each."
+        echo "   The spread placement group reserved distinct nodes, so the instances"
+        echo "   did not end up where the reservation said they would."
+        exit 1
+    fi
+    echo "✅ Placement confirmed — one instance on each of $DISTINCT physical nodes"
+fi

scripts/verify-network.sh

@@ -0,0 +1,262 @@
+#!/bin/bash
+# verify-network.sh — Assert a dev node's datapath matches its declared planes.
+#
+# smoke-test.sh proves an instance booted, got a public IP and answered SSH
+# from the host. None of that traffic leaves the box, so it passes on a node
+# whose EIPs are unreachable from the internet, whose Geneve rides the WAN
+# plane, or whose NATS listens on the public interface. This asserts the
+# things that are actually load-bearing.
+#
+# Run standalone or from reset-dev-env.sh, which exports the resolved planes.
+# Without those, planes are auto-detected the same way the reset does.
+#
+# Usage:
+#   ./scripts/verify-network.sh
+#   DEV_VPC_BRIDGE=br-vpc DEV_BIND_PLANES=1 ./scripts/verify-network.sh
+#
+# Environment:
+#   DEV_WAN_BRIDGE / DEV_LAN_BRIDGE / DEV_VPC_BRIDGE   plane bridges
+#   DEV_MGMT_BRIDGE / DEV_MGMT_CIDR                    guest→host bridge
+#   DEV_UPLINK        'bridged' (default) or 'nat'
+#   DEV_BIND_PLANES   1 to additionally assert the service bind matrix
+#   SKIP_EGRESS       1 to skip the in-guest egress probe (no instance running)
+#
+# Exits non-zero on the first failed assertion class, after running them all.
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+INVOKING_USER="${SUDO_USER:-$(id -un)}"
+INVOKING_HOME=$(getent passwd "$INVOKING_USER" | cut -d: -f6)
+aws_as_user() { sudo -u "$INVOKING_USER" env HOME="$INVOKING_HOME" AWS_PROFILE=spinifex aws "$@"; }
+
+FAILURES=0
+pass() { printf '  \033[32m✓\033[0m %s\n' "$*"; }
+fail() { printf '  \033[31m✗\033[0m %s\n' "$*"; FAILURES=$((FAILURES + 1)); }
+skip() { printf '  \033[33m-\033[0m %s\n' "$*"; }
+
+iface_ip() {
+    ip -4 -o addr show "$1" 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1
+}
+
+# --- Resolve planes (same collapse the reset applies) ---
+WAN_BRIDGE="${DEV_WAN_BRIDGE:-$(ip -4 route show default | awk '{print $5}' | head -1)}"
+LAN_BRIDGE="${DEV_LAN_BRIDGE:-$WAN_BRIDGE}"
+VPC_BRIDGE="${DEV_VPC_BRIDGE:-$LAN_BRIDGE}"
+MGMT_BRIDGE="${DEV_MGMT_BRIDGE:-br-mgmt}"
+MGMT_CIDR="${DEV_MGMT_CIDR:-10.15.8.1/24}"
+UPLINK="${DEV_UPLINK:-bridged}"
+BIND_PLANES="${DEV_BIND_PLANES:-0}"
+
+WAN_IP=$(iface_ip "$WAN_BRIDGE")
+LAN_IP=$(iface_ip "$LAN_BRIDGE")
+VPC_IP=$(iface_ip "$VPC_BRIDGE")
+
+echo "==> Planes: wan=$WAN_BRIDGE($WAN_IP) lan=$LAN_BRIDGE($LAN_IP) vpc=$VPC_BRIDGE($VPC_IP)"
+
+# --- OVN chassis configuration ---
+echo "==> OVN chassis"
+ENCAP_IP=$(sudo ovs-vsctl get Open_vSwitch . external_ids:ovn-encap-ip 2>/dev/null | tr -d '"')
+if [ "$ENCAP_IP" = "$VPC_IP" ]; then
+    pass "ovn-encap-ip is the vpc plane ($ENCAP_IP)"
+else
+    fail "ovn-encap-ip is $ENCAP_IP, expected the vpc plane $VPC_IP — Geneve is on the wrong plane"
+fi
+
+MAPPINGS=$(sudo ovs-vsctl get Open_vSwitch . external_ids:ovn-bridge-mappings 2>/dev/null | tr -d '"')
+EXT_BR="${MAPPINGS#external:}"
+if [ -z "$MAPPINGS" ]; then
+    fail "ovn-bridge-mappings is unset — no external network"
+elif sudo ovs-vsctl br-exists "$EXT_BR" 2>/dev/null; then
+    pass "ovn-bridge-mappings → $MAPPINGS"
+else
+    fail "ovn-bridge-mappings names '$EXT_BR', which is not an OVS bridge"
+fi
+
+# --- Plane bridges and the uplink ---
+echo "==> Bridges"
+for pair in "wan:$WAN_BRIDGE:$WAN_IP" "lan:$LAN_BRIDGE:$LAN_IP" "vpc:$VPC_BRIDGE:$VPC_IP"; do
+    plane="${pair%%:*}"; rest="${pair#*:}"; br="${rest%%:*}"; expect="${rest#*:}"
+    if [ -z "$expect" ]; then
+        fail "$plane plane '$br' has no IPv4 address"
+    elif ip link show "$br" 2>/dev/null | grep -q "state UP\|UP,LOWER_UP"; then
+        pass "$plane plane $br up with $expect"
+    else
+        fail "$plane plane '$br' is not up"
+    fi
+done
+
+if [ "$UPLINK" = "bridged" ] && [ "$EXT_BR" = "br-ext" ]; then
+    # A Linux WAN bridge is linked to OVS by a veth pair; the OVS end must be
+    # a port on the external bridge or nothing reaches the physical network.
+    if sudo ovs-vsctl list-ports br-ext 2>/dev/null | grep -q '^veth-wan-ovs$'; then
+        pass "veth uplink veth-wan-ovs is a port on br-ext"
+    else
+        fail "br-ext has no veth-wan-ovs port — the WAN uplink is not connected"
+    fi
+fi
+
+# --- Guest→host management bridge ---
+echo "==> Management bridge"
+if ! sudo ovs-vsctl br-exists "$MGMT_BRIDGE" 2>/dev/null; then
+    fail "$MGMT_BRIDGE is not an OVS bridge"
+elif sudo ovs-vsctl show 2>/dev/null | grep -A3 "Bridge $MGMT_BRIDGE" | grep -q "error:"; then
+    # The half-built state: OVS holds the record but a Linux link owns the
+    # name, so the internal netdev was never created.
+    fail "$MGMT_BRIDGE has a port error — a non-OVS link is holding the name"
+else
+    MGMT_ACTUAL=$(ip -4 -o addr show "$MGMT_BRIDGE" 2>/dev/null | awk '{print $4}' | head -1)
+    if [ "$MGMT_ACTUAL" = "$MGMT_CIDR" ]; then
+        pass "$MGMT_BRIDGE is OVS with $MGMT_ACTUAL"
+    else
+        fail "$MGMT_BRIDGE carries ${MGMT_ACTUAL:-no address}, expected $MGMT_CIDR — system instances cannot reach the host"
+    fi
+fi
+
+# --- No VPC gateway consumes a public address ---
+# Distributed NAT leaves the gateway LRP link-local. A router port holding a
+# pool address means centralised NAT crept back and each VPC is burning one.
+echo "==> Gateway router ports"
+if command -v ovn-nbctl >/dev/null 2>&1; then
+    GW_ADDRS=$(sudo ovn-nbctl --bare --columns=networks list Logical_Router_Port 2>/dev/null |
+               tr ' ' '\n' | grep -v '^$' || true)
+    OFFENDERS=""
+    if [ -n "$WAN_IP" ] && [ -n "$GW_ADDRS" ]; then
+        WAN_PREFIX3=$(echo "$WAN_IP" | cut -d. -f1-3)
+        OFFENDERS=$(echo "$GW_ADDRS" | grep "^${WAN_PREFIX3}\." || true)
+    fi
+    if [ -z "$OFFENDERS" ]; then
+        pass "no logical router port holds a WAN-subnet address"
+    else
+        fail "logical router ports hold WAN-subnet addresses (centralised NAT regression): $(echo "$OFFENDERS" | tr '\n' ' ')"
+    fi
+else
+    skip "ovn-nbctl not present"
+fi
+
+# --- EIP advertisement ---
+# Something must answer ARP for the EIP or inbound traffic dies. Which
+# something depends on the NAT mode, so read it off the rule rather than
+# assuming: distributed NAT advertises from the chassis hosting the VM via
+# external_mac + logical_port, centralised NAT from the gateway port's
+# nat_addresses. Asserting the centralised mechanism against a distributed
+# deployment fails a node whose datapath is entirely correct.
+echo "==> EIP advertisement"
+INSTANCE_IP=$(aws_as_user ec2 describe-instances \
+    --filters "Name=instance-state-name,Values=running" \
+    --query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null || true)
+if [ -z "$INSTANCE_IP" ] || [ "$INSTANCE_IP" = "None" ]; then
+    skip "no running instance with a public IP"
+elif ! command -v ovn-nbctl >/dev/null 2>&1; then
+    skip "ovn-nbctl not present"
+else
+    NAT_TYPE=$(sudo ovn-nbctl --bare --columns=type find NAT external_ip="$INSTANCE_IP" 2>/dev/null || true)
+    NAT_MAC=$(sudo ovn-nbctl --bare --columns=external_mac find NAT external_ip="$INSTANCE_IP" 2>/dev/null || true)
+    NAT_PORT=$(sudo ovn-nbctl --bare --columns=logical_port find NAT external_ip="$INSTANCE_IP" 2>/dev/null || true)
+
+    if [ -z "$NAT_TYPE" ]; then
+        fail "no NAT rule for EIP $INSTANCE_IP"
+    elif [ -n "$NAT_MAC" ] && [ -n "$NAT_PORT" ]; then
+        pass "EIP $INSTANCE_IP advertised by the VM's chassis ($NAT_TYPE, mac $NAT_MAC on $NAT_PORT)"
+    elif ! command -v ovn-sbctl >/dev/null 2>&1; then
+        skip "centralised NAT, but ovn-sbctl is not present to check nat_addresses"
+    else
+        # Centralised: the EIP must be on a router-type Port_Binding. OVN
+        # ignores nat_addresses on the localnet port, so nothing would ARP.
+        NAT_ADDR_TYPE=$(sudo ovn-sbctl --bare --columns=type,nat_addresses list Port_Binding 2>/dev/null |
+                        paste - - | grep -F "$INSTANCE_IP" | awk '{print $1}' | head -1)
+        if [ -z "$NAT_ADDR_TYPE" ]; then
+            fail "EIP $INSTANCE_IP has no external_mac and is in no nat_addresses — nothing will ARP for it"
+        elif [ "$NAT_ADDR_TYPE" = "localnet" ]; then
+            fail "EIP $INSTANCE_IP is advertised on the localnet port, where OVN ignores it"
+        else
+            pass "EIP $INSTANCE_IP is in nat_addresses on a $NAT_ADDR_TYPE port"
+        fi
+    fi
+fi
+
+# --- Guest egress, observed on the wire ---
+# The assertion the smoke test cannot make: traffic that actually leaves the
+# box, with the EIP as source rather than the host's own address.
+echo "==> Guest egress"
+SSH_KEY="$INVOKING_HOME/.ssh/spinifex-key"
+if [ "${SKIP_EGRESS:-0}" = "1" ]; then
+    skip "SKIP_EGRESS=1"
+elif [ -z "$INSTANCE_IP" ] || [ "$INSTANCE_IP" = "None" ] || [ ! -f "$SSH_KEY" ]; then
+    skip "no running instance or SSH key"
+elif ! command -v tcpdump >/dev/null 2>&1; then
+    skip "tcpdump not present"
+else
+    CAP=$(mktemp)
+    # Capture only the outbound leg. Matching on the instance address instead
+    # would also catch echo replies and ICMP redirects from the upstream
+    # router, which can fill the count before a single request is seen.
+    sudo timeout 15 tcpdump -l -n -i "$WAN_BRIDGE" -c 3 \
+        "icmp and dst host 8.8.8.8" > "$CAP" 2>/dev/null &
+    TCPDUMP_PID=$!
+    sleep 2
+    EGRESS_OUT=$(sudo -u "$INVOKING_USER" ssh -o StrictHostKeyChecking=no \
+        -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes \
+        -i "$SSH_KEY" "ubuntu@$INSTANCE_IP" 'ping -c 3 -W 2 8.8.8.8' 2>&1 || true)
+    wait $TCPDUMP_PID 2>/dev/null
+
+    if echo "$EGRESS_OUT" | grep -q '[1-3] received'; then
+        pass "guest reached 8.8.8.8"
+    else
+        fail "guest could not reach 8.8.8.8: $(echo "$EGRESS_OUT" | tail -2 | tr '\n' ' ')"
+    fi
+
+    # The source address on the wire is what distinguishes a 1:1 EIP from the
+    # host masquerading on the instance's behalf.
+    EGRESS_SRC=$(awk '{for (i = 1; i < NF; i++) if ($i == "IP") { print $(i+1); exit }}' "$CAP")
+    if [ "$UPLINK" = "nat" ]; then
+        skip "routed NAT masquerades behind the host address by design"
+    elif [ "$EGRESS_SRC" = "$INSTANCE_IP" ]; then
+        pass "egress leaves $WAN_BRIDGE sourced from the EIP $INSTANCE_IP"
+    elif [ -n "$EGRESS_SRC" ]; then
+        fail "egress on $WAN_BRIDGE is sourced from $EGRESS_SRC, not the EIP $INSTANCE_IP"
+    else
+        fail "no egress to 8.8.8.8 seen on $WAN_BRIDGE — traffic never left the box"
+    fi
+    rm -f "$CAP"
+fi
+
+# --- Service bind matrix ---
+echo "==> Service binding"
+if [ "$BIND_PLANES" != "1" ]; then
+    skip "DEV_BIND_PLANES != 1 (services are on the wildcard)"
+elif [ -z "$WAN_IP" ]; then
+    skip "no WAN address to test against"
+else
+    LISTENERS=$(ss -ltnH 2>/dev/null | awk '{print $4}')
+    # Internal control planes must never answer on the public address.
+    for port in 4222 6641 6642; do
+        if echo "$LISTENERS" | grep -q "^${WAN_IP}:${port}$"; then
+            fail "port $port is listening on the WAN address $WAN_IP"
+        else
+            pass "port $port is not on the WAN address"
+        fi
+    done
+    # Public surfaces must still answer there.
+    for port in 9999 8443; do
+        if echo "$LISTENERS" | grep -qE "^(0\.0\.0\.0|\*|${WAN_IP}):${port}$"; then
+            pass "port $port answers on the WAN address"
+        else
+            fail "port $port is not reachable on the WAN address $WAN_IP"
+        fi
+    done
+    if echo "$LISTENERS" | grep -qE "^(0\.0\.0\.0|\*|${WAN_IP}):53$"; then
+        pass "northstar answers :53 on the WAN address"
+    else
+        fail "northstar is not on ${WAN_IP}:53 — public DNS delegation moved off the wan plane"
+    fi
+fi
+
+echo ""
+if [ "$FAILURES" -eq 0 ]; then
+    echo "✅ Network verification passed"
+    exit 0
+fi
+echo "❌ Network verification failed ($FAILURES assertion(s))"
+exit 1

spinifex/network/external/igw.go

@@ -198,11 +198,17 @@ func (m *igwManager) AttachIGW(ctx context.Context, spec IGWSpec) error {
 		createdLRP = true
 	}
 
+	// nat-addresses is only honoured on a router-type port; OVN silently ignores
+	// it on the localnet, which leaves EIPs unadvertised on the physical network.
+	gwPortOpts := map[string]string{"router-port": gwPortName}
+	if m.gatewayOwnsNAT() {
+		gwPortOpts["nat-addresses"] = "router"
+	}
 	if err := m.ovn.CreateLogicalSwitchPort(ctx, extSwitchName, &nbdb.LogicalSwitchPort{
 		Name:      switchGWPortName,
 		Type:      "router",
 		Addresses: []string{"router"},
-		Options:   map[string]string{"router-port": gwPortName},
+		Options:   gwPortOpts,
 		ExternalIDs: map[string]string{
 			"spinifex:vpc_id": spec.VPCID,
 			"spinifex:igw_id": spec.InternetGatewayID,
@@ -360,8 +366,8 @@ func (m *igwManager) gatewayLRPIP(ctx context.Context, vpcID string) string {
 }
 
 // ensureSharedExternal idempotently creates the singleton shared external switch
-// and its single localnet port. The localnet advertises router NAT addresses in
-// centralized mode so gateways behind it are reachable from the uplink.
+// and its single localnet port. NAT advertisement lives on the per-VPC gateway
+// port, not here — OVN ignores nat-addresses on a localnet port.
 func (m *igwManager) ensureSharedExternal(ctx context.Context, switchName, portName string) error {
 	extSwitch := &nbdb.LogicalSwitch{
 		Name:        switchName,
@@ -370,18 +376,25 @@ func (m *igwManager) ensureSharedExternal(ctx context.Context, switchName, portN
 	if _, _, err := m.ovn.EnsureLogicalSwitch(ctx, extSwitch); err != nil {
 		return fmt.Errorf("ensure shared external switch %s: %w", switchName, err)
 	}
-	if _, err := m.ovn.GetLogicalSwitchPort(ctx, portName); err == nil {
+	// Converge rather than early-return: deployments created before NAT
+	// advertisement moved to the gateway port still carry a stale nat-addresses
+	// here, and it must not survive an upgrade.
+	if existing, err := m.ovn.GetLogicalSwitchPort(ctx, portName); err == nil {
+		if _, stale := existing.Options["nat-addresses"]; !stale {
+			return nil
+		}
+		delete(existing.Options, "nat-addresses")
+		if err := m.ovn.UpdateLogicalSwitchPort(ctx, existing); err != nil {
+			return fmt.Errorf("clear stale nat-addresses on %s: %w", portName, err)
+		}
+		slog.Info("external: cleared stale nat-addresses from localnet port", "port", portName)
 		return nil
 	}
-	localnetOpts := map[string]string{"network_name": "external"}
-	if m.gatewayOwnsNAT() {
-		localnetOpts["nat-addresses"] = "router"
-	}
 	if err := m.ovn.CreateLogicalSwitchPort(ctx, switchName, &nbdb.LogicalSwitchPort{
 		Name:        portName,
 		Type:        "localnet",
 		Addresses:   []string{"unknown"},
-		Options:     localnetOpts,
+		Options:     map[string]string{"network_name": "external"},
 		ExternalIDs: map[string]string{"spinifex:role": "external-localnet"},
 	}); err != nil {
 		return fmt.Errorf("create shared localnet port %s: %w", portName, err)

spinifex/network/external/igw_routed_test.go

@@ -36,9 +36,9 @@ func TestAttachIGW_Routed_AllocatesTransitIPAndInstallsSNAT(t *testing.T) {
 
 	require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"}))
 
-	localnet, err := m.GetLogicalSwitchPort(ctx, topology.ExternalLocalnetPortShared())
+	gwPort, err := m.GetLogicalSwitchPort(ctx, topology.GatewaySwitchPort("vpc-1"))
 	require.NoError(t, err)
-	assert.Equal(t, "router", localnet.Options["nat-addresses"], "routed mode must set nat-addresses=router")
+	assert.Equal(t, "router", gwPort.Options["nat-addresses"], "routed mode must set nat-addresses=router on the gateway port")
 
 	lrp, err := m.GetLogicalRouterPort(ctx, topology.GatewayRouterPort("vpc-1"))
 	require.NoError(t, err)

spinifex/network/external/igw_test.go

@@ -98,7 +98,12 @@ func TestAttachIGW_Distributed_LinkLocalLRP(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "external", localnet.Options["network_name"])
 	_, hasNat := localnet.Options["nat-addresses"]
-	assert.False(t, hasNat, "distributed mode must NOT set nat-addresses")
+	assert.False(t, hasNat, "nat-addresses is never valid on a localnet port")
+
+	gwPort, err := m.GetLogicalSwitchPort(ctx, topology.GatewaySwitchPort("vpc-1"))
+	require.NoError(t, err)
+	_, hasGwNat := gwPort.Options["nat-addresses"]
+	assert.False(t, hasGwNat, "distributed mode must NOT set nat-addresses")
 
 	// Gateway LRP exists with link-local network.
 	lrp, err := m.GetLogicalRouterPort(ctx, topology.GatewayRouterPort("vpc-1"))
@@ -148,7 +153,12 @@ func TestAttachIGW_Centralized_AllocatesGwLrpIP(t *testing.T) {
 
 	localnet, err := m.GetLogicalSwitchPort(ctx, topology.ExternalLocalnetPortShared())
 	require.NoError(t, err)
-	assert.Equal(t, "router", localnet.Options["nat-addresses"], "centralized mode must set nat-addresses=router")
+	_, hasNat := localnet.Options["nat-addresses"]
+	assert.False(t, hasNat, "nat-addresses is never valid on a localnet port")
+
+	gwPort, err := m.GetLogicalSwitchPort(ctx, topology.GatewaySwitchPort("vpc-1"))
+	require.NoError(t, err)
+	assert.Equal(t, "router", gwPort.Options["nat-addresses"], "centralized mode must set nat-addresses=router on the gateway port")
 
 	lrp, err := m.GetLogicalRouterPort(ctx, topology.GatewayRouterPort("vpc-1"))
 	require.NoError(t, err)
@@ -156,6 +166,35 @@ func TestAttachIGW_Centralized_AllocatesGwLrpIP(t *testing.T) {
 	assert.Equal(t, "192.168.1.240", lrp.ExternalIDs[gatewayIPExtIDKey])
 }
 
+// A localnet port created before NAT advertisement moved to the gateway port
+// carries a stale nat-addresses that must be cleared on the next attach.
+func TestAttachIGW_ClearsStaleLocalnetNATAddresses(t *testing.T) {
+	ctx := context.Background()
+	m := mock.New()
+	seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16")
+
+	extSwitchName := topology.ExternalSwitchShared()
+	portName := topology.ExternalLocalnetPortShared()
+	_, _, err := m.EnsureLogicalSwitch(ctx, &nbdb.LogicalSwitch{Name: extSwitchName})
+	require.NoError(t, err)
+	require.NoError(t, m.CreateLogicalSwitchPort(ctx, extSwitchName, &nbdb.LogicalSwitchPort{
+		Name:      portName,
+		Type:      "localnet",
+		Addresses: []string{"unknown"},
+		Options:   map[string]string{"network_name": "external", "nat-addresses": "router"},
+	}))
+
+	pool := &ExternalPoolConfig{Name: "p", Gateway: "192.168.1.1", PrefixLen: 24}
+	mgr, _ := newTestIGWManager(t, m, policy.NATModeDistributed, pool, LinkLocalAllocator{}, []string{"chassis-a"})
+	require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"}))
+
+	localnet, err := m.GetLogicalSwitchPort(ctx, portName)
+	require.NoError(t, err)
+	_, hasNat := localnet.Options["nat-addresses"]
+	assert.False(t, hasNat, "stale nat-addresses must be cleared from the localnet port")
+	assert.Equal(t, "external", localnet.Options["network_name"], "converge must not drop network_name")
+}
+
 // The LRP address is written once at attach, so a lease re-issued on a new IP
 // leaves the port answering ARP for an address nothing renews.
 func TestRebindGatewayIP_MovesLRPAndStamp(t *testing.T) {

spinifex/network/host/veth.go

@@ -88,7 +88,7 @@ func (v *Veth) EnsureUplinkPort(ctx context.Context) (net.HardwareAddr, error) {
 	return mac, nil
 }
 
-// UplinkMode returns UplinkModeVeth (centralised NAT).
+// UplinkMode returns UplinkModeVeth.
 func (v *Veth) UplinkMode() UplinkMode { return UplinkModeVeth }
 
 // ExternalCIDR returns the IPv4 prefix on LinuxBridge (OS-assigned).

spinifex/network/policy/nat_test.go

@@ -48,7 +48,7 @@ func countNAT(m *mock.Client, natType, logicalIP string) int {
 
 func TestNATModeFromUplinkMode(t *testing.T) {
 	assert.Equal(t, NATModeDistributed, NATModeFromUplinkMode(host.UplinkModePhysical))
-	assert.Equal(t, NATModeCentralized, NATModeFromUplinkMode(host.UplinkModeVeth))
+	assert.Equal(t, NATModeDistributed, NATModeFromUplinkMode(host.UplinkModeVeth))
 	assert.Equal(t, NATModeRouted, NATModeFromUplinkMode(host.UplinkModeRouted))
 	assert.Equal(t, NATModeUnknown, NATModeFromUplinkMode(host.UplinkModeUnknown))
 }

spinifex/network/policy/policy.go

@@ -15,11 +15,12 @@ const (
 	NATModeUnknown NATMode = iota
 
 	// NATModeDistributed sets ExternalMAC+LogicalPort so OVN processes DNAT on
-	// the VM's own chassis. Required by UplinkModePhysical.
+	// the VM's own chassis. The gateway LRP stays link-local, so a VPC consumes
+	// no address from the external pool. Used by physical and veth uplinks.
 	NATModeDistributed
 
 	// NATModeCentralized leaves ExternalMAC/LogicalPort unset; gateway chassis
-	// owns SNAT/DNAT. Required by UplinkModeVeth.
+	// owns SNAT/DNAT and its LRP takes a real pool address.
 	NATModeCentralized
 
 	// NATModeRouted is centralised-shaped: gateway chassis SNATs the VPC CIDR
@@ -45,10 +46,11 @@ func (m NATMode) String() string {
 // NATModeUnknown so misconfiguration fails loudly at construction.
 func NATModeFromUplinkMode(m host.UplinkMode) NATMode {
 	switch m {
-	case host.UplinkModePhysical:
+	case host.UplinkModePhysical, host.UplinkModeVeth:
+		// A veth uplink carries distributed NAT fine: the Linux bridge learns the
+		// per-rule external MAC on the veth port like any other MAC. Only the
+		// host's own uplink address is bridge-owned, and NAT never touches it.
 		return NATModeDistributed
-	case host.UplinkModeVeth:
-		return NATModeCentralized
 	case host.UplinkModeRouted:
 		return NATModeRouted
 	default:

spinifex/network/reconcile/apply.go

@@ -336,6 +336,11 @@ func gatewayLRPIP(lrp *nbdb.LogicalRouterPort) string {
 	if !ok {
 		return ""
 	}
+	// A distributed-NAT gateway LRP is link-local and sits on no host-reachable
+	// segment, so it is not a probe target — reporting none gates the check off.
+	if addr, err := netip.ParseAddr(ip); err == nil && addr.IsLinkLocalUnicast() {
+		return ""
+	}
 	return ip
 }
 

spinifex/network/reconcile/apply_gateway_claim_test.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"testing"
 	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb"
 )
 
 // fakeClaimVerifier scripts a sequence of GatewayPortClaimed results and counts
@@ -568,3 +570,25 @@ func TestEnsureGuestPortDatapath_ConnectedSBNeverResets(t *testing.T) {
 		t.Errorf("sbResets = %d, want 0 (connected SB must never be reset)", f.sbResets)
 	}
 }
+
+// A distributed-NAT gateway LRP is link-local, so the host cannot route to it.
+// Probing it would fail forever and report external connectivity as degraded.
+func TestGatewayLRPIP_LinkLocalIsNotAProbeTarget(t *testing.T) {
+	cases := []struct {
+		name string
+		lrp  *nbdb.LogicalRouterPort
+		want string
+	}{
+		{"nil", nil, ""},
+		{"no networks", &nbdb.LogicalRouterPort{}, ""},
+		{"routable", &nbdb.LogicalRouterPort{Networks: []string{"216.218.163.111/27"}}, "216.218.163.111"},
+		{"link-local", &nbdb.LogicalRouterPort{Networks: []string{"169.254.0.1/30"}}, ""},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := gatewayLRPIP(tc.lrp); got != tc.want {
+				t.Errorf("gatewayLRPIP() = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}