spinifex/cmd/installer/install/install.go

package install

import (
	"bufio"
	"fmt"
	"io"
	"log/slog"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/mulgadc/spinifex/cmd/installer/firstboot"
	"github.com/mulgadc/spinifex/cmd/installer/systemd"
)

// mountRoot is the install target. A variable, not a constant, so tests can
// drive the file-writing steps against a temp tree rather than a real mount.
var mountRoot = "/mnt/spinifex-install"

// efiPart is the ESP mount point inside the target. Derived on each call so it
// cannot go stale against mountRoot.
func efiPart() string { return filepath.Join(mountRoot, "boot/efi") }

// step is one named unit of work in the install sequence.
type step struct {
	name string
	fn   func() error
}

// Run executes all installation steps in order. It is intentionally sequential
// and explicit — each step is visible in logs so failures are easy to diagnose.
func Run(cfg *Config) error {
	// Re-checked here even though the UI and the headless path both validate:
	// this is the last point before anything is erased, and it is the only one
	// every caller must pass through.
	if err := cfg.Storage.Validate(); err != nil {
		return fmt.Errorf("storage: %w", err)
	}

	// The live environment may not have /sbin or /usr/sbin in PATH. Set it
	// explicitly so exec.Command's LookPath finds system binaries like grub-install.
	_ = os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")

	// Unmount unconditionally on exit so a failed step never leaves the target
	// mounted in the live environment, which would make a retry double-mount or,
	// on ZFS, fail with "pool is busy".
	defer cleanupTarget(cfg.Storage)

	steps := []step{{"partition disks", func() error { return partitionDisks(cfg.Storage) }}}
	if cfg.Storage.FS.IsZFS() {
		steps = append(steps, zfsRootSteps(cfg)...)
	} else {
		steps = append(steps, ext4RootSteps(cfg)...)
	}
	steps = append(steps,
		step{"copy rootfs", func() error { return copyRootfs(cfg.Storage) }},
		step{"verify role layout", func() error { return verifyRoleLayout(cfg.Storage) }},
		step{"create swap", func() error { return createSwap(cfg.Storage) }},
		step{"write fstab", func() error { return writeFstab(cfg.Storage) }},
		step{"install spinifex", func() error { return installSpinifex(cfg) }},
		step{"write network config", func() error { return writeNetworkConfig(cfg) }},
		step{"write firstboot service", func() error { return firstboot.Write(mountRoot, cfg.toFirstbootConfig()) }},
		step{"install bootloader", func() error { return installBootloader(cfg.Storage) }},
		step{"install CA cert", func() error { return installCACert(cfg) }},
	)

	for _, s := range steps {
		slog.Info("installer", "step", s.name)
		if err := s.fn(); err != nil {
			return fmt.Errorf("step %q: %w", s.name, err)
		}
	}

	slog.Info("installation complete")
	refreshBootToolNote(cfg.Storage)
	fireInstallCallback()
	promptRemoveUSB()
	return reboot()
}

// ext4RootSteps formats and mounts the ext4 root and any role filesystems.
func ext4RootSteps(cfg *Config) []step {
	return []step{
		{"format partitions", func() error { return formatPartitions(cfg.Storage) }},
		{"mount partitions", func() error { return mountPartitions(cfg.Storage) }},
	}
}

// zfsRootSteps builds the root pool. The ESPs are formatted here rather than
// with the bootloader so that a failure to make a filesystem on one of them is
// reported before the rootfs has been copied.
func zfsRootSteps(cfg *Config) []step {
	// Resolved once, before any step runs, so the values written to the pool,
	// to modprobe.d and to the daemon's reserve are all the same numbers.
	var opts ZFSOpts
	return []step{
		{"format ESPs", func() error { return formatESPs(cfg.Storage) }},
		{"load zfs module", loadZFSModule},
		{"create zfs pool", func() error {
			opts = resolveZFSOpts(cfg.Storage)
			slog.Info("zfs pool", "topology", cfg.Storage.FS, "disks", len(cfg.Storage.Disks),
				"ashift", opts.Ashift, "compress", opts.Compress, "arc_max_mib", opts.ARCMaxMiB)
			return createPool(cfg.Storage, opts)
		}},
		{"create zfs datasets", createDatasets},
		{"configure zfs", func() error { return writeZFSSystemConfig(opts) }},
	}
}

// cleanupTarget releases everything the install mounted, in the right order for
// the filesystem in use.
func cleanupTarget(cfg DiskConfig) {
	unbindChrootMounts()
	if cfg.FS.IsZFS() {
		// Recursive: the layout is a dozen nested datasets, and leaving one
		// mounted is enough to block the export.
		_ = runQuiet("umount", "-R", mountRoot)
		exportPool()
		return
	}
	// Deepest first: predastore nests inside spinifex, and unmounting the outer
	// filesystem fails with EBUSY while the inner one is still attached.
	for _, rm := range slices.Backward(cfg.DataMounts()) {
		_ = runQuiet("umount", targetPath(rm.Mountpoint()))
	}
	_ = runQuiet("umount", efiPart())
	_ = runQuiet("umount", mountRoot)
}

// targetPath maps a path on the installed system to its location under the
// install mount root.
func targetPath(mountpoint string) string {
	return filepath.Join(mountRoot, strings.TrimPrefix(mountpoint, "/"))
}

func formatPartitions(cfg DiskConfig) error {
	if err := formatESPs(cfg); err != nil {
		return err
	}
	if err := run("mkfs.ext4", "-F", cfg.Primary().PartitionPath(rootPartNum)); err != nil {
		return err
	}
	for _, rm := range cfg.DataMounts() {
		// Labelled with the role so `lsblk -f` identifies each drive on a
		// console that has no access to the installer log.
		if err := run("mkfs.ext4", "-F", "-L", rm.Role.Label(), rm.Disk.PartitionPath(dataPartNum)); err != nil {
			return fmt.Errorf("format %s (%s): %w", rm.Disk.Path, rm.Role, err)
		}
	}
	return nil
}

func mountPartitions(cfg DiskConfig) error {
	d := cfg.Primary()
	if err := os.MkdirAll(mountRoot, 0o755); err != nil {
		return err
	}
	if err := run("mount", d.PartitionPath(rootPartNum), mountRoot); err != nil {
		return err
	}
	if err := os.MkdirAll(efiPart(), 0o755); err != nil {
		return err
	}
	if err := run("mount", d.PartitionPath(espPartNum), efiPart()); err != nil {
		return err
	}
	// Role filesystems are mounted here, before the rootfs copy, so rsync lays
	// each tree down on its own drive with the ownership setup.sh gave it at
	// ISO build time. Mounting afterwards hides those trees and loses it.
	for _, rm := range cfg.DataMounts() {
		target := targetPath(rm.Mountpoint())
		if err := os.MkdirAll(target, 0o755); err != nil {
			return err
		}
		if err := run("mount", rm.Disk.PartitionPath(dataPartNum), target); err != nil {
			return fmt.Errorf("mount %s at %s: %w", rm.Disk.Path, rm.Mountpoint(), err)
		}
	}
	return nil
}

// mountProtectFilters stops rsync's --delete from acting across a mount
// boundary. Every mountpoint the installer created before the copy needs one,
// whether it is a ZFS dataset or a role filesystem.
func mountProtectFilters(cfg DiskConfig) []string {
	if cfg.FS.IsZFS() {
		return datasetProtectFilters(cfg)
	}
	var out []string
	for _, rm := range cfg.DataMounts() {
		out = append(out, "--filter=P "+rm.Mountpoint())
	}
	return out
}

// copyRootfs copies the live squashfs environment onto the target disk using
// rsync. This is the air-gapped alternative to debootstrap — all packages are
// already embedded in the ISO so no network access is required.
func copyRootfs(cfg DiskConfig) error {
	args := []string{
		"-aHAX", "--delete", "--info=progress2",
		"--exclude=/proc",
		"--exclude=/sys",
		"--exclude=/dev",
		"--exclude=/run",
		"--exclude=/tmp",
		"--exclude=/cdrom",
		"--exclude=/mnt",
		"--exclude=/etc/machine-id",
		"--exclude=/var/lib/dbus/",
		"--exclude=/etc/openvswitch/",
		"--exclude=/var/lib/openvswitch/",
		"--exclude=/var/lib/dhcpcd/",
		"--exclude=/etc/ssh/ssh_host_*",
		"--exclude=/lost+found",
		"--exclude=/boot/efi",
	}
	args = append(args, mountProtectFilters(cfg)...)
	args = append(args, "/", mountRoot+"/")
	if err := run("rsync", args...); err != nil {
		return err
	}

	// Verify critical paths exist before proceeding. rsync exits 0 on ENOSPC
	// for individual file writes on some filesystems, which would produce a
	// partial rootfs that boots into a panic.
	critical := []string{
		filepath.Join(mountRoot, "bin/bash"),
		filepath.Join(mountRoot, "lib/systemd/systemd"),
		filepath.Join(mountRoot, "usr/local/bin/spx"),
	}
	for _, p := range critical {
		if _, err := os.Stat(p); err != nil {
			return fmt.Errorf("copyRootfs: critical path missing after rsync (%s): %w", p, err)
		}
	}

	// rsync skips excluded paths entirely — recreate the virtual filesystem
	// mount points that systemd expects to exist on the installed system.
	mountPoints := []struct {
		path string
		mode os.FileMode
	}{
		{"proc", 0o555},
		{"sys", 0o555},
		{"dev", 0o755},
		{"run", 0o755},
		{"run/lock", 0o1777},
		{"tmp", 0o1777},
		{"mnt", 0o755},
	}
	for _, mp := range mountPoints {
		if err := os.MkdirAll(filepath.Join(mountRoot, mp.path), mp.mode); err != nil {
			return fmt.Errorf("create mountpoint /%s: %w", mp.path, err)
		}
	}
	return nil
}

// verifyRoleLayout asserts the rootfs copy landed on the role drives rather
// than on top of them. A mount that happened in the wrong order produces a node
// that boots, starts every service, and has each one writing to an empty
// directory on the wrong drive — healthy-looking and silently wrong.
func verifyRoleLayout(cfg DiskConfig) error {
	for _, rm := range cfg.DataMounts() {
		target := targetPath(rm.Mountpoint())
		dev, err := deviceOf(target)
		if err != nil {
			return fmt.Errorf("verify %s: %w", rm.Mountpoint(), err)
		}
		parentDev, err := deviceOf(filepath.Dir(target))
		if err != nil {
			return fmt.Errorf("verify %s: %w", rm.Mountpoint(), err)
		}
		if dev == parentDev {
			return fmt.Errorf("%s is not a separate filesystem — %s was not mounted before the rootfs copy",
				rm.Mountpoint(), rm.Disk.Path)
		}
		if err := matchesLiveOwnership(target, rm.Mountpoint()); err != nil {
			return err
		}
		// Logged so the layout is recoverable from the install log alone, which
		// on a headless node is the only record of where each workload landed.
		slog.Info("role mounted", "role", rm.Role, "mountpoint", rm.Mountpoint(), "disk", rm.Disk.Path)
	}
	return nil
}

// matchesLiveOwnership compares a copied directory against the live rootfs,
// which is where setup.sh set every service owner at ISO build time. Comparing
// rather than hardcoding uids keeps this correct as services are added.
func matchesLiveOwnership(target, source string) error {
	src, err := os.Stat(source)
	if err != nil {
		// Nothing to compare against; the mount check above still holds.
		slog.Warn("skipping ownership check, path absent in live environment", "path", source, "err", err)
		return nil
	}
	dst, err := os.Stat(target)
	if err != nil {
		return fmt.Errorf("verify %s: %w", source, err)
	}
	srcSt, dstSt := src.Sys().(*syscall.Stat_t), dst.Sys().(*syscall.Stat_t)
	if srcSt.Uid != dstSt.Uid || srcSt.Gid != dstSt.Gid || src.Mode().Perm() != dst.Mode().Perm() {
		return fmt.Errorf("%s copied as %d:%d %o, expected %d:%d %o — the drive was mounted after the rootfs copy",
			source, dstSt.Uid, dstSt.Gid, dst.Mode().Perm(), srcSt.Uid, srcSt.Gid, src.Mode().Perm())
	}
	return nil
}

// deviceOf returns the filesystem a path lives on, for distinguishing a real
// mount from a directory of the same name.
func deviceOf(path string) (uint64, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return 0, err
	}
	st, ok := fi.Sys().(*syscall.Stat_t)
	if !ok {
		return 0, fmt.Errorf("%s: no stat information", path)
	}
	return st.Dev, nil
}

func installSpinifex(cfg *Config) error {
	// The rootfs copy already contains spx and spinifex-installer at
	// /usr/local/bin/ — no binary copy needed. Regenerate machine-specific
	// identity files so each installed node is unique.

	// Bind-mount /dev, /proc, /sys into the chroot so PAM (chpasswd) and
	// other chroot commands work correctly. Trixie's PAM requires /proc and
	// /dev/urandom for password hashing.
	if err := bindChrootMounts(); err != nil {
		return err
	}
	defer unbindChrootMounts()

	// Generate a unique machine-id from the kernel CSPRNG. We do not use
	// systemd-machine-id-setup in the chroot: it reads the SMBIOS UUID via
	// the bind-mounted /sys, which is identical on cloned VMs or hosts with
	// absent/zeroed DMI data, and falls back to non-unique sources (MAC,
	// hostname) when that also fails — producing the same ID on every node.
	// /proc/sys/kernel/random/uuid is always unique and requires no chroot.
	machineIDPath := filepath.Join(mountRoot, "etc/machine-id")
	rawUUID, err := os.ReadFile("/proc/sys/kernel/random/uuid")
	if err != nil {
		return fmt.Errorf("installSpinifex: read kernel uuid: %w", err)
	}
	machineID := strings.ReplaceAll(strings.TrimSpace(string(rawUUID)), "-", "") + "\n"
	if err := os.WriteFile(machineIDPath, []byte(machineID), 0o444); err != nil {
		return fmt.Errorf("installSpinifex: write machine-id: %w", err)
	}
	// dbus mirrors /etc/machine-id; remove it so it is re-created from the
	// new ID on first boot rather than carrying a stale value.
	_ = os.Remove(filepath.Join(mountRoot, "var/lib/dbus/machine-id"))

	// Hostname.
	hostnamePath := filepath.Join(mountRoot, "etc/hostname")
	if err := os.WriteFile(hostnamePath, []byte(cfg.Hostname+"\n"), 0o644); err != nil {
		return err
	}

	// /etc/hosts entry for the hostname.
	hosts := fmt.Sprintf("127.0.0.1\tlocalhost\n127.0.1.1\t%s\n", cfg.Hostname)
	if err := os.WriteFile(filepath.Join(mountRoot, "etc/hosts"), []byte(hosts), 0o644); err != nil {
		return err
	}

	// Set root + spinifex passwords. We invoke chpasswd from the LIVE
	// installer (not via `chroot`), passing the target root with -R and
	// forcing -c YESCRYPT. This deliberately bypasses PAM:
	//   * `chroot ... chpasswd` uses /etc/pam.d/chpasswd → common-password →
	//     pam_unix.so with the "obscure" option, which can return
	//     "Authentication token manipulation error" inside a chroot for
	//     reasons that are awkward to diagnose (audit subsystem, locked
	//     shadow entries from useradd, etc.).
	//   * -c YESCRYPT tells chpasswd to hash locally with libcrypt and write
	//     directly to <root>/etc/shadow — no PAM stack involved. YESCRYPT
	//     matches Trixie's ENCRYPT_METHOD so subsequent logins use the same
	//     algorithm.
	//   * -R <root> opens the target's passwd/shadow directly; no bind
	//     mounts of /dev/urandom or /proc are needed for the password step.
	if cfg.RootPassword != "" {
		if err := setShadowPassword("root", cfg.RootPassword); err != nil {
			return fmt.Errorf("set root password: %w", err)
		}
		// The spinifex account is the default interactive login on the
		// node (console + SSH). Root SSH is disabled, so this is the sole
		// remote entry point. The user itself is created at rootfs build
		// time (build-rootfs.sh) — here we just set its password.
		if err := setShadowPassword("spinifex", cfg.RootPassword); err != nil {
			return fmt.Errorf("set spinifex password: %w", err)
		}
	}

	// Write /etc/spinifex/node.conf — read at runtime by spx admin banner
	// to look up the current IP dynamically (handles IP changes after install).
	// MANAGEMENT_IFACE is the bridge (br-wan), not the physical NIC.
	// MANAGEMENT_IP is empty for DHCP — banner's --boot-check fills it in at boot.
	nodeConf := fmt.Sprintf("MANAGEMENT_IP=%s\nMANAGEMENT_IFACE=br-wan\nNODE_HOSTNAME=%s\n",
		cfg.WAN.Address, cfg.Hostname)
	confDir := filepath.Join(mountRoot, "etc/spinifex")
	if err := os.MkdirAll(confDir, 0o755); err != nil {
		return err
	}
	if err := os.WriteFile(filepath.Join(confDir, "node.conf"), []byte(nodeConf), 0o644); err != nil {
		return err
	}

	// dhcpcd-base is present on the installed system (used by setup-ovn.sh
	// for WAN DHCP acquisition). Mask the standalone dhcpcd.service so it
	// never auto-starts and races with systemd-networkd's built-in DHCP
	// client on br-wan.
	if err := maskSystemdUnit(mountRoot, "dhcpcd.service"); err != nil {
		slog.Warn("installSpinifex: failed to mask dhcpcd.service", "err", err)
	}

	return nil
}

func writeNetworkConfig(cfg *Config) error {
	// IPs live on Linux bridges (br-wan, br-lan), not on the physical NICs.
	// systemd-networkd owns the full lifecycle: it creates the bridge NetDevs,
	// enslaves the physical NICs, and runs the DHCP client. This means the
	// veth pair that setup-ovn.sh adds later (veth-wan-br) is a port of a
	// networkd-known bridge, so a networkctl reload or reboot never orphans it.
	netdDir := filepath.Join(mountRoot, "etc/systemd/network")
	if err := os.MkdirAll(netdDir, 0o755); err != nil {
		return err
	}

	if err := cfg.Validate(); err != nil {
		return fmt.Errorf("network roles: %w", err)
	}

	// A physical NIC is configured once even when two roles share it via
	// VLANs, so parent-interface files are keyed by NIC rather than by plane.
	parents := map[string]*parentNIC{}
	for _, rb := range cfg.Roles() {
		p, ok := parents[rb.Role.Interface]
		if !ok {
			p = &parentNIC{}
			parents[rb.Role.Interface] = p
		}
		if rb.Role.VLAN > 0 {
			p.vlans = append(p.vlans, rb.Role.Link())
		} else {
			p.bridge = rb.Plane.Bridge()
		}
		// The parent must carry at least the largest MTU of any VLAN riding it.
		if rb.Role.MTU > p.mtu {
			p.mtu = rb.Role.MTU
		}
	}
	for iface, p := range parents {
		if err := writeParentNIC(netdDir, iface, p); err != nil {
			return err
		}
	}

	for _, rb := range cfg.Roles() {
		// Only br-wan auto-activates; the east-west bridges are brought up by
		// their own unit after network-online.target so a missing cable or a
		// DHCP timeout on them can never stall the management path.
		manual := rb.Plane != PlaneWAN
		if err := writeNetworkdBridge(netdDir, rb.Plane, rb.Role, manual); err != nil {
			return err
		}
		if manual {
			if err := systemd.WriteBridgeUnit(mountRoot, rb.Plane.Bridge()); err != nil {
				return fmt.Errorf("%s bridge unit: %w", rb.Plane, err)
			}
		}
	}

	// Unmask systemd-networkd-wait-online on the installed system and scope
	// it to br-wan only. The live ISO masks this service (build-rootfs.sh) to
	// avoid blocking the installer environment (which has no br-wan). The mask
	// is copied to the installed system by copyRootfs and must be removed here
	// so that network-online.target — and therefore spinifex-firstboot.service
	// (After=network-online.target) — does not fire before br-wan has its
	// DHCP lease and default route. Without this, setup-ovn.sh finds no default
	// route and exits early, creating no veth pair.
	//
	// The --interface=br-wan scope means br-lan (ActivationPolicy=manual) never
	// blocks the wait; --timeout=60 caps the delay on a cold switch.
	waitOnlineMask := filepath.Join(mountRoot, "etc/systemd/system/systemd-networkd-wait-online.service")
	_ = os.Remove(waitOnlineMask)
	waitOnlineDir := filepath.Join(mountRoot, "etc/systemd/system/systemd-networkd-wait-online.service.d")
	if err := os.MkdirAll(waitOnlineDir, 0o755); err != nil {
		return err
	}
	waitOnlineConf := "[Service]\nExecStart=\nExecStart=/lib/systemd/systemd-networkd-wait-online --interface=br-wan --timeout=60\n"
	if err := os.WriteFile(filepath.Join(waitOnlineDir, "spinifex-wan-only.conf"), []byte(waitOnlineConf), 0o644); err != nil {
		return err
	}

	// Disable IPv6 via sysctl — belt-and-suspenders alongside IPv6AcceptRA=no
	// in the networkd .network files.
	var bridges []string
	for _, rb := range cfg.Roles() {
		bridges = append(bridges, rb.Plane.Bridge())
	}
	var sysctl strings.Builder
	sysctl.WriteString("# Generated by Spinifex installer — IPv6 disabled on management bridges\n")
	for _, br := range bridges {
		fmt.Fprintf(&sysctl, "net.ipv6.conf.%s.disable_ipv6=1\n", br)
	}
	sysctlDir := filepath.Join(mountRoot, "etc/sysctl.d")
	if err := os.MkdirAll(sysctlDir, 0o755); err != nil {
		return err
	}
	if err := os.WriteFile(filepath.Join(sysctlDir, "99-spinifex-network.conf"), []byte(sysctl.String()), 0o644); err != nil {
		return err
	}

	// Pin each NIC name to its MAC address via udev so the installed system
	// always uses the same interface name regardless of probe order.
	udevDir := filepath.Join(mountRoot, "etc/udev/rules.d")
	if err := os.MkdirAll(udevDir, 0o755); err != nil {
		return err
	}
	var udevRules strings.Builder
	pinned := map[string]bool{}
	for _, rb := range cfg.Roles() {
		iface := rb.Role.Interface
		// Two roles can share one NIC via VLANs; pin the physical name once.
		if iface == "" || pinned[iface] {
			continue
		}
		pinned[iface] = true
		mac, err := os.ReadFile("/sys/class/net/" + iface + "/address")
		if err != nil {
			slog.Warn("writeNetworkConfig: could not read NIC MAC, skipping udev pin", "iface", iface, "err", err)
			continue
		}
		fmt.Fprintf(&udevRules, "SUBSYSTEM==\"net\", ACTION==\"add\", ATTR{address}==\"%s\", NAME=\"%s\"\n",
			strings.TrimSpace(string(mac)), iface)
	}
	if udevRules.Len() > 0 {
		return os.WriteFile(filepath.Join(udevDir, "70-spinifex-net.rules"), []byte(udevRules.String()), 0o644)
	}
	return nil
}

// parentNIC accumulates how one physical interface is used across roles. A NIC
// is either enslaved directly to a bridge (untagged) or carries VLAN
// subinterfaces — never both for the same plane.
type parentNIC struct {
	bridge string
	vlans  []string
	mtu    int
}

// writeParentNIC writes the .network file for a physical interface. An
// untagged NIC is enslaved straight to its bridge; a tagged one declares its
// VLAN subinterfaces and stays unbridged, since the VLAN devices are what get
// enslaved.
func writeParentNIC(dir, iface string, p *parentNIC) error {
	var b strings.Builder
	fmt.Fprintf(&b, "[Match]\nName=%s\n\n", iface)
	// A physical NIC here is either a bridge port or a VLAN trunk, and neither
	// carries the address — the bridge above it does. Such a link never reaches
	// the "degraded" state systemd-networkd-wait-online requires, so leaving it
	// required makes that unit fail on every boot and stall until it times out.
	b.WriteString("[Link]\nRequiredForOnline=no\n")
	if p.mtu > 0 {
		fmt.Fprintf(&b, "MTUBytes=%d\n", p.mtu)
	}
	b.WriteString("\n[Network]\n")
	if p.bridge != "" {
		fmt.Fprintf(&b, "Bridge=%s\n", p.bridge)
	}
	for _, v := range p.vlans {
		fmt.Fprintf(&b, "VLAN=%s\n", v)
	}
	b.WriteString("IPv6AcceptRA=no\n")
	return os.WriteFile(filepath.Join(dir, fmt.Sprintf("05-spinifex-nic-%s.network", iface)), []byte(b.String()), 0o644)
}

// writeNetworkdBridge writes the systemd-networkd files that put a plane's
// address on its own Linux bridge:
//
//   - 1{n}-spinifex-{plane}-vlan.netdev   VLAN subinterface (tagged roles only)
//   - 1{n}-spinifex-{plane}-vlan.network  enslaves the VLAN device to the bridge
//   - 2{n}-spinifex-{plane}.netdev        declares the bridge device
//   - 2{n}-spinifex-{plane}.network       configures IP on the bridge
//
// The physical NIC itself is written separately by writeParentNIC, because two
// roles may share one NIC and it must be configured exactly once.
//
// manual=true sets ActivationPolicy=manual so networkd creates the bridge but
// does not auto-activate it — used for br-lan and br-vpc, which their own
// spinifex-{plane}-bridge.service activates after network-online.target.
func writeNetworkdBridge(dir string, plane Plane, role NetworkRole, manual bool) error {
	name := string(plane)
	bridgeName := plane.Bridge()

	if role.VLAN > 0 {
		link := role.Link()
		vlanNetdev := fmt.Sprintf("[NetDev]\nName=%s\nKind=vlan\n\n[VLAN]\nId=%d\n", link, role.VLAN)
		if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("10-spinifex-%s-vlan.netdev", name)), []byte(vlanNetdev), 0o644); err != nil {
			return err
		}
		var v strings.Builder
		fmt.Fprintf(&v, "[Match]\nName=%s\n\n", link)
		// Enslaved to the bridge below, so it is not what "online" means here.
		v.WriteString("[Link]\nRequiredForOnline=no\n")
		if role.MTU > 0 {
			fmt.Fprintf(&v, "MTUBytes=%d\n", role.MTU)
		}
		fmt.Fprintf(&v, "\n[Network]\nBridge=%s\nIPv6AcceptRA=no\n", bridgeName)
		if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("11-spinifex-%s-vlan.network", name)), []byte(v.String()), 0o644); err != nil {
			return err
		}
	}

	// Without a policy systemd-networkd invents a MAC for the bridge, so the node
	// appears on the wire as an address that belongs to no NIC — the DHCP server
	// sees a request whose source does not match its own chaddr and the unicast
	// reply goes nowhere. "none" leaves the kernel to inherit the enslaved port's
	// address, which is what a bridged uplink is supposed to present, and keeps
	// it stable across boots for switch MAC tables and DHCP reservations.
	brNetdev := fmt.Sprintf("[NetDev]\nName=%s\nKind=bridge\nMACAddressPolicy=none\n", bridgeName)
	if role.MTU > 0 {
		brNetdev += fmt.Sprintf("MTUBytes=%d\n", role.MTU)
	}
	brNetdev += "\n[Bridge]\nSTP=no\n"
	if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("20-spinifex-%s.netdev", name)), []byte(brNetdev), 0o644); err != nil {
		return err
	}

	// Bridge .network — IP configuration.
	var b strings.Builder
	fmt.Fprintf(&b, "[Match]\nName=%s\n\n", bridgeName)
	if manual {
		b.WriteString("[Link]\nActivationPolicy=manual\nRequiredForOnline=no\n\n")
	}
	b.WriteString("[Network]\n")
	if role.DHCPMode {
		b.WriteString("DHCP=ipv4\n")
	} else {
		cidr, err := addrCIDR(role.Address, role.Mask)
		if err != nil {
			return fmt.Errorf("bridge %s: %w", bridgeName, err)
		}
		fmt.Fprintf(&b, "Address=%s\n", cidr)
		// Only the wan plane installs a default route; the east-west planes
		// are link-local to the rack.
		if role.Gateway != "" && plane == PlaneWAN {
			fmt.Fprintf(&b, "Gateway=%s\n", role.Gateway)
		}
		// Resolvers follow the default route for the same reason. A resolver
		// pinned to an east-west bridge is unreachable — those bridges are
		// link-local and activate manually — but resolved would still try it,
		// putting a timeout in front of every lookup on the node.
		if plane == PlaneWAN {
			for _, ns := range role.DNS {
				if ns = strings.TrimSpace(ns); ns != "" {
					fmt.Fprintf(&b, "DNS=%s\n", ns)
				}
			}
		}
	}
	b.WriteString("IPv6AcceptRA=no\nConfigureWithoutCarrier=yes\n")
	if manual && role.DHCPMode {
		// The east-west bridges are non-critical; fail fast on DHCP so a
		// missing cable does not stall their activation unit indefinitely.
		b.WriteString("\n[DHCP]\nTimeout=10\n")
	}
	if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("20-spinifex-%s.network", name)), []byte(b.String()), 0o644); err != nil {
		return err
	}

	if role.WiFiSSID != "" {
		if err := writeWPASupplicant(role.Interface, role.WiFiSSID, role.WiFiPass); err != nil {
			return err
		}
	}
	return nil
}

// addrCIDR converts a dotted-decimal address + subnet mask to CIDR notation.
func addrCIDR(addr, dotMask string) (string, error) {
	if net.ParseIP(addr) == nil {
		return "", fmt.Errorf("invalid IP: %s", addr)
	}
	maskIP := net.ParseIP(dotMask)
	if maskIP == nil {
		return "", fmt.Errorf("invalid mask: %s", dotMask)
	}
	m := net.IPMask(maskIP.To4())
	ones, bits := m.Size()
	if bits == 0 {
		return "", fmt.Errorf("non-contiguous or zero subnet mask: %s", dotMask)
	}
	return fmt.Sprintf("%s/%d", addr, ones), nil
}

// writeWPASupplicant writes a wpa_supplicant config for nicIface and enables
// the per-interface wpa_supplicant@{iface}.service so authentication completes
// before networkd enslaves the NIC to the bridge.
func writeWPASupplicant(nicIface, ssid, psk string) error {
	conf := fmt.Sprintf(
		"ctrl_interface=DIR=/run/wpa_supplicant GROUP=netdev\nupdate_config=1\n\nnetwork={\n\tssid=%q\n\tpsk=%q\n}\n",
		ssid, psk,
	)
	wpaDir := filepath.Join(mountRoot, "etc/wpa_supplicant")
	if err := os.MkdirAll(wpaDir, 0o755); err != nil {
		return err
	}
	if err := os.WriteFile(filepath.Join(wpaDir, "wpa_supplicant-"+nicIface+".conf"), []byte(conf), 0o600); err != nil {
		return err
	}
	// Enable via symlink into multi-user.target.wants pointing at the
	// package-provided template unit.
	wantsDir := filepath.Join(mountRoot, "etc/systemd/system/multi-user.target.wants")
	if err := os.MkdirAll(wantsDir, 0o755); err != nil {
		return err
	}
	link := filepath.Join(wantsDir, "wpa_supplicant@"+nicIface+".service")
	_ = os.Remove(link)
	return os.Symlink("/lib/systemd/system/wpa_supplicant@.service", link)
}

func installBootloader(cfg DiskConfig) error {
	// Branding assets have to be in place before either path regenerates
	// grub.cfg — the ZFS path copies them out of /boot onto each ESP.
	copySplashImage(mountRoot)
	copyGrubFont(mountRoot)

	if cfg.FS.IsZFS() {
		// The initramfs must be able to import the pool before the ESPs are
		// synced, since refresh copies whatever initrd exists at that moment.
		if err := buildZFSInitramfs(); err != nil {
			return err
		}
		return installBootToolZFS(cfg)
	}
	return installBootloaderExt4(cfg.Primary())
}

// buildZFSInitramfs regenerates the target's initramfs so it carries the ZFS
// module, the pool cache and the hostid. The image copied from the live ISO was
// built for a machine with an ext4 root and cannot import a pool.
func buildZFSInitramfs() error {
	// ZFS refuses to import a pool last touched by a different host without
	// -f, and the hostid the initramfs sees must match the one recorded on the
	// pool — otherwise every boot stops in the initramfs shell.
	if err := run("cp", "-f", "/etc/hostid", filepath.Join(mountRoot, "etc/hostid")); err != nil {
		slog.Warn("could not copy /etc/hostid, generating one in the target", "err", err)
		if err := run("zgenhostid", "-f", "-o", filepath.Join(mountRoot, "etc/hostid")); err != nil {
			return fmt.Errorf("zgenhostid: %w", err)
		}
	}

	// The pool was created with cachefile=none so the live environment would
	// not claim it. The installed system needs the cache to import at boot
	// without scanning every block device.
	cacheDir := filepath.Join(mountRoot, "etc/zfs")
	if err := os.MkdirAll(cacheDir, 0o755); err != nil {
		return err
	}
	if err := run("zpool", "set", "cachefile="+filepath.Join(cacheDir, "zpool.cache"), ZFSPoolName); err != nil {
		return fmt.Errorf("set zpool cachefile: %w", err)
	}

	if err := bindChrootMounts(); err != nil {
		return err
	}
	defer unbindChrootMounts()
	if err := run("chroot", mountRoot, "update-initramfs", "-u", "-k", "all"); err != nil {
		return fmt.Errorf("update-initramfs: %w", err)
	}
	return nil
}

func installBootloaderExt4(disk Disk) error {
	// grub-install runs in the live environment (not chroot) using the
	// grub-pc-bin and grub-efi-amd64-bin packages already present on the ISO.
	// --boot-directory points at the installed system's /boot.
	bootDir := filepath.Join(mountRoot, "boot")
	efiDir := filepath.Join(mountRoot, "boot", "efi")

	efiErr := run("grub-install",
		"--target=x86_64-efi",
		"--efi-directory="+efiDir,
		"--boot-directory="+bootDir,
		"--bootloader-id=spinifex",
		"--removable",
		"--recheck",
	)
	if efiErr != nil {
		slog.Warn("installBootloader: EFI install failed", "err", efiErr)
	}
	if biosErr := run("grub-install",
		"--target=i386-pc",
		"--boot-directory="+bootDir,
		"--recheck",
		disk.Path,
	); biosErr != nil {
		if efiErr != nil {
			// Both targets failed — the system will not boot.
			return fmt.Errorf("both bootloader targets failed (EFI: %w; BIOS: %w)", efiErr, biosErr)
		}
		return biosErr
	}

	if err := writeGrubDefaults(DiskConfig{FS: FSExt4}); err != nil {
		return err
	}

	if err := bindChrootMounts(); err != nil {
		return err
	}
	defer unbindChrootMounts()
	return run("chroot", mountRoot, "update-grub")
}

func installCACert(cfg *Config) error {
	if !cfg.HasCACert || cfg.CACert == "" {
		return nil
	}
	certPath := filepath.Join(mountRoot, "usr/local/share/ca-certificates/spinifex-ca.crt")
	if err := os.WriteFile(certPath, []byte(cfg.CACert), 0o644); err != nil {
		return err
	}
	if err := bindChrootMounts(); err != nil {
		return err
	}
	defer unbindChrootMounts()
	return run("chroot", mountRoot, "update-ca-certificates")
}

// promptRemoveUSB prints a removal reminder and waits up to 10 seconds for
// the user to press Enter before rebooting. Reading from os.Stdin works because
// spinifex-init redirects the installer's stdin from $CONSOLE_DEV.
func promptRemoveUSB() {
	fmt.Println("\n\033[1mInstallation complete.\033[0m")
	fmt.Println("Remove the USB drive, then press Enter to reboot (auto-rebooting in 10 seconds)...")

	done := make(chan struct{})
	go func() {
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Scan()
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(10 * time.Second):
	}
}

// fireInstallCallback notifies the boot controller that installation is done
// so it clears the PXE install flag before the node reboots. Without this,
// a PXE-first boot order causes the node to reinstall on every reboot until
// firstboot fires the callback — which it can never do if it never runs.
// No-op when SPINIFEX_INSTALL_CALLBACK is not set (ISO/USB installs).
func fireInstallCallback() {
	url := strings.TrimSpace(os.Getenv("SPINIFEX_INSTALL_CALLBACK"))
	if url == "" {
		return
	}
	client := &http.Client{Timeout: 10 * time.Second}
	resp, err := client.Get(url)
	if err != nil {
		slog.Warn("fireInstallCallback: request failed", "url", url, "err", err)
		return
	}
	resp.Body.Close()
	slog.Info("fireInstallCallback: notified boot controller", "url", url, "status", resp.StatusCode)
}

func reboot() error {
	// sync filesystems before reboot so nothing is lost.
	_ = run("sync")
	// Use the kernel syscall directly — the live environment runs spinifex-init
	// as PID 1 (not systemd), so the reboot(8) utility fails trying to reach D-Bus.
	return syscall.Reboot(syscall.LINUX_REBOOT_CMD_RESTART)
}

// toFirstbootConfig maps installer Config to the firstboot package's Config.
func (c *Config) toFirstbootConfig() firstboot.Config {
	// Both addresses come from their plane after collapsing, so a single-NIC
	// node resolves them to the wan address and a three-plane rack keeps
	// Geneve on vpc and cluster traffic on lan. Empty for DHCP — setup-ovn.sh
	// auto-detects the IP from the default route at boot in that case.
	return firstboot.Config{
		Hostname:        c.Hostname,
		EncapIP:         c.PlaneAddress(PlaneVPC),
		LANIP:           c.PlaneAddress(PlaneLAN),
		WANIP:           c.PlaneAddress(PlaneWAN),
		Email:           c.Email,
		InstallCallback: strings.TrimSpace(os.Getenv("SPINIFEX_INSTALL_CALLBACK")),
		SkipFormation:   c.SkipFormation,
	}
}

// writeFstab writes /etc/fstab on the installed system.
//
// On ZFS the root and every dataset are mounted by ZFS itself from properties
// stored on the pool, and the ESPs are mounted on demand by spinifex-boot-tool,
// so the file holds nothing but swap.
func writeFstab(cfg DiskConfig) error {
	fstab := "# /etc/fstab — generated by Spinifex installer\n"

	if !cfg.FS.IsZFS() {
		d := cfg.Primary()
		rootUUID, err := partUUID(d.PartitionPath(rootPartNum))
		if err != nil {
			return fmt.Errorf("get root UUID: %w", err)
		}
		efiUUID, err := partUUID(d.PartitionPath(espPartNum))
		if err != nil {
			return fmt.Errorf("get EFI UUID: %w", err)
		}
		fstab += fmt.Sprintf("UUID=%s / ext4 errors=remount-ro 0 1\nUUID=%s /boot/efi vfat umask=0077 0 1\n",
			rootUUID, efiUUID)

		// Shallowest first, so `mount -a` in a rescue shell works top-down.
		// Deliberately no nofail: a missing data drive must stop the boot for
		// an admin, not let predastore write objects onto the OS drive while
		// every service reports healthy. fsck pass 2 keeps root first.
		for _, rm := range cfg.DataMounts() {
			uuid, err := partUUID(rm.Disk.PartitionPath(dataPartNum))
			if err != nil {
				return fmt.Errorf("get %s UUID: %w", rm.Role, err)
			}
			fstab += fmt.Sprintf("UUID=%s %s ext4 errors=remount-ro 0 2\n", uuid, rm.Mountpoint())
		}
	}

	// Only claim the swap the previous step actually produced; systemd fails the
	// boot on a missing swap unit rather than ignoring the line.
	switch {
	case cfg.FS.IsZFS():
		if _, err := os.Stat("/dev/zvol/" + swapZvolName); err == nil {
			fstab += fmt.Sprintf("/dev/zvol/%s none swap discard 0 0\n", swapZvolName)
		}
	default:
		if _, err := os.Stat(filepath.Join(mountRoot, swapFileName)); err == nil {
			fstab += fmt.Sprintf("/%s none swap sw 0 0\n", swapFileName)
		}
	}
	return os.WriteFile(filepath.Join(mountRoot, "etc/fstab"), []byte(fstab), 0o644)
}

const (
	swapFileName = "swapfile"
	// Nodes below this get half their RAM, which keeps a 4 GiB test VM to a
	// 2 GiB file instead of eating its disk. At or above it the machine is a
	// real host with a real disk, so swap is worth sizing for large imports.
	swapRAMThreshold = 32 << 30
	swapLargeBytes   = 64 << 30
	// Below this there is not enough to be worth the space it costs.
	swapMinBytes = 512 << 20
	// Hard ceiling as a share of free disk, whatever RAM suggests: a 64 GiB file
	// must never fill a small root filesystem.
	swapDiskShare = 4
)

// createSwapFile lays down /swapfile on the installed root, sized to the disk.
//
// Importing an image streams the decompressed disk through viperblock into
// predastore, which a 2-4 GiB node cannot hold in RAM — the kernel OOM-kills the
// import mid-flush. Swap trades speed for completing the operation at all. It is
// a stopgap for the memory profile of that path, not a fix for it.
//
// Failure is logged and not fatal: a node that boots without swap is worse at
// large imports but is otherwise fine, and losing a whole install over it would
// be the greater harm.
// swapZvolName is the swap volume on a ZFS root. ZFS cannot host a swap *file*
// at all, so a zvol is the only option — the same one Proxmox uses.
const swapZvolName = ZFSPoolName + "/swap"

// createSwap lays down swap in whichever form the root filesystem supports.
func createSwap(cfg DiskConfig) error {
	if cfg.FS.IsZFS() {
		return createSwapZvol()
	}
	return createSwapFile()
}

// createSwapZvol creates the swap volume on a ZFS root.
//
// The properties are not tuning preferences. A zvol swapping under memory
// pressure can deadlock against the ARC, and these are the settings that make
// it survivable: page-sized blocks so a swap-out is one record, no data
// caching so swap pages are never copied into the ARC, and sync writes so a
// page is on disk before the kernel frees it.
func createSwapZvol() error {
	size, err := swapSize(mountRoot)
	if err != nil {
		slog.Warn("swap: cannot size swap volume, continuing without one", "err", err)
		return nil
	}
	if size == 0 {
		slog.Warn("swap: pool too small for a useful swap volume, continuing without one")
		return nil
	}

	if err := run("zfs", "create",
		"-V", strconv.FormatInt(size, 10),
		"-b", strconv.Itoa(os.Getpagesize()),
		"-o", "logbias=throughput",
		"-o", "sync=always",
		"-o", "primarycache=metadata",
		"-o", "secondarycache=none",
		"-o", "compression=zle",
		"-o", "com.sun:auto-snapshot=false",
		swapZvolName,
	); err != nil {
		slog.Warn("swap: could not create swap volume, continuing without swap", "err", err)
		return nil
	}

	dev := "/dev/zvol/" + swapZvolName
	if err := waitForPath(dev, time.Now().Add(10*time.Second)); err != nil {
		slog.Warn("swap: zvol device node never appeared, continuing without swap", "err", err)
		return nil
	}
	if err := run("mkswap", "-f", dev); err != nil {
		slog.Warn("swap: mkswap failed, continuing without swap", "err", err)
		return nil
	}
	slog.Info("swap: created", "zvol", swapZvolName, "bytes", size)
	return nil
}

func createSwapFile() error {
	size, err := swapSize(mountRoot)
	if err != nil {
		slog.Warn("swap: cannot size swap file, continuing without one", "err", err)
		return nil
	}
	if size == 0 {
		slog.Warn("swap: disk too small for a useful swap file, continuing without one",
			"min_bytes", swapMinBytes)
		return nil
	}

	path := filepath.Join(mountRoot, swapFileName)
	// fallocate rather than dd: on ext4 it reserves real extents, so mkswap and
	// swapon accept it, and a 32 GiB file costs no write time during install.
	if err := run("fallocate", "-l", strconv.FormatInt(size, 10), path); err != nil {
		slog.Warn("swap: fallocate failed, continuing without swap", "err", err)
		_ = os.Remove(path)
		return nil
	}
	// mkswap refuses a world-readable file, and its contents are process memory.
	if err := os.Chmod(path, 0o600); err != nil {
		slog.Warn("swap: chmod failed, continuing without swap", "err", err)
		_ = os.Remove(path)
		return nil
	}
	if err := run("mkswap", path); err != nil {
		slog.Warn("swap: mkswap failed, continuing without swap", "err", err)
		_ = os.Remove(path)
		return nil
	}
	slog.Info("swap: created", "path", "/"+swapFileName, "bytes", size)
	return nil
}

// swapSize returns the swap file size for the filesystem mounted at root, or 0
// when the disk cannot spare a useful amount. SPINIFEX_SWAP_SIZE overrides the
// calculation; 0 disables swap entirely.
//
// RAM drives the size and the disk only caps it, so a small test VM stays cheap
// on disk while a real host gets enough to carry a large import.
func swapSize(root string) (int64, error) {
	if v := strings.TrimSpace(os.Getenv("SPINIFEX_SWAP_SIZE")); v != "" {
		size, err := parseSize(v)
		if err != nil {
			return 0, fmt.Errorf("SPINIFEX_SWAP_SIZE: %w", err)
		}
		return size, nil
	}

	ram, err := totalRAM()
	if err != nil {
		return 0, err
	}
	size := ram / 2
	if ram >= swapRAMThreshold {
		size = swapLargeBytes
	}

	var st syscall.Statfs_t
	if err := syscall.Statfs(root, &st); err != nil {
		return 0, fmt.Errorf("statfs %s: %w", root, err)
	}
	// Free space, not total: the rootfs is already copied in, so this is what is
	// genuinely left to give away.
	free := int64(st.Bavail) * st.Bsize
	if diskCap := free / swapDiskShare; size > diskCap {
		size = diskCap
	}
	if size < swapMinBytes {
		return 0, nil
	}
	return size, nil
}

// totalRAM reports installed memory in bytes.
func totalRAM() (int64, error) {
	var si syscall.Sysinfo_t
	if err := syscall.Sysinfo(&si); err != nil {
		return 0, fmt.Errorf("sysinfo: %w", err)
	}
	unit := int64(si.Unit)
	if unit == 0 {
		unit = 1
	}
	return int64(si.Totalram) * unit, nil
}

// parseSize reads a plain byte count or a G/M suffixed size ("32G", "512M").
func parseSize(s string) (int64, error) {
	mult := int64(1)
	switch {
	case strings.HasSuffix(s, "G"), strings.HasSuffix(s, "g"):
		mult, s = 1<<30, s[:len(s)-1]
	case strings.HasSuffix(s, "M"), strings.HasSuffix(s, "m"):
		mult, s = 1<<20, s[:len(s)-1]
	}
	n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
	if err != nil {
		return 0, fmt.Errorf("invalid size %q", s)
	}
	if n < 0 {
		return 0, fmt.Errorf("negative size %q", s)
	}
	return n * mult, nil
}

// partUUID reads a partition's filesystem UUID. A variable so fstab generation
// can be tested without real block devices.
var partUUID = func(dev string) (string, error) {
	out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", dev).Output()
	if err != nil {
		return "", fmt.Errorf("blkid %s: %w", dev, err)
	}
	uuid := strings.TrimSpace(string(out))
	if uuid == "" {
		return "", fmt.Errorf("blkid returned no UUID for %s — partition may not have a filesystem yet", dev)
	}
	return uuid, nil
}

// copyGrubFont copies the unicode.pf2 GRUB font into the installed system's
// /boot/grub/fonts/ directory. This ensures update-grub finds the font at
// /boot/grub/fonts/unicode.pf2 — the same path the ISO's grub.cfg uses —
// so the generated grub.cfg enables gfxterm and the background image.
// Without this, grub-mkconfig falls back to /usr/share/grub/unicode.pf2,
// a path that GRUB 2.12 (trixie) may fail to resolve at boot time.
// Non-fatal — a missing source is logged and skipped.
func copyGrubFont(root string) {
	candidates := []string{
		"/cdrom/boot/grub/fonts/unicode.pf2", // ISO tree (preferred)
		"/usr/share/grub/unicode.pf2",        // live system grub-common fallback
	}
	for _, src := range candidates {
		in, err := os.Open(src)
		if err != nil {
			continue
		}
		defer in.Close()
		dstDir := filepath.Join(root, "boot/grub/fonts")
		if err := os.MkdirAll(dstDir, 0o755); err != nil {
			slog.Warn("copyGrubFont: cannot create fonts dir", "err", err)
			return
		}
		out, err := os.OpenFile(filepath.Join(dstDir, "unicode.pf2"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
		if err != nil {
			slog.Warn("copyGrubFont: cannot open destination", "err", err)
			return
		}
		defer out.Close()
		if _, err := io.Copy(out, in); err != nil {
			slog.Warn("copyGrubFont: copy failed", "err", err)
		}
		return
	}
	slog.Warn("copyGrubFont: no unicode.pf2 found, splash may not display")
}

// copySplashImage copies the GRUB splash from the live ISO (/cdrom/boot/grub/splash.png)
// into the installed system so the post-install GRUB shows the same branded background
// as the installer. Non-fatal — a missing or unreadable source is logged and skipped.
func copySplashImage(root string) {
	const src = "/usr/share/spinifex/grub-splash.png"
	in, err := os.Open(src)
	if err != nil {
		slog.Warn("copySplashImage: splash not found, skipping", "path", src)
		return
	}
	defer in.Close()

	dstDir := filepath.Join(root, "boot/grub")
	if err := os.MkdirAll(dstDir, 0o755); err != nil {
		slog.Warn("copySplashImage: cannot create boot/grub dir", "err", err)
		return
	}
	out, err := os.OpenFile(filepath.Join(dstDir, "splash.png"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
	if err != nil {
		slog.Warn("copySplashImage: cannot open destination", "err", err)
		return
	}
	defer out.Close()
	if _, err := io.Copy(out, in); err != nil {
		slog.Warn("copySplashImage: copy failed", "err", err)
	}
}

// maskSystemdUnit creates a symlink to /dev/null for the given unit, which is
// the standard way to permanently disable a unit so systemd will never start it.
func maskSystemdUnit(root, unit string) error {
	unitPath := filepath.Join(root, "etc/systemd/system", unit)
	_ = os.Remove(unitPath)
	return os.Symlink("/dev/null", unitPath)
}

// setShadowPassword sets a Unix password on the installed system without
// going through PAM. See the long comment in installSpinifex for the
// rationale.
func setShadowPassword(user, password string) error {
	cmd := exec.Command("chpasswd", "-c", "YESCRYPT", "-R", mountRoot)
	cmd.Stdin = strings.NewReader(user + ":" + password)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// chrootMountPaths lists virtual filesystems to bind-mount into the chroot.
// Order matters: unbind in reverse.
var chrootMountPaths = []string{"dev", "proc", "sys"}

// bindChrootMounts bind-mounts /dev, /proc, and /sys into the installed rootfs
// so chroot commands (chpasswd, systemd-machine-id-setup, update-grub) can
// access hardware, process info, and entropy sources. Idempotent — already-
// mounted paths are skipped.
func bindChrootMounts() error {
	for _, m := range chrootMountPaths {
		dst := filepath.Join(mountRoot, m)
		if err := os.MkdirAll(dst, 0o755); err != nil {
			return fmt.Errorf("create chroot mountpoint /%s: %w", m, err)
		}
		if err := run("mount", "--bind", "/"+m, dst); err != nil {
			return fmt.Errorf("bind-mount /%s into chroot: %w", m, err)
		}
	}
	return nil
}

// unbindChrootMounts unmounts the virtual filesystems in reverse order.
// Errors are logged but not returned — this is best-effort cleanup.
func unbindChrootMounts() {
	for _, v := range slices.Backward(chrootMountPaths) {
		// Quiet: this also runs from the deferred cleanup, where nothing is
		// mounted and three "no mount point specified" lines above a real error
		// message are pure distraction.
		_ = runQuiet("umount", filepath.Join(mountRoot, v))
	}
}

var run = func(name string, args ...string) error {
	return runEnv(nil, name, args...)
}

// runQuiet is run with output discarded, for best-effort probes whose failure
// is expected and whose stderr would otherwise look like an install fault.
var runQuiet = func(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	return cmd.Run()
}

// runEnv is run with extra environment variables appended to the installer's own.
var runEnv = func(env []string, name string, args ...string) error {
	cmd := exec.Command(name, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if len(env) > 0 {
		cmd.Env = append(os.Environ(), env...)
	}
	return cmd.Run()
}