spinifex/cmd/installer/install/boottool.go

package install

import (
	_ "embed"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
)

//go:embed assets/spinifex-boot-tool
var bootToolScript []byte

//go:embed assets/grub-install-wrapper
var grubInstallWrapper []byte

const (
	bootToolPath     = "/usr/sbin/spinifex-boot-tool"
	grubInstallPath  = "/usr/sbin/grub-install"
	grubInstallReal  = "/usr/sbin/grub-install.real"
	bootToolUUIDList = "/etc/kernel/spinifex-boot-uuids"
)

// installBootToolZFS sets up per-disk bootloaders on a ZFS root system.
//
// Nothing in the boot path is allowed to read the pool: GRUB's ZFS support
// trails OpenZFS, so a pool feature flag enabled years from now could make the
// machine unbootable. Each disk instead carries a self-contained ESP with its
// own GRUB, kernels and initramfs.
func installBootToolZFS(cfg DiskConfig) error {
	if err := writeTargetFile(bootToolPath, bootToolScript, 0o755); err != nil {
		return err
	}
	if err := writeGrubDefaults(cfg); err != nil {
		return err
	}

	if err := bindChrootMounts(); err != nil {
		return err
	}
	defer unbindChrootMounts()

	// Diverted before the first init: init runs a refresh, and refresh calls
	// grub-install. With the wrapper already in place that call has to reach
	// the real binary, which is what the diversion arranges.
	if err := divertGrubInstall(); err != nil {
		return err
	}

	// A partially-updated set of ESPs is an install that boots today and fails
	// after the next kernel upgrade, so every disk must succeed here.
	env := []string{"SPINIFEX_BOOT_TOOL_STRICT=1"}
	for _, d := range cfg.Disks {
		esp := d.PartitionPath(espPartNum)
		if err := runEnv(env, "chroot", mountRoot, bootToolPath, "init", esp); err != nil {
			return fmt.Errorf("spinifex-boot-tool init %s: %w", esp, err)
		}
	}

	// The one moment firmware boot variables are written. Refreshes run on
	// every kernel upgrade and deliberately leave NVRAM alone: repeatedly
	// rewriting boot entries has bricked more than one vendor's boards. Losing
	// the entry is survivable anyway, because each ESP also carries the
	// removable-path loader that firmware falls back to.
	if err := runEnv(append(env, "SPINIFEX_BOOT_TOOL_NVRAM=1"),
		"chroot", mountRoot, bootToolPath, "refresh"); err != nil {
		return fmt.Errorf("spinifex-boot-tool refresh: %w", err)
	}
	return verifyBootConfig(cfg)
}

// divertGrubInstall moves the real grub-install aside and puts the wrapper in
// its place. dpkg-divert makes the move survive package upgrades, which is the
// point: grub-pc's postinst runs grub-install and would otherwise quietly
// replace a working boot layout with one that reads the pool.
func divertGrubInstall() error {
	if err := run("chroot", mountRoot, "dpkg-divert", "--divert", grubInstallReal,
		"--rename", "--add", grubInstallPath); err != nil {
		return fmt.Errorf("divert grub-install: %w", err)
	}
	return writeTargetFile(grubInstallPath, grubInstallWrapper, 0o755)
}

// verifyBootConfig asserts that the config the machine will actually boot from
// names the ZFS root. grub-mkconfig degrades quietly when grub-probe cannot
// read the pool, emitting a config that gets as far as the GRUB menu and then
// panics with "unable to mount root fs" — which looks like a hardware fault.
func verifyBootConfig(cfg DiskConfig) error {
	uuids, err := os.ReadFile(filepath.Join(mountRoot, strings.TrimPrefix(bootToolUUIDList, "/")))
	if err != nil {
		return fmt.Errorf("read ESP list: %w", err)
	}
	if len(strings.Fields(string(uuids))) != len(cfg.Disks) {
		return fmt.Errorf("expected %d registered ESPs, found %d — some disks have no bootloader",
			len(cfg.Disks), len(strings.Fields(string(uuids))))
	}

	// One ESP is enough to prove grub-mkconfig produced the right thing; they
	// are all generated by the same code path in the same run.
	mp := filepath.Join(mountRoot, "var/tmp/verify-esp")
	if err := os.MkdirAll(mp, 0o755); err != nil {
		return err
	}
	defer func() { _ = os.Remove(mp) }()

	esp := cfg.Disks[0].PartitionPath(espPartNum)
	if err := run("mount", esp, mp); err != nil {
		return fmt.Errorf("mount %s for verification: %w", esp, err)
	}
	defer func() { _ = run("umount", mp) }()

	grubCfg, err := os.ReadFile(filepath.Join(mp, "grub/grub.cfg"))
	if err != nil {
		return fmt.Errorf("read grub.cfg from %s: %w", esp, err)
	}
	want := "root=ZFS=" + ZFSRootDataset
	if !strings.Contains(string(grubCfg), want) {
		return fmt.Errorf("grub.cfg on %s does not contain %q — the installed system would not find its root filesystem", esp, want)
	}
	if !strings.Contains(string(grubCfg), "vmlinuz-") {
		return fmt.Errorf("grub.cfg on %s references no kernel", esp)
	}
	return nil
}

// writeTargetFile writes a file into the install target.
func writeTargetFile(path string, content []byte, mode os.FileMode) error {
	full := filepath.Join(mountRoot, strings.TrimPrefix(path, "/"))
	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
		return err
	}
	if err := os.WriteFile(full, content, mode); err != nil {
		return fmt.Errorf("write %s: %w", path, err)
	}
	// WriteFile does not apply the mode to a file that already exists, and this
	// path is re-run on a retried install.
	return os.Chmod(full, mode)
}

// writeGrubDefaults writes /etc/default/grub and the branding snippet.
//
// Both consoles are listed: tty0 stays primary for a local install while ttyS0
// mirrors kernel output for headless racks and CI. The last console= on the
// cmdline becomes the system console, so serial-only hardware still works.
func writeGrubDefaults(cfg DiskConfig) error {
	cmdline := "console=tty0 console=ttyS0,115200 systemd.show_status=1"
	if cfg.FS.IsZFS() {
		// Stated explicitly rather than relying on grub-mkconfig's probe. It
		// appears after the root= that 10_linux computes, so it wins if the
		// probe came back with something wrong.
		cmdline = fmt.Sprintf("root=ZFS=%s boot=zfs %s", ZFSRootDataset, cmdline)
	}
	grubDefault := fmt.Sprintf(`GRUB_DEFAULT=0
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR=Spinifex
GRUB_CMDLINE_LINUX_DEFAULT=""
GRUB_CMDLINE_LINUX="%s"
`, cmdline)
	if err := os.WriteFile(filepath.Join(mountRoot, "etc/default/grub"), []byte(grubDefault), 0o644); err != nil {
		return fmt.Errorf("write /etc/default/grub: %w", err)
	}

	// gfxterm must be activated before serial is appended — background_image
	// silently does nothing in text mode. $prefix is used rather than a literal
	// /boot/grub so the same snippet is correct whether GRUB's root is the ext4
	// filesystem or a per-disk ESP.
	grubTheme := `#!/bin/sh
exec tail -n +3 $0
insmod all_video
insmod font
if loadfont $prefix/fonts/unicode.pf2; then
  set gfxmode=auto
  insmod gfxterm
  terminal_output gfxterm
fi
insmod serial
if serial --unit=0 --speed=115200 --timeout=1; then
  terminal_input  --append serial
  terminal_output --append serial
fi

# --- Branding ---
insmod png
set theme=$prefix/theme.txt
export theme
`
	if err := os.WriteFile(filepath.Join(mountRoot, "etc/grub.d/06_spinifex"), []byte(grubTheme), 0o755); err != nil {
		return fmt.Errorf("write /etc/grub.d/06_spinifex: %w", err)
	}
	return nil
}

// refreshBootToolNote logs the command an operator needs after replacing a
// disk. Nothing in the installer can do this for them later.
func refreshBootToolNote(cfg DiskConfig) {
	if cfg.FS.IsZFS() {
		slog.Info("bootloader installed on every pool member",
			"disks", len(cfg.Disks),
			"after_disk_replacement", "spinifex-boot-tool format <esp> && spinifex-boot-tool init <esp>")
	}
}