spinifex/cmd/installer/firstboot/firstboot.go

// Package firstboot writes the oneshot systemd service and configuration that
// completes Spinifex provisioning on the first real boot after installation.
package firstboot

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/mulgadc/spinifex/cmd/installer/systemd"
)

// Config holds the values the firstboot service needs to configure the node.
type Config struct {
	Hostname string
	// EncapIP is the Geneve tunnel IP for OVN, taken from the vpc plane after
	// collapsing (vpc <- lan <- wan). Empty when that plane uses DHCP —
	// setup-ovn.sh auto-detects the IP from the default route in that case.
	EncapIP string
	// LANIP is the internal cluster address, taken from the lan plane after
	// collapsing. Passed as --bind and --cluster-bind so predastore replication,
	// the NATS mesh and OVN control traffic stay off the public interface.
	// Empty leaves the 0.0.0.0 wildcard default, which is correct for a
	// single-NIC node where lan folds onto wan.
	LANIP string
	// WANIP is the public address, taken from the wan plane. Passed as
	// --advertise so northstar's :53 listener, the awsgw registry host and the
	// dial target recorded for off-host clients stay on the public interface.
	// Without it a concrete --bind is echoed back as the advertise address,
	// which would move all three onto the internal plane. Empty when wan uses
	// DHCP, leaving spx to auto-detect from the default route.
	WANIP string
	// Email is the operator email collected by the TUI or SPINIFEX_EMAIL on
	// the headless path. Passed to `spx admin init --email=<value>` when set;
	// omitted entirely when empty.
	Email string
	// GPUPassthrough enables VFIO GPU passthrough by passing --gpu-passthrough
	// to `spx admin init`, which writes gpu_passthrough = true in the daemon config.
	GPUPassthrough bool
	// InstallCallback, when non-empty, is curled once at the end of firstboot
	// after the success marker is written. Generic phone-home hook for
	// provisioning controllers (PXE/MAAS-style flows). Best-effort: non-2xx
	// and network failures are swallowed so the install is not gated on
	// controller reachability.
	InstallCallback string
	// SkipFormation skips spx admin init/join. Used when a provisioning
	// controller (e.g. bm-bootstrap.sh) owns cluster formation and will call
	// spx admin init/join itself with the correct multi-node parameters.
	SkipFormation bool
}

// Write drops the firstboot script and systemd unit into root, which should be
// the path of the installed system's root filesystem (e.g. /mnt/spinifex-install).
func Write(root string, cfg Config) error {
	if err := writeScript(root, cfg); err != nil {
		return fmt.Errorf("firstboot script: %w", err)
	}
	if err := systemd.WriteFirstbootUnit(root); err != nil {
		return fmt.Errorf("firstboot unit: %w", err)
	}
	if err := systemd.EnableUnit(root, "spinifex-firstboot.service"); err != nil {
		return err
	}
	// Banner unit is written but NOT enabled here. firstboot enables it
	// (along with spinifex.target) at the end of its run, so on the first
	// boot the only spinifex unit running is firstboot itself — no implicit
	// pull of spinifex.target via the banner's Wants= relationship, which
	// was the root cause of the boot-time race against firstboot.
	if err := systemd.WriteBannerUnit(root); err != nil {
		return fmt.Errorf("banner unit: %w", err)
	}
	return systemd.WriteGettyDropIn(root)
}

func writeScript(root string, cfg Config) error {
	clusterCmd := buildClusterCmd(cfg)

	// --encap-ip is optional: when DHCP is used the IP is unknown at install time
	// and setup-ovn.sh auto-detects it from the default route at boot.
	setupOVN := "/usr/local/bin/setup-ovn.sh --management"
	if cfg.EncapIP != "" {
		setupOVN += fmt.Sprintf(" --encap-ip=%s", cfg.EncapIP)
	}

	// Pre-start OVS and OVN central so their databases are initialised before
	// setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes
	// longer than setup-ovn.sh's internal 15-second timeout allows. Starting them
	// here and waiting until the NB DB is ready means setup-ovn.sh sees a live DB
	// the moment it starts — no races, no timeout failures.
	ovnPrestart := `systemctl start openvswitch-switch
systemctl start ovn-central
echo "Waiting for OVN NB DB to initialise..."
for _i in $(seq 1 120); do
    if ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then
        echo "OVN NB DB ready (${_i}s)"
        break
    fi
    sleep 1
done`

	// When a provisioning controller owns formation (SkipFormation) it also owns
	// the clustered OVN bring-up via setup-ovn.sh's RAFT flags. firstboot must
	// not start a standalone ovn-central here: the single-node .db it would leave
	// blocks ovn-ctl's create/join (which require a clean DB). Defer both.
	if cfg.SkipFormation {
		ovnPrestart = `echo "[firstboot] OVN bring-up deferred to provisioning controller"`
		setupOVN = `echo "[firstboot] setup-ovn deferred to provisioning controller"`
	}

	callbackBlock := ""
	if cfg.InstallCallback != "" {
		callbackBlock = fmt.Sprintf(
			"\ncurl -fsS --max-time 10 --retry 2 --retry-delay 2 -o /dev/null %s || true\n",
			shellEscapeSingle(cfg.InstallCallback),
		)
	}

	script := fmt.Sprintf(`#!/bin/bash
# Spinifex firstboot — runs once after ISO installation. On success, writes
# /var/lib/spinifex/.firstboot-done and the systemd unit's
# ConditionPathExists=! prevents re-execution on subsequent boots. A partial
# run leaves no marker, so the next reboot retries from the top — safe
# because every step below is idempotent (hostnamectl, setup-ovn.sh, spx
# admin init are all "set if not set"; systemctl enable on an already-enabled
# unit is a no-op).
set -euo pipefail

DONE_MARKER=/var/lib/spinifex/.firstboot-done

# Idempotency: bail early if a previous run completed successfully. The unit
# also has ConditionPathExists=!$DONE_MARKER so we shouldn't get here on
# subsequent boots, but defend in depth in case the unit is re-triggered
# manually before the operator notices the marker exists.
if [ -f "$DONE_MARKER" ]; then
    echo "[firstboot] already complete — skipping"
    exit 0
fi

# Set hostname
hostnamectl set-hostname %s

%s

# Create default nameservers if DHCP server returns blank
printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolvconf/resolv.conf.d/base
resolvconf -u

# Configure OVN networking.
# br-wan (and br-lan if present) are Linux bridges managed by systemd-networkd
# (declared in /etc/systemd/network/ by the installer). setup-ovn.sh
# auto-detects br-wan as the default route device and wires it to OVS via a
# veth pair — non-destructive, SSH-safe.
%s

# Write the banner (first time, spinifex-banner service will do this on reboot)
/usr/local/bin/spx admin banner --boot-check

# Cluster formation. Output goes to the journal via the systemd unit; spx
# admin init writes credentials to disk and never prints them.
%s 2>&1

# Fix ownership of files spx admin init wrote. spx runs as root under
# systemd, so /etc/spinifex/{spinifex.toml,master.key,ca.key,*.pem} and any
# per-service files written under /var/lib/spinifex/* land as root:root.
# Delegate to setup.sh's fix_file_ownership (single source of truth) — it
# knows the per-file modes (ca.key 0600, *.pem 0644, toml/key 0640) that a
# blunt recursive chmod would clobber.
SETUP_STAGES=fixown /usr/local/share/spinifex/setup.sh
# The /var/lib/spinifex/awsgw/config symlink was created at build time by
# setup.sh's create_directories — it resolves to /etc/spinifex, so
# {BaseDir}/config/master.key automatically points at /etc/spinifex/master.key.

# Bootstrap the spinifex user's SSH directory with a generated key pair so
# ~/.ssh exists and the operator can immediately use ssh-keygen / ssh-copy-id
# without hitting "No such file or directory" errors documented in the setup guide.
if [ ! -f /home/spinifex/.ssh/id_ed25519 ]; then
    mkdir -p /home/spinifex/.ssh
    chmod 700 /home/spinifex/.ssh
    ssh-keygen -t ed25519 -f /home/spinifex/.ssh/id_ed25519 -N "" -C "spinifex@$(hostname)"
    chown -R spinifex:spinifex /home/spinifex/.ssh
    chmod 600 /home/spinifex/.ssh/id_ed25519
    chmod 644 /home/spinifex/.ssh/id_ed25519.pub
fi

# Copy AWS credentials to the spinifex user's home directory.
# spx admin init runs with HOME=/root (set by the systemd unit), so credentials
# land in /root/.aws/. Copy them to the spinifex user's home so the operator
# can use the AWS CLI without sudo.
if [ -f /root/.aws/credentials ]; then
    mkdir -p /home/spinifex/.aws
    cp /root/.aws/credentials /home/spinifex/.aws/credentials
    cp /root/.aws/config /home/spinifex/.aws/config 2>/dev/null || true
    chown -R spinifex:spinifex /home/spinifex/.aws
    chmod 700 /home/spinifex/.aws
    chmod 600 /home/spinifex/.aws/credentials
    [ -f /home/spinifex/.aws/config ] && chmod 600 /home/spinifex/.aws/config
fi

# Enable + activate spinifex.target and the banner now that all configs are
# in place. "enable --now" creates the multi-user.target.wants/ symlinks (so
# they start directly on every subsequent boot — no longer dependent on
# firstboot running each time, since firstboot is condition-skipped after
# this run via the marker) and activates them in this boot.
# NOTE: Moved to firstinstall to enable
#systemctl enable --now spinifex.target spinifex-banner.service

# Enable services to start, on reboot
systemctl enable spinifex.target spinifex-banner.service
# WantedBy=timers.target only self-activates once enabled — without this the
# JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart.
systemctl enable --now spinifex-nats-watchdog.timer

# Mark complete only after every step above has succeeded. Until this point,
# any failure (set -e) leaves the marker absent and the next reboot retries
# firstboot from the top.
mkdir -p "$(dirname "$DONE_MARKER")"
touch "$DONE_MARKER"
%s
systemctl start spinifex.target

# Wait for the daemon to bring up external networking. When external_mode=pool
# is configured the daemon creates br-ext (the OVN external bridge) during
# startup; launching instances before it is ready results in no public IP being
# assigned. ip link requires no root and avoids parsing daemon logs.
if grep -q 'external_mode.*pool' /etc/spinifex/spinifex.toml 2>/dev/null; then
    echo "[firstboot] waiting for external networking (br-ext)..."
    for _i in $(seq 1 30); do
        if ip link show br-ext >/dev/null 2>&1; then
            echo "[firstboot] br-ext ready (${_i}s)"
            break
        fi
        sleep 1
    done
    if ! ip link show br-ext >/dev/null 2>&1; then
        echo "[firstboot] warning: br-ext not up after 30s — external networking may be delayed"
    fi
fi
`, cfg.Hostname, ovnPrestart, setupOVN, clusterCmd, callbackBlock)

	path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
	return os.WriteFile(path, []byte(script), 0o755)
}

func buildClusterCmd(cfg Config) string {
	if cfg.SkipFormation {
		return `echo "[firstboot] cluster formation skipped — provisioning controller will handle it"`
	}
	emailFlag := ""
	if cfg.Email != "" {
		// shellEscapeSingle keeps the email safe if it ever contains a
		// character shell treats specially — belt-and-braces; the regex
		// validator already rejects whitespace and @-chains.
		emailFlag = " --email=" + shellEscapeSingle(cfg.Email)
	}
	gpuFlag := ""
	if cfg.GPUPassthrough {
		gpuFlag = " --gpu-passthrough"
	}
	// Without these the node takes the 0.0.0.0 wildcard default and every
	// internal service resolves onto the auto-detected WAN address, which is
	// exactly what the three-plane model exists to prevent.
	bindFlags := ""
	if cfg.LANIP != "" {
		bindFlags = fmt.Sprintf(" --bind %s --cluster-bind %s", cfg.LANIP, cfg.LANIP)
	}
	// --advertise must be explicit whenever --bind is: spx echoes a concrete
	// bind address straight back as the advertise address and never reaches its
	// WAN auto-detection, which would silently publish the internal plane as
	// this node's public dial target.
	preamble := ""
	switch {
	case bindFlags == "":
		// Nothing pinned, so spx auto-detects both and a guessed advertise
		// address would only get in the way.
	case cfg.WANIP != "":
		bindFlags += " --advertise " + cfg.WANIP
	default:
		// A DHCP wan has no address at install time, but it does by the time
		// firstboot runs, so read it off the bridge instead of shipping the
		// bind address as the public one.
		preamble = wanAdvertisePreamble
		bindFlags += ` $SPX_ADVERTISE`
	}

	// Always a single-node cluster. The installer cannot form a multi-node one:
	// cluster membership decides the OVN database topology and the join token
	// only exists once the primary has booted, so neither is knowable while the
	// nodes are still being installed. Multi-node is a post-install conversion,
	// documented in the multi-node install guide.
	cmd := fmt.Sprintf("spx admin init --node %s --nodes 1%s%s%s", cfg.Hostname, bindFlags, emailFlag, gpuFlag)
	return preamble + cmd
}

// wanAdvertisePreamble resolves the wan plane's DHCP lease into $SPX_ADVERTISE
// just before formation. br-wan is always the wan bridge — that plane is the one
// role that cannot fold — so its address is the node's public identity.
//
// Falling through with SPX_ADVERTISE empty is deliberate: no lease means there
// is no public address to advertise yet, and spx echoing the bind address is
// still better than formation failing outright.
const wanAdvertisePreamble = `# The wan plane leases its address, so it is only knowable at boot. Without this
# spx would echo --bind back as the advertise address and publish the internal
# plane as this node's public dial target.
SPX_ADVERTISE=""
echo "[firstboot] waiting for the wan plane to acquire an address..."
for _i in $(seq 1 60); do
    _wan_ip=$(ip -4 -o addr show br-wan scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1)
    if [ -n "$_wan_ip" ]; then
        SPX_ADVERTISE=" --advertise $_wan_ip"
        echo "[firstboot] wan address $_wan_ip (${_i}s)"
        break
    fi
    sleep 1
done
if [ -z "$SPX_ADVERTISE" ]; then
    echo "[firstboot] warning: br-wan has no address after 60s — this node will advertise its bind address"
fi
`

// shellEscapeSingle wraps s in single quotes with any embedded single
// quotes escaped. Minimal — we only need this because the email value is
// interpolated into a shell script written by Write().
func shellEscapeSingle(s string) string {
	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}