// Package firstboot writes the oneshot systemd service and configuration that // completes Spinifex provisioning on the first real boot after installation. package firstboot import ( "fmt" "os" "path/filepath" "strings" "github.com/mulgadc/spinifex/cmd/installer/systemd" ) // Config holds the values the firstboot service needs to configure the node. type Config struct { Hostname string // EncapIP is the Geneve tunnel IP for OVN, taken from the vpc plane after // collapsing (vpc <- lan <- wan). Empty when that plane uses DHCP — // setup-ovn.sh auto-detects the IP from the default route in that case. EncapIP string // LANIP is the internal cluster address, taken from the lan plane after // collapsing. Passed as --bind and --cluster-bind so predastore replication, // the NATS mesh and OVN control traffic stay off the public interface. // Empty leaves the 0.0.0.0 wildcard default, which is correct for a // single-NIC node where lan folds onto wan. LANIP string // WANIP is the public address, taken from the wan plane. Passed as // --advertise so northstar's :53 listener, the awsgw registry host and the // dial target recorded for off-host clients stay on the public interface. // Without it a concrete --bind is echoed back as the advertise address, // which would move all three onto the internal plane. Empty when wan uses // DHCP, leaving spx to auto-detect from the default route. WANIP string // Email is the operator email collected by the TUI or SPINIFEX_EMAIL on // the headless path. Passed to `spx admin init --email=` when set; // omitted entirely when empty. Email string // GPUPassthrough enables VFIO GPU passthrough by passing --gpu-passthrough // to `spx admin init`, which writes gpu_passthrough = true in the daemon config. GPUPassthrough bool // InstallCallback, when non-empty, is curled once at the end of firstboot // after the success marker is written. Generic phone-home hook for // provisioning controllers (PXE/MAAS-style flows). Best-effort: non-2xx // and network failures are swallowed so the install is not gated on // controller reachability. InstallCallback string // SkipFormation skips spx admin init/join. Used when a provisioning // controller (e.g. bm-bootstrap.sh) owns cluster formation and will call // spx admin init/join itself with the correct multi-node parameters. SkipFormation bool } // Write drops the firstboot script and systemd unit into root, which should be // the path of the installed system's root filesystem (e.g. /mnt/spinifex-install). func Write(root string, cfg Config) error { if err := writeScript(root, cfg); err != nil { return fmt.Errorf("firstboot script: %w", err) } if err := systemd.WriteFirstbootUnit(root); err != nil { return fmt.Errorf("firstboot unit: %w", err) } if err := systemd.EnableUnit(root, "spinifex-firstboot.service"); err != nil { return err } // Banner unit is written but NOT enabled here. firstboot enables it // (along with spinifex.target) at the end of its run, so on the first // boot the only spinifex unit running is firstboot itself — no implicit // pull of spinifex.target via the banner's Wants= relationship, which // was the root cause of the boot-time race against firstboot. if err := systemd.WriteBannerUnit(root); err != nil { return fmt.Errorf("banner unit: %w", err) } return systemd.WriteGettyDropIn(root) } func writeScript(root string, cfg Config) error { clusterCmd := buildClusterCmd(cfg) // --encap-ip is optional: when DHCP is used the IP is unknown at install time // and setup-ovn.sh auto-detects it from the default route at boot. setupOVN := "/usr/local/bin/setup-ovn.sh --management" if cfg.EncapIP != "" { setupOVN += fmt.Sprintf(" --encap-ip=%s", cfg.EncapIP) } // Pre-start OVS and OVN central so their databases are initialised before // setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes // longer than setup-ovn.sh's internal 15-second timeout allows. Starting them // here and waiting until the NB DB is ready means setup-ovn.sh sees a live DB // the moment it starts — no races, no timeout failures. ovnPrestart := `systemctl start openvswitch-switch systemctl start ovn-central echo "Waiting for OVN NB DB to initialise..." for _i in $(seq 1 120); do if ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then echo "OVN NB DB ready (${_i}s)" break fi sleep 1 done` // When a provisioning controller owns formation (SkipFormation) it also owns // the clustered OVN bring-up via setup-ovn.sh's RAFT flags. firstboot must // not start a standalone ovn-central here: the single-node .db it would leave // blocks ovn-ctl's create/join (which require a clean DB). Defer both. if cfg.SkipFormation { ovnPrestart = `echo "[firstboot] OVN bring-up deferred to provisioning controller"` setupOVN = `echo "[firstboot] setup-ovn deferred to provisioning controller"` } callbackBlock := "" if cfg.InstallCallback != "" { callbackBlock = fmt.Sprintf( "\ncurl -fsS --max-time 10 --retry 2 --retry-delay 2 -o /dev/null %s || true\n", shellEscapeSingle(cfg.InstallCallback), ) } script := fmt.Sprintf(`#!/bin/bash # Spinifex firstboot — runs once after ISO installation. On success, writes # /var/lib/spinifex/.firstboot-done and the systemd unit's # ConditionPathExists=! prevents re-execution on subsequent boots. A partial # run leaves no marker, so the next reboot retries from the top — safe # because every step below is idempotent (hostnamectl, setup-ovn.sh, spx # admin init are all "set if not set"; systemctl enable on an already-enabled # unit is a no-op). set -euo pipefail DONE_MARKER=/var/lib/spinifex/.firstboot-done # Idempotency: bail early if a previous run completed successfully. The unit # also has ConditionPathExists=!$DONE_MARKER so we shouldn't get here on # subsequent boots, but defend in depth in case the unit is re-triggered # manually before the operator notices the marker exists. if [ -f "$DONE_MARKER" ]; then echo "[firstboot] already complete — skipping" exit 0 fi # Set hostname hostnamectl set-hostname %s %s # Create default nameservers if DHCP server returns blank printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolvconf/resolv.conf.d/base resolvconf -u # The RTC on a fresh machine can be badly wrong — a flat CMOS battery is routine # on refurbished hardware. Credentials and TLS certificates are minted further # down, and a certificate issued against a wrong clock carries a NotBefore that # outlives the correction, so the clock has to be right before that happens. # chronyd is already running; makestep forces an immediate correction instead of # waiting out the slew. Bounded and non-fatal: an air-gapped install must still # complete, and rtcsync writes the corrected time back so later boots start close. echo "[firstboot] syncing clock before credentials are minted" chronyc -a makestep >/dev/null 2>&1 || true if chronyc waitsync 12 0.1 0 5 >/dev/null 2>&1; then echo "[firstboot] clock synced: $(date -uIs)" else echo "[firstboot] WARNING: clock not synced, continuing with $(date -uIs)" fi # Configure OVN networking. # br-wan (and br-lan if present) are Linux bridges managed by systemd-networkd # (declared in /etc/systemd/network/ by the installer). setup-ovn.sh # auto-detects br-wan as the default route device and wires it to OVS via a # veth pair — non-destructive, SSH-safe. %s # Write the banner (first time, spinifex-banner service will do this on reboot) /usr/local/bin/spx admin banner --boot-check # Cluster formation. Output goes to the journal via the systemd unit; spx # admin init writes credentials to disk and never prints them. %s 2>&1 # Fix ownership of files spx admin init wrote. spx runs as root under # systemd, so /etc/spinifex/{spinifex.toml,master.key,ca.key,*.pem} and any # per-service files written under /var/lib/spinifex/* land as root:root. # Delegate to setup.sh's fix_file_ownership (single source of truth) — it # knows the per-file modes (ca.key 0600, *.pem 0644, toml/key 0640) that a # blunt recursive chmod would clobber. SETUP_STAGES=fixown /usr/local/share/spinifex/setup.sh # The /var/lib/spinifex/awsgw/config symlink was created at build time by # setup.sh's create_directories — it resolves to /etc/spinifex, so # {BaseDir}/config/master.key automatically points at /etc/spinifex/master.key. # Bootstrap the spinifex user's SSH directory with a generated key pair so # ~/.ssh exists and the operator can immediately use ssh-keygen / ssh-copy-id # without hitting "No such file or directory" errors documented in the setup guide. if [ ! -f /home/spinifex/.ssh/id_ed25519 ]; then mkdir -p /home/spinifex/.ssh chmod 700 /home/spinifex/.ssh ssh-keygen -t ed25519 -f /home/spinifex/.ssh/id_ed25519 -N "" -C "spinifex@$(hostname)" chown -R spinifex:spinifex /home/spinifex/.ssh chmod 600 /home/spinifex/.ssh/id_ed25519 chmod 644 /home/spinifex/.ssh/id_ed25519.pub fi # Copy AWS credentials to the spinifex user's home directory. # spx admin init runs with HOME=/root (set by the systemd unit), so credentials # land in /root/.aws/. Copy them to the spinifex user's home so the operator # can use the AWS CLI without sudo. if [ -f /root/.aws/credentials ]; then mkdir -p /home/spinifex/.aws cp /root/.aws/credentials /home/spinifex/.aws/credentials cp /root/.aws/config /home/spinifex/.aws/config 2>/dev/null || true chown -R spinifex:spinifex /home/spinifex/.aws chmod 700 /home/spinifex/.aws chmod 600 /home/spinifex/.aws/credentials [ -f /home/spinifex/.aws/config ] && chmod 600 /home/spinifex/.aws/config fi # Enable + activate spinifex.target and the banner now that all configs are # in place. "enable --now" creates the multi-user.target.wants/ symlinks (so # they start directly on every subsequent boot — no longer dependent on # firstboot running each time, since firstboot is condition-skipped after # this run via the marker) and activates them in this boot. # NOTE: Moved to firstinstall to enable #systemctl enable --now spinifex.target spinifex-banner.service # Enable services to start, on reboot systemctl enable spinifex.target spinifex-banner.service # setup.sh enables this on the package install path, but the ISO chroot returns # before that line (no systemd to talk to), so the ISO needs it here. An nft # ruleset is runtime state and does not survive a reboot; without this the unit # never reapplies the policy and the node comes back up open. Enabled, not # started: ConditionPathExists holds it inert until the daemon writes a peer file. systemctl enable spinifex-firewall.service # WantedBy=timers.target only self-activates once enabled — without this the # JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart. systemctl enable --now spinifex-nats-watchdog.timer # Mark complete only after every step above has succeeded. Until this point, # any failure (set -e) leaves the marker absent and the next reboot retries # firstboot from the top. mkdir -p "$(dirname "$DONE_MARKER")" touch "$DONE_MARKER" %s systemctl start spinifex.target # Wait for the daemon to bring up external networking. When external_mode=pool # is configured the daemon creates br-ext (the OVN external bridge) during # startup; launching instances before it is ready results in no public IP being # assigned. ip link requires no root and avoids parsing daemon logs. if grep -q 'external_mode.*pool' /etc/spinifex/spinifex.toml 2>/dev/null; then echo "[firstboot] waiting for external networking (br-ext)..." for _i in $(seq 1 30); do if ip link show br-ext >/dev/null 2>&1; then echo "[firstboot] br-ext ready (${_i}s)" break fi sleep 1 done if ! ip link show br-ext >/dev/null 2>&1; then echo "[firstboot] warning: br-ext not up after 30s — external networking may be delayed" fi fi `, cfg.Hostname, ovnPrestart, setupOVN, clusterCmd, callbackBlock) path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh") return os.WriteFile(path, []byte(script), 0o755) } func buildClusterCmd(cfg Config) string { if cfg.SkipFormation { return `echo "[firstboot] cluster formation skipped — provisioning controller will handle it"` } emailFlag := "" if cfg.Email != "" { // shellEscapeSingle keeps the email safe if it ever contains a // character shell treats specially — belt-and-braces; the regex // validator already rejects whitespace and @-chains. emailFlag = " --email=" + shellEscapeSingle(cfg.Email) } gpuFlag := "" if cfg.GPUPassthrough { gpuFlag = " --gpu-passthrough" } // Without these the node takes the 0.0.0.0 wildcard default and every // internal service resolves onto the auto-detected WAN address, which is // exactly what the three-plane model exists to prevent. bindFlags := "" if cfg.LANIP != "" { bindFlags = fmt.Sprintf(" --bind %s --cluster-bind %s", cfg.LANIP, cfg.LANIP) } // --advertise must be explicit whenever --bind is: spx echoes a concrete // bind address straight back as the advertise address and never reaches its // WAN auto-detection, which would silently publish the internal plane as // this node's public dial target. preamble := "" switch { case bindFlags == "": // Nothing pinned, so spx auto-detects both and a guessed advertise // address would only get in the way. case cfg.WANIP != "": bindFlags += " --advertise " + cfg.WANIP default: // A DHCP wan has no address at install time, but it does by the time // firstboot runs, so read it off the bridge instead of shipping the // bind address as the public one. preamble = wanAdvertisePreamble bindFlags += ` $SPX_ADVERTISE` } // Always a single-node cluster. The installer cannot form a multi-node one: // cluster membership decides the OVN database topology and the join token // only exists once the primary has booted, so neither is knowable while the // nodes are still being installed. Multi-node is a post-install conversion, // documented in the multi-node install guide. cmd := fmt.Sprintf("spx admin init --node %s --nodes 1%s%s%s", cfg.Hostname, bindFlags, emailFlag, gpuFlag) return preamble + cmd } // wanAdvertisePreamble resolves the wan plane's DHCP lease into $SPX_ADVERTISE // just before formation. br-wan is always the wan bridge — that plane is the one // role that cannot fold — so its address is the node's public identity. // // Falling through with SPX_ADVERTISE empty is deliberate: no lease means there // is no public address to advertise yet, and spx echoing the bind address is // still better than formation failing outright. const wanAdvertisePreamble = `# The wan plane leases its address, so it is only knowable at boot. Without this # spx would echo --bind back as the advertise address and publish the internal # plane as this node's public dial target. SPX_ADVERTISE="" echo "[firstboot] waiting for the wan plane to acquire an address..." for _i in $(seq 1 60); do _wan_ip=$(ip -4 -o addr show br-wan scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1) if [ -n "$_wan_ip" ]; then SPX_ADVERTISE=" --advertise $_wan_ip" echo "[firstboot] wan address $_wan_ip (${_i}s)" break fi sleep 1 done if [ -z "$SPX_ADVERTISE" ]; then echo "[firstboot] warning: br-wan has no address after 60s — this node will advertise its bind address" fi ` // shellEscapeSingle wraps s in single quotes with any embedded single // quotes escaped. Minimal — we only need this because the email value is // interpolated into a shell script written by Write(). func shellEscapeSingle(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" }