ISO tweaks

Makefile

@@ -46,8 +46,7 @@ else
   _SECQ  = 2>&1 | tee
 endif
 
-build:
-	$(MAKE) go_build
+build: go_build build-installer build-lb-agent
 
 # Build spinifex-ui frontend (requires pnpm)
 build-ui:

build/systemd/spinifex-awsgw.service

@@ -12,6 +12,7 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service awsgw start
 Restart=on-failure
 RestartSec=5
+
 OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/

cmd/installer/firstboot/firstboot.go

@@ -52,12 +52,14 @@ func Write(root string, cfg Config) error {
 	if err := systemd.EnableUnit(root, "spinifex-firstboot.service"); err != nil {
 		return err
 	}
+	// Banner unit is written but NOT enabled here. firstboot enables it
+	// (along with spinifex.target) at the end of its run, so on the first
+	// boot the only spinifex unit running is firstboot itself — no implicit
+	// pull of spinifex.target via the banner's Wants= relationship, which
+	// was the root cause of the boot-time race against firstboot.
 	if err := systemd.WriteBannerUnit(root); err != nil {
 		return fmt.Errorf("banner unit: %w", err)
 	}
-	if err := systemd.EnableUnit(root, "spinifex-banner.service"); err != nil {
-		return err
-	}
 	return systemd.WriteGettyDropIn(root)
 }
 
@@ -72,12 +74,25 @@ func writeScript(root string, cfg Config) error {
 	}
 
 	script := fmt.Sprintf(`#!/bin/bash
-# Spinifex firstboot — runs once after ISO installation, then disables itself.
+# Spinifex firstboot — runs once after ISO installation. On success, writes
+# /var/lib/spinifex/.firstboot-done and the systemd unit's
+# ConditionPathExists=! prevents re-execution on subsequent boots. A partial
+# run leaves no marker, so the next reboot retries from the top — safe
+# because every step below is idempotent (hostnamectl, setup-ovn.sh, spx
+# admin init are all "set if not set"; systemctl enable on an already-enabled
+# unit is a no-op).
 set -euo pipefail
 
-# Always disable this service on exit, even on failure, so a partial run
-# does not cause an infinite retry loop on subsequent reboots.
-trap 'systemctl disable spinifex-firstboot.service' EXIT
+DONE_MARKER=/var/lib/spinifex/.firstboot-done
+
+# Idempotency: bail early if a previous run completed successfully. The unit
+# also has ConditionPathExists=!$DONE_MARKER so we shouldn't get here on
+# subsequent boots, but defend in depth in case the unit is re-triggered
+# manually before the operator notices the marker exists.
+if [ -f "$DONE_MARKER" ]; then
+    echo "[firstboot] already complete — skipping"
+    exit 0
+fi
 
 # Set hostname
 hostnamectl set-hostname %s
@@ -114,6 +129,9 @@ done
 # and wires it to OVS via a veth pair — non-destructive, SSH-safe.
 %s
 
+# Write the banner (first time, spinifex-banner service will do this on reboot)
+/usr/local/bin/spx admin banner --boot-check
+
 # Cluster formation — capture credentials to file for display on console.
 %s 2>&1
 
@@ -154,7 +172,23 @@ if [ -f /root/.aws/credentials ]; then
     [ -f /home/spinifex/.aws/config ] && chmod 600 /home/spinifex/.aws/config
 fi
 
-# Start services
+# Enable + activate spinifex.target and the banner now that all configs are
+# in place. "enable --now" creates the multi-user.target.wants/ symlinks (so
+# they start directly on every subsequent boot — no longer dependent on
+# firstboot running each time, since firstboot is condition-skipped after
+# this run via the marker) and activates them in this boot.
+# NOTE: Moved to firstinstall to enable
+#systemctl enable --now spinifex.target spinifex-banner.service
+
+# Enable services to start, on reboot
+systemctl enable spinifex.target spinifex-banner.service
+
+# Mark complete only after every step above has succeeded. Until this point,
+# any failure (set -e) leaves the marker absent and the next reboot retries
+# firstboot from the top.
+mkdir -p "$(dirname "$DONE_MARKER")"
+touch "$DONE_MARKER"
+
 systemctl start spinifex.target
 `, cfg.Hostname, setupOVN, clusterCmd)
 

cmd/installer/systemd/systemd.go

@@ -74,23 +74,33 @@ func WriteNetworkingDropIn(root string) error {
 }
 
 // WriteFirstbootUnit writes the spinifex-firstboot.service oneshot unit that
-// runs the firstboot provisioning script on the first real boot after installation.
+// runs the firstboot provisioning script on the first real boot after
+// installation. The script writes /var/lib/spinifex/.firstboot-done on
+// success; ConditionPathExists=! on that marker condition-skips the unit on
+// every subsequent boot so it never re-runs (replacing the previous
+// "disable on EXIT" trap which couldn't distinguish success from failure).
 func WriteFirstbootUnit(root string) error {
 	unit := `[Unit]
 Description=Spinifex first-boot provisioning
 After=network-online.target
 Wants=network-online.target
 ConditionPathExists=/usr/local/bin/spinifex-firstboot.sh
+ConditionPathExists=!/var/lib/spinifex/.firstboot-done
 
 [Service]
 Type=oneshot
 Environment=HOME=/root
 ExecStart=/usr/local/bin/spinifex-firstboot.sh
 RemainAfterExit=yes
+# On non-zero exit, overwrite /etc/issue with a hint so the operator sees
+# the failure on the console — the banner unit is never enabled in that
+# case (firstboot didn't reach its enable --now line), so /etc/issue would
+# otherwise stay at the Debian default with no signal that anything is wrong.
+ExecStopPost=/bin/sh -c 'if [ "$EXIT_STATUS" != "0" ]; then printf "\n*** Spinifex firstboot FAILED ***\nLogin as spinifex, then run:\n  sudo journalctl -u spinifex-firstboot.service\n\n" > /etc/issue; fi'
 # Cap total firstboot runtime so a hang in setup-ovn.sh / spx admin init /
 # ovn-central startup cannot wedge multi-user.target and keep getty from
 # ever reaching the login prompt.
-TimeoutStartSec=180s
+TimeoutStartSec=300s
 StandardOutput=journal
 StandardError=journal
 
@@ -105,11 +115,20 @@ WantedBy=multi-user.target
 // `spx admin banner --boot-check` on every boot after spinifex.target is up.
 // Running After=spinifex.target ensures the banner reflects a settled system
 // state and that the IP check/restart happens once services are already running.
+//
+// The unit file is written by the installer but NOT enabled at install time.
+// firstboot enables it (along with spinifex.target) at the end of its first
+// run via `systemctl enable --now`. On every subsequent boot both units are
+// driven by the multi-user.target.wants/ symlinks firstboot installed.
+//
+// Note: only After=spinifex.target — no Wants=. A Wants= would implicitly
+// pull spinifex.target up at boot, which is exactly the race this whole
+// change is fixing. After= alone is a no-op when the target isn't started,
+// which is correct on first boot before firstboot has provisioned configs.
 func WriteBannerUnit(root string) error {
 	unit := `[Unit]
 Description=Spinifex console banner and boot health check
-After=spinifex.target
-Wants=spinifex.target
+After=network-online.target
 
 [Service]
 Type=oneshot
@@ -129,13 +148,21 @@ WantedBy=multi-user.target
 }
 
 // WriteGettyDropIn holds the primary consoles (tty1 and ttyS0) until
-// spinifex-banner.service completes, so the MOTD banner is visible before the
-// login prompt appears. Drop-ins are scoped to named instances (getty@tty1,
-// serial-getty@ttyS0) so tty2, tty3, etc. remain available as rescue terminals.
+// spinifex-banner.service completes (when banner is enabled), so the MOTD
+// banner is visible before the login prompt appears. Drop-ins are scoped to
+// named instances (getty@tty1, serial-getty@ttyS0) so tty2, tty3, etc.
+// remain available as rescue terminals.
+//
+// Only After=, not Wants=. A Wants= would implicitly start the (un-enabled)
+// banner on the very first boot, partially undoing the "only firstboot is
+// enabled on first boot" guarantee. On first boot, banner is not enabled,
+// so After= is a no-op and login appears immediately with the default
+// /etc/issue. Once firstboot completes and enables the banner, subsequent
+// boots have banner enabled at multi-user.target and After= correctly
+// orders getty after the banner has rendered /etc/issue.
 func WriteGettyDropIn(root string) error {
 	dropIn := `[Unit]
-After=spinifex-banner.service
-Wants=spinifex-banner.service
+After=spinifex-firstboot.service
 `
 	for _, svc := range []string{"getty@tty1", "serial-getty@ttyS0"} {
 		dropInDir := filepath.Join(root, "etc/systemd/system/"+svc+".service.d")

scripts/setup.sh

@@ -50,8 +50,8 @@ detect_os() {
 
     case "$ID" in
         debian)
-            if [ "${VERSION_ID%%.*}" -lt 12 ] 2>/dev/null; then
-                fatal "Debian $VERSION_ID is not supported. Minimum: Debian 12"
+            if [ "${VERSION_ID%%.*}" -lt 13 ] 2>/dev/null; then
+                fatal "Debian $VERSION_ID is not supported. Minimum: Debian 13"
             fi
             ;;
         ubuntu)
@@ -61,7 +61,7 @@ detect_os() {
             fi
             ;;
         *)
-            fatal "Unsupported OS: $ID $VERSION_ID. Spinifex requires Debian 12+ or Ubuntu 22.04+"
+            fatal "Unsupported OS: $ID $VERSION_ID. Spinifex requires Debian 13+ or Ubuntu 22.04+"
             ;;
     esac