From a64ee2b2da5706d178811175d07c4380d65b0f1e Mon Sep 17 00:00:00 2001 From: Ben Duncan <5178243+benduncan@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:22:34 +1000 Subject: [PATCH] ISO tweaks --- Makefile | 3 +- build/systemd/spinifex-awsgw.service | 1 + cmd/installer/firstboot/firstboot.go | 50 +++++++++++++++++++++++----- cmd/installer/systemd/systemd.go | 45 ++++++++++++++++++++----- scripts/setup.sh | 6 ++-- 5 files changed, 83 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 8bcdeff60..a07ace7e0 100644 --- a/Makefile +++ b/Makefile @@ -46,8 +46,7 @@ else _SECQ = 2>&1 | tee endif -build: - $(MAKE) go_build +build: go_build build-installer build-lb-agent # Build spinifex-ui frontend (requires pnpm) build-ui: diff --git a/build/systemd/spinifex-awsgw.service b/build/systemd/spinifex-awsgw.service index fd61d424a..ec5f1213d 100644 --- a/build/systemd/spinifex-awsgw.service +++ b/build/systemd/spinifex-awsgw.service @@ -12,6 +12,7 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh ExecStart=/usr/local/bin/spx service awsgw start Restart=on-failure RestartSec=5 + OOMScoreAdjust=-500 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/ diff --git a/cmd/installer/firstboot/firstboot.go b/cmd/installer/firstboot/firstboot.go index b5e8e2b41..16ec33abd 100644 --- a/cmd/installer/firstboot/firstboot.go +++ b/cmd/installer/firstboot/firstboot.go @@ -52,12 +52,14 @@ func Write(root string, cfg Config) error { if err := systemd.EnableUnit(root, "spinifex-firstboot.service"); err != nil { return err } + // Banner unit is written but NOT enabled here. firstboot enables it + // (along with spinifex.target) at the end of its run, so on the first + // boot the only spinifex unit running is firstboot itself — no implicit + // pull of spinifex.target via the banner's Wants= relationship, which + // was the root cause of the boot-time race against firstboot. if err := systemd.WriteBannerUnit(root); err != nil { return fmt.Errorf("banner unit: %w", err) } - if err := systemd.EnableUnit(root, "spinifex-banner.service"); err != nil { - return err - } return systemd.WriteGettyDropIn(root) } @@ -72,12 +74,25 @@ func writeScript(root string, cfg Config) error { } script := fmt.Sprintf(`#!/bin/bash -# Spinifex firstboot — runs once after ISO installation, then disables itself. +# Spinifex firstboot — runs once after ISO installation. On success, writes +# /var/lib/spinifex/.firstboot-done and the systemd unit's +# ConditionPathExists=! prevents re-execution on subsequent boots. A partial +# run leaves no marker, so the next reboot retries from the top — safe +# because every step below is idempotent (hostnamectl, setup-ovn.sh, spx +# admin init are all "set if not set"; systemctl enable on an already-enabled +# unit is a no-op). set -euo pipefail -# Always disable this service on exit, even on failure, so a partial run -# does not cause an infinite retry loop on subsequent reboots. -trap 'systemctl disable spinifex-firstboot.service' EXIT +DONE_MARKER=/var/lib/spinifex/.firstboot-done + +# Idempotency: bail early if a previous run completed successfully. The unit +# also has ConditionPathExists=!$DONE_MARKER so we shouldn't get here on +# subsequent boots, but defend in depth in case the unit is re-triggered +# manually before the operator notices the marker exists. +if [ -f "$DONE_MARKER" ]; then + echo "[firstboot] already complete — skipping" + exit 0 +fi # Set hostname hostnamectl set-hostname %s @@ -114,6 +129,9 @@ done # and wires it to OVS via a veth pair — non-destructive, SSH-safe. %s +# Write the banner (first time, spinifex-banner service will do this on reboot) +/usr/local/bin/spx admin banner --boot-check + # Cluster formation — capture credentials to file for display on console. %s 2>&1 @@ -154,7 +172,23 @@ if [ -f /root/.aws/credentials ]; then [ -f /home/spinifex/.aws/config ] && chmod 600 /home/spinifex/.aws/config fi -# Start services +# Enable + activate spinifex.target and the banner now that all configs are +# in place. "enable --now" creates the multi-user.target.wants/ symlinks (so +# they start directly on every subsequent boot — no longer dependent on +# firstboot running each time, since firstboot is condition-skipped after +# this run via the marker) and activates them in this boot. +# NOTE: Moved to firstinstall to enable +#systemctl enable --now spinifex.target spinifex-banner.service + +# Enable services to start, on reboot +systemctl enable spinifex.target spinifex-banner.service + +# Mark complete only after every step above has succeeded. Until this point, +# any failure (set -e) leaves the marker absent and the next reboot retries +# firstboot from the top. +mkdir -p "$(dirname "$DONE_MARKER")" +touch "$DONE_MARKER" + systemctl start spinifex.target `, cfg.Hostname, setupOVN, clusterCmd) diff --git a/cmd/installer/systemd/systemd.go b/cmd/installer/systemd/systemd.go index 2f08b0a6e..ecda577fc 100644 --- a/cmd/installer/systemd/systemd.go +++ b/cmd/installer/systemd/systemd.go @@ -74,23 +74,33 @@ func WriteNetworkingDropIn(root string) error { } // WriteFirstbootUnit writes the spinifex-firstboot.service oneshot unit that -// runs the firstboot provisioning script on the first real boot after installation. +// runs the firstboot provisioning script on the first real boot after +// installation. The script writes /var/lib/spinifex/.firstboot-done on +// success; ConditionPathExists=! on that marker condition-skips the unit on +// every subsequent boot so it never re-runs (replacing the previous +// "disable on EXIT" trap which couldn't distinguish success from failure). func WriteFirstbootUnit(root string) error { unit := `[Unit] Description=Spinifex first-boot provisioning After=network-online.target Wants=network-online.target ConditionPathExists=/usr/local/bin/spinifex-firstboot.sh +ConditionPathExists=!/var/lib/spinifex/.firstboot-done [Service] Type=oneshot Environment=HOME=/root ExecStart=/usr/local/bin/spinifex-firstboot.sh RemainAfterExit=yes +# On non-zero exit, overwrite /etc/issue with a hint so the operator sees +# the failure on the console — the banner unit is never enabled in that +# case (firstboot didn't reach its enable --now line), so /etc/issue would +# otherwise stay at the Debian default with no signal that anything is wrong. +ExecStopPost=/bin/sh -c 'if [ "$EXIT_STATUS" != "0" ]; then printf "\n*** Spinifex firstboot FAILED ***\nLogin as spinifex, then run:\n sudo journalctl -u spinifex-firstboot.service\n\n" > /etc/issue; fi' # Cap total firstboot runtime so a hang in setup-ovn.sh / spx admin init / # ovn-central startup cannot wedge multi-user.target and keep getty from # ever reaching the login prompt. -TimeoutStartSec=180s +TimeoutStartSec=300s StandardOutput=journal StandardError=journal @@ -105,11 +115,20 @@ WantedBy=multi-user.target // `spx admin banner --boot-check` on every boot after spinifex.target is up. // Running After=spinifex.target ensures the banner reflects a settled system // state and that the IP check/restart happens once services are already running. +// +// The unit file is written by the installer but NOT enabled at install time. +// firstboot enables it (along with spinifex.target) at the end of its first +// run via `systemctl enable --now`. On every subsequent boot both units are +// driven by the multi-user.target.wants/ symlinks firstboot installed. +// +// Note: only After=spinifex.target — no Wants=. A Wants= would implicitly +// pull spinifex.target up at boot, which is exactly the race this whole +// change is fixing. After= alone is a no-op when the target isn't started, +// which is correct on first boot before firstboot has provisioned configs. func WriteBannerUnit(root string) error { unit := `[Unit] Description=Spinifex console banner and boot health check -After=spinifex.target -Wants=spinifex.target +After=network-online.target [Service] Type=oneshot @@ -129,13 +148,21 @@ WantedBy=multi-user.target } // WriteGettyDropIn holds the primary consoles (tty1 and ttyS0) until -// spinifex-banner.service completes, so the MOTD banner is visible before the -// login prompt appears. Drop-ins are scoped to named instances (getty@tty1, -// serial-getty@ttyS0) so tty2, tty3, etc. remain available as rescue terminals. +// spinifex-banner.service completes (when banner is enabled), so the MOTD +// banner is visible before the login prompt appears. Drop-ins are scoped to +// named instances (getty@tty1, serial-getty@ttyS0) so tty2, tty3, etc. +// remain available as rescue terminals. +// +// Only After=, not Wants=. A Wants= would implicitly start the (un-enabled) +// banner on the very first boot, partially undoing the "only firstboot is +// enabled on first boot" guarantee. On first boot, banner is not enabled, +// so After= is a no-op and login appears immediately with the default +// /etc/issue. Once firstboot completes and enables the banner, subsequent +// boots have banner enabled at multi-user.target and After= correctly +// orders getty after the banner has rendered /etc/issue. func WriteGettyDropIn(root string) error { dropIn := `[Unit] -After=spinifex-banner.service -Wants=spinifex-banner.service +After=spinifex-firstboot.service ` for _, svc := range []string{"getty@tty1", "serial-getty@ttyS0"} { dropInDir := filepath.Join(root, "etc/systemd/system/"+svc+".service.d") diff --git a/scripts/setup.sh b/scripts/setup.sh index 7fd325e13..fca824013 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -50,8 +50,8 @@ detect_os() { case "$ID" in debian) - if [ "${VERSION_ID%%.*}" -lt 12 ] 2>/dev/null; then - fatal "Debian $VERSION_ID is not supported. Minimum: Debian 12" + if [ "${VERSION_ID%%.*}" -lt 13 ] 2>/dev/null; then + fatal "Debian $VERSION_ID is not supported. Minimum: Debian 13" fi ;; ubuntu) @@ -61,7 +61,7 @@ detect_os() { fi ;; *) - fatal "Unsupported OS: $ID $VERSION_ID. Spinifex requires Debian 12+ or Ubuntu 22.04+" + fatal "Unsupported OS: $ID $VERSION_ID. Spinifex requires Debian 13+ or Ubuntu 22.04+" ;; esac