feat: reconcile systemd unit files on upgrade (#771)

* feat: reconcile systemd unit files on upgrade

Unit files were written once at install time by install_systemd() in
scripts/setup.sh and never re-asserted. A node upgraded by swapping the
spx binary kept whatever units it was installed with, so any fix that
lives in a unit rather than in code never reached it.

This is not theoretical. A node whose spinifex-viperblock.service
predates KillMode=process defaults to KillMode=control-group, so on a
target stop TimeoutStopSec expires mid-flush and systemd cgroup-SIGKILLs
every nbdkit while guest QEMU survives in the daemon's own cgroup: the
guest keeps running with no storage path.

Units are now stamped with a spinifex-unit-version marker, embedded in
the binary via a generated file, and reconciled by spx admin upgrade.
Missing or version-stale units are replaced after a timestamped backup;
a unit at the current version whose content differs is reported as
operator-modified and never overwritten. Drop-ins under <unit>.d are
untouched, so systemctl edit overrides survive.

Nothing is restarted. A daemon-reload is enough for KillMode and
TimeoutStopSec to apply to the next stop, so drift is corrected without
disturbing a running guest.

Adds --dry-run, --units-only and --skip-units to spx admin upgrade.

* test: cover systemd reconcile error paths and the admin upgrade CLI flow

Diff coverage on the unit-migration PR sat at 45.8%, below the 60% gate.
Adds tests for reconcile.go's error/edge paths (unreadable unit, backup
and write failures, daemon-reload failure, malformed version markers,
downgrade-is-conflict) and drives admin_upgrade.go's CLI flow directly
via an injectable exit and a decision matrix extracted into
validateUpgradeFlags/upgradeSummary for testability.

build/systemd/regenerate-ssh-host-keys.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Regenerate SSH host keys
 ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key

build/systemd/spinifex-awsgw.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex AWS Gateway
 PartOf=spinifex.target

build/systemd/spinifex-daemon.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target

build/systemd/spinifex-guests.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex guest VM tier
 Before=slices.target

build/systemd/spinifex-nats-watchdog.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog
 PartOf=spinifex.target

build/systemd/spinifex-nats-watchdog.timer

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog timer
 PartOf=spinifex.target

build/systemd/spinifex-nats.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS Message Bus
 PartOf=spinifex.target

build/systemd/spinifex-northstar.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Northstar (Authoritative DNS)
 PartOf=spinifex.target

build/systemd/spinifex-predastore.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target

build/systemd/spinifex-qmp-collector.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex QMP Telemetry Collector (guest metrics)
 PartOf=spinifex.target

build/systemd/spinifex-shutdown.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex graceful guest drain
 PartOf=spinifex.target

build/systemd/spinifex-system.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)
 Before=slices.target

build/systemd/spinifex-ui.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Web UI
 PartOf=spinifex.target

build/systemd/spinifex-viperblock.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Viperblock (Block Storage)
 PartOf=spinifex.target

build/systemd/spinifex-vpcd.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex VPC Daemon (OVN Networking)
 PartOf=spinifex.target

build/systemd/spinifex.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex resource-bounded slice (host ceiling)
 Before=slices.target

build/systemd/spinifex.target

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Infrastructure Platform
 After=network-online.target

cmd/spinifex/cmd/admin.go

@@ -330,6 +330,9 @@ func init() {
 
 	adminCmd.AddCommand(upgradeCmd)
 	upgradeCmd.Flags().Bool("yes", false, "Apply migrations without prompting")
+	upgradeCmd.Flags().Bool("dry-run", false, "Report pending config and unit changes without applying them")
+	upgradeCmd.Flags().Bool("units-only", false, "Reconcile systemd units only, skip config migrations")
+	upgradeCmd.Flags().Bool("skip-units", false, "Apply config migrations only, skip systemd unit reconciliation")
 	certRenewCmd.Flags().StringSlice("extra-ip", nil, "Additional IP addresses to include in SANs")
 	certRenewCmd.Flags().StringSlice("extra-dns", nil, "Additional DNS names to include in SANs")
 	accountCreateCmd.Flags().String("name", "", "Account name (required)")

cmd/spinifex/cmd/admin_upgrade.go

@@ -2,22 +2,70 @@ package cmd
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
 	"os"
 	"strings"
 
 	"github.com/mulgadc/spinifex/spinifex/migrate"
+	"github.com/mulgadc/spinifex/spinifex/systemd"
 
 	"github.com/spf13/cobra"
 )
 
+// systemdUnitRoot is where installed units live. A var, not a const, so
+// tests could point it elsewhere if this command ever grows a test harness.
+var systemdUnitRoot = "/etc/systemd/system"
+
+// upgradeExit is os.Exit, overridable in tests so a failure path can be
+// exercised without killing the test binary.
+var upgradeExit = os.Exit
+
+// errUpgradeFlagsConflict is returned when both --units-only and --skip-units
+// are set, a combination that would leave nothing for the command to do.
+var errUpgradeFlagsConflict = errors.New("--units-only and --skip-units are mutually exclusive")
+
+// validateUpgradeFlags checks flag combinations that runAdminUpgrade cannot
+// proceed with, kept separate from runAdminUpgrade so the check is testable
+// without driving the whole command.
+func validateUpgradeFlags(unitsOnly, skipUnits bool) error {
+	if unitsOnly && skipUnits {
+		return errUpgradeFlagsConflict
+	}
+	return nil
+}
+
+// upgradeSummary decides whether runAdminUpgrade has anything left to do
+// after gathering config and unit status, and what to print when it does
+// not. Kept pure so the dryRun × hasConfigWork × hasUnitWork matrix is
+// testable without driving the whole command.
+func upgradeSummary(dryRun, hasConfigWork, hasUnitWork bool) (proceed bool, message string) {
+	if !hasConfigWork && !hasUnitWork {
+		if dryRun {
+			return false, "\nNothing pending — config and units are current."
+		}
+		return false, "\nNothing to do."
+	}
+	if dryRun {
+		return false, ""
+	}
+	return true, ""
+}
+
 var upgradeCmd = &cobra.Command{
 	Use:   "upgrade",
-	Short: "Apply pending config migrations",
-	Long: `Apply pending configuration file migrations for upgrades between Spinifex versions.
+	Short: "Apply pending config migrations and systemd unit updates",
+	Long: `Apply pending configuration file migrations and systemd unit reconciliation
+for upgrades between Spinifex versions.
 
-When run interactively (without --yes), shows pending migrations and prompts
+Swapping the spx binary alone does not upgrade a node: unit files (KillMode,
+TimeoutStopSec, drain ordering, ...) are installed once at setup time and
+never re-asserted otherwise. This command brings both config and units up to
+date with the binary running it.
+
+When run interactively (without --yes), shows what is pending and prompts
 for confirmation. When called from setup.sh with --yes, applies immediately.
+--dry-run reports drift without prompting or changing anything.
 
 Operators can skip migrations during install by setting INSTALL_SPINIFEX_SKIP_MIGRATE=1,
 then run 'spx admin upgrade' manually to review and apply.`,
@@ -28,19 +76,90 @@ func runAdminUpgrade(cmd *cobra.Command, _ []string) {
 	configDir, _ := cmd.Root().Flags().GetString("config-dir")
 	dataDir, _ := cmd.Root().Flags().GetString("spinifex-dir")
 	yes, _ := cmd.Flags().GetBool("yes")
+	dryRun, _ := cmd.Flags().GetBool("dry-run")
+	unitsOnly, _ := cmd.Flags().GetBool("units-only")
+	skipUnits, _ := cmd.Flags().GetBool("skip-units")
+
+	if err := validateUpgradeFlags(unitsOnly, skipUnits); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		upgradeExit(1)
+	}
 
 	// Check that the installation exists.
 	spinifexToml := configDir + "/spinifex.toml"
 	if _, err := os.Stat(spinifexToml); os.IsNotExist(err) {
 		fmt.Fprintf(os.Stderr, "No Spinifex installation found at %s\nRun 'spx admin init' first.\n", configDir)
-		os.Exit(1)
+		upgradeExit(1)
 	}
 
-	// Show current versions.
+	var pending []migrate.PendingMigration
+	if !unitsOnly {
+		pending = reportConfigStatus(configDir)
+	}
+
+	var unitResult systemd.Result
+	if !skipUnits {
+		var err error
+		unitResult, err = systemd.Reconcile(systemdUnitRoot, systemd.Options{DryRun: true})
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error checking systemd units: %v\n", err)
+			upgradeExit(1)
+		}
+		reportUnitStatus(unitResult)
+	}
+
+	hasConfigWork := !unitsOnly && len(pending) > 0
+	hasUnitWork := !skipUnits && unitResult.HasChanges()
+
+	proceed, message := upgradeSummary(dryRun, hasConfigWork, hasUnitWork)
+	if message != "" {
+		fmt.Println(message)
+	}
+	if !proceed {
+		return
+	}
+
+	if !yes {
+		fmt.Print("\nApply? [y/N] ")
+		reader := bufio.NewReader(os.Stdin)
+		answer, _ := reader.ReadString('\n')
+		answer = strings.TrimSpace(strings.ToLower(answer))
+		if answer != "y" && answer != "yes" {
+			fmt.Println("Aborted.")
+			return
+		}
+	}
+
+	if hasConfigWork {
+		if err := migrate.DefaultRegistry.RunAllConfig(configDir, dataDir); err != nil {
+			fmt.Fprintf(os.Stderr, "Config migration failed: %v\n", err)
+			upgradeExit(1)
+		}
+	}
+
+	if !skipUnits {
+		applied, err := systemd.Reconcile(systemdUnitRoot, systemd.Options{})
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "\nUnit reconciliation failed: %v\n", err)
+			if errors.Is(err, systemd.ErrRootRequired) {
+				fmt.Fprintln(os.Stderr, "Re-run as root to apply the unit changes reported above: sudo spx admin upgrade --units-only --yes")
+			}
+			upgradeExit(1)
+		}
+		reportUnitApply(applied)
+	}
+
+	fmt.Println("\nDone. daemon-reload has already picked up any unit changes for the")
+	fmt.Println("next stop; restart to apply config changes: sudo systemctl restart spinifex.target")
+}
+
+// reportConfigStatus prints current config versions and pending migrations,
+// mirroring the pre-existing `spx admin upgrade` output.
+func reportConfigStatus(configDir string) []migrate.PendingMigration {
 	versions, err := migrate.DefaultRegistry.ConfigVersions(configDir)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error reading config versions: %v\n", err)
-		os.Exit(1)
+		upgradeExit(1)
 	}
 
 	fmt.Println("Reading current config versions...")
@@ -51,43 +170,59 @@ func runAdminUpgrade(cmd *cobra.Command, _ []string) {
 			fmt.Printf("  %-20s %d\n", name+":", v)
 		}
 	}
-	fmt.Println()
 
-	// Check for pending migrations.
 	pending, err := migrate.DefaultRegistry.PendingConfig(configDir)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error checking pending migrations: %v\n", err)
-		os.Exit(1)
+		upgradeExit(1)
 	}
 
 	if len(pending) == 0 {
 		fmt.Println("No pending config migrations.")
-		return
+		return pending
 	}
 
 	fmt.Println("Pending config migrations:")
 	for _, p := range pending {
 		fmt.Printf("  [config] %s %d → %d: %s\n", p.Target, p.FromVersion, p.ToVersion, p.Description)
 	}
-	fmt.Println()
+	return pending
+}
 
-	// Prompt for confirmation unless --yes.
-	if !yes {
-		fmt.Print("Apply? [y/N] ")
-		reader := bufio.NewReader(os.Stdin)
-		answer, _ := reader.ReadString('\n')
-		answer = strings.TrimSpace(strings.ToLower(answer))
-		if answer != "y" && answer != "yes" {
-			fmt.Println("Aborted.")
-			return
+// reportUnitStatus prints the reconcile decision for each unit, computed
+// with Options{DryRun: true} so this is always safe to call before a prompt.
+func reportUnitStatus(result systemd.Result) {
+	fmt.Println("\nReading installed systemd unit versions...")
+	for _, s := range result.Statuses {
+		switch s.Action {
+		case systemd.ActionNoop:
+			fmt.Printf("  %-38s %d (up to date)\n", s.Name+":", s.InstalledVersion)
+		case systemd.ActionInstall:
+			fmt.Printf("  %-38s missing → will install (v%d)\n", s.Name+":", s.EmbeddedVersion)
+		case systemd.ActionReplace:
+			fmt.Printf("  %-38s %d → %d (stale, will replace)\n", s.Name+":", s.InstalledVersion, s.EmbeddedVersion)
+		case systemd.ActionConflict:
+			fmt.Printf("  %-38s %d (operator-modified, not touched — see `systemctl edit %s`)\n", s.Name+":", s.InstalledVersion, s.Name)
 		}
 	}
-
-	// Apply migrations.
-	if err := migrate.DefaultRegistry.RunAllConfig(configDir, dataDir); err != nil {
-		fmt.Fprintf(os.Stderr, "Migration failed: %v\n", err)
-		os.Exit(1)
+	if result.HasConflicts() {
+		fmt.Println("Operator-modified units above are left as-is; review them by hand.")
+	}
+}
+
+// reportUnitApply prints backups written and confirms daemon-reload ran.
+func reportUnitApply(result systemd.Result) {
+	for _, s := range result.Statuses {
+		if !s.Applied {
+			continue
+		}
+		if s.Backup != "" {
+			fmt.Printf("  [unit] %s replaced (backup: %s)\n", s.Name, s.Backup)
+		} else {
+			fmt.Printf("  [unit] %s installed\n", s.Name)
+		}
+	}
+	if result.ReloadRan {
+		fmt.Println("  systemctl daemon-reload complete.")
 	}
-
-	fmt.Println("\nDone. Restart services to apply: sudo systemctl restart spinifex.target")
 }

cmd/spinifex/cmd/admin_upgrade_test.go

@@ -1,11 +1,15 @@
 package cmd
 
 import (
+	"bytes"
+	"io"
 	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/mulgadc/spinifex/spinifex/migrate"
+	"github.com/mulgadc/spinifex/spinifex/systemd"
+	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -38,3 +42,391 @@ func TestUpgradeSeesEveryConfigTarget(t *testing.T) {
 	}
 	assert.Equal(t, map[string][2]int{"spinifex.toml": {3, 4}}, targets)
 }
+
+func TestValidateUpgradeFlags(t *testing.T) {
+	assert.NoError(t, validateUpgradeFlags(false, false))
+	assert.NoError(t, validateUpgradeFlags(true, false))
+	assert.NoError(t, validateUpgradeFlags(false, true))
+	assert.ErrorIs(t, validateUpgradeFlags(true, true), errUpgradeFlagsConflict)
+}
+
+func TestUpgradeSummary(t *testing.T) {
+	cases := []struct {
+		name                       string
+		dryRun, hasConfig, hasUnit bool
+		wantProceed                bool
+		wantMessage                string
+	}{
+		{"dry-run, nothing pending", true, false, false, false, "\nNothing pending — config and units are current."},
+		{"dry-run, config pending", true, true, false, false, ""},
+		{"dry-run, unit pending", true, false, true, false, ""},
+		{"dry-run, both pending", true, true, true, false, ""},
+		{"apply, nothing pending", false, false, false, false, "\nNothing to do."},
+		{"apply, config pending", false, true, false, true, ""},
+		{"apply, unit pending", false, false, true, true, ""},
+		{"apply, both pending", false, true, true, true, ""},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			proceed, message := upgradeSummary(tc.dryRun, tc.hasConfig, tc.hasUnit)
+			assert.Equal(t, tc.wantProceed, proceed)
+			assert.Equal(t, tc.wantMessage, message)
+		})
+	}
+}
+
+func TestReportUnitStatus_PrintsEachActionAndConflictFooter(t *testing.T) {
+	result := systemd.Result{Statuses: []systemd.UnitStatus{
+		{Name: "noop.service", Action: systemd.ActionNoop, InstalledVersion: 3},
+		{Name: "install.service", Action: systemd.ActionInstall, EmbeddedVersion: 2},
+		{Name: "replace.service", Action: systemd.ActionReplace, InstalledVersion: 1, EmbeddedVersion: 2},
+		{Name: "conflict.service", Action: systemd.ActionConflict, InstalledVersion: 2},
+	}}
+
+	out := captureStdout(t, func() { reportUnitStatus(result) })
+
+	for _, want := range []string{
+		"noop.service:", "3 (up to date)",
+		"install.service:", "missing → will install (v2)",
+		"replace.service:", "1 → 2 (stale, will replace)",
+		"conflict.service:", "2 (operator-modified, not touched",
+		"Operator-modified units above are left as-is",
+	} {
+		assert.Contains(t, out, want)
+	}
+}
+
+func TestReportUnitStatus_NoConflictsOmitsFooter(t *testing.T) {
+	result := systemd.Result{Statuses: []systemd.UnitStatus{{Name: "a.service", Action: systemd.ActionNoop}}}
+	out := captureStdout(t, func() { reportUnitStatus(result) })
+	assert.NotContains(t, out, "operator-modified")
+}
+
+func TestReportUnitApply_PrintsAppliedUnitsAndReload(t *testing.T) {
+	result := systemd.Result{
+		ReloadRan: true,
+		Statuses: []systemd.UnitStatus{
+			{Name: "skipped.service", Applied: false},
+			{Name: "installed.service", Applied: true},
+			{Name: "replaced.service", Applied: true, Backup: "/etc/systemd/system/replaced.service.bak"},
+		},
+	}
+	out := captureStdout(t, func() { reportUnitApply(result) })
+
+	assert.NotContains(t, out, "skipped.service", "unapplied unit must not be printed")
+	for _, want := range []string{
+		"installed.service installed",
+		"replaced.service replaced (backup: /etc/systemd/system/replaced.service.bak)",
+		"systemctl daemon-reload complete.",
+	} {
+		assert.Contains(t, out, want)
+	}
+}
+
+func TestReportUnitApply_NoReloadOmitsMessage(t *testing.T) {
+	out := captureStdout(t, func() { reportUnitApply(systemd.Result{}) })
+	assert.NotContains(t, out, "daemon-reload")
+}
+
+func TestReportConfigStatus_NoPendingMigrations(t *testing.T) {
+	configDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
+
+	var pending []migrate.PendingMigration
+	out := captureStdout(t, func() { pending = reportConfigStatus(configDir) })
+
+	assert.Empty(t, pending)
+	assert.Contains(t, out, "No pending config migrations.")
+	assert.Contains(t, out, "spinifex.toml:")
+}
+
+func TestReportConfigStatus_PendingMigrationIsListed(t *testing.T) {
+	configDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+
+	var pending []migrate.PendingMigration
+	out := captureStdout(t, func() { pending = reportConfigStatus(configDir) })
+
+	require.Len(t, pending, 1)
+	assert.Equal(t, "spinifex.toml", pending[0].Target)
+	assert.Contains(t, out, "Pending config migrations:")
+	assert.Contains(t, out, "spinifex.toml 3 → 4")
+}
+
+// captureStderr redirects os.Stderr for the duration of fn and returns what
+// was written. runAdminUpgrade's error paths print to stderr, not stdout.
+func captureStderr(t *testing.T, fn func()) string {
+	t.Helper()
+	r, w, err := os.Pipe()
+	require.NoError(t, err)
+	orig := os.Stderr
+	os.Stderr = w //nolint:reassign // test-local stderr capture, restored below before this func returns
+
+	fn()
+
+	require.NoError(t, w.Close())
+	os.Stderr = orig //nolint:reassign // restoring the real os.Stderr captured above
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, r)
+	require.NoError(t, err)
+	return buf.String()
+}
+
+// upgradeExitPanic is the sentinel withUpgradeExitCapture's fake upgradeExit
+// panics with, so a deferred recover can distinguish "the command called
+// exit" from any other panic escaping fn.
+type upgradeExitPanic struct{ code int }
+
+// withUpgradeExitCapture swaps upgradeExit for a fake that records the exit
+// code and panics with upgradeExitPanic instead of killing the test binary,
+// runs fn, and returns the recorded code (-1 if fn returned without
+// exiting). This lets runAdminUpgrade's real control flow, including its
+// error/exit branches, be exercised directly.
+func withUpgradeExitCapture(t *testing.T, fn func()) int {
+	t.Helper()
+	origExit := upgradeExit
+	t.Cleanup(func() { upgradeExit = origExit })
+
+	code := -1
+	upgradeExit = func(c int) { code = c; panic(upgradeExitPanic{c}) }
+
+	func() {
+		defer func() {
+			if r := recover(); r != nil {
+				if _, ok := r.(upgradeExitPanic); !ok {
+					panic(r)
+				}
+			}
+		}()
+		fn()
+	}()
+	return code
+}
+
+// newUpgradeTestCmd builds an isolated command tree mirroring rootCmd's and
+// upgradeCmd's flag wiring, so runAdminUpgrade can be driven directly
+// without touching the package's real global command tree (which other
+// tests in this package also mutate).
+func newUpgradeTestCmd(t *testing.T, configDir, dataDir string) *cobra.Command {
+	t.Helper()
+	root := &cobra.Command{Use: "spx"}
+	root.PersistentFlags().String("config-dir", configDir, "")
+	root.PersistentFlags().String("spinifex-dir", dataDir, "")
+
+	upgrade := &cobra.Command{Use: "upgrade", Run: runAdminUpgrade}
+	upgrade.Flags().Bool("yes", false, "")
+	upgrade.Flags().Bool("dry-run", false, "")
+	upgrade.Flags().Bool("units-only", false, "")
+	upgrade.Flags().Bool("skip-units", false, "")
+	root.AddCommand(upgrade)
+
+	// runAdminUpgrade reads config-dir/spinifex-dir via cmd.Root().Flags(),
+	// which only sees persistent flags after they've been merged in —
+	// normally done by Execute(). Parsing with no args merges them without
+	// running the command.
+	require.NoError(t, root.ParseFlags(nil))
+	return upgrade
+}
+
+// setUpgradeFlags sets each flag=value pair on cmd, failing the test on any
+// unknown flag or parse error.
+func setUpgradeFlags(t *testing.T, cmd *cobra.Command, flags map[string]string) {
+	t.Helper()
+	for name, value := range flags {
+		require.NoError(t, cmd.Flags().Set(name, value))
+	}
+}
+
+// seedUnitRoot writes every embedded unit unmodified into dir, so a
+// Reconcile pass against dir reports every unit ActionNoop.
+func seedUnitRoot(t *testing.T, dir string) {
+	t.Helper()
+	for name, content := range systemd.Units {
+		require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0644))
+	}
+}
+
+// withUnitRoot points the package's systemdUnitRoot var at dir for the
+// duration of the test, restoring the original on cleanup.
+func withUnitRoot(t *testing.T, dir string) {
+	t.Helper()
+	prev := systemdUnitRoot
+	systemdUnitRoot = dir
+	t.Cleanup(func() { systemdUnitRoot = prev })
+}
+
+func TestRunAdminUpgrade_MutuallyExclusiveFlagsExits(t *testing.T) {
+	cmd := newUpgradeTestCmd(t, t.TempDir(), t.TempDir())
+	setUpgradeFlags(t, cmd, map[string]string{"units-only": "true", "skip-units": "true"})
+
+	var code int
+	out := captureStdout(t, func() {
+		code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) })
+	})
+	assert.Equal(t, 1, code)
+	assert.Empty(t, out, "no stdout expected before the mutual-exclusivity check exits")
+}
+
+func TestRunAdminUpgrade_MissingInstallationExits(t *testing.T) {
+	configDir := t.TempDir() // no spinifex.toml written
+	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
+
+	code := withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) })
+	assert.Equal(t, 1, code)
+}
+
+func TestRunAdminUpgrade_DryRunNothingPending(t *testing.T) {
+	configDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
+	unitRoot := t.TempDir()
+	seedUnitRoot(t, unitRoot)
+	withUnitRoot(t, unitRoot)
+
+	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
+	setUpgradeFlags(t, cmd, map[string]string{"dry-run": "true"})
+
+	var code int
+	out := captureStdout(t, func() {
+		code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) })
+	})
+	assert.Equal(t, -1, code, "must not exit when everything is current")
+	assert.Contains(t, out, "Nothing pending — config and units are current.")
+}
+
+func TestRunAdminUpgrade_DryRunReportsBothPending(t *testing.T) {
+	configDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	withUnitRoot(t, t.TempDir()) // empty: every unit reports missing
+
+	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
+	setUpgradeFlags(t, cmd, map[string]string{"dry-run": "true"})
+
+	var code int
+	out := captureStdout(t, func() {
+		code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) })
+	})
+	assert.Equal(t, -1, code)
+	assert.NotContains(t, out, "Nothing pending")
+	assert.Contains(t, out, "Pending config migrations:")
+	assert.Contains(t, out, "will install")
+}
+
+func TestRunAdminUpgrade_UnitsOnlySkipsConfigReporting(t *testing.T) {
+	configDir := t.TempDir()
+	// Version 3 would normally report a pending config migration; --units-only
+	// must never look at it.
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	withUnitRoot(t, t.TempDir())
+
+	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
+	setUpgradeFlags(t, cmd, map[string]string{"dry-run": "true", "units-only": "true"})
+
+	var code int
+	out := captureStdout(t, func() {
+		code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) })
+	})
+	assert.Equal(t, -1, code)
+	assert.NotContains(t, out, "Reading current config versions", "--units-only must skip config reporting")
+	assert.Contains(t, out, "Reading installed systemd unit versions")
+}
+
+func TestRunAdminUpgrade_SkipUnitsSkipsUnitReporting(t *testing.T) {
+	configDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	// Point systemdUnitRoot at a path Reconcile must never touch: if
+	// --skip-units leaked a call through, a nonexistent root would still
+	// "work" for a dry run, so the real proof is that its output never
+	// appears at all.
+	withUnitRoot(t, filepath.Join(t.TempDir(), "never-touched"))
+
+	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
+	setUpgradeFlags(t, cmd, map[string]string{"dry-run": "true", "skip-units": "true"})
+
+	var code int
+	out := captureStdout(t, func() {
+		code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) })
+	})
+	assert.Equal(t, -1, code)
+	assert.NotContains(t, out, "Reading installed systemd unit versions", "--skip-units must skip unit reporting")
+	assert.Contains(t, out, "Pending config migrations:")
+}
+
+func TestRunAdminUpgrade_ApplyRunsConfigMigrationAndUnitReconcile(t *testing.T) {
+	configDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	// Every unit already current: the real (non-dry-run) Reconcile call
+	// below finds nothing to change, so it never touches daemon-reload.
+	unitRoot := t.TempDir()
+	seedUnitRoot(t, unitRoot)
+	withUnitRoot(t, unitRoot)
+
+	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
+	setUpgradeFlags(t, cmd, map[string]string{"yes": "true"})
+
+	var code int
+	out := captureStdout(t, func() {
+		code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) })
+	})
+	assert.Equal(t, -1, code, "the apply must succeed without exiting")
+	assert.Contains(t, out, "Done.")
+
+	migrated, err := os.ReadFile(filepath.Join(configDir, "spinifex.toml"))
+	require.NoError(t, err)
+	assert.Contains(t, string(migrated), `version = "4"`, "spinifex.toml must have been migrated to the latest version")
+}
+
+func TestRunAdminUpgrade_ApplyWithNonWritableUnitRootExitsWithRootHint(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("running as root bypasses directory permissions; this test needs an unprivileged process")
+	}
+	configDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
+
+	unitRoot := t.TempDir() // left empty: every unit reports missing
+	require.NoError(t, os.Chmod(unitRoot, 0o500))
+	t.Cleanup(func() { _ = os.Chmod(unitRoot, 0o700) })
+	withUnitRoot(t, unitRoot)
+
+	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
+	setUpgradeFlags(t, cmd, map[string]string{"yes": "true"})
+
+	var code int
+	var errOut string
+	captureStdout(t, func() {
+		errOut = captureStderr(t, func() {
+			code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) })
+		})
+	})
+	assert.Equal(t, 1, code)
+	assert.Contains(t, errOut, "Unit reconciliation failed")
+	assert.Contains(t, errOut, "Re-run as root to apply the unit changes reported above")
+}
+
+func TestRunAdminUpgrade_DryRunReconcileErrorExits(t *testing.T) {
+	configDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
+
+	unitRoot := t.TempDir()
+	// A directory in place of a unit file fails the read with something
+	// other than IsNotExist, so the very first (dry-run) Reconcile call
+	// returns an error.
+	var oneName string
+	for name := range systemd.Units {
+		oneName = name
+		break
+	}
+	require.NoError(t, os.Mkdir(filepath.Join(unitRoot, oneName), 0o755))
+	withUnitRoot(t, unitRoot)
+
+	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
+
+	var code int
+	var errOut string
+	captureStdout(t, func() {
+		errOut = captureStderr(t, func() {
+			code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) })
+		})
+	})
+	assert.Equal(t, 1, code)
+	assert.Contains(t, errOut, "Error checking systemd units")
+}

docs/COMMANDS.md

@@ -85,7 +85,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin upgrade` | `--yes` (apply without prompting), `--config-dir` (persistent, default: `~/spinifex/config`), `--spinifex-dir` (persistent, default: `~/spinifex`) | Reads current config versions from registry → computes pending config migrations (from→to per target) → prints version summary and pending list → prompts for confirmation unless `--yes` → runs `migrate.DefaultRegistry.RunAllConfig()` to apply migrations to config files. Intended for upgrades between Spinifex versions. Operators can skip migrations during install by setting `INSTALL_SPINIFEX_SKIP_MIGRATE=1`, then run `spx admin upgrade` manually to review and apply. After completion, services must be restarted with `sudo systemctl restart spinifex.target`. Invoked non-interactively by `scripts/setup.sh` with `--yes`. |
+| `spx admin upgrade` | `--yes` (apply without prompting), `--dry-run` (report pending config and unit changes without applying), `--units-only` (reconcile systemd units only, skip config migrations), `--skip-units` (apply config migrations only, skip unit reconciliation), `--config-dir` (persistent, default: `~/spinifex/config`), `--spinifex-dir` (persistent, default: `~/spinifex`) | Reads current config versions from registry → computes pending config migrations (from→to per target) → reads installed systemd unit versions and compares each against the copy embedded in the binary (`systemd.Reconcile` with `DryRun: true`) → prints both summaries → prompts for confirmation unless `--yes` or `--dry-run` → runs `migrate.DefaultRegistry.RunAllConfig()` to apply migrations to config files → runs `systemd.Reconcile` for real to install missing units and replace stale ones (after a timestamped backup), then `systemctl daemon-reload`. Units whose installed marker version matches the embedded one but whose content differs are left untouched and reported as operator-modified — never overwritten. Writing `/etc/systemd/system` requires root; without it, drift is still reported and the command exits non-zero rather than writing partially. Never restarts services — a unit change applies to that service's next stop. Intended for upgrades between Spinifex versions, including nodes that were upgraded by swapping the `spx` binary directly rather than by re-running the installer. Operators can skip migrations during install by setting `INSTALL_SPINIFEX_SKIP_MIGRATE=1`, then run `spx admin upgrade` manually to review and apply. After completion, services must be restarted with `sudo systemctl restart spinifex.target` to pick up config changes. Invoked non-interactively by `scripts/setup.sh` with `--yes`. |
 
 ### Account Management
 

docs/admin/host-lifecycle/README.md

@@ -72,7 +72,15 @@ skipped the drain and logged `Host is not shutting down; skipping guest
 drain.` The gate now also reads systemd's pending job list, which does
 distinguish the two.
 
-To check which behaviour a node has, look at its unit:
+The fastest way to check is `spx admin upgrade --dry-run`, which reports every
+unit's status against the one shipped in the running binary — including
+`spinifex-shutdown.service` — without changing anything. See
+[Checking for Unit Drift](/docs/admin/update#checking-for-unit-drift). A node
+reported stale or missing for that unit predates this fix; reconcile it with
+`sudo spx admin upgrade --units-only --yes` (this only rewrites the unit and
+runs `daemon-reload` — it does not restart anything or drain guests itself).
+
+To check by hand instead, look at the unit directly:
 
 ```bash
 systemctl cat spinifex-shutdown.service | grep ExecStop

docs/admin/update/README.md

@@ -22,6 +22,7 @@ resources:
 - [Overview](#overview)
 - [Instructions](#instructions)
 - [Manual Upgrade](#manual-upgrade)
+- [Checking for Unit Drift](#checking-for-unit-drift)
 - [Troubleshooting](#troubleshooting)
 
 ---
@@ -32,6 +33,9 @@ Updating Spinifex is the same command used to install it. The installer detects
 
 For operators who want to review migrations before they are applied, a manual upgrade path is also supported.
 
+> [!WARNING]
+> **Swapping the `spx` binary alone is not an upgrade.** Systemd unit files (`KillMode`, `TimeoutStopSec`, drain ordering, and similar) are written once at install time and never re-asserted just because a new binary is in place — a node "upgraded" by replacing `/usr/local/bin/spx` directly keeps running whatever units it was first installed with, which can silently disagree with the new binary's behaviour. Re-running the installer always reinstalls units unconditionally, so it is unaffected. `spx admin upgrade` now reconciles units too, so it closes this gap for operators who update the binary by hand. See [Checking for Unit Drift](#checking-for-unit-drift).
+
 > [!WARNING]
 > **Object storage is exempt when upgrading from v1.15.0 or earlier to v1.16.0.** Predastore's configuration schema and on-disk layout changed with the object storage cutover in v1.16.0, and no migration converts an installation from before it. `spx admin upgrade` will not report anything pending for `predastore.toml`, and updating the binary over such an installation leaves Predastore unable to start. Those clusters have to be re-initialised from scratch, which discards their stored objects — export anything you need first. Upgrades between v1.16.0 and later releases are unaffected.
 
@@ -82,15 +86,15 @@ curl -fsSL https://install.mulgadc.com | INSTALL_SPINIFEX_SKIP_MIGRATE=1 bash
 
 The installer will download the new binary and reinstall systemd units, but will **not** apply any configuration migrations.
 
-## Step 2. Review Pending Migrations
+## Step 2. Review Pending Changes
 
 ```bash
 sudo spx admin upgrade
 ```
 
-The command prints the current version of each config file, the migrations that would be applied, and a `from → to` description for each. It then prompts for confirmation before making any changes. Answer `n` to abort without touching your config.
+The command prints the current version of each config file and systemd unit, the migrations and unit replacements that would be applied, and a `from → to` description for each. It then prompts for confirmation before making any changes. Answer `n` to abort without touching config or units. Use `--dry-run` instead of the prompt to only report and never apply.
 
-## Step 3. Apply Migrations
+## Step 3. Apply Changes
 
 When you are ready, answer `y` at the prompt, or re-run with `--yes` to apply non-interactively:
 
@@ -98,9 +102,11 @@ When you are ready, answer `y` at the prompt, or re-run with `--yes` to apply no
 sudo spx admin upgrade --yes
 ```
 
+This requires root: config files are typically owned by their service user, but writing `/etc/systemd/system` needs root. Run the whole command with `sudo`, not just parts of it.
+
 ## Step 4. Restart Services
 
-Migrations modify config files on disk but do not restart running services. Apply the new config with:
+Migrations modify config files on disk but do not restart running services, and unit reconciliation deliberately never restarts anything either — it writes the unit and runs `systemctl daemon-reload`, so the fix applies to the *next* stop of that service without disturbing a running guest. Apply a config change with:
 
 ```bash
 sudo systemctl restart spinifex.target
@@ -108,6 +114,25 @@ sudo systemctl restart spinifex.target
 
 A restart preserves any running guests — they are not rebooted, and storage returns within seconds. See [Host and Guest Lifecycle](/docs/admin/host-lifecycle) for the full contract.
 
+## Checking for Unit Drift
+
+`spx admin upgrade --dry-run` reports whether a node's installed systemd units match the ones shipped in the running `spx` binary, without prompting or changing anything — the fastest way to answer "are this node's units current?" in a support conversation:
+
+```bash
+spx admin upgrade --dry-run
+```
+
+Each unit is reported as one of:
+
+- **up to date** — installed content matches the embedded copy.
+- **stale, will replace** — the installed marker version is older than the embedded one (or has no marker at all, which is version 0 — every node installed before units were versioned).
+- **missing → will install** — no unit installed under that name.
+- **operator-modified, not touched** — the installed marker version matches, but the content differs. `spx admin upgrade` never overwrites this case; review it with `systemctl cat <unit>` and reconcile by hand with `systemctl edit`.
+
+Run `sudo spx admin upgrade --units-only --yes` to reconcile units without touching config, or `--skip-units` to do the reverse. A replaced unit is backed up alongside the original as `<unit>.pre-reconcile-<from>to<to>.<unix-timestamp>` before being overwritten.
+
+This covers the 16 core units installed by `install_systemd()` (`spinifex-*.service`, `spinifex.target`, `spinifex.slice` and friends). Firstboot, banner, bridge and getty units written by the installer, and the auxiliary units Ansible manages (`wattle-wan-veth-persist`, `wattle-mgmt-bridge-persist`, `obs-agent`), are separate lifecycles with no overlap in unit names and are out of scope for this reconciler.
+
 ## Troubleshooting
 
 ### No Pending Config Migrations
@@ -174,3 +199,22 @@ Or `describe-images --image-ids` reports `InvalidAMIID.NotFound` for an AMI that
 
 1. **A node running a replaced binary**, per the previous entry. Suspect this first if the failure is intermittent — the same command succeeding on some attempts and failing on others is characteristic.
 2. **AMI metadata predating the EBS-provider decoupling**, per the warning at the top of this page. This is consistent rather than intermittent, and is resolved by re-importing the AMI.
+
+### Root Privileges Required to Write Systemd Units
+
+```
+root privileges required to write systemd units (writing to /etc/systemd/system): ...
+Re-run as root to apply the unit changes reported above: sudo spx admin upgrade --units-only --yes
+```
+
+`spx admin upgrade` computes and prints unit drift without needing root, but writing `/etc/systemd/system` does. Nothing is written when this happens — re-run the full command with `sudo`.
+
+### Operator-Modified Unit Reported, Not Replaced
+
+A unit whose installed marker version matches the embedded one but whose content differs is never overwritten — this is a deliberate safety property so a hand-tuned unit does not get silently reverted. Compare it against the shipped copy and decide whether to keep, discard or merge the local change:
+
+```bash
+systemctl cat <unit>
+```
+
+If you want the shipped version, remove the local override and re-run `spx admin upgrade`; systemd falls back to the packaged unit and it reconciles as up to date.

spinifex/systemd/generate.go

@@ -0,0 +1,73 @@
+//go:build ignore
+
+// Command generate regenerates units_gen.go from build/systemd, the single
+// source of truth for spinifex unit files. go:embed cannot reach outside
+// this package directory, so the contents are baked in as a Go map instead.
+//
+//go:generate go run generate.go
+package main
+
+import (
+	"fmt"
+	"go/format"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintln(os.Stderr, "generate:", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	selfDir, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	unitsDir := filepath.Join(selfDir, "..", "..", "build", "systemd")
+
+	ents, err := os.ReadDir(unitsDir)
+	if err != nil {
+		return fmt.Errorf("read %s: %w", unitsDir, err)
+	}
+
+	var names []string
+	for _, e := range ents {
+		n := e.Name()
+		for _, suffix := range []string{".service", ".slice", ".target", ".timer"} {
+			if strings.HasSuffix(n, suffix) {
+				names = append(names, n)
+				break
+			}
+		}
+	}
+	sort.Strings(names)
+
+	var b strings.Builder
+	b.WriteString("// Code generated by `go generate ./...` from build/systemd; DO NOT EDIT.\n\n")
+	b.WriteString("package systemd\n\n")
+	b.WriteString("// Units maps each core unit file name to its embedded contents, sourced from\n")
+	b.WriteString("// build/systemd at generation time. Regenerate with `go generate ./...` after\n")
+	b.WriteString("// editing a unit; units_invariants_test.go fails preflight if this drifts.\n")
+	b.WriteString("var Units = map[string]string{\n")
+	for _, name := range names {
+		content, err := os.ReadFile(filepath.Join(unitsDir, name))
+		if err != nil {
+			return fmt.Errorf("read %s: %w", name, err)
+		}
+		fmt.Fprintf(&b, "\t%q: %q,\n", name, string(content))
+	}
+	b.WriteString("}\n")
+
+	formatted, err := format.Source([]byte(b.String()))
+	if err != nil {
+		return fmt.Errorf("format generated source: %w", err)
+	}
+
+	outPath := filepath.Join(selfDir, "units_gen.go")
+	return os.WriteFile(outPath, formatted, 0o644)
+}

spinifex/systemd/reconcile.go

@@ -0,0 +1,216 @@
+package systemd
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Action describes what Reconcile did, or would do, for one unit.
+type Action string
+
+const (
+	ActionNoop     Action = "noop"     // installed content matches the embedded copy
+	ActionInstall  Action = "install"  // unit missing on disk
+	ActionReplace  Action = "replace"  // installed marker version is older than embedded
+	ActionConflict Action = "conflict" // marker version matches but content differs — operator-modified
+)
+
+// UnitStatus reports the reconcile decision for a single unit.
+type UnitStatus struct {
+	Name             string
+	Action           Action
+	InstalledVersion int
+	EmbeddedVersion  int
+	Backup           string // set once a replace has written a backup
+	Applied          bool   // true once Install/Replace has actually been written
+}
+
+// Options controls how Reconcile behaves.
+type Options struct {
+	// DryRun computes and returns the full status without writing anything.
+	DryRun bool
+}
+
+// Result is the outcome of a Reconcile call.
+type Result struct {
+	Statuses  []UnitStatus
+	ReloadRan bool
+}
+
+// HasChanges reports whether any unit is missing or stale — i.e. whether
+// applying (DryRun: false) would write anything to disk.
+func (r Result) HasChanges() bool {
+	for _, s := range r.Statuses {
+		if s.Action == ActionInstall || s.Action == ActionReplace {
+			return true
+		}
+	}
+	return false
+}
+
+// HasConflicts reports whether any unit is at the current version but was
+// modified on disk — drift Reconcile deliberately refuses to overwrite.
+func (r Result) HasConflicts() bool {
+	for _, s := range r.Statuses {
+		if s.Action == ActionConflict {
+			return true
+		}
+	}
+	return false
+}
+
+// ErrRootRequired is returned when Reconcile has pending changes but root
+// cannot write to the target directory. The Result is still populated and
+// safe to print — no file was written.
+var ErrRootRequired = errors.New("root privileges required to write systemd units")
+
+var unitVersionRe = regexp.MustCompile(`^#\s*spinifex-unit-version:\s*(\d+)\s*$`)
+
+// unitVersion reads the marker on a unit's first line. An absent or
+// unparsable marker is version 0, matching every node installed before
+// units were versioned.
+func unitVersion(content string) int {
+	line, _, _ := strings.Cut(content, "\n")
+	m := unitVersionRe.FindStringSubmatch(strings.TrimSpace(line))
+	if m == nil {
+		return 0
+	}
+	v, err := strconv.Atoi(m[1])
+	if err != nil {
+		return 0
+	}
+	return v
+}
+
+// systemctlDaemonReload is overridable in tests so Reconcile never shells
+// out to a real systemd instance.
+var systemctlDaemonReload = func() error {
+	return exec.Command("systemctl", "daemon-reload").Run()
+}
+
+func sortedUnitNames() []string {
+	names := make([]string, 0, len(Units))
+	for n := range Units {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	return names
+}
+
+// Reconcile compares each embedded unit (Units, generated from
+// build/systemd) against the copy installed under root. Missing or
+// version-stale units are replaced after a timestamped backup; units whose
+// marker version matches but whose content differs are left untouched and
+// reported as operator-modified. Never restarts anything — only writes
+// files and, when it wrote at least one, runs systemctl daemon-reload.
+func Reconcile(root string, opts Options) (Result, error) {
+	var result Result
+
+	for _, name := range sortedUnitNames() {
+		embedded := Units[name]
+		embeddedVersion := unitVersion(embedded)
+		path := filepath.Join(root, name)
+
+		installed, err := os.ReadFile(path)
+		if os.IsNotExist(err) {
+			result.Statuses = append(result.Statuses, UnitStatus{
+				Name: name, Action: ActionInstall, EmbeddedVersion: embeddedVersion,
+			})
+			continue
+		}
+		if err != nil {
+			return result, fmt.Errorf("read %s: %w", path, err)
+		}
+
+		installedVersion := unitVersion(string(installed))
+		status := UnitStatus{Name: name, InstalledVersion: installedVersion, EmbeddedVersion: embeddedVersion}
+		switch {
+		case installedVersion < embeddedVersion:
+			status.Action = ActionReplace
+		case string(installed) == embedded:
+			status.Action = ActionNoop
+		default:
+			status.Action = ActionConflict
+		}
+		result.Statuses = append(result.Statuses, status)
+	}
+
+	if opts.DryRun || !result.HasChanges() {
+		return result, nil
+	}
+
+	// Check writability up front so a permission problem is reported before
+	// any unit is touched, rather than after replacing some and failing on
+	// the next — the failure mode this exists to avoid.
+	if err := checkWritable(root); err != nil {
+		return result, fmt.Errorf("%w (writing to %s): %w", ErrRootRequired, root, err)
+	}
+
+	for i := range result.Statuses {
+		s := &result.Statuses[i]
+		if s.Action != ActionInstall && s.Action != ActionReplace {
+			continue
+		}
+		path := filepath.Join(root, s.Name)
+		if s.Action == ActionReplace {
+			backup, err := backupUnit(path, s.InstalledVersion, s.EmbeddedVersion)
+			if err != nil {
+				return result, fmt.Errorf("backup %s: %w", path, err)
+			}
+			s.Backup = backup
+		}
+		if err := os.WriteFile(path, []byte(Units[s.Name]), 0o644); err != nil { //nolint:gosec // systemd unit files are intentionally world-readable, matching setup.sh's install -m 0644
+			return result, fmt.Errorf("write %s: %w", path, err)
+		}
+		s.Applied = true
+	}
+
+	if err := systemctlDaemonReload(); err != nil {
+		return result, fmt.Errorf("systemctl daemon-reload: %w", err)
+	}
+	result.ReloadRan = true
+
+	return result, nil
+}
+
+// checkWritable reports whether root is writable by the current process,
+// via a throwaway probe file rather than an EUID check — Reconcile is
+// pointed at a temp directory in tests, not necessarily run as root there.
+func checkWritable(root string) error {
+	f, err := os.CreateTemp(root, ".spx-systemd-write-check-*")
+	if err != nil {
+		return err
+	}
+	name := f.Name()
+	if cerr := f.Close(); cerr != nil {
+		return cerr
+	}
+	return os.Remove(name)
+}
+
+// backupUnit mirrors migrate.BackupConfig's timestamped-backup convention,
+// scoped to unit reconciliation instead of config migration.
+func backupUnit(path string, fromVersion, toVersion int) (string, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	info, err := os.Stat(path)
+	if err != nil {
+		return "", err
+	}
+
+	backupPath := fmt.Sprintf("%s.pre-reconcile-%dto%d.%d", path, fromVersion, toVersion, time.Now().Unix())
+	if err := os.WriteFile(backupPath, data, info.Mode()); err != nil {
+		return "", err
+	}
+	return backupPath, nil
+}

spinifex/systemd/reconcile_test.go

@@ -0,0 +1,523 @@
+package systemd
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// realUnit returns the embedded content for a real production unit, so
+// reconcile tests exercise the actual shipped units rather than fixtures
+// that could drift from what Reconcile runs against in production.
+func realUnit(t *testing.T, name string) string {
+	t.Helper()
+	content, ok := Units[name]
+	if !ok {
+		t.Fatalf("no embedded unit %q", name)
+	}
+	return content
+}
+
+// stripFirstLine drops a unit's version-marker line, simulating an
+// installed copy from before units were versioned.
+func stripFirstLine(content string) string {
+	_, rest, _ := strings.Cut(content, "\n")
+	return rest
+}
+
+// setVersion rewrites a unit's version-marker line to v, keeping the body
+// unchanged — simulating an installed copy stamped at an older version.
+func setVersion(content string, v int) string {
+	return "# spinifex-unit-version: " + strconv.Itoa(v) + "\n" + stripFirstLine(content)
+}
+
+func findStatus(t *testing.T, result Result, name string) UnitStatus {
+	t.Helper()
+	for _, s := range result.Statuses {
+		if s.Name == name {
+			return s
+		}
+	}
+	t.Fatalf("no status for %s", name)
+	return UnitStatus{}
+}
+
+func stubDaemonReload(t *testing.T) func() int {
+	t.Helper()
+	calls := 0
+	prev := systemctlDaemonReload
+	systemctlDaemonReload = func() error {
+		calls++
+		return nil
+	}
+	t.Cleanup(func() { systemctlDaemonReload = prev })
+	return func() int { return calls }
+}
+
+func TestReconcile_MissingUnitIsInstalled(t *testing.T) {
+	root := t.TempDir()
+	reloadCalls := stubDaemonReload(t)
+	const name = "spinifex-ui.service"
+	embedded := realUnit(t, name)
+
+	result, err := Reconcile(root, Options{})
+	if err != nil {
+		t.Fatalf("Reconcile: %v", err)
+	}
+
+	st := findStatus(t, result, name)
+	if st.Action != ActionInstall {
+		t.Errorf("Action = %s, want %s", st.Action, ActionInstall)
+	}
+	if !st.Applied {
+		t.Error("Applied = false, want true")
+	}
+	if st.Backup != "" {
+		t.Errorf("Backup = %q, want empty — nothing existed to back up", st.Backup)
+	}
+
+	got, err := os.ReadFile(filepath.Join(root, name))
+	if err != nil {
+		t.Fatalf("read installed unit: %v", err)
+	}
+	if string(got) != embedded {
+		t.Error("installed content does not match embedded unit")
+	}
+	if reloadCalls() != 1 {
+		t.Errorf("daemon-reload calls = %d, want 1", reloadCalls())
+	}
+}
+
+func TestReconcile_OlderVersionIsReplacedWithBackup(t *testing.T) {
+	root := t.TempDir()
+	stubDaemonReload(t)
+	const name = "spinifex-daemon.service"
+	embedded := realUnit(t, name)
+	stale := setVersion(embedded, 0)
+	writeFile(t, root, name, stale)
+
+	result, err := Reconcile(root, Options{})
+	if err != nil {
+		t.Fatalf("Reconcile: %v", err)
+	}
+
+	st := findStatus(t, result, name)
+	if st.Action != ActionReplace {
+		t.Errorf("Action = %s, want %s", st.Action, ActionReplace)
+	}
+	if st.InstalledVersion != 0 {
+		t.Errorf("InstalledVersion = %d, want 0", st.InstalledVersion)
+	}
+	if st.Backup == "" {
+		t.Fatal("Backup path empty, want a timestamped backup")
+	}
+
+	backupContent, err := os.ReadFile(st.Backup)
+	if err != nil {
+		t.Fatalf("read backup: %v", err)
+	}
+	if string(backupContent) != stale {
+		t.Error("backup content does not match the pre-replace installed content")
+	}
+
+	got, err := os.ReadFile(filepath.Join(root, name))
+	if err != nil {
+		t.Fatalf("read replaced unit: %v", err)
+	}
+	if string(got) != embedded {
+		t.Error("replaced content does not match embedded unit")
+	}
+}
+
+func TestReconcile_NoMarkerIsTreatedAsVersionZeroAndReplaced(t *testing.T) {
+	root := t.TempDir()
+	stubDaemonReload(t)
+	const name = "spinifex-viperblock.service"
+	embedded := realUnit(t, name)
+	noMarker := stripFirstLine(embedded)
+	writeFile(t, root, name, noMarker)
+
+	result, err := Reconcile(root, Options{})
+	if err != nil {
+		t.Fatalf("Reconcile: %v", err)
+	}
+
+	st := findStatus(t, result, name)
+	if st.Action != ActionReplace {
+		t.Errorf("Action = %s, want %s", st.Action, ActionReplace)
+	}
+	if st.InstalledVersion != 0 {
+		t.Errorf("InstalledVersion = %d, want 0 (absent marker)", st.InstalledVersion)
+	}
+	if st.Backup == "" {
+		t.Error("Backup path empty, want a timestamped backup")
+	}
+}
+
+func TestReconcile_IdenticalContentIsNoop(t *testing.T) {
+	root := t.TempDir()
+	reloadCalls := stubDaemonReload(t)
+	// Seed every embedded unit as already-current, so the only thing left
+	// for Reconcile to decide on is the one unit under test — otherwise the
+	// other 15 real units would still be "missing" and trigger a reload.
+	seedAllUnits(t, root)
+	const name = "spinifex-northstar.service"
+
+	result, err := Reconcile(root, Options{})
+	if err != nil {
+		t.Fatalf("Reconcile: %v", err)
+	}
+
+	st := findStatus(t, result, name)
+	if st.Action != ActionNoop {
+		t.Errorf("Action = %s, want %s", st.Action, ActionNoop)
+	}
+	if st.Applied {
+		t.Error("Applied = true, want false — nothing should change")
+	}
+
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, e := range entries {
+		if strings.Contains(e.Name(), "pre-reconcile") {
+			t.Errorf("unexpected backup file for a no-op unit: %s", e.Name())
+		}
+	}
+	if reloadCalls() != 0 {
+		t.Errorf("daemon-reload calls = %d, want 0 — nothing changed", reloadCalls())
+	}
+}
+
+func TestReconcile_OperatorModifiedAtCurrentVersionIsLeftAlone(t *testing.T) {
+	root := t.TempDir()
+	stubDaemonReload(t)
+	const name = "spinifex-vpcd.service"
+	embedded := realUnit(t, name)
+	modified := embedded + "# operator hand-edit, not from the shipped unit\n"
+	writeFile(t, root, name, modified)
+
+	result, err := Reconcile(root, Options{})
+	if err != nil {
+		t.Fatalf("Reconcile: %v", err)
+	}
+
+	st := findStatus(t, result, name)
+	if st.Action != ActionConflict {
+		t.Errorf("Action = %s, want %s", st.Action, ActionConflict)
+	}
+	if st.Applied {
+		t.Error("Applied = true, want false — a same-version conflict must never be overwritten")
+	}
+	if st.Backup != "" {
+		t.Errorf("Backup = %q, want empty — nothing was written", st.Backup)
+	}
+
+	got, err := os.ReadFile(filepath.Join(root, name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != modified {
+		t.Error("operator-modified unit must be untouched byte-for-byte")
+	}
+	if !result.HasConflicts() {
+		t.Error("HasConflicts() = false, want true")
+	}
+}
+
+func TestReconcile_DryRunChangesNothing(t *testing.T) {
+	root := t.TempDir()
+	reloadCalls := stubDaemonReload(t)
+	const staleName = "spinifex-awsgw.service"
+	stale := setVersion(realUnit(t, staleName), 0)
+	writeFile(t, root, staleName, stale)
+
+	result, err := Reconcile(root, Options{DryRun: true})
+	if err != nil {
+		t.Fatalf("Reconcile: %v", err)
+	}
+	if !result.HasChanges() {
+		t.Fatal("HasChanges() = false, want true — a stale unit and missing units are pending")
+	}
+
+	for _, s := range result.Statuses {
+		if s.Applied {
+			t.Errorf("%s: Applied = true during a dry run", s.Name)
+		}
+	}
+
+	got, err := os.ReadFile(filepath.Join(root, staleName))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != stale {
+		t.Error("dry run must not modify the stale unit on disk")
+	}
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 1 {
+		t.Errorf("dry run wrote %d files to root, want exactly the 1 pre-seeded file", len(entries))
+	}
+	if reloadCalls() != 0 {
+		t.Errorf("daemon-reload calls = %d, want 0 during a dry run", reloadCalls())
+	}
+}
+
+func TestReconcile_NonWritableRootReportsWithoutPartialWrites(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("running as root bypasses directory permissions; this test needs an unprivileged process")
+	}
+
+	root := t.TempDir()
+	if err := os.Chmod(root, 0o500); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = os.Chmod(root, 0o700) })
+
+	result, err := Reconcile(root, Options{})
+	if err == nil {
+		t.Fatal("Reconcile: want an error on a non-writable root, got nil")
+	}
+	if !errors.Is(err, ErrRootRequired) {
+		t.Errorf("err = %v, want it to wrap ErrRootRequired", err)
+	}
+	if len(result.Statuses) == 0 {
+		t.Error("Statuses empty — drift must still be reported even when it cannot be applied")
+	}
+	for _, s := range result.Statuses {
+		if s.Applied {
+			t.Errorf("%s: Applied = true despite a non-writable root", s.Name)
+		}
+	}
+}
+
+func TestReconcile_ReloadRunsOnceThenSettles(t *testing.T) {
+	root := t.TempDir()
+	reloadCalls := stubDaemonReload(t)
+
+	if _, err := Reconcile(root, Options{}); err != nil {
+		t.Fatalf("first Reconcile: %v", err)
+	}
+	if reloadCalls() != 1 {
+		t.Fatalf("daemon-reload calls after first apply = %d, want 1", reloadCalls())
+	}
+
+	result, err := Reconcile(root, Options{})
+	if err != nil {
+		t.Fatalf("second Reconcile: %v", err)
+	}
+	if result.HasChanges() {
+		t.Error("HasChanges() = true on a second pass over an already-reconciled root")
+	}
+	if reloadCalls() != 1 {
+		t.Errorf("daemon-reload calls after idempotent second apply = %d, want still 1", reloadCalls())
+	}
+}
+
+func writeFile(t *testing.T, root, name, content string) {
+	t.Helper()
+	if err := os.WriteFile(filepath.Join(root, name), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// seedAllUnits pre-populates root with every embedded unit at its current
+// content, so a Reconcile pass has nothing pending except what the test
+// deliberately alters afterward.
+func seedAllUnits(t *testing.T, root string) {
+	t.Helper()
+	for name, content := range Units {
+		writeFile(t, root, name, content)
+	}
+}
+
+// withFakeUnit temporarily adds name to the package-level Units map, so a
+// test can control the exact path Reconcile computes (including a name with
+// a path separator, to point at a directory that does or doesn't exist)
+// without touching the generated production unit set.
+func withFakeUnit(t *testing.T, name, content string) {
+	t.Helper()
+	prev, existed := Units[name]
+	Units[name] = content
+	t.Cleanup(func() {
+		if existed {
+			Units[name] = prev
+		} else {
+			delete(Units, name)
+		}
+	})
+}
+
+func TestHasConflicts_NoConflictsIsFalse(t *testing.T) {
+	result := Result{Statuses: []UnitStatus{
+		{Name: "a.service", Action: ActionNoop},
+		{Name: "b.service", Action: ActionInstall},
+	}}
+	if result.HasConflicts() {
+		t.Error("HasConflicts() = true, want false — no unit is ActionConflict")
+	}
+}
+
+func TestUnitVersion_OverflowingMarkerIsVersionZero(t *testing.T) {
+	// A marker value too large for strconv.Atoi must be treated the same as
+	// no marker at all, not propagate the parse error.
+	content := "# spinifex-unit-version: 99999999999999999999999999\nExecStart=/usr/bin/true\n"
+	if v := unitVersion(content); v != 0 {
+		t.Errorf("unitVersion() = %d, want 0 for an overflowing marker value", v)
+	}
+}
+
+func TestUnitVersion_MarkerNotOnFirstLineIsIgnored(t *testing.T) {
+	content := "[Unit]\n# spinifex-unit-version: 5\nExecStart=/usr/bin/true\n"
+	if v := unitVersion(content); v != 0 {
+		t.Errorf("unitVersion() = %d, want 0 — a marker must be on the first line to count", v)
+	}
+}
+
+func TestSystemctlDaemonReload_MissingBinaryReturnsError(t *testing.T) {
+	// Exercises the real (non-stubbed) production implementation: an empty
+	// PATH means systemctl cannot be found, so the command fails cleanly
+	// instead of touching a real systemd instance.
+	t.Setenv("PATH", t.TempDir())
+	if err := systemctlDaemonReload(); err == nil {
+		t.Fatal("systemctlDaemonReload(): want an error when systemctl is not on PATH, got nil")
+	}
+}
+
+func TestReconcile_UnreadableInstalledUnitReturnsError(t *testing.T) {
+	root := t.TempDir()
+	stubDaemonReload(t)
+	const name = "spinifex-ui.service"
+	// A directory in place of the unit file fails the read with something
+	// other than IsNotExist, exercising the general read-error path
+	// distinct from "missing unit".
+	if err := os.Mkdir(filepath.Join(root, name), 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err := Reconcile(root, Options{})
+	if err == nil {
+		t.Fatal("Reconcile: want an error when an installed unit path is unreadable, got nil")
+	}
+	if !strings.Contains(err.Error(), name) {
+		t.Errorf("err = %v, want it to name the unreadable unit %s", err, name)
+	}
+}
+
+func TestReconcile_WriteFailureIsSurfaced(t *testing.T) {
+	root := t.TempDir()
+	stubDaemonReload(t)
+
+	// A unit whose path has a parent directory that does not exist: the
+	// read reports NotExist (classified Install, like any missing unit),
+	// but the later write into that missing directory fails. Deterministic
+	// and root-safe, unlike a permission-based probe.
+	const name = "missing-parent/fake-install.service"
+	withFakeUnit(t, name, "# spinifex-unit-version: 1\nExecStart=/usr/bin/true\n")
+
+	_, err := Reconcile(root, Options{})
+	if err == nil {
+		t.Fatal("Reconcile: want an error when the write's parent directory is missing, got nil")
+	}
+	if !strings.Contains(err.Error(), "write") {
+		t.Errorf("err = %v, want it to mention the write step", err)
+	}
+}
+
+func TestReconcile_BackupFailureIsSurfaced(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("running as root bypasses directory permissions; this test needs an unprivileged process")
+	}
+	root := t.TempDir()
+	stubDaemonReload(t)
+
+	const name = "nested/fake-backup.service"
+	nested := filepath.Join(root, "nested")
+	if err := os.Mkdir(nested, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	embedded := "# spinifex-unit-version: 2\nExecStart=/usr/bin/true\n"
+	withFakeUnit(t, name, embedded)
+	stale := setVersion(embedded, 0)
+	writeFile(t, root, name, stale)
+
+	// Lock the nested directory only after seeding it: the initial read
+	// (classification) still succeeds, but backupUnit's write of the
+	// backup copy into the same directory fails.
+	if err := os.Chmod(nested, 0o555); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = os.Chmod(nested, 0o755) })
+
+	_, err := Reconcile(root, Options{})
+	if err == nil {
+		t.Fatal("Reconcile: want an error when the backup write fails, got nil")
+	}
+	if !strings.Contains(err.Error(), "backup") {
+		t.Errorf("err = %v, want it to mention the backup step", err)
+	}
+}
+
+func TestReconcile_DaemonReloadFailureIsSurfaced(t *testing.T) {
+	root := t.TempDir()
+	prevReload := systemctlDaemonReload
+	wantErr := errors.New("boom: reload failed")
+	systemctlDaemonReload = func() error { return wantErr }
+	t.Cleanup(func() { systemctlDaemonReload = prevReload })
+
+	result, err := Reconcile(root, Options{})
+	if err == nil {
+		t.Fatal("Reconcile: want an error when daemon-reload fails, got nil")
+	}
+	if !errors.Is(err, wantErr) {
+		t.Errorf("err = %v, want it to wrap %v", err, wantErr)
+	}
+	if result.ReloadRan {
+		t.Error("ReloadRan = true despite daemon-reload failing")
+	}
+}
+
+func TestReconcile_DowngradeMarkerIsConflictNotClobbered(t *testing.T) {
+	root := t.TempDir()
+	stubDaemonReload(t)
+	const name = "spinifex-vpcd.service"
+	embedded := realUnit(t, name)
+	// Simulate a downgrade: the installed unit is stamped at a version
+	// newer than what this binary embeds. Reconcile must never clobber a
+	// newer file with an older one — ActionReplace only ever moves forward.
+	future := setVersion(embedded, unitVersion(embedded)+1) + "# from a future release\n"
+	writeFile(t, root, name, future)
+
+	result, err := Reconcile(root, Options{})
+	if err != nil {
+		t.Fatalf("Reconcile: %v", err)
+	}
+
+	st := findStatus(t, result, name)
+	if st.Action != ActionConflict {
+		t.Errorf("Action = %s, want %s — a newer installed marker must never be replaced", st.Action, ActionConflict)
+	}
+	if st.Applied {
+		t.Error("Applied = true, want false — a downgrade must not clobber a newer unit")
+	}
+	got, err := os.ReadFile(filepath.Join(root, name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != future {
+		t.Error("downgrade case: installed unit content must be untouched")
+	}
+}
+
+func TestBackupUnit_UnreadableSourceReturnsError(t *testing.T) {
+	_, err := backupUnit(filepath.Join(t.TempDir(), "does-not-exist.service"), 0, 1)
+	if err == nil {
+		t.Fatal("backupUnit: want an error for a nonexistent source path, got nil")
+	}
+}

spinifex/systemd/units_gen.go

@@ -0,0 +1,26 @@
+// Code generated by `go generate ./...` from build/systemd; DO NOT EDIT.
+
+package systemd
+
+// Units maps each core unit file name to its embedded contents, sourced from
+// build/systemd at generation time. Regenerate with `go generate ./...` after
+// editing a unit; units_invariants_test.go fails preflight if this drifts.
+var Units = map[string]string{
+	"regenerate-ssh-host-keys.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Regenerate SSH host keys\nConditionPathExists=!/etc/ssh/ssh_host_ed25519_key\nBefore=ssh.service\n\n[Service]\nType=oneshot\nExecStart=/usr/bin/ssh-keygen -A\nRemainAfterExit=yes\n# No ExecStop: the keygen process has already exited by the time this unit\n# is stopped, so there is nothing to signal.\nTimeoutStopSec=5\n\n[Install]\nWantedBy=multi-user.target\n",
+	"spinifex-awsgw.service":           "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex AWS Gateway\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-gw\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service awsgw start\nRestart=on-failure\nRestartSec=5\n# No SIGTERM handler in this process: it exits immediately on signal, so this\n# only bounds an unexpectedly wedged process before SIGKILL.\nTimeoutStopSec=10\n\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/\nEnvironment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/awsgw /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex-daemon.service":          "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Daemon (VM Orchestration)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service\n# Wants (not Requires): stay bootable when openvswitch-ipsec is absent\nWants=openvswitch-ipsec.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see\nEnvironmentFile=-/etc/spinifex/host.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-daemon\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex start\nRestart=on-failure\nRestartSec=5\n# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)\nKillMode=process\nTimeoutStopSec=15\n# RG-4: infra tier — reaped only after every guest\nOOMScoreAdjust=-500\n# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead\nMemoryHigh=75%\nMemoryMax=85%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\nEnvironment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/\n# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,\n# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.\nAmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE\nProtectSystem=full\nPrivateDevices=no\nDevicePolicy=closed\nDeviceAllow=/dev/kvm rw\nDeviceAllow=/dev/net/tun rw\nDeviceAllow=char-vfio rw\nDeviceAllow=/dev/vfio/vfio rw\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\n# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which\n# ProtectKernelTunables=yes would otherwise keep read-only.\nReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n# LimitMEMLOCK=infinity: VFIO pins all guest RAM\nLimitMEMLOCK=infinity\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex-guests.slice":            "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex guest VM tier\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-guests -> spinifex), sibling of\n# spinifex-system. Parent for the per-instance spinifex-instance-<id>.slice\n# leaves (0002 RG-7). Inherits the spinifex.slice ceiling; no extra limit here.\n",
+	"spinifex-nats-watchdog.service":   "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS JetStream liveness watchdog\nPartOf=spinifex.target\nAfter=spinifex-nats.service\n\n[Service]\nType=oneshot\n# Runs as root: the probes are a local curl to the NATS monitoring port and a\n# canary KV round-trip via spx, and the remedy is `systemctl try-restart\n# spinifex-nats` when JetStream stops accepting writes while the process stays\n# alive (which Restart=on-failure cannot catch).\nExecStart=/var/lib/spinifex/nats-js-watchdog.sh\n# Sized for the probe loop's own worst case (3 samples x (5s curl + 5s canary)\n# + 2 gaps x 2s = 34s): a stop mid-cycle lets the script finish rather than\n# cutting it off. Only the failing path runs that long; a healthy node exits on\n# the first sample.\nTimeoutStopSec=45\n\n# Hardening (kept compatible with talking to PID1 over /run/systemd).\nNoNewPrivileges=yes\nProtectHome=yes\nPrivateTmp=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectControlGroups=yes\nRestrictRealtime=yes\nLockPersonality=yes\nRestrictSUIDSGID=yes\n",
+	"spinifex-nats-watchdog.timer":     "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS JetStream liveness watchdog timer\nPartOf=spinifex.target\n\n[Timer]\n# Delay the first probe so NATS startup + JetStream cluster quorum settle (a\n# forming cluster reports JS not-current transiently, which is not a fault).\nOnBootSec=60s\nOnUnitActiveSec=30s\nAccuracySec=5s\n\n[Install]\nWantedBy=timers.target\n",
+	"spinifex-nats.service":            "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS Message Bus\nPartOf=spinifex.target\nBefore=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-nats\nGroup=spinifex\nExecStart=/usr/local/bin/spx service nats start\nRestart=on-failure\nRestartSec=5\n# nats-server installs its own SIGTERM handler (JetStream store shutdown, raft\n# stepdown, client drain) with no internal deadline; every dependent service\n# stops before nats does, so size this for nats's own graceful-stop needs.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/nats /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/nats /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex-northstar.service":       "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Northstar (Authoritative DNS)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-predastore.service\nWants=spinifex-predastore.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-northstar\nGroup=spinifex\nExecStart=/usr/local/bin/spx service northstar start\nRestart=on-failure\nRestartSec=5\n# server.Shutdown() is called with an internal 5s context; margin covers the\n# NATS unsubscribe/close and process exit that follow it.\nTimeoutStopSec=10\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/northstar/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/northstar /etc/spinifex/spinifex.toml /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/northstar /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\n# RG-9: exactly CAP_NET_BIND_SERVICE so the unprivileged user binds :53 (no root).\nAmbientCapabilities=CAP_NET_BIND_SERVICE\nCapabilityBoundingSet=CAP_NET_BIND_SERVICE\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex-predastore.service":      "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Predastore (S3-compatible Object Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# Guest block I/O arrives with no caller trace; sample those roots at 5% so\n# they don't drown request traces. Request-linked spans are always kept.\nEnvironment=MULGA_ROOT_TRACE_RATIO=0.05\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-storage\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service predastore start\nRestart=on-failure\nRestartSec=5\n# Matches viperblock's drain budget: viperblock's WAL flush writes into\n# predastore, and After= already stops viperblock first, but this is the\n# backstop if that ordering is ever bypassed (direct stop, future edit).\nTimeoutStopSec=30\nOOMScoreAdjust=-500\n# Legitimately the most memory-hungry infra service (S3 buffering/caching);\n# High/Max stay well above idle so normal load isn't throttled, while still\n# bounding a leak short of the slice-wide cap.\nMemoryHigh=40%\nMemoryMax=55%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml\nEnvironment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/\nEnvironment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex-qmp-collector.service":   "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex QMP Telemetry Collector (guest metrics)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\n# spinifex-daemon user: the collector reads the daemon's telemetry QMP\n# sockets in /run/spinifex and the daemon-owned QEMU /proc entries.\nUser=spinifex-daemon\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service qmp-collector start\nRestart=on-failure\nRestartSec=5\n# Poll loop selects on ctx.Done() directly and closes NATS locally; shutdown\n# is near-instant, so this only bounds an unexpectedly wedged process.\nTimeoutStopSec=10\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\n# Same runtime dir as the daemon: telemetry sockets + metadata live there\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: read-mostly telemetry tier — no capabilities, no device access\nProtectSystem=full\nPrivateDevices=yes\nNoNewPrivileges=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/spinifex /run/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\n# ProtectProc must stay default: hidepid=invisible hides the QEMU vCPU threads\n# (dumpable=0) this collector must read from /proc/<tid>/stat, even same-uid.\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=512\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex-shutdown.service":        "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex graceful guest drain\nPartOf=spinifex.target\n# After= all services so it stops FIRST on target/host shutdown: the drain runs\n# while nats/predastore/viperblock/daemon are still up (QMP, unmount, WAL flush).\nAfter=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/true\n# ExecStop fires on ANY target stop, including the stop half of `systemctl\n# restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless\n# of whether the host is actually going down. --unless-restarting drains on a\n# real shutdown/reboot and on a plain `systemctl stop spinifex.target`; it\n# skips only when a restart of the stack is already queued, so QEMU/nbdkit\n# survive (KillMode=process) and the daemon reconnects on the way back up.\nExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --unless-restarting\nTimeoutStopSec=180\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex-system.slice":            "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-system -> spinifex), inherits the\n# parent ceiling, no extra limit. Holds the 7 control/infra services. Guest QEMU\n# moves to the sibling spinifex-guests.slice (per-instance placement lands in\n# 0002 RG-7); until then the daemon-forked guests stay under spinifex-daemon.service.\n",
+	"spinifex-ui.service":              "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Web UI\nPartOf=spinifex.target\nAfter=spinifex-awsgw.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-ui\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex-ui start\nRestart=on-failure\nRestartSec=5\n# server.Shutdown() uses an internal 30s context (WriteTimeout=120s covers\n# long streaming proxy responses, e.g. console passthrough); add margin above it.\nTimeoutStopSec=40\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key\nReadWritePaths=/var/lib/spinifex/spinifex-ui /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex-viperblock.service":      "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Viperblock (Block Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-predastore.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-viperblock\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service viperblock start\nRestart=on-failure\nRestartSec=5\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=15%\nMemoryMax=25%\n# KillMode=process: nbdkit still serving a guest survives a restart here, the\n# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts\n# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.\nKillMode=process\n# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)\n# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must\n# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.\nTimeoutStopSec=150\nEnvironmentFile=/etc/spinifex/systemd.env\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /run/spinifex/nbd\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex-vpcd.service":            "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex VPC Daemon (OVN Networking)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service\nWants=spinifex-nats.service ovn-controller.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-vpcd\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service vpcd start\nRestart=on-failure\nRestartSec=5\n# Waits for an in-flight OVN drift-reconcile pass to return before exiting;\n# no sub-timeout bounds that call today, so this covers a normal pass.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling: bounds a leak here instead of the slice-wide cap.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/\n\n# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no\n# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem\n# sandbox is restored.\n#\n# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and\n# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR\n# update that retires the last escalation path.\nAmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nCapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nNoNewPrivileges=no\nProtectSystem=full\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\n# AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex.slice":                   "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex resource-bounded slice (host ceiling)\nBefore=slices.target\n\n[Slice]\n# RG-1: no CPUWeight — slice roots are siblings; capping below the default would\n# let user.slice out-prioritize the VM fleet. Operator CPU priority lives on\n# system.slice (system.slice.d/spinifex-reserve.conf).\n# RG-1/RG-5: throttle+reclaim at High, hard wall at Max (% of physical RAM).\nMemoryHigh=80%\nMemoryMax=90%\n# RG-5: spill to the full swap device before the MemoryMax wall; matched to swap size.\nMemorySwapMax=8G\n",
+	"spinifex.target":                  "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Infrastructure Platform\nAfter=network-online.target\nWants=network-online.target\nWants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service\n\n[Install]\nWantedBy=multi-user.target\n",
+}

spinifex/systemd/units_invariants_test.go

@@ -1,6 +1,8 @@
 package systemd
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -55,6 +57,27 @@ func unitFiles(t *testing.T, dir string) []string {
 	return out
 }
 
+// allUnitFiles lists every unit in dir regardless of type — .service,
+// .slice, .target and .timer — for invariants that apply to all 16.
+func allUnitFiles(t *testing.T, dir string) []string {
+	t.Helper()
+	ents, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var out []string
+	for _, e := range ents {
+		n := e.Name()
+		for _, suffix := range []string{".service", ".slice", ".target", ".timer"} {
+			if strings.HasSuffix(n, suffix) {
+				out = append(out, n)
+				break
+			}
+		}
+	}
+	return out
+}
+
 // timeoutStopSec parses a unit's TimeoutStopSec= value (this repo always
 // writes plain seconds, no unit suffix) into a time.Duration.
 func timeoutStopSec(t *testing.T, unit string) time.Duration {
@@ -287,6 +310,11 @@ func TestRG11_LeanUnits(t *testing.T) {
 			if !strings.HasPrefix(ls, "#") {
 				continue
 			}
+			// The version marker is structural metadata, not rationale prose,
+			// so it does not count against the comment budget.
+			if unitVersionRe.MatchString(ls) {
+				continue
+			}
 			comments++
 			if m := planRef.FindString(ls); m != "" {
 				t.Errorf("unit comment references external planning or CI material (%q): %s", m, ls)
@@ -355,3 +383,92 @@ func TestApplicationUnitsExportTelemetry(t *testing.T) {
 		}
 	}
 }
+
+// TestGeneratedUnitsMatchSource asserts units_gen.go (the embedded copy
+// Reconcile ships and compares against) is byte-identical to build/systemd.
+// A stale generated file would make `spx admin upgrade` write the wrong
+// content; regenerate with `go generate ./...` when this fails.
+func TestGeneratedUnitsMatchSource(t *testing.T) {
+	dir := unitsDir(t)
+	source := allUnitFiles(t, dir)
+
+	if len(Units) != len(source) {
+		t.Errorf("units_gen.go has %d units, build/systemd has %d — run `go generate ./...`", len(Units), len(source))
+	}
+	for _, name := range source {
+		want := readUnit(t, dir, name)
+		got, ok := Units[name]
+		if !ok {
+			t.Errorf("units_gen.go missing %s — run `go generate ./...`", name)
+			continue
+		}
+		if got != want {
+			t.Errorf("units_gen.go for %s is stale — run `go generate ./...`", name)
+		}
+	}
+}
+
+// unitBodyHashes pins the sha256 of each unit's body (everything after the
+// `# spinifex-unit-version:` line) for its current version. Editing a unit's
+// body without bumping the marker and adding a matching entry here fails
+// TestUnitBodyPinnedToVersion — the forcing function that keeps a running
+// node's reconciled units honest about what changed.
+var unitBodyHashes = map[string]map[int]string{
+	"regenerate-ssh-host-keys.service": {1: "5b2cf4c2ba5d5799c790e32896928c235db92a069583a87004801f58df390e55"},
+	"spinifex-awsgw.service":           {1: "dd662cebca19acca9dbcd0a6f29e6a40125805bdfeabcb94c02fbae07d1f2d6e"},
+	"spinifex-daemon.service":          {1: "04b0f23b9cda322e4012d589acc196abe466b619dec5ac0d832897b9af2f926c"},
+	"spinifex-guests.slice":            {1: "22dac772e81a9a716db98415a4cf590885c9cf19d4290cc21e645e5fe15bc793"},
+	"spinifex-nats-watchdog.service":   {1: "b778e398da37c4cb3c170781f074c76555cb6a137a7f96f8dc692457b12aa9b8"},
+	"spinifex-nats-watchdog.timer":     {1: "9d43a8d2aa4fd80ab5f944d7620c666fd2a255850de2e6494479b8746922c979"},
+	"spinifex-nats.service":            {1: "f7f9900b95e364dc2575684a9bf16fa1f96a903a3a4aa773a34c04d761106838"},
+	"spinifex-northstar.service":       {1: "030c382aed20f4383acdb7df7a830d48dd76dc29be5965b595cc3842ae43a3f1"},
+	"spinifex-predastore.service":      {1: "d0f6415b8f0e6ff3c045f2d3ce2794c347bf141066d7e0bd85fcec48797854d8"},
+	"spinifex-qmp-collector.service":   {1: "beb18e6dd9351901f19d992cb2f757fb0e0e4a4d986402ccdb0ebb0a449f225c"},
+	"spinifex-shutdown.service":        {1: "bcdc455916f35aa7494b2fe25e691339e8f1e22f031dfd9fd95203a9aa4bdaa4"},
+	"spinifex-system.slice":            {1: "ca450c2b28a8b13dd767957fa9469bd74bd222d7abed79945e83d564d5ce16dd"},
+	"spinifex-ui.service":              {1: "9d5ec3785bf730405f23ec9df79675bf65ecbca9df47c15f5a4b4639393afbc5"},
+	"spinifex-viperblock.service":      {1: "5c8cdd2004abf8e5725cbce0200565b9b29be7ab3210a4e0a2cdfe37ec5facb9"},
+	"spinifex-vpcd.service":            {1: "1b722640310145767cd34e87f4804852e63f46d2145ed20f8cc5f5400ebc5965"},
+	"spinifex.slice":                   {1: "f73d9343e0e1bedd647835c8bb0c80fb3a3bd66474661234ecac23a4caafc24f"},
+	"spinifex.target":                  {1: "0ffba9faee5a477f8ff7466a6bccb4dc7e04f5cf92a405553c242e6548402078"},
+}
+
+// TestUnitBodyPinnedToVersion asserts each unit's body hash matches the
+// pinned hash for its current version marker. A body edit without a version
+// bump changes the hash at the same version number and fails; bumping the
+// version without adding a new table entry also fails, since no entry
+// exists for the new version yet.
+func TestUnitBodyPinnedToVersion(t *testing.T) {
+	dir := unitsDir(t)
+	for _, name := range allUnitFiles(t, dir) {
+		u := readUnit(t, dir, name)
+		version := unitVersion(u)
+		body := stripVersionMarkerLine(u)
+		sum := sha256.Sum256([]byte(body))
+		hash := hex.EncodeToString(sum[:])
+
+		byVersion, ok := unitBodyHashes[name]
+		if !ok {
+			t.Errorf("%s: no entries in unitBodyHashes — add one for version %d", name, version)
+			continue
+		}
+		want, ok := byVersion[version]
+		if !ok {
+			t.Errorf("%s: no unitBodyHashes entry for version %d — bumped the marker without pinning its new body hash", name, version)
+			continue
+		}
+		if want != hash {
+			t.Errorf("%s: body hash for version %d changed (got %s, want %s) — bump # spinifex-unit-version and add a new unitBodyHashes entry", name, version, hash, want)
+		}
+	}
+}
+
+// stripVersionMarkerLine drops the first line when it is the unit-version
+// marker, so the body hash covers content only, not the marker itself.
+func stripVersionMarkerLine(content string) string {
+	first, rest, found := strings.Cut(content, "\n")
+	if found && unitVersionRe.MatchString(strings.TrimSpace(first)) {
+		return rest
+	}
+	return content
+}