diff --git a/build/systemd/regenerate-ssh-host-keys.service b/build/systemd/regenerate-ssh-host-keys.service index f771a5d2e..aeb31d10d 100644 --- a/build/systemd/regenerate-ssh-host-keys.service +++ b/build/systemd/regenerate-ssh-host-keys.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Regenerate SSH host keys ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key diff --git a/build/systemd/spinifex-awsgw.service b/build/systemd/spinifex-awsgw.service index 5efc21605..3d38c69f8 100644 --- a/build/systemd/spinifex-awsgw.service +++ b/build/systemd/spinifex-awsgw.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex AWS Gateway PartOf=spinifex.target diff --git a/build/systemd/spinifex-daemon.service b/build/systemd/spinifex-daemon.service index 505ed6a73..3e0715024 100644 --- a/build/systemd/spinifex-daemon.service +++ b/build/systemd/spinifex-daemon.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex Daemon (VM Orchestration) PartOf=spinifex.target diff --git a/build/systemd/spinifex-guests.slice b/build/systemd/spinifex-guests.slice index 515b52e28..7189fa0f0 100644 --- a/build/systemd/spinifex-guests.slice +++ b/build/systemd/spinifex-guests.slice @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex guest VM tier Before=slices.target diff --git a/build/systemd/spinifex-nats-watchdog.service b/build/systemd/spinifex-nats-watchdog.service index f414f5ca2..46d51c0b4 100644 --- a/build/systemd/spinifex-nats-watchdog.service +++ b/build/systemd/spinifex-nats-watchdog.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex NATS JetStream liveness watchdog PartOf=spinifex.target diff --git a/build/systemd/spinifex-nats-watchdog.timer b/build/systemd/spinifex-nats-watchdog.timer index 33d79697b..46fa85c7e 100644 --- a/build/systemd/spinifex-nats-watchdog.timer +++ b/build/systemd/spinifex-nats-watchdog.timer @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex NATS JetStream liveness watchdog timer PartOf=spinifex.target diff --git a/build/systemd/spinifex-nats.service b/build/systemd/spinifex-nats.service index 583ed3d11..66ad4565b 100644 --- a/build/systemd/spinifex-nats.service +++ b/build/systemd/spinifex-nats.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex NATS Message Bus PartOf=spinifex.target diff --git a/build/systemd/spinifex-northstar.service b/build/systemd/spinifex-northstar.service index f65bb2507..b6c521358 100644 --- a/build/systemd/spinifex-northstar.service +++ b/build/systemd/spinifex-northstar.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex Northstar (Authoritative DNS) PartOf=spinifex.target diff --git a/build/systemd/spinifex-predastore.service b/build/systemd/spinifex-predastore.service index 82afbd983..e5f209e52 100644 --- a/build/systemd/spinifex-predastore.service +++ b/build/systemd/spinifex-predastore.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex Predastore (S3-compatible Object Storage) PartOf=spinifex.target diff --git a/build/systemd/spinifex-qmp-collector.service b/build/systemd/spinifex-qmp-collector.service index 0cdb59842..b732bf5bb 100644 --- a/build/systemd/spinifex-qmp-collector.service +++ b/build/systemd/spinifex-qmp-collector.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex QMP Telemetry Collector (guest metrics) PartOf=spinifex.target diff --git a/build/systemd/spinifex-shutdown.service b/build/systemd/spinifex-shutdown.service index b78f28a50..c39506b6d 100644 --- a/build/systemd/spinifex-shutdown.service +++ b/build/systemd/spinifex-shutdown.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex graceful guest drain PartOf=spinifex.target diff --git a/build/systemd/spinifex-system.slice b/build/systemd/spinifex-system.slice index bf02a0771..8f3e08a9c 100644 --- a/build/systemd/spinifex-system.slice +++ b/build/systemd/spinifex-system.slice @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui) Before=slices.target diff --git a/build/systemd/spinifex-ui.service b/build/systemd/spinifex-ui.service index 600f5641a..6bb06aa8d 100644 --- a/build/systemd/spinifex-ui.service +++ b/build/systemd/spinifex-ui.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex Web UI PartOf=spinifex.target diff --git a/build/systemd/spinifex-viperblock.service b/build/systemd/spinifex-viperblock.service index 3e9c424c9..c262b23d2 100644 --- a/build/systemd/spinifex-viperblock.service +++ b/build/systemd/spinifex-viperblock.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex Viperblock (Block Storage) PartOf=spinifex.target diff --git a/build/systemd/spinifex-vpcd.service b/build/systemd/spinifex-vpcd.service index 3ef8606ed..3b5cfe510 100644 --- a/build/systemd/spinifex-vpcd.service +++ b/build/systemd/spinifex-vpcd.service @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex VPC Daemon (OVN Networking) PartOf=spinifex.target diff --git a/build/systemd/spinifex.slice b/build/systemd/spinifex.slice index 42edceba2..6be97092d 100644 --- a/build/systemd/spinifex.slice +++ b/build/systemd/spinifex.slice @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex resource-bounded slice (host ceiling) Before=slices.target diff --git a/build/systemd/spinifex.target b/build/systemd/spinifex.target index 127133c71..7f1427296 100644 --- a/build/systemd/spinifex.target +++ b/build/systemd/spinifex.target @@ -1,3 +1,4 @@ +# spinifex-unit-version: 1 [Unit] Description=Spinifex Infrastructure Platform After=network-online.target diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 3e43b1433..ac19902fe 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -330,6 +330,9 @@ func init() { adminCmd.AddCommand(upgradeCmd) upgradeCmd.Flags().Bool("yes", false, "Apply migrations without prompting") + upgradeCmd.Flags().Bool("dry-run", false, "Report pending config and unit changes without applying them") + upgradeCmd.Flags().Bool("units-only", false, "Reconcile systemd units only, skip config migrations") + upgradeCmd.Flags().Bool("skip-units", false, "Apply config migrations only, skip systemd unit reconciliation") certRenewCmd.Flags().StringSlice("extra-ip", nil, "Additional IP addresses to include in SANs") certRenewCmd.Flags().StringSlice("extra-dns", nil, "Additional DNS names to include in SANs") accountCreateCmd.Flags().String("name", "", "Account name (required)") diff --git a/cmd/spinifex/cmd/admin_upgrade.go b/cmd/spinifex/cmd/admin_upgrade.go index 2f8d2687f..06094f3e5 100644 --- a/cmd/spinifex/cmd/admin_upgrade.go +++ b/cmd/spinifex/cmd/admin_upgrade.go @@ -2,22 +2,70 @@ package cmd import ( "bufio" + "errors" "fmt" "os" "strings" "github.com/mulgadc/spinifex/spinifex/migrate" + "github.com/mulgadc/spinifex/spinifex/systemd" "github.com/spf13/cobra" ) +// systemdUnitRoot is where installed units live. A var, not a const, so +// tests could point it elsewhere if this command ever grows a test harness. +var systemdUnitRoot = "/etc/systemd/system" + +// upgradeExit is os.Exit, overridable in tests so a failure path can be +// exercised without killing the test binary. +var upgradeExit = os.Exit + +// errUpgradeFlagsConflict is returned when both --units-only and --skip-units +// are set, a combination that would leave nothing for the command to do. +var errUpgradeFlagsConflict = errors.New("--units-only and --skip-units are mutually exclusive") + +// validateUpgradeFlags checks flag combinations that runAdminUpgrade cannot +// proceed with, kept separate from runAdminUpgrade so the check is testable +// without driving the whole command. +func validateUpgradeFlags(unitsOnly, skipUnits bool) error { + if unitsOnly && skipUnits { + return errUpgradeFlagsConflict + } + return nil +} + +// upgradeSummary decides whether runAdminUpgrade has anything left to do +// after gathering config and unit status, and what to print when it does +// not. Kept pure so the dryRun × hasConfigWork × hasUnitWork matrix is +// testable without driving the whole command. +func upgradeSummary(dryRun, hasConfigWork, hasUnitWork bool) (proceed bool, message string) { + if !hasConfigWork && !hasUnitWork { + if dryRun { + return false, "\nNothing pending — config and units are current." + } + return false, "\nNothing to do." + } + if dryRun { + return false, "" + } + return true, "" +} + var upgradeCmd = &cobra.Command{ Use: "upgrade", - Short: "Apply pending config migrations", - Long: `Apply pending configuration file migrations for upgrades between Spinifex versions. + Short: "Apply pending config migrations and systemd unit updates", + Long: `Apply pending configuration file migrations and systemd unit reconciliation +for upgrades between Spinifex versions. -When run interactively (without --yes), shows pending migrations and prompts +Swapping the spx binary alone does not upgrade a node: unit files (KillMode, +TimeoutStopSec, drain ordering, ...) are installed once at setup time and +never re-asserted otherwise. This command brings both config and units up to +date with the binary running it. + +When run interactively (without --yes), shows what is pending and prompts for confirmation. When called from setup.sh with --yes, applies immediately. +--dry-run reports drift without prompting or changing anything. Operators can skip migrations during install by setting INSTALL_SPINIFEX_SKIP_MIGRATE=1, then run 'spx admin upgrade' manually to review and apply.`, @@ -28,19 +76,90 @@ func runAdminUpgrade(cmd *cobra.Command, _ []string) { configDir, _ := cmd.Root().Flags().GetString("config-dir") dataDir, _ := cmd.Root().Flags().GetString("spinifex-dir") yes, _ := cmd.Flags().GetBool("yes") + dryRun, _ := cmd.Flags().GetBool("dry-run") + unitsOnly, _ := cmd.Flags().GetBool("units-only") + skipUnits, _ := cmd.Flags().GetBool("skip-units") + + if err := validateUpgradeFlags(unitsOnly, skipUnits); err != nil { + fmt.Fprintln(os.Stderr, err) + upgradeExit(1) + } // Check that the installation exists. spinifexToml := configDir + "/spinifex.toml" if _, err := os.Stat(spinifexToml); os.IsNotExist(err) { fmt.Fprintf(os.Stderr, "No Spinifex installation found at %s\nRun 'spx admin init' first.\n", configDir) - os.Exit(1) + upgradeExit(1) } - // Show current versions. + var pending []migrate.PendingMigration + if !unitsOnly { + pending = reportConfigStatus(configDir) + } + + var unitResult systemd.Result + if !skipUnits { + var err error + unitResult, err = systemd.Reconcile(systemdUnitRoot, systemd.Options{DryRun: true}) + if err != nil { + fmt.Fprintf(os.Stderr, "Error checking systemd units: %v\n", err) + upgradeExit(1) + } + reportUnitStatus(unitResult) + } + + hasConfigWork := !unitsOnly && len(pending) > 0 + hasUnitWork := !skipUnits && unitResult.HasChanges() + + proceed, message := upgradeSummary(dryRun, hasConfigWork, hasUnitWork) + if message != "" { + fmt.Println(message) + } + if !proceed { + return + } + + if !yes { + fmt.Print("\nApply? [y/N] ") + reader := bufio.NewReader(os.Stdin) + answer, _ := reader.ReadString('\n') + answer = strings.TrimSpace(strings.ToLower(answer)) + if answer != "y" && answer != "yes" { + fmt.Println("Aborted.") + return + } + } + + if hasConfigWork { + if err := migrate.DefaultRegistry.RunAllConfig(configDir, dataDir); err != nil { + fmt.Fprintf(os.Stderr, "Config migration failed: %v\n", err) + upgradeExit(1) + } + } + + if !skipUnits { + applied, err := systemd.Reconcile(systemdUnitRoot, systemd.Options{}) + if err != nil { + fmt.Fprintf(os.Stderr, "\nUnit reconciliation failed: %v\n", err) + if errors.Is(err, systemd.ErrRootRequired) { + fmt.Fprintln(os.Stderr, "Re-run as root to apply the unit changes reported above: sudo spx admin upgrade --units-only --yes") + } + upgradeExit(1) + } + reportUnitApply(applied) + } + + fmt.Println("\nDone. daemon-reload has already picked up any unit changes for the") + fmt.Println("next stop; restart to apply config changes: sudo systemctl restart spinifex.target") +} + +// reportConfigStatus prints current config versions and pending migrations, +// mirroring the pre-existing `spx admin upgrade` output. +func reportConfigStatus(configDir string) []migrate.PendingMigration { versions, err := migrate.DefaultRegistry.ConfigVersions(configDir) if err != nil { fmt.Fprintf(os.Stderr, "Error reading config versions: %v\n", err) - os.Exit(1) + upgradeExit(1) } fmt.Println("Reading current config versions...") @@ -51,43 +170,59 @@ func runAdminUpgrade(cmd *cobra.Command, _ []string) { fmt.Printf(" %-20s %d\n", name+":", v) } } - fmt.Println() - // Check for pending migrations. pending, err := migrate.DefaultRegistry.PendingConfig(configDir) if err != nil { fmt.Fprintf(os.Stderr, "Error checking pending migrations: %v\n", err) - os.Exit(1) + upgradeExit(1) } if len(pending) == 0 { fmt.Println("No pending config migrations.") - return + return pending } fmt.Println("Pending config migrations:") for _, p := range pending { fmt.Printf(" [config] %s %d → %d: %s\n", p.Target, p.FromVersion, p.ToVersion, p.Description) } - fmt.Println() + return pending +} - // Prompt for confirmation unless --yes. - if !yes { - fmt.Print("Apply? [y/N] ") - reader := bufio.NewReader(os.Stdin) - answer, _ := reader.ReadString('\n') - answer = strings.TrimSpace(strings.ToLower(answer)) - if answer != "y" && answer != "yes" { - fmt.Println("Aborted.") - return +// reportUnitStatus prints the reconcile decision for each unit, computed +// with Options{DryRun: true} so this is always safe to call before a prompt. +func reportUnitStatus(result systemd.Result) { + fmt.Println("\nReading installed systemd unit versions...") + for _, s := range result.Statuses { + switch s.Action { + case systemd.ActionNoop: + fmt.Printf(" %-38s %d (up to date)\n", s.Name+":", s.InstalledVersion) + case systemd.ActionInstall: + fmt.Printf(" %-38s missing → will install (v%d)\n", s.Name+":", s.EmbeddedVersion) + case systemd.ActionReplace: + fmt.Printf(" %-38s %d → %d (stale, will replace)\n", s.Name+":", s.InstalledVersion, s.EmbeddedVersion) + case systemd.ActionConflict: + fmt.Printf(" %-38s %d (operator-modified, not touched — see `systemctl edit %s`)\n", s.Name+":", s.InstalledVersion, s.Name) } } - - // Apply migrations. - if err := migrate.DefaultRegistry.RunAllConfig(configDir, dataDir); err != nil { - fmt.Fprintf(os.Stderr, "Migration failed: %v\n", err) - os.Exit(1) + if result.HasConflicts() { + fmt.Println("Operator-modified units above are left as-is; review them by hand.") + } +} + +// reportUnitApply prints backups written and confirms daemon-reload ran. +func reportUnitApply(result systemd.Result) { + for _, s := range result.Statuses { + if !s.Applied { + continue + } + if s.Backup != "" { + fmt.Printf(" [unit] %s replaced (backup: %s)\n", s.Name, s.Backup) + } else { + fmt.Printf(" [unit] %s installed\n", s.Name) + } + } + if result.ReloadRan { + fmt.Println(" systemctl daemon-reload complete.") } - - fmt.Println("\nDone. Restart services to apply: sudo systemctl restart spinifex.target") } diff --git a/cmd/spinifex/cmd/admin_upgrade_test.go b/cmd/spinifex/cmd/admin_upgrade_test.go index 2b7aa9938..24d5e41fd 100644 --- a/cmd/spinifex/cmd/admin_upgrade_test.go +++ b/cmd/spinifex/cmd/admin_upgrade_test.go @@ -1,11 +1,15 @@ package cmd import ( + "bytes" + "io" "os" "path/filepath" "testing" "github.com/mulgadc/spinifex/spinifex/migrate" + "github.com/mulgadc/spinifex/spinifex/systemd" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -38,3 +42,391 @@ func TestUpgradeSeesEveryConfigTarget(t *testing.T) { } assert.Equal(t, map[string][2]int{"spinifex.toml": {3, 4}}, targets) } + +func TestValidateUpgradeFlags(t *testing.T) { + assert.NoError(t, validateUpgradeFlags(false, false)) + assert.NoError(t, validateUpgradeFlags(true, false)) + assert.NoError(t, validateUpgradeFlags(false, true)) + assert.ErrorIs(t, validateUpgradeFlags(true, true), errUpgradeFlagsConflict) +} + +func TestUpgradeSummary(t *testing.T) { + cases := []struct { + name string + dryRun, hasConfig, hasUnit bool + wantProceed bool + wantMessage string + }{ + {"dry-run, nothing pending", true, false, false, false, "\nNothing pending — config and units are current."}, + {"dry-run, config pending", true, true, false, false, ""}, + {"dry-run, unit pending", true, false, true, false, ""}, + {"dry-run, both pending", true, true, true, false, ""}, + {"apply, nothing pending", false, false, false, false, "\nNothing to do."}, + {"apply, config pending", false, true, false, true, ""}, + {"apply, unit pending", false, false, true, true, ""}, + {"apply, both pending", false, true, true, true, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + proceed, message := upgradeSummary(tc.dryRun, tc.hasConfig, tc.hasUnit) + assert.Equal(t, tc.wantProceed, proceed) + assert.Equal(t, tc.wantMessage, message) + }) + } +} + +func TestReportUnitStatus_PrintsEachActionAndConflictFooter(t *testing.T) { + result := systemd.Result{Statuses: []systemd.UnitStatus{ + {Name: "noop.service", Action: systemd.ActionNoop, InstalledVersion: 3}, + {Name: "install.service", Action: systemd.ActionInstall, EmbeddedVersion: 2}, + {Name: "replace.service", Action: systemd.ActionReplace, InstalledVersion: 1, EmbeddedVersion: 2}, + {Name: "conflict.service", Action: systemd.ActionConflict, InstalledVersion: 2}, + }} + + out := captureStdout(t, func() { reportUnitStatus(result) }) + + for _, want := range []string{ + "noop.service:", "3 (up to date)", + "install.service:", "missing → will install (v2)", + "replace.service:", "1 → 2 (stale, will replace)", + "conflict.service:", "2 (operator-modified, not touched", + "Operator-modified units above are left as-is", + } { + assert.Contains(t, out, want) + } +} + +func TestReportUnitStatus_NoConflictsOmitsFooter(t *testing.T) { + result := systemd.Result{Statuses: []systemd.UnitStatus{{Name: "a.service", Action: systemd.ActionNoop}}} + out := captureStdout(t, func() { reportUnitStatus(result) }) + assert.NotContains(t, out, "operator-modified") +} + +func TestReportUnitApply_PrintsAppliedUnitsAndReload(t *testing.T) { + result := systemd.Result{ + ReloadRan: true, + Statuses: []systemd.UnitStatus{ + {Name: "skipped.service", Applied: false}, + {Name: "installed.service", Applied: true}, + {Name: "replaced.service", Applied: true, Backup: "/etc/systemd/system/replaced.service.bak"}, + }, + } + out := captureStdout(t, func() { reportUnitApply(result) }) + + assert.NotContains(t, out, "skipped.service", "unapplied unit must not be printed") + for _, want := range []string{ + "installed.service installed", + "replaced.service replaced (backup: /etc/systemd/system/replaced.service.bak)", + "systemctl daemon-reload complete.", + } { + assert.Contains(t, out, want) + } +} + +func TestReportUnitApply_NoReloadOmitsMessage(t *testing.T) { + out := captureStdout(t, func() { reportUnitApply(systemd.Result{}) }) + assert.NotContains(t, out, "daemon-reload") +} + +func TestReportConfigStatus_NoPendingMigrations(t *testing.T) { + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640)) + + var pending []migrate.PendingMigration + out := captureStdout(t, func() { pending = reportConfigStatus(configDir) }) + + assert.Empty(t, pending) + assert.Contains(t, out, "No pending config migrations.") + assert.Contains(t, out, "spinifex.toml:") +} + +func TestReportConfigStatus_PendingMigrationIsListed(t *testing.T) { + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640)) + + var pending []migrate.PendingMigration + out := captureStdout(t, func() { pending = reportConfigStatus(configDir) }) + + require.Len(t, pending, 1) + assert.Equal(t, "spinifex.toml", pending[0].Target) + assert.Contains(t, out, "Pending config migrations:") + assert.Contains(t, out, "spinifex.toml 3 → 4") +} + +// captureStderr redirects os.Stderr for the duration of fn and returns what +// was written. runAdminUpgrade's error paths print to stderr, not stdout. +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + r, w, err := os.Pipe() + require.NoError(t, err) + orig := os.Stderr + os.Stderr = w //nolint:reassign // test-local stderr capture, restored below before this func returns + + fn() + + require.NoError(t, w.Close()) + os.Stderr = orig //nolint:reassign // restoring the real os.Stderr captured above + var buf bytes.Buffer + _, err = io.Copy(&buf, r) + require.NoError(t, err) + return buf.String() +} + +// upgradeExitPanic is the sentinel withUpgradeExitCapture's fake upgradeExit +// panics with, so a deferred recover can distinguish "the command called +// exit" from any other panic escaping fn. +type upgradeExitPanic struct{ code int } + +// withUpgradeExitCapture swaps upgradeExit for a fake that records the exit +// code and panics with upgradeExitPanic instead of killing the test binary, +// runs fn, and returns the recorded code (-1 if fn returned without +// exiting). This lets runAdminUpgrade's real control flow, including its +// error/exit branches, be exercised directly. +func withUpgradeExitCapture(t *testing.T, fn func()) int { + t.Helper() + origExit := upgradeExit + t.Cleanup(func() { upgradeExit = origExit }) + + code := -1 + upgradeExit = func(c int) { code = c; panic(upgradeExitPanic{c}) } + + func() { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(upgradeExitPanic); !ok { + panic(r) + } + } + }() + fn() + }() + return code +} + +// newUpgradeTestCmd builds an isolated command tree mirroring rootCmd's and +// upgradeCmd's flag wiring, so runAdminUpgrade can be driven directly +// without touching the package's real global command tree (which other +// tests in this package also mutate). +func newUpgradeTestCmd(t *testing.T, configDir, dataDir string) *cobra.Command { + t.Helper() + root := &cobra.Command{Use: "spx"} + root.PersistentFlags().String("config-dir", configDir, "") + root.PersistentFlags().String("spinifex-dir", dataDir, "") + + upgrade := &cobra.Command{Use: "upgrade", Run: runAdminUpgrade} + upgrade.Flags().Bool("yes", false, "") + upgrade.Flags().Bool("dry-run", false, "") + upgrade.Flags().Bool("units-only", false, "") + upgrade.Flags().Bool("skip-units", false, "") + root.AddCommand(upgrade) + + // runAdminUpgrade reads config-dir/spinifex-dir via cmd.Root().Flags(), + // which only sees persistent flags after they've been merged in — + // normally done by Execute(). Parsing with no args merges them without + // running the command. + require.NoError(t, root.ParseFlags(nil)) + return upgrade +} + +// setUpgradeFlags sets each flag=value pair on cmd, failing the test on any +// unknown flag or parse error. +func setUpgradeFlags(t *testing.T, cmd *cobra.Command, flags map[string]string) { + t.Helper() + for name, value := range flags { + require.NoError(t, cmd.Flags().Set(name, value)) + } +} + +// seedUnitRoot writes every embedded unit unmodified into dir, so a +// Reconcile pass against dir reports every unit ActionNoop. +func seedUnitRoot(t *testing.T, dir string) { + t.Helper() + for name, content := range systemd.Units { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0644)) + } +} + +// withUnitRoot points the package's systemdUnitRoot var at dir for the +// duration of the test, restoring the original on cleanup. +func withUnitRoot(t *testing.T, dir string) { + t.Helper() + prev := systemdUnitRoot + systemdUnitRoot = dir + t.Cleanup(func() { systemdUnitRoot = prev }) +} + +func TestRunAdminUpgrade_MutuallyExclusiveFlagsExits(t *testing.T) { + cmd := newUpgradeTestCmd(t, t.TempDir(), t.TempDir()) + setUpgradeFlags(t, cmd, map[string]string{"units-only": "true", "skip-units": "true"}) + + var code int + out := captureStdout(t, func() { + code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) }) + }) + assert.Equal(t, 1, code) + assert.Empty(t, out, "no stdout expected before the mutual-exclusivity check exits") +} + +func TestRunAdminUpgrade_MissingInstallationExits(t *testing.T) { + configDir := t.TempDir() // no spinifex.toml written + cmd := newUpgradeTestCmd(t, configDir, t.TempDir()) + + code := withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) }) + assert.Equal(t, 1, code) +} + +func TestRunAdminUpgrade_DryRunNothingPending(t *testing.T) { + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640)) + unitRoot := t.TempDir() + seedUnitRoot(t, unitRoot) + withUnitRoot(t, unitRoot) + + cmd := newUpgradeTestCmd(t, configDir, t.TempDir()) + setUpgradeFlags(t, cmd, map[string]string{"dry-run": "true"}) + + var code int + out := captureStdout(t, func() { + code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) }) + }) + assert.Equal(t, -1, code, "must not exit when everything is current") + assert.Contains(t, out, "Nothing pending — config and units are current.") +} + +func TestRunAdminUpgrade_DryRunReportsBothPending(t *testing.T) { + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640)) + withUnitRoot(t, t.TempDir()) // empty: every unit reports missing + + cmd := newUpgradeTestCmd(t, configDir, t.TempDir()) + setUpgradeFlags(t, cmd, map[string]string{"dry-run": "true"}) + + var code int + out := captureStdout(t, func() { + code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) }) + }) + assert.Equal(t, -1, code) + assert.NotContains(t, out, "Nothing pending") + assert.Contains(t, out, "Pending config migrations:") + assert.Contains(t, out, "will install") +} + +func TestRunAdminUpgrade_UnitsOnlySkipsConfigReporting(t *testing.T) { + configDir := t.TempDir() + // Version 3 would normally report a pending config migration; --units-only + // must never look at it. + require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640)) + withUnitRoot(t, t.TempDir()) + + cmd := newUpgradeTestCmd(t, configDir, t.TempDir()) + setUpgradeFlags(t, cmd, map[string]string{"dry-run": "true", "units-only": "true"}) + + var code int + out := captureStdout(t, func() { + code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) }) + }) + assert.Equal(t, -1, code) + assert.NotContains(t, out, "Reading current config versions", "--units-only must skip config reporting") + assert.Contains(t, out, "Reading installed systemd unit versions") +} + +func TestRunAdminUpgrade_SkipUnitsSkipsUnitReporting(t *testing.T) { + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640)) + // Point systemdUnitRoot at a path Reconcile must never touch: if + // --skip-units leaked a call through, a nonexistent root would still + // "work" for a dry run, so the real proof is that its output never + // appears at all. + withUnitRoot(t, filepath.Join(t.TempDir(), "never-touched")) + + cmd := newUpgradeTestCmd(t, configDir, t.TempDir()) + setUpgradeFlags(t, cmd, map[string]string{"dry-run": "true", "skip-units": "true"}) + + var code int + out := captureStdout(t, func() { + code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) }) + }) + assert.Equal(t, -1, code) + assert.NotContains(t, out, "Reading installed systemd unit versions", "--skip-units must skip unit reporting") + assert.Contains(t, out, "Pending config migrations:") +} + +func TestRunAdminUpgrade_ApplyRunsConfigMigrationAndUnitReconcile(t *testing.T) { + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640)) + // Every unit already current: the real (non-dry-run) Reconcile call + // below finds nothing to change, so it never touches daemon-reload. + unitRoot := t.TempDir() + seedUnitRoot(t, unitRoot) + withUnitRoot(t, unitRoot) + + cmd := newUpgradeTestCmd(t, configDir, t.TempDir()) + setUpgradeFlags(t, cmd, map[string]string{"yes": "true"}) + + var code int + out := captureStdout(t, func() { + code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) }) + }) + assert.Equal(t, -1, code, "the apply must succeed without exiting") + assert.Contains(t, out, "Done.") + + migrated, err := os.ReadFile(filepath.Join(configDir, "spinifex.toml")) + require.NoError(t, err) + assert.Contains(t, string(migrated), `version = "4"`, "spinifex.toml must have been migrated to the latest version") +} + +func TestRunAdminUpgrade_ApplyWithNonWritableUnitRootExitsWithRootHint(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root bypasses directory permissions; this test needs an unprivileged process") + } + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640)) + + unitRoot := t.TempDir() // left empty: every unit reports missing + require.NoError(t, os.Chmod(unitRoot, 0o500)) + t.Cleanup(func() { _ = os.Chmod(unitRoot, 0o700) }) + withUnitRoot(t, unitRoot) + + cmd := newUpgradeTestCmd(t, configDir, t.TempDir()) + setUpgradeFlags(t, cmd, map[string]string{"yes": "true"}) + + var code int + var errOut string + captureStdout(t, func() { + errOut = captureStderr(t, func() { + code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) }) + }) + }) + assert.Equal(t, 1, code) + assert.Contains(t, errOut, "Unit reconciliation failed") + assert.Contains(t, errOut, "Re-run as root to apply the unit changes reported above") +} + +func TestRunAdminUpgrade_DryRunReconcileErrorExits(t *testing.T) { + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640)) + + unitRoot := t.TempDir() + // A directory in place of a unit file fails the read with something + // other than IsNotExist, so the very first (dry-run) Reconcile call + // returns an error. + var oneName string + for name := range systemd.Units { + oneName = name + break + } + require.NoError(t, os.Mkdir(filepath.Join(unitRoot, oneName), 0o755)) + withUnitRoot(t, unitRoot) + + cmd := newUpgradeTestCmd(t, configDir, t.TempDir()) + + var code int + var errOut string + captureStdout(t, func() { + errOut = captureStderr(t, func() { + code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) }) + }) + }) + assert.Equal(t, 1, code) + assert.Contains(t, errOut, "Error checking systemd units") +} diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index e710621e3..a69050e7d 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -85,7 +85,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t | Command | Flags | Description | |---------|-------|-------------| -| `spx admin upgrade` | `--yes` (apply without prompting), `--config-dir` (persistent, default: `~/spinifex/config`), `--spinifex-dir` (persistent, default: `~/spinifex`) | Reads current config versions from registry → computes pending config migrations (from→to per target) → prints version summary and pending list → prompts for confirmation unless `--yes` → runs `migrate.DefaultRegistry.RunAllConfig()` to apply migrations to config files. Intended for upgrades between Spinifex versions. Operators can skip migrations during install by setting `INSTALL_SPINIFEX_SKIP_MIGRATE=1`, then run `spx admin upgrade` manually to review and apply. After completion, services must be restarted with `sudo systemctl restart spinifex.target`. Invoked non-interactively by `scripts/setup.sh` with `--yes`. | +| `spx admin upgrade` | `--yes` (apply without prompting), `--dry-run` (report pending config and unit changes without applying), `--units-only` (reconcile systemd units only, skip config migrations), `--skip-units` (apply config migrations only, skip unit reconciliation), `--config-dir` (persistent, default: `~/spinifex/config`), `--spinifex-dir` (persistent, default: `~/spinifex`) | Reads current config versions from registry → computes pending config migrations (from→to per target) → reads installed systemd unit versions and compares each against the copy embedded in the binary (`systemd.Reconcile` with `DryRun: true`) → prints both summaries → prompts for confirmation unless `--yes` or `--dry-run` → runs `migrate.DefaultRegistry.RunAllConfig()` to apply migrations to config files → runs `systemd.Reconcile` for real to install missing units and replace stale ones (after a timestamped backup), then `systemctl daemon-reload`. Units whose installed marker version matches the embedded one but whose content differs are left untouched and reported as operator-modified — never overwritten. Writing `/etc/systemd/system` requires root; without it, drift is still reported and the command exits non-zero rather than writing partially. Never restarts services — a unit change applies to that service's next stop. Intended for upgrades between Spinifex versions, including nodes that were upgraded by swapping the `spx` binary directly rather than by re-running the installer. Operators can skip migrations during install by setting `INSTALL_SPINIFEX_SKIP_MIGRATE=1`, then run `spx admin upgrade` manually to review and apply. After completion, services must be restarted with `sudo systemctl restart spinifex.target` to pick up config changes. Invoked non-interactively by `scripts/setup.sh` with `--yes`. | ### Account Management diff --git a/docs/admin/host-lifecycle/README.md b/docs/admin/host-lifecycle/README.md index 62eb8e20f..1f71b917c 100644 --- a/docs/admin/host-lifecycle/README.md +++ b/docs/admin/host-lifecycle/README.md @@ -72,7 +72,15 @@ skipped the drain and logged `Host is not shutting down; skipping guest drain.` The gate now also reads systemd's pending job list, which does distinguish the two. -To check which behaviour a node has, look at its unit: +The fastest way to check is `spx admin upgrade --dry-run`, which reports every +unit's status against the one shipped in the running binary — including +`spinifex-shutdown.service` — without changing anything. See +[Checking for Unit Drift](/docs/admin/update#checking-for-unit-drift). A node +reported stale or missing for that unit predates this fix; reconcile it with +`sudo spx admin upgrade --units-only --yes` (this only rewrites the unit and +runs `daemon-reload` — it does not restart anything or drain guests itself). + +To check by hand instead, look at the unit directly: ```bash systemctl cat spinifex-shutdown.service | grep ExecStop diff --git a/docs/admin/update/README.md b/docs/admin/update/README.md index 01158a62e..2e30974d3 100644 --- a/docs/admin/update/README.md +++ b/docs/admin/update/README.md @@ -22,6 +22,7 @@ resources: - [Overview](#overview) - [Instructions](#instructions) - [Manual Upgrade](#manual-upgrade) +- [Checking for Unit Drift](#checking-for-unit-drift) - [Troubleshooting](#troubleshooting) --- @@ -32,6 +33,9 @@ Updating Spinifex is the same command used to install it. The installer detects For operators who want to review migrations before they are applied, a manual upgrade path is also supported. +> [!WARNING] +> **Swapping the `spx` binary alone is not an upgrade.** Systemd unit files (`KillMode`, `TimeoutStopSec`, drain ordering, and similar) are written once at install time and never re-asserted just because a new binary is in place — a node "upgraded" by replacing `/usr/local/bin/spx` directly keeps running whatever units it was first installed with, which can silently disagree with the new binary's behaviour. Re-running the installer always reinstalls units unconditionally, so it is unaffected. `spx admin upgrade` now reconciles units too, so it closes this gap for operators who update the binary by hand. See [Checking for Unit Drift](#checking-for-unit-drift). + > [!WARNING] > **Object storage is exempt when upgrading from v1.15.0 or earlier to v1.16.0.** Predastore's configuration schema and on-disk layout changed with the object storage cutover in v1.16.0, and no migration converts an installation from before it. `spx admin upgrade` will not report anything pending for `predastore.toml`, and updating the binary over such an installation leaves Predastore unable to start. Those clusters have to be re-initialised from scratch, which discards their stored objects — export anything you need first. Upgrades between v1.16.0 and later releases are unaffected. @@ -82,15 +86,15 @@ curl -fsSL https://install.mulgadc.com | INSTALL_SPINIFEX_SKIP_MIGRATE=1 bash The installer will download the new binary and reinstall systemd units, but will **not** apply any configuration migrations. -## Step 2. Review Pending Migrations +## Step 2. Review Pending Changes ```bash sudo spx admin upgrade ``` -The command prints the current version of each config file, the migrations that would be applied, and a `from → to` description for each. It then prompts for confirmation before making any changes. Answer `n` to abort without touching your config. +The command prints the current version of each config file and systemd unit, the migrations and unit replacements that would be applied, and a `from → to` description for each. It then prompts for confirmation before making any changes. Answer `n` to abort without touching config or units. Use `--dry-run` instead of the prompt to only report and never apply. -## Step 3. Apply Migrations +## Step 3. Apply Changes When you are ready, answer `y` at the prompt, or re-run with `--yes` to apply non-interactively: @@ -98,9 +102,11 @@ When you are ready, answer `y` at the prompt, or re-run with `--yes` to apply no sudo spx admin upgrade --yes ``` +This requires root: config files are typically owned by their service user, but writing `/etc/systemd/system` needs root. Run the whole command with `sudo`, not just parts of it. + ## Step 4. Restart Services -Migrations modify config files on disk but do not restart running services. Apply the new config with: +Migrations modify config files on disk but do not restart running services, and unit reconciliation deliberately never restarts anything either — it writes the unit and runs `systemctl daemon-reload`, so the fix applies to the *next* stop of that service without disturbing a running guest. Apply a config change with: ```bash sudo systemctl restart spinifex.target @@ -108,6 +114,25 @@ sudo systemctl restart spinifex.target A restart preserves any running guests — they are not rebooted, and storage returns within seconds. See [Host and Guest Lifecycle](/docs/admin/host-lifecycle) for the full contract. +## Checking for Unit Drift + +`spx admin upgrade --dry-run` reports whether a node's installed systemd units match the ones shipped in the running `spx` binary, without prompting or changing anything — the fastest way to answer "are this node's units current?" in a support conversation: + +```bash +spx admin upgrade --dry-run +``` + +Each unit is reported as one of: + +- **up to date** — installed content matches the embedded copy. +- **stale, will replace** — the installed marker version is older than the embedded one (or has no marker at all, which is version 0 — every node installed before units were versioned). +- **missing → will install** — no unit installed under that name. +- **operator-modified, not touched** — the installed marker version matches, but the content differs. `spx admin upgrade` never overwrites this case; review it with `systemctl cat ` and reconcile by hand with `systemctl edit`. + +Run `sudo spx admin upgrade --units-only --yes` to reconcile units without touching config, or `--skip-units` to do the reverse. A replaced unit is backed up alongside the original as `.pre-reconcile-to.` before being overwritten. + +This covers the 16 core units installed by `install_systemd()` (`spinifex-*.service`, `spinifex.target`, `spinifex.slice` and friends). Firstboot, banner, bridge and getty units written by the installer, and the auxiliary units Ansible manages (`wattle-wan-veth-persist`, `wattle-mgmt-bridge-persist`, `obs-agent`), are separate lifecycles with no overlap in unit names and are out of scope for this reconciler. + ## Troubleshooting ### No Pending Config Migrations @@ -174,3 +199,22 @@ Or `describe-images --image-ids` reports `InvalidAMIID.NotFound` for an AMI that 1. **A node running a replaced binary**, per the previous entry. Suspect this first if the failure is intermittent — the same command succeeding on some attempts and failing on others is characteristic. 2. **AMI metadata predating the EBS-provider decoupling**, per the warning at the top of this page. This is consistent rather than intermittent, and is resolved by re-importing the AMI. + +### Root Privileges Required to Write Systemd Units + +``` +root privileges required to write systemd units (writing to /etc/systemd/system): ... +Re-run as root to apply the unit changes reported above: sudo spx admin upgrade --units-only --yes +``` + +`spx admin upgrade` computes and prints unit drift without needing root, but writing `/etc/systemd/system` does. Nothing is written when this happens — re-run the full command with `sudo`. + +### Operator-Modified Unit Reported, Not Replaced + +A unit whose installed marker version matches the embedded one but whose content differs is never overwritten — this is a deliberate safety property so a hand-tuned unit does not get silently reverted. Compare it against the shipped copy and decide whether to keep, discard or merge the local change: + +```bash +systemctl cat +``` + +If you want the shipped version, remove the local override and re-run `spx admin upgrade`; systemd falls back to the packaged unit and it reconciles as up to date. diff --git a/spinifex/systemd/generate.go b/spinifex/systemd/generate.go new file mode 100644 index 000000000..87eacc598 --- /dev/null +++ b/spinifex/systemd/generate.go @@ -0,0 +1,73 @@ +//go:build ignore + +// Command generate regenerates units_gen.go from build/systemd, the single +// source of truth for spinifex unit files. go:embed cannot reach outside +// this package directory, so the contents are baked in as a Go map instead. +// +//go:generate go run generate.go +package main + +import ( + "fmt" + "go/format" + "os" + "path/filepath" + "sort" + "strings" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "generate:", err) + os.Exit(1) + } +} + +func run() error { + selfDir, err := os.Getwd() + if err != nil { + return err + } + unitsDir := filepath.Join(selfDir, "..", "..", "build", "systemd") + + ents, err := os.ReadDir(unitsDir) + if err != nil { + return fmt.Errorf("read %s: %w", unitsDir, err) + } + + var names []string + for _, e := range ents { + n := e.Name() + for _, suffix := range []string{".service", ".slice", ".target", ".timer"} { + if strings.HasSuffix(n, suffix) { + names = append(names, n) + break + } + } + } + sort.Strings(names) + + var b strings.Builder + b.WriteString("// Code generated by `go generate ./...` from build/systemd; DO NOT EDIT.\n\n") + b.WriteString("package systemd\n\n") + b.WriteString("// Units maps each core unit file name to its embedded contents, sourced from\n") + b.WriteString("// build/systemd at generation time. Regenerate with `go generate ./...` after\n") + b.WriteString("// editing a unit; units_invariants_test.go fails preflight if this drifts.\n") + b.WriteString("var Units = map[string]string{\n") + for _, name := range names { + content, err := os.ReadFile(filepath.Join(unitsDir, name)) + if err != nil { + return fmt.Errorf("read %s: %w", name, err) + } + fmt.Fprintf(&b, "\t%q: %q,\n", name, string(content)) + } + b.WriteString("}\n") + + formatted, err := format.Source([]byte(b.String())) + if err != nil { + return fmt.Errorf("format generated source: %w", err) + } + + outPath := filepath.Join(selfDir, "units_gen.go") + return os.WriteFile(outPath, formatted, 0o644) +} diff --git a/spinifex/systemd/reconcile.go b/spinifex/systemd/reconcile.go new file mode 100644 index 000000000..83f2aa36c --- /dev/null +++ b/spinifex/systemd/reconcile.go @@ -0,0 +1,216 @@ +package systemd + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// Action describes what Reconcile did, or would do, for one unit. +type Action string + +const ( + ActionNoop Action = "noop" // installed content matches the embedded copy + ActionInstall Action = "install" // unit missing on disk + ActionReplace Action = "replace" // installed marker version is older than embedded + ActionConflict Action = "conflict" // marker version matches but content differs — operator-modified +) + +// UnitStatus reports the reconcile decision for a single unit. +type UnitStatus struct { + Name string + Action Action + InstalledVersion int + EmbeddedVersion int + Backup string // set once a replace has written a backup + Applied bool // true once Install/Replace has actually been written +} + +// Options controls how Reconcile behaves. +type Options struct { + // DryRun computes and returns the full status without writing anything. + DryRun bool +} + +// Result is the outcome of a Reconcile call. +type Result struct { + Statuses []UnitStatus + ReloadRan bool +} + +// HasChanges reports whether any unit is missing or stale — i.e. whether +// applying (DryRun: false) would write anything to disk. +func (r Result) HasChanges() bool { + for _, s := range r.Statuses { + if s.Action == ActionInstall || s.Action == ActionReplace { + return true + } + } + return false +} + +// HasConflicts reports whether any unit is at the current version but was +// modified on disk — drift Reconcile deliberately refuses to overwrite. +func (r Result) HasConflicts() bool { + for _, s := range r.Statuses { + if s.Action == ActionConflict { + return true + } + } + return false +} + +// ErrRootRequired is returned when Reconcile has pending changes but root +// cannot write to the target directory. The Result is still populated and +// safe to print — no file was written. +var ErrRootRequired = errors.New("root privileges required to write systemd units") + +var unitVersionRe = regexp.MustCompile(`^#\s*spinifex-unit-version:\s*(\d+)\s*$`) + +// unitVersion reads the marker on a unit's first line. An absent or +// unparsable marker is version 0, matching every node installed before +// units were versioned. +func unitVersion(content string) int { + line, _, _ := strings.Cut(content, "\n") + m := unitVersionRe.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil { + return 0 + } + v, err := strconv.Atoi(m[1]) + if err != nil { + return 0 + } + return v +} + +// systemctlDaemonReload is overridable in tests so Reconcile never shells +// out to a real systemd instance. +var systemctlDaemonReload = func() error { + return exec.Command("systemctl", "daemon-reload").Run() +} + +func sortedUnitNames() []string { + names := make([]string, 0, len(Units)) + for n := range Units { + names = append(names, n) + } + sort.Strings(names) + return names +} + +// Reconcile compares each embedded unit (Units, generated from +// build/systemd) against the copy installed under root. Missing or +// version-stale units are replaced after a timestamped backup; units whose +// marker version matches but whose content differs are left untouched and +// reported as operator-modified. Never restarts anything — only writes +// files and, when it wrote at least one, runs systemctl daemon-reload. +func Reconcile(root string, opts Options) (Result, error) { + var result Result + + for _, name := range sortedUnitNames() { + embedded := Units[name] + embeddedVersion := unitVersion(embedded) + path := filepath.Join(root, name) + + installed, err := os.ReadFile(path) + if os.IsNotExist(err) { + result.Statuses = append(result.Statuses, UnitStatus{ + Name: name, Action: ActionInstall, EmbeddedVersion: embeddedVersion, + }) + continue + } + if err != nil { + return result, fmt.Errorf("read %s: %w", path, err) + } + + installedVersion := unitVersion(string(installed)) + status := UnitStatus{Name: name, InstalledVersion: installedVersion, EmbeddedVersion: embeddedVersion} + switch { + case installedVersion < embeddedVersion: + status.Action = ActionReplace + case string(installed) == embedded: + status.Action = ActionNoop + default: + status.Action = ActionConflict + } + result.Statuses = append(result.Statuses, status) + } + + if opts.DryRun || !result.HasChanges() { + return result, nil + } + + // Check writability up front so a permission problem is reported before + // any unit is touched, rather than after replacing some and failing on + // the next — the failure mode this exists to avoid. + if err := checkWritable(root); err != nil { + return result, fmt.Errorf("%w (writing to %s): %w", ErrRootRequired, root, err) + } + + for i := range result.Statuses { + s := &result.Statuses[i] + if s.Action != ActionInstall && s.Action != ActionReplace { + continue + } + path := filepath.Join(root, s.Name) + if s.Action == ActionReplace { + backup, err := backupUnit(path, s.InstalledVersion, s.EmbeddedVersion) + if err != nil { + return result, fmt.Errorf("backup %s: %w", path, err) + } + s.Backup = backup + } + if err := os.WriteFile(path, []byte(Units[s.Name]), 0o644); err != nil { //nolint:gosec // systemd unit files are intentionally world-readable, matching setup.sh's install -m 0644 + return result, fmt.Errorf("write %s: %w", path, err) + } + s.Applied = true + } + + if err := systemctlDaemonReload(); err != nil { + return result, fmt.Errorf("systemctl daemon-reload: %w", err) + } + result.ReloadRan = true + + return result, nil +} + +// checkWritable reports whether root is writable by the current process, +// via a throwaway probe file rather than an EUID check — Reconcile is +// pointed at a temp directory in tests, not necessarily run as root there. +func checkWritable(root string) error { + f, err := os.CreateTemp(root, ".spx-systemd-write-check-*") + if err != nil { + return err + } + name := f.Name() + if cerr := f.Close(); cerr != nil { + return cerr + } + return os.Remove(name) +} + +// backupUnit mirrors migrate.BackupConfig's timestamped-backup convention, +// scoped to unit reconciliation instead of config migration. +func backupUnit(path string, fromVersion, toVersion int) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + info, err := os.Stat(path) + if err != nil { + return "", err + } + + backupPath := fmt.Sprintf("%s.pre-reconcile-%dto%d.%d", path, fromVersion, toVersion, time.Now().Unix()) + if err := os.WriteFile(backupPath, data, info.Mode()); err != nil { + return "", err + } + return backupPath, nil +} diff --git a/spinifex/systemd/reconcile_test.go b/spinifex/systemd/reconcile_test.go new file mode 100644 index 000000000..6963a085c --- /dev/null +++ b/spinifex/systemd/reconcile_test.go @@ -0,0 +1,523 @@ +package systemd + +import ( + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// realUnit returns the embedded content for a real production unit, so +// reconcile tests exercise the actual shipped units rather than fixtures +// that could drift from what Reconcile runs against in production. +func realUnit(t *testing.T, name string) string { + t.Helper() + content, ok := Units[name] + if !ok { + t.Fatalf("no embedded unit %q", name) + } + return content +} + +// stripFirstLine drops a unit's version-marker line, simulating an +// installed copy from before units were versioned. +func stripFirstLine(content string) string { + _, rest, _ := strings.Cut(content, "\n") + return rest +} + +// setVersion rewrites a unit's version-marker line to v, keeping the body +// unchanged — simulating an installed copy stamped at an older version. +func setVersion(content string, v int) string { + return "# spinifex-unit-version: " + strconv.Itoa(v) + "\n" + stripFirstLine(content) +} + +func findStatus(t *testing.T, result Result, name string) UnitStatus { + t.Helper() + for _, s := range result.Statuses { + if s.Name == name { + return s + } + } + t.Fatalf("no status for %s", name) + return UnitStatus{} +} + +func stubDaemonReload(t *testing.T) func() int { + t.Helper() + calls := 0 + prev := systemctlDaemonReload + systemctlDaemonReload = func() error { + calls++ + return nil + } + t.Cleanup(func() { systemctlDaemonReload = prev }) + return func() int { return calls } +} + +func TestReconcile_MissingUnitIsInstalled(t *testing.T) { + root := t.TempDir() + reloadCalls := stubDaemonReload(t) + const name = "spinifex-ui.service" + embedded := realUnit(t, name) + + result, err := Reconcile(root, Options{}) + if err != nil { + t.Fatalf("Reconcile: %v", err) + } + + st := findStatus(t, result, name) + if st.Action != ActionInstall { + t.Errorf("Action = %s, want %s", st.Action, ActionInstall) + } + if !st.Applied { + t.Error("Applied = false, want true") + } + if st.Backup != "" { + t.Errorf("Backup = %q, want empty — nothing existed to back up", st.Backup) + } + + got, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatalf("read installed unit: %v", err) + } + if string(got) != embedded { + t.Error("installed content does not match embedded unit") + } + if reloadCalls() != 1 { + t.Errorf("daemon-reload calls = %d, want 1", reloadCalls()) + } +} + +func TestReconcile_OlderVersionIsReplacedWithBackup(t *testing.T) { + root := t.TempDir() + stubDaemonReload(t) + const name = "spinifex-daemon.service" + embedded := realUnit(t, name) + stale := setVersion(embedded, 0) + writeFile(t, root, name, stale) + + result, err := Reconcile(root, Options{}) + if err != nil { + t.Fatalf("Reconcile: %v", err) + } + + st := findStatus(t, result, name) + if st.Action != ActionReplace { + t.Errorf("Action = %s, want %s", st.Action, ActionReplace) + } + if st.InstalledVersion != 0 { + t.Errorf("InstalledVersion = %d, want 0", st.InstalledVersion) + } + if st.Backup == "" { + t.Fatal("Backup path empty, want a timestamped backup") + } + + backupContent, err := os.ReadFile(st.Backup) + if err != nil { + t.Fatalf("read backup: %v", err) + } + if string(backupContent) != stale { + t.Error("backup content does not match the pre-replace installed content") + } + + got, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatalf("read replaced unit: %v", err) + } + if string(got) != embedded { + t.Error("replaced content does not match embedded unit") + } +} + +func TestReconcile_NoMarkerIsTreatedAsVersionZeroAndReplaced(t *testing.T) { + root := t.TempDir() + stubDaemonReload(t) + const name = "spinifex-viperblock.service" + embedded := realUnit(t, name) + noMarker := stripFirstLine(embedded) + writeFile(t, root, name, noMarker) + + result, err := Reconcile(root, Options{}) + if err != nil { + t.Fatalf("Reconcile: %v", err) + } + + st := findStatus(t, result, name) + if st.Action != ActionReplace { + t.Errorf("Action = %s, want %s", st.Action, ActionReplace) + } + if st.InstalledVersion != 0 { + t.Errorf("InstalledVersion = %d, want 0 (absent marker)", st.InstalledVersion) + } + if st.Backup == "" { + t.Error("Backup path empty, want a timestamped backup") + } +} + +func TestReconcile_IdenticalContentIsNoop(t *testing.T) { + root := t.TempDir() + reloadCalls := stubDaemonReload(t) + // Seed every embedded unit as already-current, so the only thing left + // for Reconcile to decide on is the one unit under test — otherwise the + // other 15 real units would still be "missing" and trigger a reload. + seedAllUnits(t, root) + const name = "spinifex-northstar.service" + + result, err := Reconcile(root, Options{}) + if err != nil { + t.Fatalf("Reconcile: %v", err) + } + + st := findStatus(t, result, name) + if st.Action != ActionNoop { + t.Errorf("Action = %s, want %s", st.Action, ActionNoop) + } + if st.Applied { + t.Error("Applied = true, want false — nothing should change") + } + + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.Contains(e.Name(), "pre-reconcile") { + t.Errorf("unexpected backup file for a no-op unit: %s", e.Name()) + } + } + if reloadCalls() != 0 { + t.Errorf("daemon-reload calls = %d, want 0 — nothing changed", reloadCalls()) + } +} + +func TestReconcile_OperatorModifiedAtCurrentVersionIsLeftAlone(t *testing.T) { + root := t.TempDir() + stubDaemonReload(t) + const name = "spinifex-vpcd.service" + embedded := realUnit(t, name) + modified := embedded + "# operator hand-edit, not from the shipped unit\n" + writeFile(t, root, name, modified) + + result, err := Reconcile(root, Options{}) + if err != nil { + t.Fatalf("Reconcile: %v", err) + } + + st := findStatus(t, result, name) + if st.Action != ActionConflict { + t.Errorf("Action = %s, want %s", st.Action, ActionConflict) + } + if st.Applied { + t.Error("Applied = true, want false — a same-version conflict must never be overwritten") + } + if st.Backup != "" { + t.Errorf("Backup = %q, want empty — nothing was written", st.Backup) + } + + got, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatal(err) + } + if string(got) != modified { + t.Error("operator-modified unit must be untouched byte-for-byte") + } + if !result.HasConflicts() { + t.Error("HasConflicts() = false, want true") + } +} + +func TestReconcile_DryRunChangesNothing(t *testing.T) { + root := t.TempDir() + reloadCalls := stubDaemonReload(t) + const staleName = "spinifex-awsgw.service" + stale := setVersion(realUnit(t, staleName), 0) + writeFile(t, root, staleName, stale) + + result, err := Reconcile(root, Options{DryRun: true}) + if err != nil { + t.Fatalf("Reconcile: %v", err) + } + if !result.HasChanges() { + t.Fatal("HasChanges() = false, want true — a stale unit and missing units are pending") + } + + for _, s := range result.Statuses { + if s.Applied { + t.Errorf("%s: Applied = true during a dry run", s.Name) + } + } + + got, err := os.ReadFile(filepath.Join(root, staleName)) + if err != nil { + t.Fatal(err) + } + if string(got) != stale { + t.Error("dry run must not modify the stale unit on disk") + } + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Errorf("dry run wrote %d files to root, want exactly the 1 pre-seeded file", len(entries)) + } + if reloadCalls() != 0 { + t.Errorf("daemon-reload calls = %d, want 0 during a dry run", reloadCalls()) + } +} + +func TestReconcile_NonWritableRootReportsWithoutPartialWrites(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root bypasses directory permissions; this test needs an unprivileged process") + } + + root := t.TempDir() + if err := os.Chmod(root, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(root, 0o700) }) + + result, err := Reconcile(root, Options{}) + if err == nil { + t.Fatal("Reconcile: want an error on a non-writable root, got nil") + } + if !errors.Is(err, ErrRootRequired) { + t.Errorf("err = %v, want it to wrap ErrRootRequired", err) + } + if len(result.Statuses) == 0 { + t.Error("Statuses empty — drift must still be reported even when it cannot be applied") + } + for _, s := range result.Statuses { + if s.Applied { + t.Errorf("%s: Applied = true despite a non-writable root", s.Name) + } + } +} + +func TestReconcile_ReloadRunsOnceThenSettles(t *testing.T) { + root := t.TempDir() + reloadCalls := stubDaemonReload(t) + + if _, err := Reconcile(root, Options{}); err != nil { + t.Fatalf("first Reconcile: %v", err) + } + if reloadCalls() != 1 { + t.Fatalf("daemon-reload calls after first apply = %d, want 1", reloadCalls()) + } + + result, err := Reconcile(root, Options{}) + if err != nil { + t.Fatalf("second Reconcile: %v", err) + } + if result.HasChanges() { + t.Error("HasChanges() = true on a second pass over an already-reconciled root") + } + if reloadCalls() != 1 { + t.Errorf("daemon-reload calls after idempotent second apply = %d, want still 1", reloadCalls()) + } +} + +func writeFile(t *testing.T, root, name, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(root, name), []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// seedAllUnits pre-populates root with every embedded unit at its current +// content, so a Reconcile pass has nothing pending except what the test +// deliberately alters afterward. +func seedAllUnits(t *testing.T, root string) { + t.Helper() + for name, content := range Units { + writeFile(t, root, name, content) + } +} + +// withFakeUnit temporarily adds name to the package-level Units map, so a +// test can control the exact path Reconcile computes (including a name with +// a path separator, to point at a directory that does or doesn't exist) +// without touching the generated production unit set. +func withFakeUnit(t *testing.T, name, content string) { + t.Helper() + prev, existed := Units[name] + Units[name] = content + t.Cleanup(func() { + if existed { + Units[name] = prev + } else { + delete(Units, name) + } + }) +} + +func TestHasConflicts_NoConflictsIsFalse(t *testing.T) { + result := Result{Statuses: []UnitStatus{ + {Name: "a.service", Action: ActionNoop}, + {Name: "b.service", Action: ActionInstall}, + }} + if result.HasConflicts() { + t.Error("HasConflicts() = true, want false — no unit is ActionConflict") + } +} + +func TestUnitVersion_OverflowingMarkerIsVersionZero(t *testing.T) { + // A marker value too large for strconv.Atoi must be treated the same as + // no marker at all, not propagate the parse error. + content := "# spinifex-unit-version: 99999999999999999999999999\nExecStart=/usr/bin/true\n" + if v := unitVersion(content); v != 0 { + t.Errorf("unitVersion() = %d, want 0 for an overflowing marker value", v) + } +} + +func TestUnitVersion_MarkerNotOnFirstLineIsIgnored(t *testing.T) { + content := "[Unit]\n# spinifex-unit-version: 5\nExecStart=/usr/bin/true\n" + if v := unitVersion(content); v != 0 { + t.Errorf("unitVersion() = %d, want 0 — a marker must be on the first line to count", v) + } +} + +func TestSystemctlDaemonReload_MissingBinaryReturnsError(t *testing.T) { + // Exercises the real (non-stubbed) production implementation: an empty + // PATH means systemctl cannot be found, so the command fails cleanly + // instead of touching a real systemd instance. + t.Setenv("PATH", t.TempDir()) + if err := systemctlDaemonReload(); err == nil { + t.Fatal("systemctlDaemonReload(): want an error when systemctl is not on PATH, got nil") + } +} + +func TestReconcile_UnreadableInstalledUnitReturnsError(t *testing.T) { + root := t.TempDir() + stubDaemonReload(t) + const name = "spinifex-ui.service" + // A directory in place of the unit file fails the read with something + // other than IsNotExist, exercising the general read-error path + // distinct from "missing unit". + if err := os.Mkdir(filepath.Join(root, name), 0o755); err != nil { + t.Fatal(err) + } + + _, err := Reconcile(root, Options{}) + if err == nil { + t.Fatal("Reconcile: want an error when an installed unit path is unreadable, got nil") + } + if !strings.Contains(err.Error(), name) { + t.Errorf("err = %v, want it to name the unreadable unit %s", err, name) + } +} + +func TestReconcile_WriteFailureIsSurfaced(t *testing.T) { + root := t.TempDir() + stubDaemonReload(t) + + // A unit whose path has a parent directory that does not exist: the + // read reports NotExist (classified Install, like any missing unit), + // but the later write into that missing directory fails. Deterministic + // and root-safe, unlike a permission-based probe. + const name = "missing-parent/fake-install.service" + withFakeUnit(t, name, "# spinifex-unit-version: 1\nExecStart=/usr/bin/true\n") + + _, err := Reconcile(root, Options{}) + if err == nil { + t.Fatal("Reconcile: want an error when the write's parent directory is missing, got nil") + } + if !strings.Contains(err.Error(), "write") { + t.Errorf("err = %v, want it to mention the write step", err) + } +} + +func TestReconcile_BackupFailureIsSurfaced(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root bypasses directory permissions; this test needs an unprivileged process") + } + root := t.TempDir() + stubDaemonReload(t) + + const name = "nested/fake-backup.service" + nested := filepath.Join(root, "nested") + if err := os.Mkdir(nested, 0o755); err != nil { + t.Fatal(err) + } + embedded := "# spinifex-unit-version: 2\nExecStart=/usr/bin/true\n" + withFakeUnit(t, name, embedded) + stale := setVersion(embedded, 0) + writeFile(t, root, name, stale) + + // Lock the nested directory only after seeding it: the initial read + // (classification) still succeeds, but backupUnit's write of the + // backup copy into the same directory fails. + if err := os.Chmod(nested, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(nested, 0o755) }) + + _, err := Reconcile(root, Options{}) + if err == nil { + t.Fatal("Reconcile: want an error when the backup write fails, got nil") + } + if !strings.Contains(err.Error(), "backup") { + t.Errorf("err = %v, want it to mention the backup step", err) + } +} + +func TestReconcile_DaemonReloadFailureIsSurfaced(t *testing.T) { + root := t.TempDir() + prevReload := systemctlDaemonReload + wantErr := errors.New("boom: reload failed") + systemctlDaemonReload = func() error { return wantErr } + t.Cleanup(func() { systemctlDaemonReload = prevReload }) + + result, err := Reconcile(root, Options{}) + if err == nil { + t.Fatal("Reconcile: want an error when daemon-reload fails, got nil") + } + if !errors.Is(err, wantErr) { + t.Errorf("err = %v, want it to wrap %v", err, wantErr) + } + if result.ReloadRan { + t.Error("ReloadRan = true despite daemon-reload failing") + } +} + +func TestReconcile_DowngradeMarkerIsConflictNotClobbered(t *testing.T) { + root := t.TempDir() + stubDaemonReload(t) + const name = "spinifex-vpcd.service" + embedded := realUnit(t, name) + // Simulate a downgrade: the installed unit is stamped at a version + // newer than what this binary embeds. Reconcile must never clobber a + // newer file with an older one — ActionReplace only ever moves forward. + future := setVersion(embedded, unitVersion(embedded)+1) + "# from a future release\n" + writeFile(t, root, name, future) + + result, err := Reconcile(root, Options{}) + if err != nil { + t.Fatalf("Reconcile: %v", err) + } + + st := findStatus(t, result, name) + if st.Action != ActionConflict { + t.Errorf("Action = %s, want %s — a newer installed marker must never be replaced", st.Action, ActionConflict) + } + if st.Applied { + t.Error("Applied = true, want false — a downgrade must not clobber a newer unit") + } + got, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatal(err) + } + if string(got) != future { + t.Error("downgrade case: installed unit content must be untouched") + } +} + +func TestBackupUnit_UnreadableSourceReturnsError(t *testing.T) { + _, err := backupUnit(filepath.Join(t.TempDir(), "does-not-exist.service"), 0, 1) + if err == nil { + t.Fatal("backupUnit: want an error for a nonexistent source path, got nil") + } +} diff --git a/spinifex/systemd/units_gen.go b/spinifex/systemd/units_gen.go new file mode 100644 index 000000000..04b2221c2 --- /dev/null +++ b/spinifex/systemd/units_gen.go @@ -0,0 +1,26 @@ +// Code generated by `go generate ./...` from build/systemd; DO NOT EDIT. + +package systemd + +// Units maps each core unit file name to its embedded contents, sourced from +// build/systemd at generation time. Regenerate with `go generate ./...` after +// editing a unit; units_invariants_test.go fails preflight if this drifts. +var Units = map[string]string{ + "regenerate-ssh-host-keys.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Regenerate SSH host keys\nConditionPathExists=!/etc/ssh/ssh_host_ed25519_key\nBefore=ssh.service\n\n[Service]\nType=oneshot\nExecStart=/usr/bin/ssh-keygen -A\nRemainAfterExit=yes\n# No ExecStop: the keygen process has already exited by the time this unit\n# is stopped, so there is nothing to signal.\nTimeoutStopSec=5\n\n[Install]\nWantedBy=multi-user.target\n", + "spinifex-awsgw.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex AWS Gateway\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-gw\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service awsgw start\nRestart=on-failure\nRestartSec=5\n# No SIGTERM handler in this process: it exits immediately on signal, so this\n# only bounds an unexpectedly wedged process before SIGKILL.\nTimeoutStopSec=10\n\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/\nEnvironment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/awsgw /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-daemon.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Daemon (VM Orchestration)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service\n# Wants (not Requires): stay bootable when openvswitch-ipsec is absent\nWants=openvswitch-ipsec.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see\nEnvironmentFile=-/etc/spinifex/host.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-daemon\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex start\nRestart=on-failure\nRestartSec=5\n# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)\nKillMode=process\nTimeoutStopSec=15\n# RG-4: infra tier — reaped only after every guest\nOOMScoreAdjust=-500\n# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead\nMemoryHigh=75%\nMemoryMax=85%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\nEnvironment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/\n# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,\n# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.\nAmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE\nProtectSystem=full\nPrivateDevices=no\nDevicePolicy=closed\nDeviceAllow=/dev/kvm rw\nDeviceAllow=/dev/net/tun rw\nDeviceAllow=char-vfio rw\nDeviceAllow=/dev/vfio/vfio rw\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\n# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which\n# ProtectKernelTunables=yes would otherwise keep read-only.\nReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n# LimitMEMLOCK=infinity: VFIO pins all guest RAM\nLimitMEMLOCK=infinity\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-guests.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex guest VM tier\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-guests -> spinifex), sibling of\n# spinifex-system. Parent for the per-instance spinifex-instance-.slice\n# leaves (0002 RG-7). Inherits the spinifex.slice ceiling; no extra limit here.\n", + "spinifex-nats-watchdog.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS JetStream liveness watchdog\nPartOf=spinifex.target\nAfter=spinifex-nats.service\n\n[Service]\nType=oneshot\n# Runs as root: the probes are a local curl to the NATS monitoring port and a\n# canary KV round-trip via spx, and the remedy is `systemctl try-restart\n# spinifex-nats` when JetStream stops accepting writes while the process stays\n# alive (which Restart=on-failure cannot catch).\nExecStart=/var/lib/spinifex/nats-js-watchdog.sh\n# Sized for the probe loop's own worst case (3 samples x (5s curl + 5s canary)\n# + 2 gaps x 2s = 34s): a stop mid-cycle lets the script finish rather than\n# cutting it off. Only the failing path runs that long; a healthy node exits on\n# the first sample.\nTimeoutStopSec=45\n\n# Hardening (kept compatible with talking to PID1 over /run/systemd).\nNoNewPrivileges=yes\nProtectHome=yes\nPrivateTmp=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectControlGroups=yes\nRestrictRealtime=yes\nLockPersonality=yes\nRestrictSUIDSGID=yes\n", + "spinifex-nats-watchdog.timer": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS JetStream liveness watchdog timer\nPartOf=spinifex.target\n\n[Timer]\n# Delay the first probe so NATS startup + JetStream cluster quorum settle (a\n# forming cluster reports JS not-current transiently, which is not a fault).\nOnBootSec=60s\nOnUnitActiveSec=30s\nAccuracySec=5s\n\n[Install]\nWantedBy=timers.target\n", + "spinifex-nats.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS Message Bus\nPartOf=spinifex.target\nBefore=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-nats\nGroup=spinifex\nExecStart=/usr/local/bin/spx service nats start\nRestart=on-failure\nRestartSec=5\n# nats-server installs its own SIGTERM handler (JetStream store shutdown, raft\n# stepdown, client drain) with no internal deadline; every dependent service\n# stops before nats does, so size this for nats's own graceful-stop needs.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/nats /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/nats /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-northstar.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Northstar (Authoritative DNS)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-predastore.service\nWants=spinifex-predastore.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-northstar\nGroup=spinifex\nExecStart=/usr/local/bin/spx service northstar start\nRestart=on-failure\nRestartSec=5\n# server.Shutdown() is called with an internal 5s context; margin covers the\n# NATS unsubscribe/close and process exit that follow it.\nTimeoutStopSec=10\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/northstar/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/northstar /etc/spinifex/spinifex.toml /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/northstar /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\n# RG-9: exactly CAP_NET_BIND_SERVICE so the unprivileged user binds :53 (no root).\nAmbientCapabilities=CAP_NET_BIND_SERVICE\nCapabilityBoundingSet=CAP_NET_BIND_SERVICE\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-predastore.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Predastore (S3-compatible Object Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# Guest block I/O arrives with no caller trace; sample those roots at 5% so\n# they don't drown request traces. Request-linked spans are always kept.\nEnvironment=MULGA_ROOT_TRACE_RATIO=0.05\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-storage\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service predastore start\nRestart=on-failure\nRestartSec=5\n# Matches viperblock's drain budget: viperblock's WAL flush writes into\n# predastore, and After= already stops viperblock first, but this is the\n# backstop if that ordering is ever bypassed (direct stop, future edit).\nTimeoutStopSec=30\nOOMScoreAdjust=-500\n# Legitimately the most memory-hungry infra service (S3 buffering/caching);\n# High/Max stay well above idle so normal load isn't throttled, while still\n# bounding a leak short of the slice-wide cap.\nMemoryHigh=40%\nMemoryMax=55%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml\nEnvironment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/\nEnvironment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-qmp-collector.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex QMP Telemetry Collector (guest metrics)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\n# spinifex-daemon user: the collector reads the daemon's telemetry QMP\n# sockets in /run/spinifex and the daemon-owned QEMU /proc entries.\nUser=spinifex-daemon\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service qmp-collector start\nRestart=on-failure\nRestartSec=5\n# Poll loop selects on ctx.Done() directly and closes NATS locally; shutdown\n# is near-instant, so this only bounds an unexpectedly wedged process.\nTimeoutStopSec=10\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\n# Same runtime dir as the daemon: telemetry sockets + metadata live there\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: read-mostly telemetry tier — no capabilities, no device access\nProtectSystem=full\nPrivateDevices=yes\nNoNewPrivileges=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/spinifex /run/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\n# ProtectProc must stay default: hidepid=invisible hides the QEMU vCPU threads\n# (dumpable=0) this collector must read from /proc//stat, even same-uid.\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=512\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-shutdown.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex graceful guest drain\nPartOf=spinifex.target\n# After= all services so it stops FIRST on target/host shutdown: the drain runs\n# while nats/predastore/viperblock/daemon are still up (QMP, unmount, WAL flush).\nAfter=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/true\n# ExecStop fires on ANY target stop, including the stop half of `systemctl\n# restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless\n# of whether the host is actually going down. --unless-restarting drains on a\n# real shutdown/reboot and on a plain `systemctl stop spinifex.target`; it\n# skips only when a restart of the stack is already queued, so QEMU/nbdkit\n# survive (KillMode=process) and the daemon reconnects on the way back up.\nExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --unless-restarting\nTimeoutStopSec=180\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-system.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-system -> spinifex), inherits the\n# parent ceiling, no extra limit. Holds the 7 control/infra services. Guest QEMU\n# moves to the sibling spinifex-guests.slice (per-instance placement lands in\n# 0002 RG-7); until then the daemon-forked guests stay under spinifex-daemon.service.\n", + "spinifex-ui.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Web UI\nPartOf=spinifex.target\nAfter=spinifex-awsgw.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-ui\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex-ui start\nRestart=on-failure\nRestartSec=5\n# server.Shutdown() uses an internal 30s context (WriteTimeout=120s covers\n# long streaming proxy responses, e.g. console passthrough); add margin above it.\nTimeoutStopSec=40\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key\nReadWritePaths=/var/lib/spinifex/spinifex-ui /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-viperblock.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Viperblock (Block Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-predastore.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-viperblock\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service viperblock start\nRestart=on-failure\nRestartSec=5\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=15%\nMemoryMax=25%\n# KillMode=process: nbdkit still serving a guest survives a restart here, the\n# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts\n# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.\nKillMode=process\n# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)\n# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must\n# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.\nTimeoutStopSec=150\nEnvironmentFile=/etc/spinifex/systemd.env\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /run/spinifex/nbd\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-vpcd.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex VPC Daemon (OVN Networking)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service\nWants=spinifex-nats.service ovn-controller.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-vpcd\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service vpcd start\nRestart=on-failure\nRestartSec=5\n# Waits for an in-flight OVN drift-reconcile pass to return before exiting;\n# no sub-timeout bounds that call today, so this covers a normal pass.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling: bounds a leak here instead of the slice-wide cap.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/\n\n# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no\n# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem\n# sandbox is restored.\n#\n# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and\n# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR\n# update that retires the last escalation path.\nAmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nCapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nNoNewPrivileges=no\nProtectSystem=full\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\n# AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex resource-bounded slice (host ceiling)\nBefore=slices.target\n\n[Slice]\n# RG-1: no CPUWeight — slice roots are siblings; capping below the default would\n# let user.slice out-prioritize the VM fleet. Operator CPU priority lives on\n# system.slice (system.slice.d/spinifex-reserve.conf).\n# RG-1/RG-5: throttle+reclaim at High, hard wall at Max (% of physical RAM).\nMemoryHigh=80%\nMemoryMax=90%\n# RG-5: spill to the full swap device before the MemoryMax wall; matched to swap size.\nMemorySwapMax=8G\n", + "spinifex.target": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Infrastructure Platform\nAfter=network-online.target\nWants=network-online.target\nWants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service\n\n[Install]\nWantedBy=multi-user.target\n", +} diff --git a/spinifex/systemd/units_invariants_test.go b/spinifex/systemd/units_invariants_test.go index 37a395c75..575b9a478 100644 --- a/spinifex/systemd/units_invariants_test.go +++ b/spinifex/systemd/units_invariants_test.go @@ -1,6 +1,8 @@ package systemd import ( + "crypto/sha256" + "encoding/hex" "os" "path/filepath" "regexp" @@ -55,6 +57,27 @@ func unitFiles(t *testing.T, dir string) []string { return out } +// allUnitFiles lists every unit in dir regardless of type — .service, +// .slice, .target and .timer — for invariants that apply to all 16. +func allUnitFiles(t *testing.T, dir string) []string { + t.Helper() + ents, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + var out []string + for _, e := range ents { + n := e.Name() + for _, suffix := range []string{".service", ".slice", ".target", ".timer"} { + if strings.HasSuffix(n, suffix) { + out = append(out, n) + break + } + } + } + return out +} + // timeoutStopSec parses a unit's TimeoutStopSec= value (this repo always // writes plain seconds, no unit suffix) into a time.Duration. func timeoutStopSec(t *testing.T, unit string) time.Duration { @@ -287,6 +310,11 @@ func TestRG11_LeanUnits(t *testing.T) { if !strings.HasPrefix(ls, "#") { continue } + // The version marker is structural metadata, not rationale prose, + // so it does not count against the comment budget. + if unitVersionRe.MatchString(ls) { + continue + } comments++ if m := planRef.FindString(ls); m != "" { t.Errorf("unit comment references external planning or CI material (%q): %s", m, ls) @@ -355,3 +383,92 @@ func TestApplicationUnitsExportTelemetry(t *testing.T) { } } } + +// TestGeneratedUnitsMatchSource asserts units_gen.go (the embedded copy +// Reconcile ships and compares against) is byte-identical to build/systemd. +// A stale generated file would make `spx admin upgrade` write the wrong +// content; regenerate with `go generate ./...` when this fails. +func TestGeneratedUnitsMatchSource(t *testing.T) { + dir := unitsDir(t) + source := allUnitFiles(t, dir) + + if len(Units) != len(source) { + t.Errorf("units_gen.go has %d units, build/systemd has %d — run `go generate ./...`", len(Units), len(source)) + } + for _, name := range source { + want := readUnit(t, dir, name) + got, ok := Units[name] + if !ok { + t.Errorf("units_gen.go missing %s — run `go generate ./...`", name) + continue + } + if got != want { + t.Errorf("units_gen.go for %s is stale — run `go generate ./...`", name) + } + } +} + +// unitBodyHashes pins the sha256 of each unit's body (everything after the +// `# spinifex-unit-version:` line) for its current version. Editing a unit's +// body without bumping the marker and adding a matching entry here fails +// TestUnitBodyPinnedToVersion — the forcing function that keeps a running +// node's reconciled units honest about what changed. +var unitBodyHashes = map[string]map[int]string{ + "regenerate-ssh-host-keys.service": {1: "5b2cf4c2ba5d5799c790e32896928c235db92a069583a87004801f58df390e55"}, + "spinifex-awsgw.service": {1: "dd662cebca19acca9dbcd0a6f29e6a40125805bdfeabcb94c02fbae07d1f2d6e"}, + "spinifex-daemon.service": {1: "04b0f23b9cda322e4012d589acc196abe466b619dec5ac0d832897b9af2f926c"}, + "spinifex-guests.slice": {1: "22dac772e81a9a716db98415a4cf590885c9cf19d4290cc21e645e5fe15bc793"}, + "spinifex-nats-watchdog.service": {1: "b778e398da37c4cb3c170781f074c76555cb6a137a7f96f8dc692457b12aa9b8"}, + "spinifex-nats-watchdog.timer": {1: "9d43a8d2aa4fd80ab5f944d7620c666fd2a255850de2e6494479b8746922c979"}, + "spinifex-nats.service": {1: "f7f9900b95e364dc2575684a9bf16fa1f96a903a3a4aa773a34c04d761106838"}, + "spinifex-northstar.service": {1: "030c382aed20f4383acdb7df7a830d48dd76dc29be5965b595cc3842ae43a3f1"}, + "spinifex-predastore.service": {1: "d0f6415b8f0e6ff3c045f2d3ce2794c347bf141066d7e0bd85fcec48797854d8"}, + "spinifex-qmp-collector.service": {1: "beb18e6dd9351901f19d992cb2f757fb0e0e4a4d986402ccdb0ebb0a449f225c"}, + "spinifex-shutdown.service": {1: "bcdc455916f35aa7494b2fe25e691339e8f1e22f031dfd9fd95203a9aa4bdaa4"}, + "spinifex-system.slice": {1: "ca450c2b28a8b13dd767957fa9469bd74bd222d7abed79945e83d564d5ce16dd"}, + "spinifex-ui.service": {1: "9d5ec3785bf730405f23ec9df79675bf65ecbca9df47c15f5a4b4639393afbc5"}, + "spinifex-viperblock.service": {1: "5c8cdd2004abf8e5725cbce0200565b9b29be7ab3210a4e0a2cdfe37ec5facb9"}, + "spinifex-vpcd.service": {1: "1b722640310145767cd34e87f4804852e63f46d2145ed20f8cc5f5400ebc5965"}, + "spinifex.slice": {1: "f73d9343e0e1bedd647835c8bb0c80fb3a3bd66474661234ecac23a4caafc24f"}, + "spinifex.target": {1: "0ffba9faee5a477f8ff7466a6bccb4dc7e04f5cf92a405553c242e6548402078"}, +} + +// TestUnitBodyPinnedToVersion asserts each unit's body hash matches the +// pinned hash for its current version marker. A body edit without a version +// bump changes the hash at the same version number and fails; bumping the +// version without adding a new table entry also fails, since no entry +// exists for the new version yet. +func TestUnitBodyPinnedToVersion(t *testing.T) { + dir := unitsDir(t) + for _, name := range allUnitFiles(t, dir) { + u := readUnit(t, dir, name) + version := unitVersion(u) + body := stripVersionMarkerLine(u) + sum := sha256.Sum256([]byte(body)) + hash := hex.EncodeToString(sum[:]) + + byVersion, ok := unitBodyHashes[name] + if !ok { + t.Errorf("%s: no entries in unitBodyHashes — add one for version %d", name, version) + continue + } + want, ok := byVersion[version] + if !ok { + t.Errorf("%s: no unitBodyHashes entry for version %d — bumped the marker without pinning its new body hash", name, version) + continue + } + if want != hash { + t.Errorf("%s: body hash for version %d changed (got %s, want %s) — bump # spinifex-unit-version and add a new unitBodyHashes entry", name, version, hash, want) + } + } +} + +// stripVersionMarkerLine drops the first line when it is the unit-version +// marker, so the body hash covers content only, not the marker itself. +func stripVersionMarkerLine(content string) string { + first, rest, found := strings.Cut(content, "\n") + if found && unitVersionRe.MatchString(strings.TrimSpace(first)) { + return rest + } + return content +}