diff --git a/build/systemd/spinifex-daemon.service b/build/systemd/spinifex-daemon.service index e09381cad..fd0efdd2b 100644 --- a/build/systemd/spinifex-daemon.service +++ b/build/systemd/spinifex-daemon.service @@ -23,6 +23,17 @@ Group=spinifex ExecStart=/usr/local/bin/spx service spinifex start Restart=on-failure RestartSec=5 +# DDIL Tier 1 (mulga-siv-58): KillMode=process so systemd only signals the +# daemon's main PID, not the cgroup. QEMU children stay alive across +# `systemctl restart spinifex-daemon` and the new daemon reattaches to them +# via the persisted local state file + pidfile + QMP socket. With the +# default control-group mode, restart would terminate every guest VM — +# defeating Scenario B's "daemon restart preserves running instances". +KillMode=process +# 15s upper bound on the SIGTERM grace window. The new SIGTERM handler +# skips VM teardown (it now just persists state and exits), so 15s is +# plenty of slack for the cluster-manager + NATS unsubscribe path. +TimeoutStopSec=15 OOMScoreAdjust=-500 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/ diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index f960532fb..9cd8e9aaf 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -1744,16 +1744,18 @@ func (d *Daemon) setupShutdown() { // Cancel context to stop heartbeat and other goroutines d.cancel() - // If coordinated shutdown already handled VMs (DRAIN phase), skip the - // per-instance teardown. Otherwise, set the flag now so crash handlers - // and restart schedulers know to bail out during SIGTERM-based shutdown. + // DDIL Tier 1 (mulga-siv-58): SIGTERM alone never stops local VMs. + // `systemctl restart spinifex-daemon` must preserve every QEMU child so + // the new daemon can reattach via the persisted local state file + // (Scenario B). VMs are only stopped when coordinated cluster shutdown + // has explicitly drained them — handleShutdownDrain sets shuttingDown + // after calling StopAll, and crash/restart handlers also gate on this + // flag to bail out. if d.shuttingDown.Load() { slog.Info("Coordinated shutdown in progress, skipping VM stop (already handled by DRAIN phase)") } else { + slog.Info("SIGTERM with no coordinated drain — leaving local VMs running for restart recovery") d.shuttingDown.Store(true) - if err := d.vmMgr.StopAll(); err != nil { - slog.Error("Failed to stop instances during shutdown", "err", err) - } } // Stop ELBv2 background goroutines @@ -1783,8 +1785,12 @@ func (d *Daemon) setupShutdown() { slog.Error("Failed to write state", "err", err) } - // Close NATS connection - d.natsConn.Close() + // Close NATS connection. natsConn is nil when the daemon was started + // with NATS unreachable and never managed an initial connect — that + // is the DDIL Scenario B path (daemon restart while NATS is down). + if d.natsConn != nil { + d.natsConn.Close() + } // Shutdown cluster manager if d.clusterServer != nil {