From a4274f95b537237baeb8cd853bc93abc2675be5f Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Sun, 2 Aug 2026 11:28:20 +1000 Subject: [PATCH] fix(systemd): keep in-use nbdkit alive across a viperblock restart (#648) shutdownVolumes deliberately spares nbdkit that a guest is still writing through, because tearing one out mid-write corrupts that guest's filesystem. The unit left KillMode at the default, so the cgroup SIGTERMed those children anyway and SIGKILLed them at TimeoutStopSec -- causing both a 30s stop stall and the corruption the skip exists to prevent. KillMode=process matches spinifex-daemon, where guest QEMU already survives a restart for DDIL reattach. Idle nbdkit is still reaped explicitly. Measured on env12 with a guest attached: control-group stopped in 30.08s with 'Failed with result timeout' and SIGKILLed nbdkit, leaving QEMU running with zero ESTAB nbd sockets. KillMode=process stops sub-second, both nbdkit survive, and the guest keeps its device. --- build/systemd/spinifex-viperblock.service | 9 +++++---- spinifex/systemd/units_invariants_test.go | 8 ++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/build/systemd/spinifex-viperblock.service b/build/systemd/spinifex-viperblock.service index 472490c2d..3c83933f4 100644 --- a/build/systemd/spinifex-viperblock.service +++ b/build/systemd/spinifex-viperblock.service @@ -15,10 +15,11 @@ ExecStart=/usr/local/bin/spx service viperblock start Restart=on-failure RestartSec=5 OOMScoreAdjust=-500 -# Backstop for the in-use nbdkit children SIGTERMed with this cgroup. Their -# plugin bounds its own shutdown drain at 20s, so 30s lets that fire first and -# keeps SIGKILL exceptional; the 90s default let a slow backend stall the whole -# target-stop transaction. +# KillMode=process: nbdkit still serving a guest survives a restart here, the +# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts +# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown. +KillMode=process +# Bounds this process's own WAL drain, which the plugin caps at 20s. TimeoutStopSec=30 EnvironmentFile=/etc/spinifex/systemd.env Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml diff --git a/spinifex/systemd/units_invariants_test.go b/spinifex/systemd/units_invariants_test.go index c409b0236..343adc9f9 100644 --- a/spinifex/systemd/units_invariants_test.go +++ b/spinifex/systemd/units_invariants_test.go @@ -228,6 +228,14 @@ func TestGracefulDrainOrdering(t *testing.T) { t.Error("spinifex-daemon.service must keep KillMode=process — guests survive daemon restart (DDIL)") } + // shutdownVolumes deliberately spares nbdkit that a guest is still writing + // through. Without KillMode=process the cgroup SIGTERMs then SIGKILLs it + // anyway at TimeoutStopSec, which is the corruption that skip exists to avoid. + viperblock := readUnit(t, dir, "spinifex-viperblock.service") + if !hasDirective(viperblock, "KillMode=process") { + t.Error("spinifex-viperblock.service must keep KillMode=process — in-use nbdkit survives a viperblock restart (DDIL)") + } + target := readUnit(t, dir, "spinifex.target") if !strings.Contains(target, "spinifex-shutdown.service") { t.Error("spinifex.target Wants= must include spinifex-shutdown.service")