feat(systemd): cgroup-confine spinifex to a resource-limited slice + host swap (#324)

* feat(systemd): bound spinifex to a resource-limited slice

Add a spinifex.slice parent ceiling (MemoryHigh=80%, MemoryMax=90%,
MemorySwapMax=0) with a spinifex-system.slice child holding the seven
control/infra services, so a runaway hits the slice instead of the host.
A system.slice drop-in (MemoryMin=1G, CPUWeight=300) reserves RAM and CPU
for sshd/journald/the operator, keeping the box reachable when spinifex is
maxed.

The drop-in is generated in setup.sh install_systemd() rather than shipped
as a staged file, since the packaging globs flatten the systemd/ dir; this
mirrors the existing sshd-keygen drop-in pattern. The two .slice files flow
through every staging path unchanged.

Within the single effective cgroup, a slice MemoryMax OOM picks the highest
oom_score victim — guest QEMU (+500) before any service (-500) — so a slice
OOM degrades by killing one guest while the daemon, storage tier, NATS, and
the host all survive.

* feat(setup): provision 8G host swapfile + vm.swappiness=10

Add a setup_swap stage that creates an 8G /swapfile, enables it, persists it
to fstab, and writes vm.swappiness=10 so the kernel only swaps under real
pressure. Backs spinifex.slice MemorySwapMax so the slice can spill to swap
for graceful degradation instead of OOMing a guest. Idempotent; skipped under
ISO_BUILD (firstboot provisions swap on ISO hosts). Reverses the prior swap=0
assumption.

build/systemd/spinifex-awsgw.service

@@ -6,6 +6,7 @@ Wants=spinifex-nats.service
 
 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-gw
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh

build/systemd/spinifex-daemon.service

@@ -16,6 +16,7 @@ Wants=openvswitch-ipsec.service
 
 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-daemon
 Group=spinifex
 # DDIL Tier 1 (siv-17): no ExecStartPre wait-for-nats. The daemon starts in

build/systemd/spinifex-nats.service

@@ -5,6 +5,7 @@ Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.servic
 
 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-nats
 Group=spinifex
 ExecStart=/usr/local/bin/spx service nats start

build/systemd/spinifex-predastore.service

@@ -6,6 +6,7 @@ Wants=spinifex-nats.service
 
 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-storage
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh

build/systemd/spinifex-system.slice

@@ -0,0 +1,10 @@
+[Unit]
+Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)
+Before=slices.target
+
+[Slice]
+# Child of spinifex.slice by name (spinifex-system -> spinifex) — inherits the
+# parent ceiling, no extra limit. Holds the 7 control/infra services (and, until
+# the guest sub-slice lands, the daemon-forked QEMU guests under
+# spinifex-daemon.service). Exists now so a future spinifex-guests.slice can
+# become a sibling without re-editing the service units.

build/systemd/spinifex-ui.service

@@ -5,6 +5,7 @@ After=spinifex-awsgw.service
 
 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-ui
 Group=spinifex
 ExecStart=/usr/local/bin/spx service spinifex-ui start

build/systemd/spinifex-viperblock.service

@@ -6,6 +6,7 @@ Wants=spinifex-nats.service
 
 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-viperblock
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh

build/systemd/spinifex-vpcd.service

@@ -6,6 +6,7 @@ Wants=spinifex-nats.service ovn-controller.service
 
 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-vpcd
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh

build/systemd/spinifex.slice

@@ -0,0 +1,23 @@
+[Unit]
+Description=Spinifex resource-bounded slice (host ceiling)
+Before=slices.target
+
+[Slice]
+# No CPUWeight — inherits the default 100. The slice roots
+# (system/user/spinifex/init.scope) are siblings; capping spinifex below the
+# default would let a stray user.slice job out-prioritize the VM fleet under
+# contention, and CPUWeight only bites under contention anyway (idle host =>
+# spinifex still gets 100% of cores). Operator/sshd CPU priority is bought on
+# system.slice instead (see system.slice.d/spinifex-reserve.conf).
+#
+# Memory: throttle+reclaim at High, hard wall at Max. Percentages are of
+# physical RAM (systemd native, auto-scales with host size). Override per host
+# via a spinifex.slice.d/*.conf drop-in + `systemctl daemon-reload`.
+MemoryHigh=80%
+MemoryMax=90%
+# Host swap is an 8G swapfile (provisioned by setup.sh). Let spinifex spill to
+# the full swap device for graceful degradation before the MemoryMax wall. With
+# vm.swappiness=10 the kernel only swaps under real pressure, so this is a safety
+# buffer, not a routine path. Matched to the swap device size; raise both
+# together if the swapfile grows.
+MemorySwapMax=8G

scripts/setup.sh

@@ -599,6 +599,20 @@ install_systemd() {
         info "  /etc/systemd/system/$(basename "$unit")"
     done
 
+    # Reserve RAM + CPU priority for system.slice (sshd, journald, the operator)
+    # so a maxed spinifex.slice cannot starve them — the "stay sshable" guarantee.
+    # Generated here rather than shipped as a staged file because the packaging
+    # globs flatten the systemd/ dir and would skip a nested drop-in directory;
+    # this mirrors the sshd-keygen drop-in pattern in build-rootfs.sh. MemoryMin
+    # is a guaranteed-unreclaimable floor, not a cap.
+    $SUDO mkdir -p /etc/systemd/system/system.slice.d
+    $SUDO tee /etc/systemd/system/system.slice.d/spinifex-reserve.conf > /dev/null << 'EOF'
+[Slice]
+MemoryMin=1G
+CPUWeight=300
+EOF
+    info "  /etc/systemd/system/system.slice.d/spinifex-reserve.conf"
+
     # daemon-reload / enable require a running systemd — skip inside the ISO
     # chroot. Unit files are still dropped into place; firstboot enables them.
     if [ "${ISO_BUILD:-0}" = "1" ]; then
@@ -704,6 +718,48 @@ print_summary() {
     echo ""
 }
 
+# --- Configure host swap (mulga-siv-251) ---
+# Provisions an 8G swapfile and lowers vm.swappiness so spinifex.slice
+# (MemorySwapMax=100%) has a backing device for graceful degradation under
+# memory pressure. Reverses the historical swap=0 assumption. Idempotent.
+setup_swap() {
+    stage "configuring host swap"
+
+    # ISO build runs in a chroot — cannot swapon, and baking an 8G file into the
+    # rootfs bloats the image. ISO hosts provision swap at firstboot instead.
+    if [ "${ISO_BUILD:-0}" = "1" ]; then
+        info "Swap setup skipped (ISO_BUILD=1; firstboot provisions swap)"
+        return
+    fi
+
+    local swapfile=/swapfile size_mb=8192
+
+    # Swap is a safety buffer, not a routine path: reclaim page cache first.
+    $SUDO tee /etc/sysctl.d/99-spinifex-swap.conf > /dev/null << 'EOF'
+# Spinifex: minimise swapping; swap backs spinifex.slice graceful degradation.
+vm.swappiness = 10
+EOF
+    $SUDO chmod 0644 /etc/sysctl.d/99-spinifex-swap.conf
+    $SUDO sysctl -q -w vm.swappiness=10 || true
+
+    if swapon --show=NAME --noheadings 2>/dev/null | grep -qx "$swapfile"; then
+        info "Swap already active ($swapfile)"
+        return
+    fi
+
+    if [ ! -f "$swapfile" ]; then
+        info "Creating ${size_mb}MiB $swapfile"
+        $SUDO fallocate -l "${size_mb}M" "$swapfile" 2>/dev/null \
+            || $SUDO dd if=/dev/zero of="$swapfile" bs=1M count="$size_mb" status=none
+        $SUDO chmod 0600 "$swapfile"
+        $SUDO mkswap "$swapfile" > /dev/null
+    fi
+    $SUDO swapon "$swapfile"
+    grep -q "^$swapfile " /etc/fstab 2>/dev/null \
+        || echo "$swapfile none swap sw 0 0" | $SUDO tee -a /etc/fstab > /dev/null
+    info "Swap enabled: $swapfile (${size_mb}MiB), vm.swappiness=10"
+}
+
 # --- Main ---
 main() {
     info "Spinifex installer"
@@ -739,6 +795,7 @@ main() {
     stage_enabled systemd    && install_systemd
     stage_enabled logrotate  && install_logrotate
     stage_enabled udev       && install_udev
+    stage_enabled swap       && setup_swap
 
     # Migrations are only safe on a live system (need a running NATS and a
     # persisted config file). Skip under ISO_BUILD and under any explicit