From 2ea3e22db62dfc43eafb13f4ac2916ed0ee80626 Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Tue, 9 Jun 2026 15:21:13 +1000 Subject: [PATCH] feat(systemd): cgroup-confine spinifex to a resource-limited slice + host swap (#324) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(systemd): bound spinifex to a resource-limited slice Add a spinifex.slice parent ceiling (MemoryHigh=80%, MemoryMax=90%, MemorySwapMax=0) with a spinifex-system.slice child holding the seven control/infra services, so a runaway hits the slice instead of the host. A system.slice drop-in (MemoryMin=1G, CPUWeight=300) reserves RAM and CPU for sshd/journald/the operator, keeping the box reachable when spinifex is maxed. The drop-in is generated in setup.sh install_systemd() rather than shipped as a staged file, since the packaging globs flatten the systemd/ dir; this mirrors the existing sshd-keygen drop-in pattern. The two .slice files flow through every staging path unchanged. Within the single effective cgroup, a slice MemoryMax OOM picks the highest oom_score victim — guest QEMU (+500) before any service (-500) — so a slice OOM degrades by killing one guest while the daemon, storage tier, NATS, and the host all survive. * feat(setup): provision 8G host swapfile + vm.swappiness=10 Add a setup_swap stage that creates an 8G /swapfile, enables it, persists it to fstab, and writes vm.swappiness=10 so the kernel only swaps under real pressure. Backs spinifex.slice MemorySwapMax so the slice can spill to swap for graceful degradation instead of OOMing a guest. Idempotent; skipped under ISO_BUILD (firstboot provisions swap on ISO hosts). Reverses the prior swap=0 assumption. --- build/systemd/spinifex-awsgw.service | 1 + build/systemd/spinifex-daemon.service | 1 + build/systemd/spinifex-nats.service | 1 + build/systemd/spinifex-predastore.service | 1 + build/systemd/spinifex-system.slice | 10 ++++ build/systemd/spinifex-ui.service | 1 + build/systemd/spinifex-viperblock.service | 1 + build/systemd/spinifex-vpcd.service | 1 + build/systemd/spinifex.slice | 23 +++++++++ scripts/setup.sh | 57 +++++++++++++++++++++++ 10 files changed, 97 insertions(+) create mode 100644 build/systemd/spinifex-system.slice create mode 100644 build/systemd/spinifex.slice diff --git a/build/systemd/spinifex-awsgw.service b/build/systemd/spinifex-awsgw.service index 494e6dd4e..7a7dd745e 100644 --- a/build/systemd/spinifex-awsgw.service +++ b/build/systemd/spinifex-awsgw.service @@ -6,6 +6,7 @@ Wants=spinifex-nats.service [Service] Type=simple +Slice=spinifex-system.slice User=spinifex-gw Group=spinifex ExecStartPre=/var/lib/spinifex/wait-for-nats.sh diff --git a/build/systemd/spinifex-daemon.service b/build/systemd/spinifex-daemon.service index b7106fec1..ef1cf94a6 100644 --- a/build/systemd/spinifex-daemon.service +++ b/build/systemd/spinifex-daemon.service @@ -16,6 +16,7 @@ Wants=openvswitch-ipsec.service [Service] Type=simple +Slice=spinifex-system.slice User=spinifex-daemon Group=spinifex # DDIL Tier 1 (siv-17): no ExecStartPre wait-for-nats. The daemon starts in diff --git a/build/systemd/spinifex-nats.service b/build/systemd/spinifex-nats.service index 48036dc09..ea03f1e23 100644 --- a/build/systemd/spinifex-nats.service +++ b/build/systemd/spinifex-nats.service @@ -5,6 +5,7 @@ Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.servic [Service] Type=simple +Slice=spinifex-system.slice User=spinifex-nats Group=spinifex ExecStart=/usr/local/bin/spx service nats start diff --git a/build/systemd/spinifex-predastore.service b/build/systemd/spinifex-predastore.service index 71ef6aad8..d2b08d622 100644 --- a/build/systemd/spinifex-predastore.service +++ b/build/systemd/spinifex-predastore.service @@ -6,6 +6,7 @@ Wants=spinifex-nats.service [Service] Type=simple +Slice=spinifex-system.slice User=spinifex-storage Group=spinifex ExecStartPre=/var/lib/spinifex/wait-for-nats.sh diff --git a/build/systemd/spinifex-system.slice b/build/systemd/spinifex-system.slice new file mode 100644 index 000000000..1c52d8282 --- /dev/null +++ b/build/systemd/spinifex-system.slice @@ -0,0 +1,10 @@ +[Unit] +Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui) +Before=slices.target + +[Slice] +# Child of spinifex.slice by name (spinifex-system -> spinifex) — inherits the +# parent ceiling, no extra limit. Holds the 7 control/infra services (and, until +# the guest sub-slice lands, the daemon-forked QEMU guests under +# spinifex-daemon.service). Exists now so a future spinifex-guests.slice can +# become a sibling without re-editing the service units. diff --git a/build/systemd/spinifex-ui.service b/build/systemd/spinifex-ui.service index 270ddccfa..9bb4d877a 100644 --- a/build/systemd/spinifex-ui.service +++ b/build/systemd/spinifex-ui.service @@ -5,6 +5,7 @@ After=spinifex-awsgw.service [Service] Type=simple +Slice=spinifex-system.slice User=spinifex-ui Group=spinifex ExecStart=/usr/local/bin/spx service spinifex-ui start diff --git a/build/systemd/spinifex-viperblock.service b/build/systemd/spinifex-viperblock.service index f06c0f3d1..4f18d0b56 100644 --- a/build/systemd/spinifex-viperblock.service +++ b/build/systemd/spinifex-viperblock.service @@ -6,6 +6,7 @@ Wants=spinifex-nats.service [Service] Type=simple +Slice=spinifex-system.slice User=spinifex-viperblock Group=spinifex ExecStartPre=/var/lib/spinifex/wait-for-nats.sh diff --git a/build/systemd/spinifex-vpcd.service b/build/systemd/spinifex-vpcd.service index 9e68bd7cc..9cefa87d7 100644 --- a/build/systemd/spinifex-vpcd.service +++ b/build/systemd/spinifex-vpcd.service @@ -6,6 +6,7 @@ Wants=spinifex-nats.service ovn-controller.service [Service] Type=simple +Slice=spinifex-system.slice User=spinifex-vpcd Group=spinifex ExecStartPre=/var/lib/spinifex/wait-for-nats.sh diff --git a/build/systemd/spinifex.slice b/build/systemd/spinifex.slice new file mode 100644 index 000000000..859095f70 --- /dev/null +++ b/build/systemd/spinifex.slice @@ -0,0 +1,23 @@ +[Unit] +Description=Spinifex resource-bounded slice (host ceiling) +Before=slices.target + +[Slice] +# No CPUWeight — inherits the default 100. The slice roots +# (system/user/spinifex/init.scope) are siblings; capping spinifex below the +# default would let a stray user.slice job out-prioritize the VM fleet under +# contention, and CPUWeight only bites under contention anyway (idle host => +# spinifex still gets 100% of cores). Operator/sshd CPU priority is bought on +# system.slice instead (see system.slice.d/spinifex-reserve.conf). +# +# Memory: throttle+reclaim at High, hard wall at Max. Percentages are of +# physical RAM (systemd native, auto-scales with host size). Override per host +# via a spinifex.slice.d/*.conf drop-in + `systemctl daemon-reload`. +MemoryHigh=80% +MemoryMax=90% +# Host swap is an 8G swapfile (provisioned by setup.sh). Let spinifex spill to +# the full swap device for graceful degradation before the MemoryMax wall. With +# vm.swappiness=10 the kernel only swaps under real pressure, so this is a safety +# buffer, not a routine path. Matched to the swap device size; raise both +# together if the swapfile grows. +MemorySwapMax=8G diff --git a/scripts/setup.sh b/scripts/setup.sh index b2bd5111a..cc4c1ca15 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -599,6 +599,20 @@ install_systemd() { info " /etc/systemd/system/$(basename "$unit")" done + # Reserve RAM + CPU priority for system.slice (sshd, journald, the operator) + # so a maxed spinifex.slice cannot starve them — the "stay sshable" guarantee. + # Generated here rather than shipped as a staged file because the packaging + # globs flatten the systemd/ dir and would skip a nested drop-in directory; + # this mirrors the sshd-keygen drop-in pattern in build-rootfs.sh. MemoryMin + # is a guaranteed-unreclaimable floor, not a cap. + $SUDO mkdir -p /etc/systemd/system/system.slice.d + $SUDO tee /etc/systemd/system/system.slice.d/spinifex-reserve.conf > /dev/null << 'EOF' +[Slice] +MemoryMin=1G +CPUWeight=300 +EOF + info " /etc/systemd/system/system.slice.d/spinifex-reserve.conf" + # daemon-reload / enable require a running systemd — skip inside the ISO # chroot. Unit files are still dropped into place; firstboot enables them. if [ "${ISO_BUILD:-0}" = "1" ]; then @@ -704,6 +718,48 @@ print_summary() { echo "" } +# --- Configure host swap (mulga-siv-251) --- +# Provisions an 8G swapfile and lowers vm.swappiness so spinifex.slice +# (MemorySwapMax=100%) has a backing device for graceful degradation under +# memory pressure. Reverses the historical swap=0 assumption. Idempotent. +setup_swap() { + stage "configuring host swap" + + # ISO build runs in a chroot — cannot swapon, and baking an 8G file into the + # rootfs bloats the image. ISO hosts provision swap at firstboot instead. + if [ "${ISO_BUILD:-0}" = "1" ]; then + info "Swap setup skipped (ISO_BUILD=1; firstboot provisions swap)" + return + fi + + local swapfile=/swapfile size_mb=8192 + + # Swap is a safety buffer, not a routine path: reclaim page cache first. + $SUDO tee /etc/sysctl.d/99-spinifex-swap.conf > /dev/null << 'EOF' +# Spinifex: minimise swapping; swap backs spinifex.slice graceful degradation. +vm.swappiness = 10 +EOF + $SUDO chmod 0644 /etc/sysctl.d/99-spinifex-swap.conf + $SUDO sysctl -q -w vm.swappiness=10 || true + + if swapon --show=NAME --noheadings 2>/dev/null | grep -qx "$swapfile"; then + info "Swap already active ($swapfile)" + return + fi + + if [ ! -f "$swapfile" ]; then + info "Creating ${size_mb}MiB $swapfile" + $SUDO fallocate -l "${size_mb}M" "$swapfile" 2>/dev/null \ + || $SUDO dd if=/dev/zero of="$swapfile" bs=1M count="$size_mb" status=none + $SUDO chmod 0600 "$swapfile" + $SUDO mkswap "$swapfile" > /dev/null + fi + $SUDO swapon "$swapfile" + grep -q "^$swapfile " /etc/fstab 2>/dev/null \ + || echo "$swapfile none swap sw 0 0" | $SUDO tee -a /etc/fstab > /dev/null + info "Swap enabled: $swapfile (${size_mb}MiB), vm.swappiness=10" +} + # --- Main --- main() { info "Spinifex installer" @@ -739,6 +795,7 @@ main() { stage_enabled systemd && install_systemd stage_enabled logrotate && install_logrotate stage_enabled udev && install_udev + stage_enabled swap && setup_swap # Migrations are only safe on a live system (need a running NATS and a # persisted config file). Skip under ISO_BUILD and under any explicit