diff --git a/build/systemd/spinifex-predastore.service b/build/systemd/spinifex-predastore.service index e5f209e52..f589ed153 100644 --- a/build/systemd/spinifex-predastore.service +++ b/build/systemd/spinifex-predastore.service @@ -1,4 +1,4 @@ -# spinifex-unit-version: 1 +# spinifex-unit-version: 2 [Unit] Description=Spinifex Predastore (S3-compatible Object Storage) PartOf=spinifex.target @@ -18,9 +18,9 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh ExecStart=/usr/local/bin/spx service predastore start Restart=on-failure RestartSec=5 -# Matches viperblock's drain budget: viperblock's WAL flush writes into -# predastore, and After= already stops viperblock first, but this is the -# backstop if that ordering is ever bypassed (direct stop, future edit). +# Bounds predastore's own graceful stop. viperblock's WAL flush writes into +# predastore, but After= already stops viperblock first (within its own 150s +# budget); this is the backstop if that ordering is ever bypassed. TimeoutStopSec=30 OOMScoreAdjust=-500 # Legitimately the most memory-hungry infra service (S3 buffering/caching); diff --git a/spinifex/systemd/units_gen.go b/spinifex/systemd/units_gen.go index ec4697882..eb4ccdad7 100644 --- a/spinifex/systemd/units_gen.go +++ b/spinifex/systemd/units_gen.go @@ -15,7 +15,7 @@ var Units = map[string]string{ "spinifex-nats-watchdog.timer": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS JetStream liveness watchdog timer\nPartOf=spinifex.target\n\n[Timer]\n# Delay the first probe so NATS startup + JetStream cluster quorum settle (a\n# forming cluster reports JS not-current transiently, which is not a fault).\nOnBootSec=60s\nOnUnitActiveSec=30s\nAccuracySec=5s\n\n[Install]\nWantedBy=timers.target\n", "spinifex-nats.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS Message Bus\nPartOf=spinifex.target\nBefore=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-nats\nGroup=spinifex\nExecStart=/usr/local/bin/spx service nats start\nRestart=on-failure\nRestartSec=5\n# nats-server installs its own SIGTERM handler (JetStream store shutdown, raft\n# stepdown, client drain) with no internal deadline; every dependent service\n# stops before nats does, so size this for nats's own graceful-stop needs.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/nats /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/nats /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", "spinifex-northstar.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Northstar (Authoritative DNS)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-predastore.service\nWants=spinifex-predastore.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-northstar\nGroup=spinifex\nExecStart=/usr/local/bin/spx service northstar start\nRestart=on-failure\nRestartSec=5\n# server.Shutdown() is called with an internal 5s context; margin covers the\n# NATS unsubscribe/close and process exit that follow it.\nTimeoutStopSec=10\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/northstar/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/northstar /etc/spinifex/spinifex.toml /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/northstar /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\n# RG-9: exactly CAP_NET_BIND_SERVICE so the unprivileged user binds :53 (no root).\nAmbientCapabilities=CAP_NET_BIND_SERVICE\nCapabilityBoundingSet=CAP_NET_BIND_SERVICE\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", - "spinifex-predastore.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Predastore (S3-compatible Object Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# Guest block I/O arrives with no caller trace; sample those roots at 5% so\n# they don't drown request traces. Request-linked spans are always kept.\nEnvironment=MULGA_ROOT_TRACE_RATIO=0.05\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-storage\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service predastore start\nRestart=on-failure\nRestartSec=5\n# Matches viperblock's drain budget: viperblock's WAL flush writes into\n# predastore, and After= already stops viperblock first, but this is the\n# backstop if that ordering is ever bypassed (direct stop, future edit).\nTimeoutStopSec=30\nOOMScoreAdjust=-500\n# Legitimately the most memory-hungry infra service (S3 buffering/caching);\n# High/Max stay well above idle so normal load isn't throttled, while still\n# bounding a leak short of the slice-wide cap.\nMemoryHigh=40%\nMemoryMax=55%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml\nEnvironment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/\nEnvironment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-predastore.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Predastore (S3-compatible Object Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# Guest block I/O arrives with no caller trace; sample those roots at 5% so\n# they don't drown request traces. Request-linked spans are always kept.\nEnvironment=MULGA_ROOT_TRACE_RATIO=0.05\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-storage\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service predastore start\nRestart=on-failure\nRestartSec=5\n# Bounds predastore's own graceful stop. viperblock's WAL flush writes into\n# predastore, but After= already stops viperblock first (within its own 150s\n# budget); this is the backstop if that ordering is ever bypassed.\nTimeoutStopSec=30\nOOMScoreAdjust=-500\n# Legitimately the most memory-hungry infra service (S3 buffering/caching);\n# High/Max stay well above idle so normal load isn't throttled, while still\n# bounding a leak short of the slice-wide cap.\nMemoryHigh=40%\nMemoryMax=55%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml\nEnvironment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/\nEnvironment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", "spinifex-qmp-collector.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex QMP Telemetry Collector (guest metrics)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\n# spinifex-daemon user: the collector reads the daemon's telemetry QMP\n# sockets in /run/spinifex and the daemon-owned QEMU /proc entries.\nUser=spinifex-daemon\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service qmp-collector start\nRestart=on-failure\nRestartSec=5\n# Poll loop selects on ctx.Done() directly and closes NATS locally; shutdown\n# is near-instant, so this only bounds an unexpectedly wedged process.\nTimeoutStopSec=10\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\n# Same runtime dir as the daemon: telemetry sockets + metadata live there\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: read-mostly telemetry tier — no capabilities, no device access\nProtectSystem=full\nPrivateDevices=yes\nNoNewPrivileges=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/spinifex /run/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\n# ProtectProc must stay default: hidepid=invisible hides the QEMU vCPU threads\n# (dumpable=0) this collector must read from /proc//stat, even same-uid.\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=512\n\n[Install]\nWantedBy=spinifex.target\n", "spinifex-shutdown.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex graceful guest drain\nPartOf=spinifex.target\n# After= all services so it stops FIRST on target/host shutdown: the drain runs\n# while nats/predastore/viperblock/daemon are still up (QMP, unmount, WAL flush).\nAfter=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/true\n# ExecStop fires on ANY target stop, including the stop half of `systemctl\n# restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless\n# of whether the host is actually going down. --unless-restarting drains on a\n# real shutdown/reboot and on a plain `systemctl stop spinifex.target`; it\n# skips only when a restart of the stack is already queued, so QEMU/nbdkit\n# survive (KillMode=process) and the daemon reconnects on the way back up.\nExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --unless-restarting\nTimeoutStopSec=180\n\n[Install]\nWantedBy=spinifex.target\n", "spinifex-system.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-system -> spinifex), inherits the\n# parent ceiling, no extra limit. Holds the 7 control/infra services. Guest QEMU\n# moves to the sibling spinifex-guests.slice (per-instance placement lands in\n# 0002 RG-7); until then the daemon-forked guests stay under spinifex-daemon.service.\n", diff --git a/spinifex/systemd/units_invariants_test.go b/spinifex/systemd/units_invariants_test.go index 16a221680..fcae50676 100644 --- a/spinifex/systemd/units_invariants_test.go +++ b/spinifex/systemd/units_invariants_test.go @@ -426,7 +426,10 @@ var unitBodyHashes = map[string]map[int]string{ "spinifex-nats-watchdog.timer": {1: "9d43a8d2aa4fd80ab5f944d7620c666fd2a255850de2e6494479b8746922c979"}, "spinifex-nats.service": {1: "f7f9900b95e364dc2575684a9bf16fa1f96a903a3a4aa773a34c04d761106838"}, "spinifex-northstar.service": {1: "030c382aed20f4383acdb7df7a830d48dd76dc29be5965b595cc3842ae43a3f1"}, - "spinifex-predastore.service": {1: "d0f6415b8f0e6ff3c045f2d3ce2794c347bf141066d7e0bd85fcec48797854d8"}, + "spinifex-predastore.service": { + 1: "d0f6415b8f0e6ff3c045f2d3ce2794c347bf141066d7e0bd85fcec48797854d8", + 2: "157e9a7683ac58760ad96679cad9f94121294f34d5cb668d1e586ba0686b4968", + }, "spinifex-qmp-collector.service": {1: "beb18e6dd9351901f19d992cb2f757fb0e0e4a4d986402ccdb0ebb0a449f225c"}, "spinifex-shutdown.service": {1: "bcdc455916f35aa7494b2fe25e691339e8f1e22f031dfd9fd95203a9aa4bdaa4"}, "spinifex-system.slice": {1: "ca450c2b28a8b13dd767957fa9469bd74bd222d7abed79945e83d564d5ce16dd"}, diff --git a/tests/e2e/run-bm-multinode-e2e.sh b/tests/e2e/run-bm-multinode-e2e.sh index a0a56ce11..0a6ccab3e 100755 --- a/tests/e2e/run-bm-multinode-e2e.sh +++ b/tests/e2e/run-bm-multinode-e2e.sh @@ -979,11 +979,20 @@ echo "" # ========================================================================== echo "Phase 8: Node Failure" echo "========================================" -echo "Stopping services on node2 ($NODE2_IP) to simulate node failure..." +echo "Abruptly killing spinifex services on node2 ($NODE2_IP) to simulate an unclean node failure..." -# Stop services on node2 only — SPINIFEX_FORCE_LOCAL_STOP prevents coordinated -# cluster shutdown which would kill all nodes via NATS -peer_ssh "$NODE2_IP" "sudo systemctl stop spinifex.target" || { +# spinifex.target has no process of its own and PartOf= only propagates +# stop/restart jobs (not kill), so SIGKILL each unit directly. --kill-whom=main +# hits only the service's own process, leaving qemu/nbdkit orphaned for reattach. +NODE2_KILL_UNITS="spinifex-daemon.service spinifex-viperblock.service spinifex-predastore.service spinifex-nats.service spinifex-awsgw.service spinifex-vpcd.service spinifex-northstar.service spinifex-qmp-collector.service spinifex-ui.service" +peer_ssh "$NODE2_IP" "sudo systemctl kill -s SIGKILL --kill-whom=main $NODE2_KILL_UNITS" || { + echo " WARNING: systemctl kill returned non-zero (may be expected)" +} + +# Cancels each unit's queued Restart=on-failure job so the outage holds rather +# than self-healing within RestartSec=5. The main process is already dead, so +# this settles the unit instead of draining it — spinifex.target stays untouched. +peer_ssh "$NODE2_IP" "sudo systemctl stop $NODE2_KILL_UNITS" || { echo " WARNING: systemctl stop returned non-zero (may be expected)" } @@ -1017,15 +1026,21 @@ else echo " Node3 not present in this cluster — skipping node3 survival check" fi -# Check NATS degraded state (should have 1 route instead of 2) +# Check NATS degraded state: node1 should see every peer except itself and +# the now-dead node2. The stop above holds node2 down for the rest of this +# phase, so this is a real assertion now, not a best-effort guess. +EXPECTED_DEGRADED_PEERS=$((NODE_COUNT - 2)) +if [ "$EXPECTED_DEGRADED_PEERS" -lt 0 ]; then + EXPECTED_DEGRADED_PEERS=0 +fi NATS_DEGRADED=$(curl -s "http://127.0.0.1:${NATS_MONITOR_PORT}/routez" 2>/dev/null) DEGRADED_PEERS=$(echo "$NATS_DEGRADED" | jq -r '[.routes[].remote_name] | unique | length' 2>/dev/null || echo "0") -echo " NATS peers during failure: $DEGRADED_PEERS (expected: 1)" -if [ "$DEGRADED_PEERS" -eq 1 ]; then +echo " NATS peers during failure: $DEGRADED_PEERS (expected: $EXPECTED_DEGRADED_PEERS)" +if [ "$DEGRADED_PEERS" -eq "$EXPECTED_DEGRADED_PEERS" ]; then pass_test "NATS degraded mode" else - echo " WARNING: Expected 1 NATS peer during node2 failure, got $DEGRADED_PEERS" - # Not fatal — NATS might take a moment to detect + echo " ERROR: Expected $EXPECTED_DEGRADED_PEERS NATS peer(s) during node2 failure, got $DEGRADED_PEERS" + fail_test "NATS degraded mode" fi # Verify describe-instances still works from surviving nodes @@ -1048,6 +1063,8 @@ echo "Phase 9: Node Recovery" echo "========================================" echo "Restarting services on node2 ($NODE2_IP)..." +# Phase 8 stopped every unit it killed, so node2 has been genuinely down +# since then; this start is the actual recovery, not a backstop. peer_ssh "$NODE2_IP" "sudo systemctl start spinifex.target" || { echo " ERROR: Failed to restart services on node2" fail_test "Node2 restart"