From ab1b1bacf6b2c3baf675f1fe45f7bde6cafa397e Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Tue, 30 Jun 2026 08:41:10 +1000 Subject: [PATCH] fix: auto-restart NATS when JetStream is disabled while the process lives (#445) When NATS disables JetStream at runtime (e.g. a filesystem permission error flushing stream state) it keeps the process alive and serves KV as read-only. That is not a process failure, so spinifex-nats.service's Restart=on-failure never fires, and KV-backed startup (awsgw IAM bucket init) stalls for minutes until an operator restarts NATS by hand. Add a JetStream liveness watchdog: - build/scripts/nats-js-watchdog.sh probes the NATS monitoring endpoint (http://127.0.0.1:8222/healthz?js-enabled-only=true). Only when the service is active and JS fails every sample of a short multi-sample window (so transient startup/quorum 503s are ignored) does it systemctl try-restart spinifex-nats. - spinifex-nats-watchdog.{service,timer} run it as a oneshot every 30s, with a 60s boot delay so cluster quorum settles first. - spinifex.target Wants= the timer so it starts with the platform. The NATS-internal permission race itself is a server quirk outside spinifex's control; this restores service automatically when it recurs. --- build/scripts/nats-js-watchdog.sh | 50 ++++++++++++++++++++ build/systemd/spinifex-nats-watchdog.service | 22 +++++++++ build/systemd/spinifex-nats-watchdog.timer | 13 +++++ build/systemd/spinifex.target | 2 +- 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100755 build/scripts/nats-js-watchdog.sh create mode 100644 build/systemd/spinifex-nats-watchdog.service create mode 100644 build/systemd/spinifex-nats-watchdog.timer diff --git a/build/scripts/nats-js-watchdog.sh b/build/scripts/nats-js-watchdog.sh new file mode 100755 index 000000000..bccd71c30 --- /dev/null +++ b/build/scripts/nats-js-watchdog.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# JetStream liveness watchdog for spinifex-nats. +# +# NATS can disable JetStream at runtime (e.g. a filesystem permission error while +# flushing stream state) and keep the process alive serving KV as read-only. That +# is not a process failure, so Restart=on-failure never fires and KV-backed +# services (awsgw IAM bucket init, etc.) stall until NATS is restarted by hand. +# +# This probes the NATS monitoring endpoint's JS health and restarts the service +# only when JetStream is sustainedly unhealthy while the service is running. The +# multi-sample window avoids restarting during transient startup/quorum 503s. + +set -eu + +HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true" +SERVICE="spinifex-nats.service" +SAMPLES=3 # consecutive failing samples required before acting +SAMPLE_GAP=2 # seconds between samples + +# Only act on a running service: a stopped/activating unit is systemd's to manage, +# and try-restart on it would either no-op or fight an in-flight (re)start. +if ! systemctl is-active --quiet "$SERVICE"; then + exit 0 +fi + +# curl is the probe transport; if it is unavailable, do nothing rather than +# false-restart a healthy server. +if ! command -v curl >/dev/null 2>&1; then + echo "nats-js-watchdog: curl not found, skipping JS health probe" >&2 + exit 0 +fi + +# A single 200 means JetStream is healthy — exit fast on the common case. +js_healthy() { + curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null +} + +i=1 +while [ "$i" -le "$SAMPLES" ]; do + if js_healthy; then + exit 0 + fi + if [ "$i" -lt "$SAMPLES" ]; then + sleep "$SAMPLE_GAP" + fi + i=$((i + 1)) +done + +echo "nats-js-watchdog: JetStream unhealthy on $SAMPLES consecutive probes ($HEALTH_URL); restarting $SERVICE" >&2 +systemctl try-restart "$SERVICE" diff --git a/build/systemd/spinifex-nats-watchdog.service b/build/systemd/spinifex-nats-watchdog.service new file mode 100644 index 000000000..b29cd667c --- /dev/null +++ b/build/systemd/spinifex-nats-watchdog.service @@ -0,0 +1,22 @@ +[Unit] +Description=Spinifex NATS JetStream liveness watchdog +PartOf=spinifex.target +After=spinifex-nats.service + +[Service] +Type=oneshot +# Runs as root: the probe is a local curl to the NATS monitoring port, and the +# remedy is `systemctl try-restart spinifex-nats` when JetStream is disabled while +# the process stays alive (which Restart=on-failure cannot catch). +ExecStart=/var/lib/spinifex/nats-js-watchdog.sh + +# Hardening (kept compatible with talking to PID1 over /run/systemd). +NoNewPrivileges=yes +ProtectHome=yes +PrivateTmp=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectControlGroups=yes +RestrictRealtime=yes +LockPersonality=yes +RestrictSUIDSGID=yes diff --git a/build/systemd/spinifex-nats-watchdog.timer b/build/systemd/spinifex-nats-watchdog.timer new file mode 100644 index 000000000..33d79697b --- /dev/null +++ b/build/systemd/spinifex-nats-watchdog.timer @@ -0,0 +1,13 @@ +[Unit] +Description=Spinifex NATS JetStream liveness watchdog timer +PartOf=spinifex.target + +[Timer] +# Delay the first probe so NATS startup + JetStream cluster quorum settle (a +# forming cluster reports JS not-current transiently, which is not a fault). +OnBootSec=60s +OnUnitActiveSec=30s +AccuracySec=5s + +[Install] +WantedBy=timers.target diff --git a/build/systemd/spinifex.target b/build/systemd/spinifex.target index ba6d3d67c..9977e1c84 100644 --- a/build/systemd/spinifex.target +++ b/build/systemd/spinifex.target @@ -2,7 +2,7 @@ Description=Spinifex Infrastructure Platform After=network-online.target Wants=network-online.target -Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-shutdown.service openvswitch-switch.service ovn-controller.service +Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service [Install] WantedBy=multi-user.target