fix: auto-restart NATS when JetStream is disabled while the process lives (#445)

When NATS disables JetStream at runtime (e.g. a filesystem permission error
flushing stream state) it keeps the process alive and serves KV as read-only.
That is not a process failure, so spinifex-nats.service's Restart=on-failure
never fires, and KV-backed startup (awsgw IAM bucket init) stalls for minutes
until an operator restarts NATS by hand.

Add a JetStream liveness watchdog:

- build/scripts/nats-js-watchdog.sh probes the NATS monitoring endpoint
  (http://127.0.0.1:8222/healthz?js-enabled-only=true). Only when the service
  is active and JS fails every sample of a short multi-sample window (so
  transient startup/quorum 503s are ignored) does it systemctl try-restart
  spinifex-nats.
- spinifex-nats-watchdog.{service,timer} run it as a oneshot every 30s, with a
  60s boot delay so cluster quorum settles first.
- spinifex.target Wants= the timer so it starts with the platform.

The NATS-internal permission race itself is a server quirk outside spinifex's
control; this restores service automatically when it recurs.

build/scripts/nats-js-watchdog.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+# JetStream liveness watchdog for spinifex-nats.
+#
+# NATS can disable JetStream at runtime (e.g. a filesystem permission error while
+# flushing stream state) and keep the process alive serving KV as read-only. That
+# is not a process failure, so Restart=on-failure never fires and KV-backed
+# services (awsgw IAM bucket init, etc.) stall until NATS is restarted by hand.
+#
+# This probes the NATS monitoring endpoint's JS health and restarts the service
+# only when JetStream is sustainedly unhealthy while the service is running. The
+# multi-sample window avoids restarting during transient startup/quorum 503s.
+
+set -eu
+
+HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
+SERVICE="spinifex-nats.service"
+SAMPLES=3        # consecutive failing samples required before acting
+SAMPLE_GAP=2     # seconds between samples
+
+# Only act on a running service: a stopped/activating unit is systemd's to manage,
+# and try-restart on it would either no-op or fight an in-flight (re)start.
+if ! systemctl is-active --quiet "$SERVICE"; then
+    exit 0
+fi
+
+# curl is the probe transport; if it is unavailable, do nothing rather than
+# false-restart a healthy server.
+if ! command -v curl >/dev/null 2>&1; then
+    echo "nats-js-watchdog: curl not found, skipping JS health probe" >&2
+    exit 0
+fi
+
+# A single 200 means JetStream is healthy — exit fast on the common case.
+js_healthy() {
+    curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
+}
+
+i=1
+while [ "$i" -le "$SAMPLES" ]; do
+    if js_healthy; then
+        exit 0
+    fi
+    if [ "$i" -lt "$SAMPLES" ]; then
+        sleep "$SAMPLE_GAP"
+    fi
+    i=$((i + 1))
+done
+
+echo "nats-js-watchdog: JetStream unhealthy on $SAMPLES consecutive probes ($HEALTH_URL); restarting $SERVICE" >&2
+systemctl try-restart "$SERVICE"

build/systemd/spinifex-nats-watchdog.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Spinifex NATS JetStream liveness watchdog
+PartOf=spinifex.target
+After=spinifex-nats.service
+
+[Service]
+Type=oneshot
+# Runs as root: the probe is a local curl to the NATS monitoring port, and the
+# remedy is `systemctl try-restart spinifex-nats` when JetStream is disabled while
+# the process stays alive (which Restart=on-failure cannot catch).
+ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
+
+# Hardening (kept compatible with talking to PID1 over /run/systemd).
+NoNewPrivileges=yes
+ProtectHome=yes
+PrivateTmp=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectControlGroups=yes
+RestrictRealtime=yes
+LockPersonality=yes
+RestrictSUIDSGID=yes

build/systemd/spinifex-nats-watchdog.timer

@@ -0,0 +1,13 @@
+[Unit]
+Description=Spinifex NATS JetStream liveness watchdog timer
+PartOf=spinifex.target
+
+[Timer]
+# Delay the first probe so NATS startup + JetStream cluster quorum settle (a
+# forming cluster reports JS not-current transiently, which is not a fault).
+OnBootSec=60s
+OnUnitActiveSec=30s
+AccuracySec=5s
+
+[Install]
+WantedBy=timers.target

build/systemd/spinifex.target

@@ -2,7 +2,7 @@
 Description=Spinifex Infrastructure Platform
 After=network-online.target
 Wants=network-online.target
-Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-shutdown.service openvswitch-switch.service ovn-controller.service
+Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
 
 [Install]
 WantedBy=multi-user.target