Merge branch 'feat/iso-installer' of github.com:mulgadc/spinifex into feat/iso-installer

scripts/check-coverage.sh

@@ -13,7 +13,12 @@ if [[ ! -s "$PROFILE" ]]; then
     exit 1
 fi
 
-TOTAL=$(go tool cover -func="$PROFILE" | tail -1 | awk '{print $NF}' | tr -d '%')
+# Filter out packages excluded from coverage requirements (e.g. migrations)
+FILTERED=$(mktemp)
+trap 'rm -f "$FILTERED"' EXIT
+awk 'NR==1 || !/\/migrate\//' "$PROFILE" > "$FILTERED"
+
+TOTAL=$(go tool cover -func="$FILTERED" | tail -1 | awk '{print $NF}' | tr -d '%')
 
 if [[ -z "$TOTAL" ]]; then
     echo "ERROR: No coverage data generated — tests may have failed to compile"

scripts/create-multi-node.sh

@@ -39,7 +39,26 @@ echo "Starting init node (127.0.0.1) and joining nodes (127.0.0.2, 127.0.0.3)...
 INIT_PID=$!
 
 # Wait for formation server to start
-sleep 2
+echo "Waiting for formation server..."
+for i in $(seq 1 30); do
+  if curl -sk "https://127.0.0.1:4432/formation/health" > /dev/null 2>&1; then
+    echo "Formation server ready"
+    break
+  fi
+  if [ "$i" -eq 30 ]; then
+    echo "ERROR: Formation server failed to start"
+    kill $INIT_PID 2>/dev/null || true
+    exit 1
+  fi
+  sleep 1
+done
+
+# Read join token
+JOIN_TOKEN=$(cat ~/node1/config/join-token)
+if [ -z "$JOIN_TOKEN" ]; then
+  echo "ERROR: Join token file is empty"
+  exit 1
+fi
 
 # Join nodes concurrently
 ./bin/spx admin join \
@@ -47,6 +66,7 @@ sleep 2
   --bind 127.0.0.2 \
   --cluster-bind 127.0.0.2 \
   --host 127.0.0.1:4432 \
+  --token "$JOIN_TOKEN" \
   --data-dir ~/node2/ \
   --config-dir ~/node2/config/ \
   --region ap-southeast-2 \
@@ -58,6 +78,7 @@ JOIN2_PID=$!
   --bind 127.0.0.3 \
   --cluster-bind 127.0.0.3 \
   --host 127.0.0.1:4432 \
+  --token "$JOIN_TOKEN" \
   --data-dir ~/node3/ \
   --config-dir ~/node3/config/ \
   --region ap-southeast-2 \

scripts/diff-coverage.sh

@@ -66,7 +66,7 @@ trap 'rm -rf "$TMPDIR"' EXIT
 # Only non-test .go files that were added or modified.
 # -M enables rename detection so file splits/moves only count truly changed lines.
 # -C enables copy detection for the same reason (file split = partial copy).
-git diff "${BASE_REF}" HEAD --unified=0 -M -C --diff-filter=AM -- '*.go' ':!*_test.go' | awk '
+git diff "${BASE_REF}" HEAD --unified=0 -M -C --diff-filter=AM -- '*.go' ':!*_test.go' ':!spinifex/migrate/*' | awk '
     /^\+\+\+ / {
         file = substr($2, 3)
         next

scripts/iac/configure-cluster.sh

@@ -121,6 +121,14 @@ while ! curl -sk --connect-timeout 3 "https://$NODE1_IP:4432/formation/health" >
 done
 echo "    Formation server ready"
 
+# Read join token from init node
+JOIN_TOKEN=$(remote "$NODE1_IP" "cat ~/spinifex/config/join-token")
+if [ -z "$JOIN_TOKEN" ]; then
+    echo "Error: Failed to read join token from $NODE1_IP"
+    exit 1
+fi
+echo "    Join token loaded from node1"
+
 # Nodes 2..N: join
 if [ "$NODE_COUNT" -gt 1 ]; then
     JOIN_PIDS=()
@@ -134,6 +142,7 @@ if [ "$NODE_COUNT" -gt 1 ]; then
             --cluster-bind $ip \
             --cluster-routes $NODE1_IP:4248 \
             --host $NODE1_IP:4432 \
+            --token $JOIN_TOKEN \
             --data-dir ~/spinifex/ \
             --config-dir ~/spinifex/config/ \
             --region $REGION \

scripts/start-dev.sh

@@ -271,19 +271,33 @@ if has_service "nats"; then
     # Wait for NATS JetStream to be ready before Predastore starts.
     # Predastore lazily opens IAM KV buckets — it starts with config-only auth
     # and activates IAM auth once the spinifex daemon creates the KV buckets.
-    echo "🔍 Waiting for NATS JetStream..."
-    NATS_JS_READY=false
-    for i in $(seq 1 15); do
-        # Send a NATS INFO probe — JetStream-enabled servers include "jetstream" in the INFO response
-        if echo "" | nc -w 1 "$NATS_CHECK_HOST" 4222 2>/dev/null | grep -q "jetstream"; then
-            echo "   ✅ NATS JetStream is ready"
-            NATS_JS_READY=true
-            break
+    # Wait for NATS JetStream to be ready before Predastore starts.
+    # Predastore lazily opens IAM KV buckets — it starts with config-only auth
+    # and activates IAM auth once the spinifex daemon creates the KV buckets.
+    # Use the monitoring HTTP endpoint (plaintext nc probe doesn't work with TLS).
+    # Monitoring is only on the primary node; secondary nodes skip this check.
+    NATS_HAS_MONITORING=false
+    if [ -f "$CONFIG_DIR/nats/nats.conf" ] && grep -q '^http:' "$CONFIG_DIR/nats/nats.conf"; then
+        NATS_HAS_MONITORING=true
+    fi
+
+    if [ "$NATS_HAS_MONITORING" = "true" ]; then
+        echo "🔍 Waiting for NATS JetStream..."
+        NATS_JS_READY=false
+        for i in $(seq 1 15); do
+            if curl -sf "http://127.0.0.1:8222/jsz" > /dev/null 2>&1; then
+                echo "   ✅ NATS JetStream is ready"
+                NATS_JS_READY=true
+                break
+            fi
+            sleep 1
+        done
+        if [ "$NATS_JS_READY" = "false" ]; then
+            echo "   ⚠️  NATS JetStream not confirmed ready — Predastore may fall back to config-only auth"
         fi
-        sleep 1
-    done
-    if [ "$NATS_JS_READY" = "false" ]; then
-        echo "   ⚠️  NATS JetStream not confirmed ready — Predastore may fall back to config-only auth"
+    else
+        echo "🔍 NATS monitoring not configured — skipping JetStream readiness check"
+        sleep 2
     fi
 else
     echo "1️⃣  Skipping NATS (not a local service)"
@@ -329,7 +343,11 @@ if has_service "predastore"; then
     PREDASTORE_CMD="./bin/spx service predastore start"
     start_service "predastore" "$PREDASTORE_CMD"
     set_oom_score "predastore" "-500"
-    check_service "Predastore" "$SPINIFEX_PREDASTORE_HOST" "$SPINIFEX_PREDASTORE_PORT"
+    if is_multinode; then
+        echo "   ⏭️  Skipping Predastore connectivity check (multi-node: needs quorum from peer nodes)"
+    else
+        check_service "Predastore" "$SPINIFEX_PREDASTORE_HOST" "$SPINIFEX_PREDASTORE_PORT"
+    fi
 else
     echo "2️⃣  Skipping Predastore (not a local service)"
 fi