Improved multi-server node testing and tweaks

scripts/create-multi-node.sh

@@ -1,67 +1,101 @@
-# simulate node1
+#!/bin/bash
+
+# Simulated multi-node cluster on a single machine using loopback addresses.
+# Uses ~/node{1,2,3}/ for separate data directories.
+# Usage: ./scripts/create-multi-node.sh
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+cd "$PROJECT_ROOT"
+
 echo "Multi-node simulation environment"
-make
+echo "================================="
 
-echo "Adding simulated IPs: 10.11.12.1, 10.11.12.2, 10.11.12.3"
-
-sudo ip addr add 10.11.12.1/24 dev eth0
-
-# simulate node2
-sudo ip addr add 10.11.12.2/24 dev eth0
-
-# simulate node3
-sudo ip addr add 10.11.12.3/24 dev eth0
+# Build first
+make build
 
+echo ""
 echo "Removing old node directories..."
+rm -rf ~/node1/ ~/node2/ ~/node3/
 
-rm -rf ~/node1/
-rm -rf ~/node2/
-rm -rf ~/node3/
+echo ""
+echo "Step 1: Cluster Formation"
+echo "========================="
+echo "Starting init node (127.0.0.1) and joining nodes (127.0.0.2, 127.0.0.3)..."
 
-echo "Node1 Setup:"
-
-# Initialize node1
+# Init blocks until all nodes join, so run in background
 ./bin/hive admin init \
---region ap-southeast-2 \
---az ap-southeast-2a \
---node node1 \
---bind 10.11.12.1 \
---cluster-bind 10.11.12.1 \
---port 4432 \
---hive-dir ~/node1/ \
---config-dir ~/node1/config/
+  --node node1 \
+  --nodes 3 \
+  --bind 127.0.0.1 \
+  --cluster-bind 127.0.0.1 \
+  --port 4432 \
+  --hive-dir ~/node1/ \
+  --config-dir ~/node1/config/ \
+  --region ap-southeast-2 \
+  --az ap-southeast-2a &
+INIT_PID=$!
 
-# Start node1
-./scripts/start-dev.sh ~/node1/
+# Wait for formation server to start
+sleep 2
 
-echo "Node2 Setup:"
-
-# Join cluster
+# Join nodes concurrently
 ./bin/hive admin join \
---region ap-southeast-2 \
---az ap-southeast-2a \
---node node2 \
---bind 10.11.12.2 \
---cluster-bind 10.11.12.2 \
---cluster-routes 10.11.12.1:4248 \
---host 10.11.12.1:4432 \
---data-dir ~/node2/ \
---config-dir ~/node2/config/ \
-
-# Start node2
-./scripts/start-dev.sh ~/node2/
-
-echo "Node3 Setup:"
+  --node node2 \
+  --bind 127.0.0.2 \
+  --cluster-bind 127.0.0.2 \
+  --host 127.0.0.1:4432 \
+  --data-dir ~/node2/ \
+  --config-dir ~/node2/config/ \
+  --region ap-southeast-2 \
+  --az ap-southeast-2a &
+JOIN2_PID=$!
 
 ./bin/hive admin join \
---region ap-southeast-2 \
---az ap-southeast-2a \
---node node3 \
---bind 10.11.12.3 \
---cluster-bind 10.11.12.3 \
---cluster-routes 10.11.12.1:4248 \
---host 10.11.12.1:4432 \
---data-dir ~/node3/ \
---config-dir ~/node3/config/
+  --node node3 \
+  --bind 127.0.0.3 \
+  --cluster-bind 127.0.0.3 \
+  --host 127.0.0.1:4432 \
+  --data-dir ~/node3/ \
+  --config-dir ~/node3/config/ \
+  --region ap-southeast-2 \
+  --az ap-southeast-2a &
+JOIN3_PID=$!
 
-./scripts/start-dev.sh ~/node3/
+# Wait for all formation processes to complete
+wait $INIT_PID $JOIN2_PID $JOIN3_PID
+echo ""
+echo "Cluster formation complete!"
+
+echo ""
+echo "Step 2: Starting Services"
+echo "========================="
+
+# Start node 1 first (Predastore needs quorum)
+HIVE_SKIP_BUILD=true UI=false ./scripts/start-dev.sh ~/node1/ &
+sleep 5
+
+# Start nodes 2 and 3
+HIVE_SKIP_BUILD=true UI=false ./scripts/start-dev.sh ~/node2/ &
+sleep 2
+HIVE_SKIP_BUILD=true UI=false ./scripts/start-dev.sh ~/node3/ &
+
+# Wait for all start scripts to finish
+wait
+
+echo ""
+echo "Step 3: Verification"
+echo "===================="
+sleep 5
+
+echo "Daemon health:"
+for IP in 127.0.0.1 127.0.0.2 127.0.0.3; do
+  echo "  $IP: $(curl -s http://$IP:4432/health | head -c 100)"
+done
+
+echo ""
+echo "Multi-node cluster is running!"
+echo "  Stop with: ./scripts/stop-multi-node.sh"
+echo "  Logs: ~/node{1,2,3}/logs/"