gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improved multi-server node testing and tweaks

Browse Source
This commit is contained in:
Ben Duncan
2026-02-13 20:31:45 +10:00
parent cf12a3272e
commit db0d46ca5b
20 changed files with 2185 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/create-multi-node.sh
+88 -54
View File
@@ -1,67 +1,101 @@
# simulate node1
#!/bin/bash
# Simulated multi-node cluster on a single machine using loopback addresses.
# Uses ~/node{1,2,3}/ for separate data directories.
# Usage: ./scripts/create-multi-node.sh
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_ROOT"
echo "Multi-node simulation environment"
make
echo "================================="
echo "Adding simulated IPs: 10.11.12.1, 10.11.12.2, 10.11.12.3"
sudo ip addr add 10.11.12.1/24 dev eth0
# simulate node2
sudo ip addr add 10.11.12.2/24 dev eth0
# simulate node3
sudo ip addr add 10.11.12.3/24 dev eth0
# Build first
make build
echo ""
echo "Removing old node directories..."
rm -rf ~/node1/ ~/node2/ ~/node3/
rm -rf ~/node1/
rm -rf ~/node2/
rm -rf ~/node3/
echo ""
echo "Step 1: Cluster Formation"
echo "========================="
echo "Starting init node (127.0.0.1) and joining nodes (127.0.0.2, 127.0.0.3)..."
echo "Node1 Setup:"
# Initialize node1
# Init blocks until all nodes join, so run in background
./bin/hive admin init \
--region ap-southeast-2 \
--az ap-southeast-2a \
--node node1 \
--bind 10.11.12.1 \
--cluster-bind 10.11.12.1 \
--port 4432 \
--hive-dir ~/node1/ \
--config-dir ~/node1/config/
--node node1 \
--nodes 3 \
--bind 127.0.0.1 \
--cluster-bind 127.0.0.1 \
--port 4432 \
--hive-dir ~/node1/ \
--config-dir ~/node1/config/ \
--region ap-southeast-2 \
--az ap-southeast-2a &
INIT_PID=$!
# Start node1
./scripts/start-dev.sh ~/node1/
# Wait for formation server to start
sleep 2
echo "Node2 Setup:"
# Join cluster
# Join nodes concurrently
./bin/hive admin join \
--region ap-southeast-2 \
--az ap-southeast-2a \
--node node2 \
--bind 10.11.12.2 \
--cluster-bind 10.11.12.2 \
--cluster-routes 10.11.12.1:4248 \
--host 10.11.12.1:4432 \
--data-dir ~/node2/ \
--config-dir ~/node2/config/ \
# Start node2
./scripts/start-dev.sh ~/node2/
echo "Node3 Setup:"
--node node2 \
--bind 127.0.0.2 \
--cluster-bind 127.0.0.2 \
--host 127.0.0.1:4432 \
--data-dir ~/node2/ \
--config-dir ~/node2/config/ \
--region ap-southeast-2 \
--az ap-southeast-2a &
JOIN2_PID=$!
./bin/hive admin join \
--region ap-southeast-2 \
--az ap-southeast-2a \
--node node3 \
--bind 10.11.12.3 \
--cluster-bind 10.11.12.3 \
--cluster-routes 10.11.12.1:4248 \
--host 10.11.12.1:4432 \
--data-dir ~/node3/ \
--config-dir ~/node3/config/
--node node3 \
--bind 127.0.0.3 \
--cluster-bind 127.0.0.3 \
--host 127.0.0.1:4432 \
--data-dir ~/node3/ \
--config-dir ~/node3/config/ \
--region ap-southeast-2 \
--az ap-southeast-2a &
JOIN3_PID=$!
./scripts/start-dev.sh ~/node3/
# Wait for all formation processes to complete
wait $INIT_PID $JOIN2_PID $JOIN3_PID
echo ""
echo "Cluster formation complete!"
echo ""
echo "Step 2: Starting Services"
echo "========================="
# Start node 1 first (Predastore needs quorum)
HIVE_SKIP_BUILD=true UI=false ./scripts/start-dev.sh ~/node1/ &
sleep 5
# Start nodes 2 and 3
HIVE_SKIP_BUILD=true UI=false ./scripts/start-dev.sh ~/node2/ &
sleep 2
HIVE_SKIP_BUILD=true UI=false ./scripts/start-dev.sh ~/node3/ &
# Wait for all start scripts to finish
wait
echo ""
echo "Step 3: Verification"
echo "===================="
sleep 5
echo "Daemon health:"
for IP in 127.0.0.1 127.0.0.2 127.0.0.3; do
echo " $IP: $(curl -s http://$IP:4432/health | head -c 100)"
done
echo ""
echo "Multi-node cluster is running!"
echo " Stop with: ./scripts/stop-multi-node.sh"
echo " Logs: ~/node{1,2,3}/logs/"
scripts/openvswitch.sh
+56
View File
@@ -0,0 +1,56 @@
#!/bin/sh
### Setup openvswitch
echo "Install and enable OVS:"
sudo apt update
sudo apt install -y openvswitch-switch
sudo systemctl enable --now openvswitch-switch
echo "Sysctls (overlay-friendly, avoids rp_filter pain):"
sudo tee /etc/sysctl.d/99-mulga-vpc.conf >/dev/null <<'EOF'
net.ipv4.ip_forward=1
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
EOF
sudo sysctl --system
echo "1) Create br-int on BOTH hosts"
sudo ovs-vsctl --may-exist add-br br-int
sudo ip link set br-int up
echo "Creating TAP for tap-vm1"
# Consider, adduser hive, for all services used.
sudo ip tuntap add dev tap-vm1 mode tap user $USER
sudo ip link set tap-vm1 mtu 1450
sudo ip link set tap-vm1 up
ip link show tap-vm1
echo "Creating flows on host1"
sudo ovs-ofctl del-flows br-int
sudo ovs-vsctl --may-exist add-port br-int tap-vm1
# VM -> Geneve (set VNI)
sudo ovs-ofctl add-flow br-int "in_port=tap-vm1,actions=set_tunnel:10302,output:geneve-h2"
# Geneve -> VM (only that VNI)
sudo ovs-ofctl add-flow br-int "in_port=geneve-h2,tun_id=10302,actions=output:tap-vm1"
# Creating
#echo "Creating flows on host2"
#sudo ovs-vsctl --may-exist add-port br-int tap-vm1
#sudo ovs-ofctl del-flows br-int
# VM -> Geneve (set VNI)
#sudo ovs-ofctl add-flow br-int "in_port=tap-vm1,actions=set_tunnel:10302,output:geneve-h1"
# Geneve -> VM (only that VNI)
#sudo ovs-ofctl add-flow br-int "in_port=geneve-h1,tun_id=10302,actions=output:tap-vm1"
scripts/stop-multi-node.sh
+13 -12
View File
@@ -1,17 +1,18 @@
#!/bin/bash
#sudo ip addr del 10.11.12.1/24 dev eth0
#sudo ip addr del 10.11.12.2/24 dev eth0
#sudo ip addr del 10.11.12.3/24 dev eth0
# Stop simulated multi-node cluster.
# Stops services in reverse order (node3 → node2 → node1).
# Usage: ./scripts/stop-multi-node.sh
# TODO: Reverse order, predastore running on node1
# BUG: Need to wait for each QEMU to flush/commit WAL to disk > S3, before shutdown, else corruption may occur
# BUG: nbdkit: ramdisk.3: debug: sending error reply: Cannot send after transport endpoint shutdown
# TODO: Fallback, if WAL on local disk not flushed, on boot, viperblock should commit and verify. Requires UT and test-cases
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
./scripts/stop-dev.sh ~/node3/
./scripts/stop-dev.sh ~/node2/
./scripts/stop-dev.sh ~/node1/
echo "Stopping multi-node cluster..."
# Stop in reverse order — node1 last since it hosts the Predastore leader
"$SCRIPT_DIR/stop-dev.sh" ~/node3/
"$SCRIPT_DIR/stop-dev.sh" ~/node2/
"$SCRIPT_DIR/stop-dev.sh" ~/node1/
echo ""
echo "Multi-node cluster stopped."
echo "To remove all data: rm -rf ~/node1/ ~/node2/ ~/node3/"