feat: multi-node certs (#11)

2026-10-15 17:11:30 +00:00 · 2026-01-28 11:36:09 +11:00
parent 772da5a181
commit 8f0e8d7ead
8 changed files with 446 additions and 122 deletions
@@ -9,13 +9,13 @@
 - Implement VPC support using Open vSwitch across multiple nodes, core VPC functionality included
  - Add support for NVIDIA Bluefield DPU with Open vSwitch
 - Implement basic IAM using NATS Jetstream as KV store, vs IAM/access-keys in local config/TOML files for beta.
-  - Move `daemon.go` instances.json state to Jetstream KV
+  - [DONE] Move `daemon.go` instances.json state to Jetstream KV
 - Add support using the `hive` CLI tool to provision a new user with AWS access-keys/IAM.
  - Support multi-tenant operations and isolation
 - Add support to include capabilities when adding a new hardware node to MulgaOS (e.g EC2 target, S3, EBS, NATs, etc) - Features can be turned on/off depending on hardware scope.
 - [DONE] Add simple Web UI console, using the AWS JS SDK, communicating to local AWS gateway.
-  - Implement ShadCNblocks for UI framework
-  - Simple Go webserver, static files, easy build process.
+  - [DONE] Implement ShadCNblocks for UI framework
+  - [DONE] Simple Go webserver, static files, easy build process.

 ### Implementation gaps:

@@ -529,8 +529,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	fmt.Printf("   Secret Key: %s\n", secretKey)
 	fmt.Printf("   Account ID: %s\n", accountID)

-	// Generate SSL certificates
-	certPath := admin.GenerateCertificatesIfNeeded(configDir, force)
+	// Generate SSL certificates (with bind IP in SANs for multi-node support)
+	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP)

 	// Generate NATS token
 	natsToken := admin.GenerateNATSToken()
@@ -804,10 +804,34 @@ func runAdminJoin(cmd *cobra.Command, args []string) {

 	fmt.Printf("✅ Config saved to: %s\n\n", hiveTomlPath)

-	// Generate certificates if needed
-	certPath := admin.GenerateCertificatesIfNeeded(configDir, false)
+	// Handle CA and server certificate generation
+	// If leader provided CA files, use those; otherwise fall back to generating our own
+	if joinResp.CACert != "" && joinResp.CAKey != "" {
+		// Write CA files received from leader
+		caCertPath := filepath.Join(configDir, "ca.pem")
+		caKeyPath := filepath.Join(configDir, "ca.key")

-	fmt.Printf("✅ SSL certificates available at: %s\n\n", certPath)
+		if err := os.WriteFile(caCertPath, []byte(joinResp.CACert), 0644); err != nil {
+			fmt.Fprintf(os.Stderr, "Error writing CA cert: %v\n", err)
+			os.Exit(1)
+		}
+		if err := os.WriteFile(caKeyPath, []byte(joinResp.CAKey), 0600); err != nil {
+			fmt.Fprintf(os.Stderr, "Error writing CA key: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("✅ CA certificate received from leader: %s\n", caCertPath)
+
+		// Generate server cert signed by CA with this node's bind IP
+		if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil {
+			fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("✅ Server certificate generated with bind IP: %s\n\n", bindIP)
+	} else {
+		// Fallback: generate own CA if leader didn't provide one (backwards compatibility)
+		certPath := admin.GenerateCertificatesIfNeeded(configDir, false, bindIP)
+		fmt.Printf("✅ SSL certificates available at: %s\n\n", certPath)
+	}

 	// Write individual node config files
 	portStr := fmt.Sprintf("%d", port)
@@ -78,7 +78,7 @@ func GenerateConfigFile(configPath string, configTemplate string, configSettings
 	return nil
 }

-func GenerateCertificatesIfNeeded(configDir string, force bool) (caCertPath string) {
+func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (caCertPath string) {
 	// Certificate paths
 	caCertPath = filepath.Join(configDir, "ca.pem")
 	caKeyPath := filepath.Join(configDir, "ca.key")
@@ -102,8 +102,8 @@ func GenerateCertificatesIfNeeded(configDir string, force bool) (caCertPath stri
 		fmt.Printf("   CA Certificate: %s\n", caCertPath)
 		fmt.Printf("   CA Key: %s\n", caKeyPath)

-		// Step 2: Generate server certificate signed by CA
-		if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath); err != nil {
+		// Step 2: Generate server certificate signed by CA (with bind IP in SANs)
+		if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP); err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
 			os.Exit(1)
 		}
@@ -123,6 +123,23 @@ func GenerateCertificatesIfNeeded(configDir string, force bool) (caCertPath stri
 	return caCertPath
 }

+// GenerateServerCertOnly generates a server certificate signed by an existing CA.
+// Used by joining nodes that receive the CA from the leader.
+func GenerateServerCertOnly(configDir string, bindIP string) error {
+	caCertPath := filepath.Join(configDir, "ca.pem")
+	caKeyPath := filepath.Join(configDir, "ca.key")
+	serverCertPath := filepath.Join(configDir, "server.pem")
+	serverKeyPath := filepath.Join(configDir, "server.key")
+
+	// Verify CA files exist
+	if !FileExists(caCertPath) || !FileExists(caKeyPath) {
+		return fmt.Errorf("CA files not found in %s", configDir)
+	}
+
+	// Generate server cert signed by CA with this node's bind IP
+	return GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP)
+}
+
 func CreateServiceDirectories(hiveRoot string) {

 	// Create additional directories
@@ -303,8 +320,9 @@ func GenerateCACert(caCertPath, caKeyPath string) error {
 	return nil
 }

-// GenerateSignedCert generates a server certificate signed by the CA
-func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string) error {
+// GenerateSignedCert generates a server certificate signed by the CA.
+// extraIPs are additional IP addresses to include in the certificate's SANs.
+func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIPs ...string) error {
 	// Load CA certificate
 	caCertPEM, err := os.ReadFile(caCertPath)
 	if err != nil {
@@ -357,6 +375,16 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string) error {
 		return fmt.Errorf("failed to generate serial number: %w", err)
 	}

+	// Build IP list: localhost IPs + any extra IPs (e.g., bind IP for multi-node)
+	ipAddresses := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}
+	for _, ip := range extraIPs {
+		if ip != "" && ip != "127.0.0.1" && ip != "::1" && ip != "0.0.0.0" {
+			if parsed := net.ParseIP(ip); parsed != nil {
+				ipAddresses = append(ipAddresses, parsed)
+			}
+		}
+	}
+
 	serverTemplate := x509.Certificate{
 		SerialNumber: serialNumber,
 		Subject: pkix.Name{
@@ -369,7 +397,7 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string) error {
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		BasicConstraintsValid: true,
 		DNSNames:              []string{"localhost"},
-		IPAddresses:           []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
+		IPAddresses:           ipAddresses,
 	}

 	// Sign the server certificate with the CA
@@ -175,6 +175,9 @@ type NodeJoinResponse struct {
 	SharedData  *SharedClusterData `json:"shared_data,omitempty"`
 	ConfigHash  string             `json:"config_hash,omitempty"`
 	JoiningNode string             `json:"joining_node,omitempty"` // The node name that is joining
+	// CA distribution for per-node certificate generation
+	CACert string `json:"ca_cert,omitempty"` // PEM-encoded CA certificate
+	CAKey  string `json:"ca_key,omitempty"`  // PEM-encoded CA private key
 }

 type NodeHealthResponse struct {
@@ -13,6 +13,7 @@ import (
 	"os"
 	"os/exec"
 	"os/signal"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"strconv"
@@ -784,12 +785,31 @@ func (d *Daemon) ClusterManager() error {
 			Nodes:   d.clusterConfig.Nodes,
 		}

+		// Read CA certificate and key to share with joining node for per-node cert generation
+		caCertPath := filepath.Join(d.config.BaseDir, "config", "ca.pem")
+		caKeyPath := filepath.Join(d.config.BaseDir, "config", "ca.key")
+
+		var caCert, caKey string
+		if caCertPEM, err := os.ReadFile(caCertPath); err == nil {
+			caCert = string(caCertPEM)
+		} else {
+			slog.Warn("Could not read CA cert for join response", "error", err)
+		}
+
+		if caKeyPEM, err := os.ReadFile(caKeyPath); err == nil {
+			caKey = string(caKeyPEM)
+		} else {
+			slog.Warn("Could not read CA key for join response", "error", err)
+		}
+
 		return c.JSON(config.NodeJoinResponse{
 			Success:     true,
 			Message:     fmt.Sprintf("node %s successfully joined cluster", req.Node),
 			SharedData:  sharedData,
 			ConfigHash:  configHash,
 			JoiningNode: req.Node,
+			CACert:      caCert,
+			CAKey:       caKey,
 		})
 	})

@@ -240,6 +240,160 @@ check_instance_distribution() {
    echo "  Total instances: $total"
 }

+# Find the SSH port for an instance from the QEMU process
+# Usage: get_ssh_port <instance_id>
+# Returns: the SSH port number, or empty string if not found
+get_ssh_port() {
+    local instance_id="$1"
+    local max_attempts="${2:-60}"
+    local attempt=0
+
+    while [ $attempt -lt $max_attempts ]; do
+        local qemu_cmd
+        qemu_cmd=$(ps auxw | grep "$instance_id" | grep qemu-system | grep -v grep || true)
+
+        if [ -n "$qemu_cmd" ]; then
+            # Extract port from hostfwd=tcp:127.0.0.1:PORT-:22 or hostfwd=tcp::PORT-:22
+            local ssh_port
+            ssh_port=$(echo "$qemu_cmd" | sed -n 's/.*hostfwd=tcp:[^:]*:\([0-9]*\)-:22.*/\1/p')
+
+            if [ -n "$ssh_port" ]; then
+                echo "$ssh_port"
+                return 0
+            fi
+        fi
+
+        attempt=$((attempt + 1))
+        if [ $attempt -lt $max_attempts ]; then
+            sleep 1
+        fi
+    done
+
+    return 1
+}
+
+# Find which node an instance is running on (for multi-node setups)
+# Usage: get_instance_node <instance_id>
+# Returns: node number (1, 2, or 3), or empty if not found
+get_instance_node() {
+    local instance_id="$1"
+
+    for i in 1 2 3; do
+        local data_dir="$HOME/node$i"
+        local instances_file="$data_dir/hive/instances.json"
+
+        if [ -f "$instances_file" ]; then
+            if jq -e ".[\"$instance_id\"]" "$instances_file" > /dev/null 2>&1; then
+                echo "$i"
+                return 0
+            fi
+        fi
+    done
+
+    return 1
+}
+
+# Wait for SSH to be ready on an instance
+# Usage: wait_for_ssh <host> <port> <key_file> [max_attempts]
+# Returns: 0 if SSH is ready, 1 if timeout
+wait_for_ssh() {
+    local host="$1"
+    local port="$2"
+    local key_file="$3"
+    local max_attempts="${4:-120}"
+    local attempt=0
+
+    echo "  Waiting for SSH to be ready on $host:$port..."
+
+    while [ $attempt -lt $max_attempts ]; do
+        # Add host key to known_hosts (suppress errors)
+        ssh-keyscan -p "$port" "$host" >> ~/.ssh/known_hosts 2>/dev/null || true
+
+        # Try to connect with short timeout
+        if ssh -o StrictHostKeyChecking=no \
+               -o UserKnownHostsFile=/dev/null \
+               -o ConnectTimeout=2 \
+               -o BatchMode=yes \
+               -p "$port" \
+               -i "$key_file" \
+               ec2-user@"$host" 'echo ready' > /dev/null 2>&1; then
+            echo "  SSH is ready"
+            return 0
+        fi
+
+        attempt=$((attempt + 1))
+        if [ $attempt -lt $max_attempts ]; then
+            if [ $((attempt % 10)) -eq 0 ]; then
+                echo "  Waiting for SSH... ($attempt/$max_attempts)"
+            fi
+            sleep 1
+        fi
+    done
+
+    echo "  ERROR: SSH not ready after $max_attempts attempts"
+    return 1
+}
+
+# Test SSH connectivity by running 'id' command and verifying ec2-user in output
+# Usage: test_ssh_connectivity <host> <port> <key_file>
+# Returns: 0 if successful and ec2-user found, 1 otherwise
+test_ssh_connectivity() {
+    local host="$1"
+    local port="$2"
+    local key_file="$3"
+
+    echo "  Testing SSH connectivity (running 'id' command)..."
+
+    local id_output
+    id_output=$(ssh -o StrictHostKeyChecking=no \
+                    -o UserKnownHostsFile=/dev/null \
+                    -o ConnectTimeout=5 \
+                    -o BatchMode=yes \
+                    -p "$port" \
+                    -i "$key_file" \
+                    ec2-user@"$host" 'id' 2>&1) || {
+        echo "  ERROR: SSH command failed"
+        echo "  Output: $id_output"
+        return 1
+    }
+
+    echo "  SSH 'id' output: $id_output"
+
+    if echo "$id_output" | grep -q "ec2-user"; then
+        echo "  SSH connectivity test passed (ec2-user confirmed)"
+        return 0
+    else
+        echo "  ERROR: Expected 'ec2-user' in id output"
+        return 1
+    fi
+}
+
+# Verify SSH is NOT reachable (used after instance termination)
+# Usage: verify_ssh_unreachable <host> <port> <key_file>
+# Returns: 0 if SSH is unreachable (expected), 1 if SSH is still reachable (unexpected)
+verify_ssh_unreachable() {
+    local host="$1"
+    local port="$2"
+    local key_file="$3"
+
+    echo "  Verifying SSH is no longer reachable..."
+
+    # Try to connect with 1 second timeout - should fail
+    if ssh -o StrictHostKeyChecking=no \
+           -o UserKnownHostsFile=/dev/null \
+           -o ConnectTimeout=1 \
+           -o BatchMode=yes \
+           -p "$port" \
+           -i "$key_file" \
+           ec2-user@"$host" 'echo connected' > /dev/null 2>&1; then
+        echo "  ERROR: SSH is still reachable after termination"
+        return 1
+    else
+        echo "  SSH is no longer reachable (as expected)"
+        return 0
+    fi
+}
+
 # Dump logs from all nodes (for debugging failures)
 dump_all_node_logs() {
    echo ""
@@ -1,6 +1,9 @@
 #!/bin/bash
 set -e

+# Source helper functions for SSH testing
+source ./tests/e2e/lib/multinode-helpers.sh
+
 # Ensure services are stopped on exit and print logs on failure
 cleanup() {
    EXIT_CODE=$?
@@ -61,6 +64,11 @@ fi

 ./bin/hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1

+# Trust the Hive CA certificate for AWS CLI SSL verification
+echo "Adding Hive CA certificate to system trust store..."
+sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt
+sudo update-ca-certificates
+
 # Start all services
 # Ensure logs directory exists for start-dev.sh
 mkdir -p ~/hive/logs
@@ -71,8 +79,8 @@ HIVE_SKIP_BUILD=true ./scripts/start-dev.sh
 echo "Waiting for AWS Gateway..."
 MAX_RETRIES=30
 COUNT=0
-# Using -k because of self-signed certs
-until curl -k -s https://localhost:9999 > /dev/null || [ $COUNT -eq $MAX_RETRIES ]; do
+
+until curl -s https://localhost:9999 > /dev/null || [ $COUNT -eq $MAX_RETRIES ]; do
    echo "Waiting for gateway... ($COUNT/$MAX_RETRIES)"
    sleep 2
    COUNT=$((COUNT + 1))
@@ -235,6 +243,20 @@ if [ "$STATE" != "running" ]; then
    exit 1
 fi

+# Phase 5a: SSH Connectivity Test
+echo "Phase 5a: SSH Connectivity Test"
+echo "Finding SSH port for instance $INSTANCE_ID..."
+SSH_PORT=$(get_ssh_port "$INSTANCE_ID" 60)
+if [ -z "$SSH_PORT" ]; then
+    echo "ERROR: Could not find SSH port for instance"
+    exit 1
+fi
+echo "Found SSH port: $SSH_PORT"
+
+# Wait for SSH to be ready and test connectivity
+wait_for_ssh "127.0.0.1" "$SSH_PORT" "test-key-1.pem" 120
+test_ssh_connectivity "127.0.0.1" "$SSH_PORT" "test-key-1.pem"
+
 # Verify root volume attached to the instance (describe-volumes)
 VOLUME_ID=$($AWS_EC2 describe-volumes --query 'Volumes[0].VolumeId' --output text)
 if [ -z "$VOLUME_ID" ] || [ "$VOLUME_ID" == "None" ]; then
@@ -281,6 +303,18 @@ if [ "$STATE" != "running" ]; then
    exit 1
 fi

+# Test SSH connectivity after restart
+echo "Testing SSH connectivity after restart..."
+# SSH port may have changed after restart, re-detect it
+SSH_PORT=$(get_ssh_port "$INSTANCE_ID" 60)
+if [ -z "$SSH_PORT" ]; then
+    echo "ERROR: Could not find SSH port for instance after restart"
+    exit 1
+fi
+echo "Found SSH port after restart: $SSH_PORT"
+wait_for_ssh "127.0.0.1" "$SSH_PORT" "test-key-1.pem" 120
+test_ssh_connectivity "127.0.0.1" "$SSH_PORT" "test-key-1.pem"
+
 # Terminate instance (terminate-instances) and verify termination (describe-instances)
 echo "Terminating instance..."
 $AWS_EC2 terminate-instances --instance-ids "$INSTANCE_ID"
@@ -295,5 +329,9 @@ while [ $COUNT -lt 30 ]; do
    COUNT=$((COUNT + 1))
 done

+# Verify SSH is no longer reachable after termination
+echo "Phase 5b: Verify SSH Unreachable After Termination"
+verify_ssh_unreachable "127.0.0.1" "$SSH_PORT" "test-key-1.pem"
+
 echo "E2E Test Completed Successfully"
 exit 0
@@ -91,6 +91,12 @@ echo "========================================"
 echo ""
 init_leader_node

+# Trust the Hive CA certificate for AWS CLI SSL verification
+echo ""
+echo "Adding Hive CA certificate to system trust store..."
+sudo cp ~/node1/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt
+sudo update-ca-certificates
+
 # Start node1 services first (leader must be running for join)
 echo ""
 echo "Starting node1 services..."
@@ -203,130 +209,181 @@ aws_json $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | se
 echo "  AMI verified"

 # Phase 5: Multi-Node Instance Tests
-# echo ""
-# echo "Phase 5: Multi-Node Instance Tests"
-# echo "========================================"
+echo ""
+echo "Phase 5: Multi-Node Instance Tests"
+echo "========================================"

-# # Test 1: Instance Distribution
-# echo ""
-# echo "Test 1: Instance Distribution"
-# echo "----------------------------------------"
-# echo "Launching 3 instances to test distribution across nodes..."
+# Test 1: Instance Distribution
+echo ""
+echo "Test 1: Instance Distribution"
+echo "----------------------------------------"
+echo "Launching 3 instances to test distribution across nodes..."

-# INSTANCE_IDS=()
-# for i in 1 2 3; do
-#     echo "  Launching instance $i..."
-#     RUN_OUTPUT=$(aws_json $AWS_EC2 run-instances \
-#         --image-id "$AMI_ID" \
-#         --instance-type "$INSTANCE_TYPE" \
-#         --key-name multinode-test-key)
+INSTANCE_IDS=()
+for i in 1 2 3; do
+    echo "  Launching instance $i..."
+    RUN_OUTPUT=$(aws_json $AWS_EC2 run-instances \
+        --image-id "$AMI_ID" \
+        --instance-type "$INSTANCE_TYPE" \
+        --key-name multinode-test-key)

-#     INSTANCE_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId')
-#     if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "null" ]; then
-#         echo "  ERROR: Failed to launch instance $i"
-#         echo "  Output: $RUN_OUTPUT"
-#         exit 1
-#     fi
-#     echo "  Launched: $INSTANCE_ID"
-#     INSTANCE_IDS+=("$INSTANCE_ID")
+    INSTANCE_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId')
+    if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "null" ]; then
+        echo "  ERROR: Failed to launch instance $i"
+        echo "  Output: $RUN_OUTPUT"
+        exit 1
+    fi
+    echo "  Launched: $INSTANCE_ID"
+    INSTANCE_IDS+=("$INSTANCE_ID")

-#     # Small delay between launches to encourage distribution
-#     sleep 2
-# done
+    # Small delay between launches to encourage distribution
+    sleep 2
+done

-# # Wait for all instances to be running
-# echo ""
-# echo "Waiting for instances to reach running state..."
-# for instance_id in "${INSTANCE_IDS[@]}"; do
-#     wait_for_instance_state "$instance_id" "running" 60 || {
-#         echo "ERROR: Instance $instance_id failed to start"
-#         exit 1
-#     }
-# done
+# Wait for all instances to be running
+echo ""
+echo "Waiting for instances to reach running state..."
+for instance_id in "${INSTANCE_IDS[@]}"; do
+    wait_for_instance_state "$instance_id" "running" 60 || {
+        echo "ERROR: Instance $instance_id failed to start"
+        exit 1
+    }
+done

-# # Check distribution
-# echo ""
-# check_instance_distribution
+# Check distribution
+echo ""
+check_instance_distribution

-# # Test 2: DescribeInstances Aggregation
-# echo ""
-# echo "Test 2: DescribeInstances Aggregation"
-# echo "----------------------------------------"
-# echo "Verifying all instances are returned via fan-out query..."
+# Test 1b: SSH Connectivity
+echo ""
+echo "Test 1b: SSH Connectivity"
+echo "----------------------------------------"
+echo "Testing SSH connectivity to first instance..."

-# DESCRIBE_OUTPUT=$(aws_json $AWS_EC2 describe-instances --query 'Reservations[*].Instances[*].InstanceId' --output text)
-# DESCRIBED_COUNT=$(echo "$DESCRIBE_OUTPUT" | wc -w)
+# Test SSH on first instance
+SSH_TEST_INSTANCE="${INSTANCE_IDS[0]}"
+echo "  Test instance: $SSH_TEST_INSTANCE"

-# echo "  Launched: ${#INSTANCE_IDS[@]} instances"
-# echo "  Described: $DESCRIBED_COUNT instances"
+# Find which node the instance is on and get its SSH port
+NODE_NUM=$(get_instance_node "$SSH_TEST_INSTANCE")
+if [ -z "$NODE_NUM" ]; then
+    # Single-node fallback - instance is local
+    SSH_HOST="127.0.0.1"
+    echo "  Instance location: local (single-node mode)"
+else
+    SSH_HOST="${SIMULATED_NETWORK}.$NODE_NUM"
+    echo "  Instance location: node$NODE_NUM ($SSH_HOST)"
+fi

-# if [ "$DESCRIBED_COUNT" -lt "${#INSTANCE_IDS[@]}" ]; then
-#     echo "ERROR: DescribeInstances did not return all instances"
-#     echo "  Expected: ${#INSTANCE_IDS[@]}, Got: $DESCRIBED_COUNT"
-#     exit 1
-# fi
-# echo "  Aggregation test passed"
+# Get SSH port from QEMU process
+SSH_PORT=$(get_ssh_port "$SSH_TEST_INSTANCE" 60)
+if [ -z "$SSH_PORT" ]; then
+    echo "  ERROR: Could not find SSH port for instance"
+    exit 1
+fi
+echo "  SSH port: $SSH_PORT"

-# # Test 3: Cross-Node Operations
-# echo ""
-# echo "Test 3: Cross-Node Operations"
-# echo "----------------------------------------"
-# echo "Testing stop/start/terminate via gateway regardless of instance location..."
+# Wait for SSH and test connectivity
+wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 120
+test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem"

-# # Pick first instance for cross-node operations
-# TEST_INSTANCE="${INSTANCE_IDS[0]}"
-# echo "  Test instance: $TEST_INSTANCE"
+# Test 2: DescribeInstances Aggregation
+echo ""
+echo "Test 2: DescribeInstances Aggregation"
+echo "----------------------------------------"
+echo "Verifying all instances are returned via fan-out query..."

-# # Stop instance
-# echo "  Stopping instance..."
-# aws_json $AWS_EC2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null
-# wait_for_instance_state "$TEST_INSTANCE" "stopped" 30
+DESCRIBE_OUTPUT=$(aws_json $AWS_EC2 describe-instances --query 'Reservations[*].Instances[*].InstanceId' --output text)
+DESCRIBED_COUNT=$(echo "$DESCRIBE_OUTPUT" | wc -w)

-# # Start instance
-# echo "  Starting instance..."
-# aws_json $AWS_EC2 start-instances --instance-ids "$TEST_INSTANCE" > /dev/null
-# wait_for_instance_state "$TEST_INSTANCE" "running" 30
+echo "  Launched: ${#INSTANCE_IDS[@]} instances"
+echo "  Described: $DESCRIBED_COUNT instances"

-# echo "  Cross-node operations test passed"
+if [ "$DESCRIBED_COUNT" -lt "${#INSTANCE_IDS[@]}" ]; then
+    echo "ERROR: DescribeInstances did not return all instances"
+    echo "  Expected: ${#INSTANCE_IDS[@]}, Got: $DESCRIBED_COUNT"
+    exit 1
+fi
+echo "  Aggregation test passed"

-# # Test 4: NATS Cluster Health (Post-Operations)
-# echo ""
-# echo "Test 4: NATS Cluster Health (Post-Operations)"
-# echo "----------------------------------------"
-# echo "Verifying NATS cluster is still healthy after operations..."
+# Test 3: Cross-Node Operations
+echo ""
+echo "Test 3: Cross-Node Operations"
+echo "----------------------------------------"
+echo "Testing stop/start/terminate via gateway regardless of instance location..."

-# verify_nats_cluster 3 || {
-#     echo "WARNING: NATS cluster verification failed after operations"
-# }
+# Pick first instance for cross-node operations
+TEST_INSTANCE="${INSTANCE_IDS[0]}"
+echo "  Test instance: $TEST_INSTANCE"

-# # Cleanup: Terminate all test instances
-# echo ""
-# echo "Cleanup: Terminating test instances"
-# echo "----------------------------------------"
-# for instance_id in "${INSTANCE_IDS[@]}"; do
-#     echo "  Terminating $instance_id..."
-#     aws_json $AWS_EC2 terminate-instances --instance-ids "$instance_id" > /dev/null
-# done
+# Stop instance
+echo "  Stopping instance..."
+aws_json $AWS_EC2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null
+wait_for_instance_state "$TEST_INSTANCE" "stopped" 30

-# # Wait for termination - track failures
-# echo "  Waiting for termination..."
-# TERMINATION_FAILED=0
-# for instance_id in "${INSTANCE_IDS[@]}"; do
-#     if ! wait_for_instance_state "$instance_id" "terminated" 30; then
-#         echo "  WARNING: Failed to confirm termination of $instance_id"
-#         TERMINATION_FAILED=1
-#     fi
-# done
+# Start instance
+echo "  Starting instance..."
+aws_json $AWS_EC2 start-instances --instance-ids "$TEST_INSTANCE" > /dev/null
+wait_for_instance_state "$TEST_INSTANCE" "running" 30

-# if [ $TERMINATION_FAILED -ne 0 ]; then
-#     echo ""
-#     echo "ERROR: Some instances failed to terminate properly"
-#     dump_all_node_logs
-#     exit 1
-# fi
+# Test SSH after cross-node restart
+echo "  Testing SSH after cross-node restart..."
+# Re-detect SSH port (may have changed)
+SSH_PORT=$(get_ssh_port "$TEST_INSTANCE" 60)
+if [ -z "$SSH_PORT" ]; then
+    echo "  ERROR: Could not find SSH port after restart"
+    exit 1
+fi
+echo "  SSH port after restart: $SSH_PORT"
+wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 120
+test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem"

-# echo ""
-# echo "========================================"
-# echo "Multi-Node E2E Tests Completed Successfully"
-# echo "========================================"
-# exit 0
+echo "  Cross-node operations test passed"
+
+# Test 4: NATS Cluster Health (Post-Operations)
+echo ""
+echo "Test 4: NATS Cluster Health (Post-Operations)"
+echo "----------------------------------------"
+echo "Verifying NATS cluster is still healthy after operations..."
+
+verify_nats_cluster 3 || {
+    echo "WARNING: NATS cluster verification failed after operations"
+}
+
+# Cleanup: Terminate all test instances
+echo ""
+echo "Cleanup: Terminating test instances"
+echo "----------------------------------------"
+for instance_id in "${INSTANCE_IDS[@]}"; do
+    echo "  Terminating $instance_id..."
+    aws_json $AWS_EC2 terminate-instances --instance-ids "$instance_id" > /dev/null
+done
+
+# Wait for termination - track failures
+echo "  Waiting for termination..."
+TERMINATION_FAILED=0
+for instance_id in "${INSTANCE_IDS[@]}"; do
+    if ! wait_for_instance_state "$instance_id" "terminated" 30; then
+        echo "  WARNING: Failed to confirm termination of $instance_id"
+        TERMINATION_FAILED=1
+    fi
+done
+
+if [ $TERMINATION_FAILED -ne 0 ]; then
+    echo ""
+    echo "ERROR: Some instances failed to terminate properly"
+    dump_all_node_logs
+    exit 1
+fi
+
+# Verify SSH is no longer reachable after termination
+echo ""
+echo "Test 5: SSH Unreachable After Termination"
+echo "----------------------------------------"
+verify_ssh_unreachable "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem"
+
+echo ""
+echo "========================================"
+echo "Multi-Node E2E Tests Completed Successfully"
+echo "========================================"
+exit 0