diff --git a/TODO.md b/TODO.md index 0effcf57d..3aab8730b 100644 --- a/TODO.md +++ b/TODO.md @@ -9,13 +9,13 @@ - Implement VPC support using Open vSwitch across multiple nodes, core VPC functionality included - Add support for NVIDIA Bluefield DPU with Open vSwitch - Implement basic IAM using NATS Jetstream as KV store, vs IAM/access-keys in local config/TOML files for beta. - - Move `daemon.go` instances.json state to Jetstream KV + - [DONE] Move `daemon.go` instances.json state to Jetstream KV - Add support using the `hive` CLI tool to provision a new user with AWS access-keys/IAM. - Support multi-tenant operations and isolation - Add support to include capabilities when adding a new hardware node to MulgaOS (e.g EC2 target, S3, EBS, NATs, etc) - Features can be turned on/off depending on hardware scope. - [DONE] Add simple Web UI console, using the AWS JS SDK, communicating to local AWS gateway. - - Implement ShadCNblocks for UI framework - - Simple Go webserver, static files, easy build process. + - [DONE] Implement ShadCNblocks for UI framework + - [DONE] Simple Go webserver, static files, easy build process. ### Implementation gaps: diff --git a/cmd/hive/cmd/admin.go b/cmd/hive/cmd/admin.go index 940be1fc8..61a7e4c66 100644 --- a/cmd/hive/cmd/admin.go +++ b/cmd/hive/cmd/admin.go @@ -529,8 +529,8 @@ func runAdminInit(cmd *cobra.Command, args []string) { fmt.Printf(" Secret Key: %s\n", secretKey) fmt.Printf(" Account ID: %s\n", accountID) - // Generate SSL certificates - certPath := admin.GenerateCertificatesIfNeeded(configDir, force) + // Generate SSL certificates (with bind IP in SANs for multi-node support) + certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP) // Generate NATS token natsToken := admin.GenerateNATSToken() @@ -804,10 +804,34 @@ func runAdminJoin(cmd *cobra.Command, args []string) { fmt.Printf("✅ Config saved to: %s\n\n", hiveTomlPath) - // Generate certificates if needed - certPath := admin.GenerateCertificatesIfNeeded(configDir, false) + // Handle CA and server certificate generation + // If leader provided CA files, use those; otherwise fall back to generating our own + if joinResp.CACert != "" && joinResp.CAKey != "" { + // Write CA files received from leader + caCertPath := filepath.Join(configDir, "ca.pem") + caKeyPath := filepath.Join(configDir, "ca.key") - fmt.Printf("✅ SSL certificates available at: %s\n\n", certPath) + if err := os.WriteFile(caCertPath, []byte(joinResp.CACert), 0644); err != nil { + fmt.Fprintf(os.Stderr, "Error writing CA cert: %v\n", err) + os.Exit(1) + } + if err := os.WriteFile(caKeyPath, []byte(joinResp.CAKey), 0600); err != nil { + fmt.Fprintf(os.Stderr, "Error writing CA key: %v\n", err) + os.Exit(1) + } + fmt.Printf("✅ CA certificate received from leader: %s\n", caCertPath) + + // Generate server cert signed by CA with this node's bind IP + if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil { + fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err) + os.Exit(1) + } + fmt.Printf("✅ Server certificate generated with bind IP: %s\n\n", bindIP) + } else { + // Fallback: generate own CA if leader didn't provide one (backwards compatibility) + certPath := admin.GenerateCertificatesIfNeeded(configDir, false, bindIP) + fmt.Printf("✅ SSL certificates available at: %s\n\n", certPath) + } // Write individual node config files portStr := fmt.Sprintf("%d", port) diff --git a/hive/admin/admin.go b/hive/admin/admin.go index 9416afe43..5aabaa567 100644 --- a/hive/admin/admin.go +++ b/hive/admin/admin.go @@ -78,7 +78,7 @@ func GenerateConfigFile(configPath string, configTemplate string, configSettings return nil } -func GenerateCertificatesIfNeeded(configDir string, force bool) (caCertPath string) { +func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (caCertPath string) { // Certificate paths caCertPath = filepath.Join(configDir, "ca.pem") caKeyPath := filepath.Join(configDir, "ca.key") @@ -102,8 +102,8 @@ func GenerateCertificatesIfNeeded(configDir string, force bool) (caCertPath stri fmt.Printf(" CA Certificate: %s\n", caCertPath) fmt.Printf(" CA Key: %s\n", caKeyPath) - // Step 2: Generate server certificate signed by CA - if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath); err != nil { + // Step 2: Generate server certificate signed by CA (with bind IP in SANs) + if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP); err != nil { fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err) os.Exit(1) } @@ -123,6 +123,23 @@ func GenerateCertificatesIfNeeded(configDir string, force bool) (caCertPath stri return caCertPath } +// GenerateServerCertOnly generates a server certificate signed by an existing CA. +// Used by joining nodes that receive the CA from the leader. +func GenerateServerCertOnly(configDir string, bindIP string) error { + caCertPath := filepath.Join(configDir, "ca.pem") + caKeyPath := filepath.Join(configDir, "ca.key") + serverCertPath := filepath.Join(configDir, "server.pem") + serverKeyPath := filepath.Join(configDir, "server.key") + + // Verify CA files exist + if !FileExists(caCertPath) || !FileExists(caKeyPath) { + return fmt.Errorf("CA files not found in %s", configDir) + } + + // Generate server cert signed by CA with this node's bind IP + return GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP) +} + func CreateServiceDirectories(hiveRoot string) { // Create additional directories @@ -303,8 +320,9 @@ func GenerateCACert(caCertPath, caKeyPath string) error { return nil } -// GenerateSignedCert generates a server certificate signed by the CA -func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string) error { +// GenerateSignedCert generates a server certificate signed by the CA. +// extraIPs are additional IP addresses to include in the certificate's SANs. +func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIPs ...string) error { // Load CA certificate caCertPEM, err := os.ReadFile(caCertPath) if err != nil { @@ -357,6 +375,16 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string) error { return fmt.Errorf("failed to generate serial number: %w", err) } + // Build IP list: localhost IPs + any extra IPs (e.g., bind IP for multi-node) + ipAddresses := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")} + for _, ip := range extraIPs { + if ip != "" && ip != "127.0.0.1" && ip != "::1" && ip != "0.0.0.0" { + if parsed := net.ParseIP(ip); parsed != nil { + ipAddresses = append(ipAddresses, parsed) + } + } + } + serverTemplate := x509.Certificate{ SerialNumber: serialNumber, Subject: pkix.Name{ @@ -369,7 +397,7 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string) error { ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, BasicConstraintsValid: true, DNSNames: []string{"localhost"}, - IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + IPAddresses: ipAddresses, } // Sign the server certificate with the CA diff --git a/hive/config/config.go b/hive/config/config.go index 7c87f8271..0922c3128 100644 --- a/hive/config/config.go +++ b/hive/config/config.go @@ -175,6 +175,9 @@ type NodeJoinResponse struct { SharedData *SharedClusterData `json:"shared_data,omitempty"` ConfigHash string `json:"config_hash,omitempty"` JoiningNode string `json:"joining_node,omitempty"` // The node name that is joining + // CA distribution for per-node certificate generation + CACert string `json:"ca_cert,omitempty"` // PEM-encoded CA certificate + CAKey string `json:"ca_key,omitempty"` // PEM-encoded CA private key } type NodeHealthResponse struct { diff --git a/hive/daemon/daemon.go b/hive/daemon/daemon.go index 6f24544f9..9772f1419 100644 --- a/hive/daemon/daemon.go +++ b/hive/daemon/daemon.go @@ -13,6 +13,7 @@ import ( "os" "os/exec" "os/signal" + "path/filepath" "regexp" "runtime" "strconv" @@ -784,12 +785,31 @@ func (d *Daemon) ClusterManager() error { Nodes: d.clusterConfig.Nodes, } + // Read CA certificate and key to share with joining node for per-node cert generation + caCertPath := filepath.Join(d.config.BaseDir, "config", "ca.pem") + caKeyPath := filepath.Join(d.config.BaseDir, "config", "ca.key") + + var caCert, caKey string + if caCertPEM, err := os.ReadFile(caCertPath); err == nil { + caCert = string(caCertPEM) + } else { + slog.Warn("Could not read CA cert for join response", "error", err) + } + + if caKeyPEM, err := os.ReadFile(caKeyPath); err == nil { + caKey = string(caKeyPEM) + } else { + slog.Warn("Could not read CA key for join response", "error", err) + } + return c.JSON(config.NodeJoinResponse{ Success: true, Message: fmt.Sprintf("node %s successfully joined cluster", req.Node), SharedData: sharedData, ConfigHash: configHash, JoiningNode: req.Node, + CACert: caCert, + CAKey: caKey, }) }) diff --git a/tests/e2e/lib/multinode-helpers.sh b/tests/e2e/lib/multinode-helpers.sh index f4992365a..aaef66389 100755 --- a/tests/e2e/lib/multinode-helpers.sh +++ b/tests/e2e/lib/multinode-helpers.sh @@ -240,6 +240,160 @@ check_instance_distribution() { echo " Total instances: $total" } +# Find the SSH port for an instance from the QEMU process +# Usage: get_ssh_port +# Returns: the SSH port number, or empty string if not found +get_ssh_port() { + local instance_id="$1" + local max_attempts="${2:-60}" + local attempt=0 + + while [ $attempt -lt $max_attempts ]; do + local qemu_cmd + qemu_cmd=$(ps auxw | grep "$instance_id" | grep qemu-system | grep -v grep || true) + + if [ -n "$qemu_cmd" ]; then + # Extract port from hostfwd=tcp:127.0.0.1:PORT-:22 or hostfwd=tcp::PORT-:22 + local ssh_port + ssh_port=$(echo "$qemu_cmd" | sed -n 's/.*hostfwd=tcp:[^:]*:\([0-9]*\)-:22.*/\1/p') + + if [ -n "$ssh_port" ]; then + echo "$ssh_port" + return 0 + fi + fi + + attempt=$((attempt + 1)) + if [ $attempt -lt $max_attempts ]; then + sleep 1 + fi + done + + return 1 +} + +# Find which node an instance is running on (for multi-node setups) +# Usage: get_instance_node +# Returns: node number (1, 2, or 3), or empty if not found +get_instance_node() { + local instance_id="$1" + + for i in 1 2 3; do + local data_dir="$HOME/node$i" + local instances_file="$data_dir/hive/instances.json" + + if [ -f "$instances_file" ]; then + if jq -e ".[\"$instance_id\"]" "$instances_file" > /dev/null 2>&1; then + echo "$i" + return 0 + fi + fi + done + + return 1 +} + +# Wait for SSH to be ready on an instance +# Usage: wait_for_ssh [max_attempts] +# Returns: 0 if SSH is ready, 1 if timeout +wait_for_ssh() { + local host="$1" + local port="$2" + local key_file="$3" + local max_attempts="${4:-120}" + local attempt=0 + + echo " Waiting for SSH to be ready on $host:$port..." + + while [ $attempt -lt $max_attempts ]; do + # Add host key to known_hosts (suppress errors) + ssh-keyscan -p "$port" "$host" >> ~/.ssh/known_hosts 2>/dev/null || true + + # Try to connect with short timeout + if ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=2 \ + -o BatchMode=yes \ + -p "$port" \ + -i "$key_file" \ + ec2-user@"$host" 'echo ready' > /dev/null 2>&1; then + echo " SSH is ready" + return 0 + fi + + attempt=$((attempt + 1)) + if [ $attempt -lt $max_attempts ]; then + if [ $((attempt % 10)) -eq 0 ]; then + echo " Waiting for SSH... ($attempt/$max_attempts)" + fi + sleep 1 + fi + done + + echo " ERROR: SSH not ready after $max_attempts attempts" + return 1 +} + +# Test SSH connectivity by running 'id' command and verifying ec2-user in output +# Usage: test_ssh_connectivity +# Returns: 0 if successful and ec2-user found, 1 otherwise +test_ssh_connectivity() { + local host="$1" + local port="$2" + local key_file="$3" + + echo " Testing SSH connectivity (running 'id' command)..." + + local id_output + id_output=$(ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 \ + -o BatchMode=yes \ + -p "$port" \ + -i "$key_file" \ + ec2-user@"$host" 'id' 2>&1) || { + echo " ERROR: SSH command failed" + echo " Output: $id_output" + return 1 + } + + echo " SSH 'id' output: $id_output" + + if echo "$id_output" | grep -q "ec2-user"; then + echo " SSH connectivity test passed (ec2-user confirmed)" + return 0 + else + echo " ERROR: Expected 'ec2-user' in id output" + return 1 + fi +} + +# Verify SSH is NOT reachable (used after instance termination) +# Usage: verify_ssh_unreachable +# Returns: 0 if SSH is unreachable (expected), 1 if SSH is still reachable (unexpected) +verify_ssh_unreachable() { + local host="$1" + local port="$2" + local key_file="$3" + + echo " Verifying SSH is no longer reachable..." + + # Try to connect with 1 second timeout - should fail + if ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=1 \ + -o BatchMode=yes \ + -p "$port" \ + -i "$key_file" \ + ec2-user@"$host" 'echo connected' > /dev/null 2>&1; then + echo " ERROR: SSH is still reachable after termination" + return 1 + else + echo " SSH is no longer reachable (as expected)" + return 0 + fi +} + # Dump logs from all nodes (for debugging failures) dump_all_node_logs() { echo "" diff --git a/tests/e2e/run-e2e.sh b/tests/e2e/run-e2e.sh index afc712da5..fb34113e2 100755 --- a/tests/e2e/run-e2e.sh +++ b/tests/e2e/run-e2e.sh @@ -1,6 +1,9 @@ #!/bin/bash set -e +# Source helper functions for SSH testing +source ./tests/e2e/lib/multinode-helpers.sh + # Ensure services are stopped on exit and print logs on failure cleanup() { EXIT_CODE=$? @@ -61,6 +64,11 @@ fi ./bin/hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1 +# Trust the Hive CA certificate for AWS CLI SSL verification +echo "Adding Hive CA certificate to system trust store..." +sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt +sudo update-ca-certificates + # Start all services # Ensure logs directory exists for start-dev.sh mkdir -p ~/hive/logs @@ -71,8 +79,8 @@ HIVE_SKIP_BUILD=true ./scripts/start-dev.sh echo "Waiting for AWS Gateway..." MAX_RETRIES=30 COUNT=0 -# Using -k because of self-signed certs -until curl -k -s https://localhost:9999 > /dev/null || [ $COUNT -eq $MAX_RETRIES ]; do + +until curl -s https://localhost:9999 > /dev/null || [ $COUNT -eq $MAX_RETRIES ]; do echo "Waiting for gateway... ($COUNT/$MAX_RETRIES)" sleep 2 COUNT=$((COUNT + 1)) @@ -235,6 +243,20 @@ if [ "$STATE" != "running" ]; then exit 1 fi +# Phase 5a: SSH Connectivity Test +echo "Phase 5a: SSH Connectivity Test" +echo "Finding SSH port for instance $INSTANCE_ID..." +SSH_PORT=$(get_ssh_port "$INSTANCE_ID" 60) +if [ -z "$SSH_PORT" ]; then + echo "ERROR: Could not find SSH port for instance" + exit 1 +fi +echo "Found SSH port: $SSH_PORT" + +# Wait for SSH to be ready and test connectivity +wait_for_ssh "127.0.0.1" "$SSH_PORT" "test-key-1.pem" 120 +test_ssh_connectivity "127.0.0.1" "$SSH_PORT" "test-key-1.pem" + # Verify root volume attached to the instance (describe-volumes) VOLUME_ID=$($AWS_EC2 describe-volumes --query 'Volumes[0].VolumeId' --output text) if [ -z "$VOLUME_ID" ] || [ "$VOLUME_ID" == "None" ]; then @@ -281,6 +303,18 @@ if [ "$STATE" != "running" ]; then exit 1 fi +# Test SSH connectivity after restart +echo "Testing SSH connectivity after restart..." +# SSH port may have changed after restart, re-detect it +SSH_PORT=$(get_ssh_port "$INSTANCE_ID" 60) +if [ -z "$SSH_PORT" ]; then + echo "ERROR: Could not find SSH port for instance after restart" + exit 1 +fi +echo "Found SSH port after restart: $SSH_PORT" +wait_for_ssh "127.0.0.1" "$SSH_PORT" "test-key-1.pem" 120 +test_ssh_connectivity "127.0.0.1" "$SSH_PORT" "test-key-1.pem" + # Terminate instance (terminate-instances) and verify termination (describe-instances) echo "Terminating instance..." $AWS_EC2 terminate-instances --instance-ids "$INSTANCE_ID" @@ -295,5 +329,9 @@ while [ $COUNT -lt 30 ]; do COUNT=$((COUNT + 1)) done +# Verify SSH is no longer reachable after termination +echo "Phase 5b: Verify SSH Unreachable After Termination" +verify_ssh_unreachable "127.0.0.1" "$SSH_PORT" "test-key-1.pem" + echo "E2E Test Completed Successfully" exit 0 diff --git a/tests/e2e/run-multinode-e2e.sh b/tests/e2e/run-multinode-e2e.sh index 8777df30f..3556bbd60 100755 --- a/tests/e2e/run-multinode-e2e.sh +++ b/tests/e2e/run-multinode-e2e.sh @@ -91,6 +91,12 @@ echo "========================================" echo "" init_leader_node +# Trust the Hive CA certificate for AWS CLI SSL verification +echo "" +echo "Adding Hive CA certificate to system trust store..." +sudo cp ~/node1/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt +sudo update-ca-certificates + # Start node1 services first (leader must be running for join) echo "" echo "Starting node1 services..." @@ -203,130 +209,181 @@ aws_json $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | se echo " AMI verified" # Phase 5: Multi-Node Instance Tests -# echo "" -# echo "Phase 5: Multi-Node Instance Tests" -# echo "========================================" +echo "" +echo "Phase 5: Multi-Node Instance Tests" +echo "========================================" -# # Test 1: Instance Distribution -# echo "" -# echo "Test 1: Instance Distribution" -# echo "----------------------------------------" -# echo "Launching 3 instances to test distribution across nodes..." +# Test 1: Instance Distribution +echo "" +echo "Test 1: Instance Distribution" +echo "----------------------------------------" +echo "Launching 3 instances to test distribution across nodes..." -# INSTANCE_IDS=() -# for i in 1 2 3; do -# echo " Launching instance $i..." -# RUN_OUTPUT=$(aws_json $AWS_EC2 run-instances \ -# --image-id "$AMI_ID" \ -# --instance-type "$INSTANCE_TYPE" \ -# --key-name multinode-test-key) +INSTANCE_IDS=() +for i in 1 2 3; do + echo " Launching instance $i..." + RUN_OUTPUT=$(aws_json $AWS_EC2 run-instances \ + --image-id "$AMI_ID" \ + --instance-type "$INSTANCE_TYPE" \ + --key-name multinode-test-key) -# INSTANCE_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId') -# if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "null" ]; then -# echo " ERROR: Failed to launch instance $i" -# echo " Output: $RUN_OUTPUT" -# exit 1 -# fi -# echo " Launched: $INSTANCE_ID" -# INSTANCE_IDS+=("$INSTANCE_ID") + INSTANCE_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId') + if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "null" ]; then + echo " ERROR: Failed to launch instance $i" + echo " Output: $RUN_OUTPUT" + exit 1 + fi + echo " Launched: $INSTANCE_ID" + INSTANCE_IDS+=("$INSTANCE_ID") -# # Small delay between launches to encourage distribution -# sleep 2 -# done + # Small delay between launches to encourage distribution + sleep 2 +done -# # Wait for all instances to be running -# echo "" -# echo "Waiting for instances to reach running state..." -# for instance_id in "${INSTANCE_IDS[@]}"; do -# wait_for_instance_state "$instance_id" "running" 60 || { -# echo "ERROR: Instance $instance_id failed to start" -# exit 1 -# } -# done +# Wait for all instances to be running +echo "" +echo "Waiting for instances to reach running state..." +for instance_id in "${INSTANCE_IDS[@]}"; do + wait_for_instance_state "$instance_id" "running" 60 || { + echo "ERROR: Instance $instance_id failed to start" + exit 1 + } +done -# # Check distribution -# echo "" -# check_instance_distribution +# Check distribution +echo "" +check_instance_distribution -# # Test 2: DescribeInstances Aggregation -# echo "" -# echo "Test 2: DescribeInstances Aggregation" -# echo "----------------------------------------" -# echo "Verifying all instances are returned via fan-out query..." +# Test 1b: SSH Connectivity +echo "" +echo "Test 1b: SSH Connectivity" +echo "----------------------------------------" +echo "Testing SSH connectivity to first instance..." -# DESCRIBE_OUTPUT=$(aws_json $AWS_EC2 describe-instances --query 'Reservations[*].Instances[*].InstanceId' --output text) -# DESCRIBED_COUNT=$(echo "$DESCRIBE_OUTPUT" | wc -w) +# Test SSH on first instance +SSH_TEST_INSTANCE="${INSTANCE_IDS[0]}" +echo " Test instance: $SSH_TEST_INSTANCE" -# echo " Launched: ${#INSTANCE_IDS[@]} instances" -# echo " Described: $DESCRIBED_COUNT instances" +# Find which node the instance is on and get its SSH port +NODE_NUM=$(get_instance_node "$SSH_TEST_INSTANCE") +if [ -z "$NODE_NUM" ]; then + # Single-node fallback - instance is local + SSH_HOST="127.0.0.1" + echo " Instance location: local (single-node mode)" +else + SSH_HOST="${SIMULATED_NETWORK}.$NODE_NUM" + echo " Instance location: node$NODE_NUM ($SSH_HOST)" +fi -# if [ "$DESCRIBED_COUNT" -lt "${#INSTANCE_IDS[@]}" ]; then -# echo "ERROR: DescribeInstances did not return all instances" -# echo " Expected: ${#INSTANCE_IDS[@]}, Got: $DESCRIBED_COUNT" -# exit 1 -# fi -# echo " Aggregation test passed" +# Get SSH port from QEMU process +SSH_PORT=$(get_ssh_port "$SSH_TEST_INSTANCE" 60) +if [ -z "$SSH_PORT" ]; then + echo " ERROR: Could not find SSH port for instance" + exit 1 +fi +echo " SSH port: $SSH_PORT" -# # Test 3: Cross-Node Operations -# echo "" -# echo "Test 3: Cross-Node Operations" -# echo "----------------------------------------" -# echo "Testing stop/start/terminate via gateway regardless of instance location..." +# Wait for SSH and test connectivity +wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 120 +test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" -# # Pick first instance for cross-node operations -# TEST_INSTANCE="${INSTANCE_IDS[0]}" -# echo " Test instance: $TEST_INSTANCE" +# Test 2: DescribeInstances Aggregation +echo "" +echo "Test 2: DescribeInstances Aggregation" +echo "----------------------------------------" +echo "Verifying all instances are returned via fan-out query..." -# # Stop instance -# echo " Stopping instance..." -# aws_json $AWS_EC2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null -# wait_for_instance_state "$TEST_INSTANCE" "stopped" 30 +DESCRIBE_OUTPUT=$(aws_json $AWS_EC2 describe-instances --query 'Reservations[*].Instances[*].InstanceId' --output text) +DESCRIBED_COUNT=$(echo "$DESCRIBE_OUTPUT" | wc -w) -# # Start instance -# echo " Starting instance..." -# aws_json $AWS_EC2 start-instances --instance-ids "$TEST_INSTANCE" > /dev/null -# wait_for_instance_state "$TEST_INSTANCE" "running" 30 +echo " Launched: ${#INSTANCE_IDS[@]} instances" +echo " Described: $DESCRIBED_COUNT instances" -# echo " Cross-node operations test passed" +if [ "$DESCRIBED_COUNT" -lt "${#INSTANCE_IDS[@]}" ]; then + echo "ERROR: DescribeInstances did not return all instances" + echo " Expected: ${#INSTANCE_IDS[@]}, Got: $DESCRIBED_COUNT" + exit 1 +fi +echo " Aggregation test passed" -# # Test 4: NATS Cluster Health (Post-Operations) -# echo "" -# echo "Test 4: NATS Cluster Health (Post-Operations)" -# echo "----------------------------------------" -# echo "Verifying NATS cluster is still healthy after operations..." +# Test 3: Cross-Node Operations +echo "" +echo "Test 3: Cross-Node Operations" +echo "----------------------------------------" +echo "Testing stop/start/terminate via gateway regardless of instance location..." -# verify_nats_cluster 3 || { -# echo "WARNING: NATS cluster verification failed after operations" -# } +# Pick first instance for cross-node operations +TEST_INSTANCE="${INSTANCE_IDS[0]}" +echo " Test instance: $TEST_INSTANCE" -# # Cleanup: Terminate all test instances -# echo "" -# echo "Cleanup: Terminating test instances" -# echo "----------------------------------------" -# for instance_id in "${INSTANCE_IDS[@]}"; do -# echo " Terminating $instance_id..." -# aws_json $AWS_EC2 terminate-instances --instance-ids "$instance_id" > /dev/null -# done +# Stop instance +echo " Stopping instance..." +aws_json $AWS_EC2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null +wait_for_instance_state "$TEST_INSTANCE" "stopped" 30 -# # Wait for termination - track failures -# echo " Waiting for termination..." -# TERMINATION_FAILED=0 -# for instance_id in "${INSTANCE_IDS[@]}"; do -# if ! wait_for_instance_state "$instance_id" "terminated" 30; then -# echo " WARNING: Failed to confirm termination of $instance_id" -# TERMINATION_FAILED=1 -# fi -# done +# Start instance +echo " Starting instance..." +aws_json $AWS_EC2 start-instances --instance-ids "$TEST_INSTANCE" > /dev/null +wait_for_instance_state "$TEST_INSTANCE" "running" 30 -# if [ $TERMINATION_FAILED -ne 0 ]; then -# echo "" -# echo "ERROR: Some instances failed to terminate properly" -# dump_all_node_logs -# exit 1 -# fi +# Test SSH after cross-node restart +echo " Testing SSH after cross-node restart..." +# Re-detect SSH port (may have changed) +SSH_PORT=$(get_ssh_port "$TEST_INSTANCE" 60) +if [ -z "$SSH_PORT" ]; then + echo " ERROR: Could not find SSH port after restart" + exit 1 +fi +echo " SSH port after restart: $SSH_PORT" +wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 120 +test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" -# echo "" -# echo "========================================" -# echo "Multi-Node E2E Tests Completed Successfully" -# echo "========================================" -# exit 0 +echo " Cross-node operations test passed" + +# Test 4: NATS Cluster Health (Post-Operations) +echo "" +echo "Test 4: NATS Cluster Health (Post-Operations)" +echo "----------------------------------------" +echo "Verifying NATS cluster is still healthy after operations..." + +verify_nats_cluster 3 || { + echo "WARNING: NATS cluster verification failed after operations" +} + +# Cleanup: Terminate all test instances +echo "" +echo "Cleanup: Terminating test instances" +echo "----------------------------------------" +for instance_id in "${INSTANCE_IDS[@]}"; do + echo " Terminating $instance_id..." + aws_json $AWS_EC2 terminate-instances --instance-ids "$instance_id" > /dev/null +done + +# Wait for termination - track failures +echo " Waiting for termination..." +TERMINATION_FAILED=0 +for instance_id in "${INSTANCE_IDS[@]}"; do + if ! wait_for_instance_state "$instance_id" "terminated" 30; then + echo " WARNING: Failed to confirm termination of $instance_id" + TERMINATION_FAILED=1 + fi +done + +if [ $TERMINATION_FAILED -ne 0 ]; then + echo "" + echo "ERROR: Some instances failed to terminate properly" + dump_all_node_logs + exit 1 +fi + +# Verify SSH is no longer reachable after termination +echo "" +echo "Test 5: SSH Unreachable After Termination" +echo "----------------------------------------" +verify_ssh_unreachable "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" + +echo "" +echo "========================================" +echo "Multi-Node E2E Tests Completed Successfully" +echo "========================================" +exit 0