diff --git a/scripts/dev-setup.sh b/scripts/dev-setup.sh index bd4b429f3..b8ca21ea0 100755 --- a/scripts/dev-setup.sh +++ b/scripts/dev-setup.sh @@ -13,17 +13,9 @@ MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)" NATS_PORT=4222 PREDASTORE_PORT=8443 SPINIFEX_GATEWAY_PORT=9999 -DATA_DIR="$HOME/spinifex" -# Use CONFIG_DIR environment variable if set, otherwise default to ~/spinifex/config -CONFIG_DIR="${CONFIG_DIR:-$HOME/spinifex/config}" echo "๐Ÿš€ Setting up Spinifex development environment..." echo "Project root: $PROJECT_ROOT" -echo "Data directory: $DATA_DIR" - -# Create necessary directories -mkdir -p "$DATA_DIR"/{nats,predastore,viperblock,logs,spinifex} -mkdir -p "$CONFIG_DIR" # Function to check if command exists command_exists() { @@ -148,19 +140,6 @@ cd "$PROJECT_ROOT" make build echo "โœ… Spinifex built successfully" -# Initialize Spinifex configuration using admin init -#echo "" -#echo "๐Ÿ” Initializing Spinifex configuration..." - -#if [[ ! -f "$CONFIG_DIR/spinifex.toml" ]]; then -# echo "๐Ÿ“‹ Running spx admin init..." -# ./bin/spx admin init --config-dir "$CONFIG_DIR" -# echo "โœ… Spinifex configuration initialized" -#else -# echo "โœ… Spinifex configuration already exists" -# echo " To re-initialize, run: ./bin/spx admin init --force" -#fi - # Build components echo "" echo "๐Ÿ”จ Building components..." @@ -188,23 +167,26 @@ fi echo "" echo "๐ŸŽ‰ Development environment setup complete!" echo "" -echo "When running Spinifex for the first time, run the init function to create the" -echo "default directories for data, config files and layout required." -echo "./bin/spx admin init" +echo "๐Ÿ“‹ Next steps:" echo "" -echo "๐Ÿš€ To start the development environment:" -echo " ./scripts/start-dev.sh" +echo " 1. Setup OVN networking:" +echo " ./scripts/setup-ovn.sh --management" echo "" -echo "๐Ÿ›‘ To stop the development environment:" -echo " ./scripts/stop-dev.sh" +echo " 2. Initialize Spinifex (creates ~/spinifex/ directories, config, and certs):" +echo " ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1" +echo "" +echo " 3. Trust the CA certificate:" +echo " sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt" +echo " sudo update-ca-certificates" +echo "" +echo " 4. Start services:" +echo " ./scripts/start-dev.sh" echo "" echo "๐Ÿ”ง Development endpoints:" echo " - Spinifex Gateway: https://localhost:$SPINIFEX_GATEWAY_PORT" echo " - Predastore S3: https://localhost:$PREDASTORE_PORT" echo " - NATS: nats://localhost:$NATS_PORT" echo "" -echo "๐Ÿ“Š Monitor logs:" -echo " tail -f $DATA_DIR/logs/*.log" -echo "" echo "๐Ÿงช Test with AWS CLI:" -echo " aws --endpoint-url https://localhost:$SPINIFEX_GATEWAY_PORT --no-verify-ssl ec2 describe-instances" +echo " export AWS_PROFILE=spinifex" +echo " aws ec2 describe-instances" diff --git a/tests/e2e/lib/multinode-helpers.sh b/tests/e2e/lib/multinode-helpers.sh index ba57c18fa..8394079e2 100755 --- a/tests/e2e/lib/multinode-helpers.sh +++ b/tests/e2e/lib/multinode-helpers.sh @@ -553,7 +553,7 @@ wait_for_instance_recovery() { while [ $attempt -lt $max_attempts ]; do local state - state=$(aws --endpoint-url https://${NODE1_IP}:${AWSGW_PORT} ec2 describe-instances \ + state=$(aws ec2 describe-instances \ --instance-ids "$instance_id" \ --query 'Reservations[0].Instances[0].State.Name' \ --output text 2>/dev/null) || { diff --git a/tests/e2e/run-multinode-e2e.sh b/tests/e2e/run-multinode-e2e.sh index f48269822..be829ad61 100755 --- a/tests/e2e/run-multinode-e2e.sh +++ b/tests/e2e/run-multinode-e2e.sh @@ -910,9 +910,223 @@ fi echo "" # ========================================================================== -# Phase 10: Cleanup +# Phase 10: VPC Networking # ========================================================================== -echo "Phase 10: Cleanup" +echo "Phase 10: VPC Networking" +echo "========================================" +echo "Testing VPC instance launch, PrivateIpAddress, and stop/start IP persistence..." + +# Create a fresh VPC + subnet (don't reuse bootstrap's to avoid interference) +echo "" +echo "Creating test VPC + subnet..." +VPC_OUTPUT=$($AWS_EC2 create-vpc --cidr-block 10.200.0.0/16) +VPC_ID=$(echo "$VPC_OUTPUT" | jq -r '.Vpc.VpcId') +if [ -z "$VPC_ID" ] || [ "$VPC_ID" == "null" ]; then + echo " ERROR: Failed to create VPC" + fail_test "VPC create" +else + echo " Created VPC: $VPC_ID (10.200.0.0/16)" + pass_test "VPC create" + + SUBNET_OUTPUT=$($AWS_EC2 create-subnet --vpc-id "$VPC_ID" --cidr-block 10.200.1.0/24) + SUBNET_ID=$(echo "$SUBNET_OUTPUT" | jq -r '.Subnet.SubnetId') + if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" == "null" ]; then + echo " ERROR: Failed to create subnet" + fail_test "Subnet create" + else + echo " Created Subnet: $SUBNET_ID (10.200.1.0/24)" + pass_test "Subnet create" + + # Brief pause for OVN topology programming + sleep 2 + + # Launch 3 instances with subnet + echo "" + echo "Launching 3 VPC instances..." + VPC_INSTANCE_IDS=() + VPC_LAUNCH_OK=true + for i in 1 2 3; do + echo " Launching VPC instance $i with subnet $SUBNET_ID..." + RUN_OUTPUT=$($AWS_EC2 run-instances \ + --image-id "$AMI_ID" \ + --instance-type "$INSTANCE_TYPE" \ + --key-name spinifex-key \ + --subnet-id "$SUBNET_ID") + + VPC_INST_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId') + VPC_INST_IP=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].PrivateIpAddress // empty') + + if [ -z "$VPC_INST_ID" ] || [ "$VPC_INST_ID" == "null" ]; then + echo " ERROR: Failed to launch VPC instance $i" + VPC_LAUNCH_OK=false + break + fi + echo " Launched: $VPC_INST_ID (PrivateIpAddress: ${VPC_INST_IP:-not yet assigned})" + VPC_INSTANCE_IDS+=("$VPC_INST_ID") + sleep 1 + done + + if [ "$VPC_LAUNCH_OK" = true ] && [ ${#VPC_INSTANCE_IDS[@]} -eq 3 ]; then + pass_test "VPC instance launch" + + # Wait for all VPC instances to be running + echo "" + echo "Waiting for VPC instances to reach running state..." + VPC_ALL_RUNNING=true + for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + wait_for_instance_state "$vpc_inst" "running" 60 || { + echo " ERROR: VPC instance $vpc_inst failed to start" + VPC_ALL_RUNNING=false + } + done + + if [ "$VPC_ALL_RUNNING" = true ]; then + # Verify PrivateIpAddress in DescribeInstances + echo "" + echo "Verifying PrivateIpAddress in DescribeInstances..." + VPC_PRIVATE_IPS=() + VPC_IP_OK=true + for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + DESCRIBE_OUT=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst") + PRIVATE_IP=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].PrivateIpAddress // empty') + INST_SUBNET=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].SubnetId // empty') + INST_VPC=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].VpcId // empty') + + if [ -z "$PRIVATE_IP" ]; then + echo " ERROR: $vpc_inst has no PrivateIpAddress" + VPC_IP_OK=false + continue + fi + + echo " $vpc_inst: IP=$PRIVATE_IP, Subnet=$INST_SUBNET, VPC=$INST_VPC" + + if [ "$INST_SUBNET" != "$SUBNET_ID" ]; then + echo " ERROR: SubnetId mismatch (expected $SUBNET_ID, got $INST_SUBNET)" + VPC_IP_OK=false + fi + if [ "$INST_VPC" != "$VPC_ID" ]; then + echo " ERROR: VpcId mismatch (expected $VPC_ID, got $INST_VPC)" + VPC_IP_OK=false + fi + + VPC_PRIVATE_IPS+=("$PRIVATE_IP") + done + + if [ "$VPC_IP_OK" = true ]; then + # Verify all IPs are unique and in the subnet range (10.200.1.x) + UNIQUE_IPS=$(printf '%s\n' "${VPC_PRIVATE_IPS[@]}" | sort -u | wc -l) + IP_RANGE_OK=true + if [ "$UNIQUE_IPS" -ne "${#VPC_PRIVATE_IPS[@]}" ]; then + echo " ERROR: Duplicate IPs detected: ${VPC_PRIVATE_IPS[*]}" + IP_RANGE_OK=false + fi + for ip in "${VPC_PRIVATE_IPS[@]}"; do + if ! echo "$ip" | grep -qE '^10\.200\.1\.[0-9]+$'; then + echo " ERROR: IP $ip not in expected subnet 10.200.1.0/24" + IP_RANGE_OK=false + fi + done + + if [ "$IP_RANGE_OK" = true ]; then + echo " All IPs unique and in correct subnet: ${VPC_PRIVATE_IPS[*]}" + pass_test "VPC IP allocation" + else + fail_test "VPC IP allocation" + fi + + # Stop/Start IP persistence + echo "" + echo "Testing stop/start IP persistence..." + echo " IPs before stop: ${VPC_PRIVATE_IPS[*]}" + + # Stop all VPC instances + for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + $AWS_EC2 stop-instances --instance-ids "$vpc_inst" > /dev/null + done + VPC_ALL_STOPPED=true + for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + wait_for_instance_state "$vpc_inst" "stopped" 60 || { + echo " ERROR: VPC instance $vpc_inst failed to stop" + VPC_ALL_STOPPED=false + } + done + + if [ "$VPC_ALL_STOPPED" = true ]; then + # Verify IPs persist while stopped + IP_PERSIST_OK=true + for idx in "${!VPC_INSTANCE_IDS[@]}"; do + vpc_inst="${VPC_INSTANCE_IDS[$idx]}" + expected_ip="${VPC_PRIVATE_IPS[$idx]}" + STOPPED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \ + --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text) + if [ "$STOPPED_IP" != "$expected_ip" ]; then + echo " ERROR: $vpc_inst IP changed while stopped (expected $expected_ip, got $STOPPED_IP)" + IP_PERSIST_OK=false + fi + done + + # Restart all VPC instances + for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + $AWS_EC2 start-instances --instance-ids "$vpc_inst" > /dev/null + done + for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + wait_for_instance_state "$vpc_inst" "running" 60 || true + done + + # Verify IPs persist after restart + for idx in "${!VPC_INSTANCE_IDS[@]}"; do + vpc_inst="${VPC_INSTANCE_IDS[$idx]}" + expected_ip="${VPC_PRIVATE_IPS[$idx]}" + RESTARTED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \ + --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text) + if [ "$RESTARTED_IP" != "$expected_ip" ]; then + echo " ERROR: $vpc_inst IP changed after restart (expected $expected_ip, got $RESTARTED_IP)" + IP_PERSIST_OK=false + else + echo " $vpc_inst: IP=$RESTARTED_IP (matches pre-stop)" + fi + done + + if [ "$IP_PERSIST_OK" = true ]; then + pass_test "VPC stop/start IP persistence" + else + fail_test "VPC stop/start IP persistence" + fi + fi + else + fail_test "VPC IP verification" + fi + fi + + # Cleanup VPC instances + echo "" + echo "Cleaning up VPC instances..." + terminate_and_wait "${VPC_INSTANCE_IDS[@]}" || true + else + fail_test "VPC instance launch" + # Cleanup any partially-launched VPC instances + if [ ${#VPC_INSTANCE_IDS[@]} -gt 0 ]; then + echo " Cleaning up partially-launched VPC instances..." + terminate_and_wait "${VPC_INSTANCE_IDS[@]}" || true + fi + fi + + # Cleanup subnet + echo " Deleting subnet $SUBNET_ID..." + $AWS_EC2 delete-subnet --subnet-id "$SUBNET_ID" 2>/dev/null || true + fi + + # Cleanup VPC + echo " Deleting VPC $VPC_ID..." + $AWS_EC2 delete-vpc --vpc-id "$VPC_ID" 2>/dev/null || true +fi + +echo "" + +# ========================================================================== +# Phase 11: Cleanup +# ========================================================================== +echo "Phase 11: Cleanup" echo "========================================" echo "Terminating all instances..." diff --git a/tests/e2e/run-pseudo-multinode-e2e.sh b/tests/e2e/run-pseudo-multinode-e2e.sh index ffaad9188..a72f79e94 100755 --- a/tests/e2e/run-pseudo-multinode-e2e.sh +++ b/tests/e2e/run-pseudo-multinode-e2e.sh @@ -276,44 +276,6 @@ echo " AMI ID: $AMI_ID" $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.ImageId==\"$AMI_ID\")" > /dev/null echo " AMI verified" -# Phase 4b: Multi-Node Key Pair Operations -echo "" -echo "Phase 4b: Multi-Node Key Pair Operations" -echo "========================================" -echo "Testing key pair CRUD across the cluster..." - -# Import a key pair (goes to a random node via queue group) -echo " Generating local RSA key for import..." -ssh-keygen -t rsa -b 2048 -f multinode-test-key-2-local -N "" -$AWS_EC2 import-key-pair --key-name multinode-test-key-2 --public-key-material "fileb://multinode-test-key-2-local.pub" -echo " Imported key: multinode-test-key-2" - -# Describe key pairs โ€” verify both keys are visible -echo " Verifying both keys via describe-key-pairs..." -KEY_LIST=$($AWS_EC2 describe-key-pairs --query 'KeyPairs[*].KeyName' --output text) -echo " Keys found: $KEY_LIST" -echo "$KEY_LIST" | grep -q "multinode-test-key" || { - echo " ERROR: multinode-test-key not found in describe-key-pairs" - exit 1 -} -echo "$KEY_LIST" | grep -q "multinode-test-key-2" || { - echo " ERROR: multinode-test-key-2 not found in describe-key-pairs" - exit 1 -} -echo " Both keys visible" - -# Delete the imported key -echo " Deleting multinode-test-key-2..." -$AWS_EC2 delete-key-pair --key-name multinode-test-key-2 - -# Verify deletion -REMAINING_KEYS=$($AWS_EC2 describe-key-pairs --query 'KeyPairs[*].KeyName' --output text) -if echo "$REMAINING_KEYS" | grep -q "multinode-test-key-2"; then - echo " ERROR: multinode-test-key-2 was not deleted" - exit 1 -fi -echo " Key deletion verified" -echo " Multi-node key pair operations passed" # Phase 5: Multi-Node Instance Tests echo "" @@ -463,478 +425,6 @@ done echo "" echo " SSH connectivity and volume verification passed for all instances" -# Test 1b: Volume Lifecycle (Attach/Detach) -echo "" -echo "Test 1b: Volume Lifecycle (Attach/Detach)" -echo "----------------------------------------" -echo "Testing volume create -> resize -> attach -> detach -> delete..." - -# Create a test volume -echo " Creating 10GB volume in ${SPINIFEX_AZ}..." -CREATE_OUTPUT=$($AWS_EC2 create-volume --size 10 --availability-zone "$SPINIFEX_AZ") -TEST_VOLUME_ID=$(echo "$CREATE_OUTPUT" | jq -r '.VolumeId') - -if [ -z "$TEST_VOLUME_ID" ] || [ "$TEST_VOLUME_ID" == "null" ]; then - echo " ERROR: Failed to create test volume" - echo " Output: $CREATE_OUTPUT" - exit 1 -fi -echo " Created volume: $TEST_VOLUME_ID" - -# Resize to 20GB -NEW_SIZE=20 -echo " Modifying volume to ${NEW_SIZE}GB..." -$AWS_EC2 modify-volume --volume-id "$TEST_VOLUME_ID" --size "$NEW_SIZE" - -# Verify resize -echo " Verifying resize..." -COUNT=0 -while [ $COUNT -lt 30 ]; do - VOLUME_SIZE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].Size' --output text) - - if [ "$VOLUME_SIZE" -eq "$NEW_SIZE" ]; then - echo " Volume resized successfully to ${NEW_SIZE}GB" - break - fi - - sleep 1 - COUNT=$((COUNT + 1)) -done - -if [ "$VOLUME_SIZE" -ne "$NEW_SIZE" ]; then - echo " ERROR: Volume failed to resize to ${NEW_SIZE}GB (current: ${VOLUME_SIZE}GB)" - exit 1 -fi - -# Attach volume to the first running instance -echo " Attaching volume $TEST_VOLUME_ID to instance ${INSTANCE_IDS[0]}..." -$AWS_EC2 attach-volume --volume-id "$TEST_VOLUME_ID" --instance-id "${INSTANCE_IDS[0]}" --device /dev/sdf - -# Verify attachment -echo " Verifying volume attachment..." -COUNT=0 -while [ $COUNT -lt 30 ]; do - ATTACH_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].Attachments[0].State' --output text) - ATTACH_INSTANCE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].Attachments[0].InstanceId' --output text) - VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].State' --output text) - - if [ "$VOL_STATE" == "in-use" ] && [ "$ATTACH_STATE" == "attached" ] && [ "$ATTACH_INSTANCE" == "${INSTANCE_IDS[0]}" ]; then - echo " Volume attached successfully (State=$VOL_STATE, AttachState=$ATTACH_STATE, Instance=$ATTACH_INSTANCE)" - break - fi - - sleep 1 - COUNT=$((COUNT + 1)) -done - -if [ "$ATTACH_STATE" != "attached" ] || [ "$ATTACH_INSTANCE" != "${INSTANCE_IDS[0]}" ]; then - echo " ERROR: Volume attachment verification failed (AttachState=$ATTACH_STATE, Instance=$ATTACH_INSTANCE)" - exit 1 -fi - -# Detach volume (without --instance-id to test gateway resolution path) -echo " Detaching volume $TEST_VOLUME_ID..." -$AWS_EC2 detach-volume --volume-id "$TEST_VOLUME_ID" - -# Verify detachment -echo " Verifying volume detachment..." -COUNT=0 -while [ $COUNT -lt 30 ]; do - VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].State' --output text) - - if [ "$VOL_STATE" == "available" ]; then - echo " Volume detached successfully (State=$VOL_STATE)" - break - fi - - sleep 1 - COUNT=$((COUNT + 1)) -done - -if [ "$VOL_STATE" != "available" ]; then - echo " ERROR: Volume detachment verification failed (State=$VOL_STATE)" - exit 1 -fi - -# Delete the test volume -echo " Deleting test volume $TEST_VOLUME_ID..." -$AWS_EC2 delete-volume --volume-id "$TEST_VOLUME_ID" - -# Verify deletion -echo " Verifying volume deletion..." -COUNT=0 -while [ $COUNT -lt 30 ]; do - set +e - VOLUME_CHECK=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].VolumeId' --output text 2>&1) - DESCRIBE_EXIT=$? - set -e - - if [ $DESCRIBE_EXIT -ne 0 ] || [ "$VOLUME_CHECK" == "None" ] || [ -z "$VOLUME_CHECK" ]; then - echo " Volume deleted successfully" - break - fi - - sleep 1 - COUNT=$((COUNT + 1)) -done - -if [ $COUNT -ge 30 ]; then - echo " ERROR: Volume deletion verification timed out" - exit 1 -fi - -echo " Volume lifecycle test passed (create -> resize -> attach -> detach -> delete)" - -# Test 1c: Snapshot Lifecycle -echo "" -echo "Test 1c: Snapshot Lifecycle" -echo "----------------------------------------" -echo "Testing snapshot create -> describe -> copy -> delete..." - -# Use the root volume of the first instance โ€” it's already attached and mounted -# in viperblockd, which is required for create-snapshot. -SNAP_VOL_ID=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[0]}" \ - --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) -echo " Using root volume $SNAP_VOL_ID (attached to ${INSTANCE_IDS[0]})" -SNAP_VOL_SIZE=$($AWS_EC2 describe-volumes --volume-ids "$SNAP_VOL_ID" \ - --query 'Volumes[0].Size' --output text) - -# Create a snapshot -echo " Creating snapshot from volume $SNAP_VOL_ID..." -SNAP_OUTPUT=$($AWS_EC2 create-snapshot --volume-id "$SNAP_VOL_ID" --description "multinode-e2e-snapshot") -SNAPSHOT_ID=$(echo "$SNAP_OUTPUT" | jq -r '.SnapshotId') - -if [ -z "$SNAPSHOT_ID" ] || [ "$SNAPSHOT_ID" == "null" ]; then - echo " ERROR: Failed to create snapshot" - echo " Output: $SNAP_OUTPUT" - exit 1 -fi -echo " Created snapshot: $SNAPSHOT_ID" - -# Verify create response fields -SNAP_STATE=$(echo "$SNAP_OUTPUT" | jq -r '.State') -SNAP_VOL_REF=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeId') -SNAP_SIZE=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeSize') - -if [ "$SNAP_VOL_REF" != "$SNAP_VOL_ID" ]; then - echo " ERROR: Snapshot VolumeId mismatch: expected $SNAP_VOL_ID, got $SNAP_VOL_REF" - exit 1 -fi -if [ "$SNAP_SIZE" -ne "$SNAP_VOL_SIZE" ]; then - echo " ERROR: Snapshot VolumeSize mismatch: expected $SNAP_VOL_SIZE, got $SNAP_SIZE" - exit 1 -fi -echo " Create response verified (State=$SNAP_STATE, VolumeId=$SNAP_VOL_REF, Size=$SNAP_SIZE)" - -# Poll until completed -echo " Waiting for snapshot to complete..." -COUNT=0 -while [ $COUNT -lt 30 ]; do - SNAP_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ - --query 'Snapshots[0].State' --output text) - - if [ "$SNAP_STATE" == "completed" ]; then - echo " Snapshot completed" - break - fi - - sleep 1 - COUNT=$((COUNT + 1)) -done - -if [ "$SNAP_STATE" != "completed" ]; then - echo " ERROR: Snapshot failed to reach completed state (State=$SNAP_STATE)" - exit 1 -fi - -# Describe by ID and verify -echo " Verifying snapshot via describe-snapshots..." -DESCRIBE_SNAP=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID") -DESC_VOL_ID=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeId') -DESC_SIZE=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeSize') -DESC_DESC=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].Description') - -if [ "$DESC_VOL_ID" != "$SNAP_VOL_ID" ]; then - echo " ERROR: Describe VolumeId mismatch: expected $SNAP_VOL_ID, got $DESC_VOL_ID" - exit 1 -fi -if [ "$DESC_SIZE" -ne "$SNAP_VOL_SIZE" ]; then - echo " ERROR: Describe VolumeSize mismatch: expected $SNAP_VOL_SIZE, got $DESC_SIZE" - exit 1 -fi -if [ "$DESC_DESC" != "multinode-e2e-snapshot" ]; then - echo " ERROR: Describe Description mismatch: expected 'multinode-e2e-snapshot', got '$DESC_DESC'" - exit 1 -fi -echo " Describe verified (VolumeId=$DESC_VOL_ID, Size=$DESC_SIZE, Description=$DESC_DESC)" - -# Copy the snapshot -echo " Copying snapshot $SNAPSHOT_ID..." -COPY_OUTPUT=$($AWS_EC2 copy-snapshot --source-snapshot-id "$SNAPSHOT_ID" --source-region "$SPINIFEX_REGION" --description "multinode-e2e-copy") -COPY_SNAPSHOT_ID=$(echo "$COPY_OUTPUT" | jq -r '.SnapshotId') - -if [ -z "$COPY_SNAPSHOT_ID" ] || [ "$COPY_SNAPSHOT_ID" == "null" ]; then - echo " ERROR: Failed to copy snapshot" - echo " Output: $COPY_OUTPUT" - exit 1 -fi -echo " Copied snapshot: $COPY_SNAPSHOT_ID" - -if [ "$COPY_SNAPSHOT_ID" == "$SNAPSHOT_ID" ]; then - echo " ERROR: Copy snapshot ID should differ from original" - exit 1 -fi - -# Verify both exist -TOTAL_SNAPS=$($AWS_EC2 describe-snapshots \ - --snapshot-ids "$SNAPSHOT_ID" "$COPY_SNAPSHOT_ID" \ - --query 'length(Snapshots)' --output text) - -if [ "$TOTAL_SNAPS" -ne 2 ]; then - echo " ERROR: Expected 2 snapshots, got $TOTAL_SNAPS" - exit 1 -fi -echo " Both snapshots visible via describe-snapshots" - -# Verify copy description -COPY_DESC=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \ - --query 'Snapshots[0].Description' --output text) -if [ "$COPY_DESC" != "multinode-e2e-copy" ]; then - echo " ERROR: Copy description mismatch: expected 'multinode-e2e-copy', got '$COPY_DESC'" - exit 1 -fi - -# Delete original -echo " Deleting original snapshot $SNAPSHOT_ID..." -$AWS_EC2 delete-snapshot --snapshot-id "$SNAPSHOT_ID" - -# Verify original gone, copy remains -echo " Verifying snapshot deletion..." -COUNT=0 -while [ $COUNT -lt 30 ]; do - set +e - SNAP_CHECK=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ - --query 'Snapshots[0].SnapshotId' --output text 2>&1) - SNAP_EXIT=$? - set -e - - if [ $SNAP_EXIT -ne 0 ] || [ "$SNAP_CHECK" == "None" ] || [ -z "$SNAP_CHECK" ]; then - echo " Original snapshot deleted successfully" - break - fi - - sleep 1 - COUNT=$((COUNT + 1)) -done - -if [ $COUNT -ge 30 ]; then - echo " ERROR: Snapshot deletion verification timed out" - exit 1 -fi - -# Verify copy still exists -COPY_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \ - --query 'Snapshots[0].State' --output text) -if [ "$COPY_STATE" != "completed" ]; then - echo " ERROR: Copy snapshot should still exist (State=$COPY_STATE)" - exit 1 -fi -echo " Copy snapshot intact after original deletion" - -# Delete copy -echo " Deleting copy snapshot $COPY_SNAPSHOT_ID..." -$AWS_EC2 delete-snapshot --snapshot-id "$COPY_SNAPSHOT_ID" - -echo " Snapshot lifecycle test passed (create -> describe -> copy -> delete)" - -# Test 1c-ii: Verify Snapshot-Backed Instance Launch -echo "" -echo "Test 1c-ii: Verify Snapshot-Backed Instance Launch" -echo "----------------------------------------" -echo "All run-instances calls go through cloneAMIToVolume() -> OpenFromSnapshot()," -echo "so the Test 1 instances are already snapshot-backed. Verify their volume configs." - -AWS_S3="aws --endpoint-url https://${NODE1_IP}:${PREDASTORE_PORT} s3" - -# Verify the AMI snapshot exists in Predastore -echo " Checking AMI snapshot in Predastore..." -SNAP_PREFIX="snap-$AMI_ID" -SNAP_FILES=$($AWS_S3 ls "s3://predastore/$SNAP_PREFIX/" 2>&1 || echo "") -if echo "$SNAP_FILES" | grep -q "config.json"; then - echo " AMI snapshot config found at $SNAP_PREFIX/" -else - echo " ERROR: AMI snapshot config not found at $SNAP_PREFIX/" - exit 1 -fi - -# Verify the first instance's root volume has SnapshotID and SourceVolumeName -SNAP_ROOT_VOL=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[0]}" \ - --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) -echo " Verifying root volume $SNAP_ROOT_VOL is snapshot-backed via Predastore config..." -VOL_CONFIG=$($AWS_S3 cp "s3://predastore/$SNAP_ROOT_VOL/config.json" - 2>/dev/null || echo "{}") -VOL_SNAPSHOT_ID=$(echo "$VOL_CONFIG" | jq -r '.SnapshotID // empty') -VOL_SOURCE_NAME=$(echo "$VOL_CONFIG" | jq -r '.SourceVolumeName // empty') - -if [ -z "$VOL_SNAPSHOT_ID" ]; then - echo " ERROR: Volume config missing SnapshotID โ€” launch was NOT snapshot-backed" - exit 1 -fi -if [ -z "$VOL_SOURCE_NAME" ]; then - echo " ERROR: Volume config missing SourceVolumeName โ€” launch was NOT snapshot-backed" - exit 1 -fi -echo " Volume is snapshot-backed (SnapshotID=$VOL_SNAPSHOT_ID, SourceVolumeName=$VOL_SOURCE_NAME)" - -echo " Snapshot-backed instance launch verified" - -# Test 1d: Tag Management -echo "" -echo "Test 1d: Tag Management" -echo "----------------------------------------" -echo "Testing create-tags -> describe-tags -> delete-tags..." - -# Use the first instance for tag tests -TAG_INSTANCE="${INSTANCE_IDS[0]}" - -# Create tags on instance -echo " Creating tags on instance $TAG_INSTANCE..." -$AWS_EC2 create-tags --resources "$TAG_INSTANCE" --tags Key=Name,Value=multinode-test Key=Environment,Value=testing Key=DeleteMe,Value=please - -# Verify tags with describe-tags (resource-id filter) -echo " Verifying tags on instance..." -TAG_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$TAG_COUNT" -ne 3 ]; then - echo " ERROR: Expected 3 tags on instance, got $TAG_COUNT" - exit 1 -fi -echo " Instance has $TAG_COUNT tags" - -# Filter by key -echo " Testing key filter..." -ENV_TAGS=$($AWS_EC2 describe-tags --filters "Name=key,Values=Environment" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$ENV_TAGS" -lt 1 ]; then - echo " ERROR: Expected at least 1 'Environment' tag, got $ENV_TAGS" - exit 1 -fi -echo " Key filter returned $ENV_TAGS tags" - -# Filter by resource-type -echo " Testing resource-type filter..." -INSTANCE_TAGS=$($AWS_EC2 describe-tags --filters "Name=resource-type,Values=instance" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$INSTANCE_TAGS" -lt 3 ]; then - echo " ERROR: Expected at least 3 instance tags, got $INSTANCE_TAGS" - exit 1 -fi -echo " Resource-type filter returned $INSTANCE_TAGS instance tags" - -# Overwrite a tag value -echo " Overwriting Name tag..." -$AWS_EC2 create-tags --resources "$TAG_INSTANCE" --tags Key=Name,Value=multinode-updated -UPDATED_NAME=$($AWS_EC2 describe-tags \ - --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Name" \ - --query 'Tags[0].Value' --output text) -if [ "$UPDATED_NAME" != "multinode-updated" ]; then - echo " ERROR: Tag overwrite failed: expected 'multinode-updated', got '$UPDATED_NAME'" - exit 1 -fi -echo " Tag overwrite verified" - -# Delete tag by key (unconditional) -echo " Deleting DeleteMe tag unconditionally..." -$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=DeleteMe -REMAINING=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$REMAINING" -ne 2 ]; then - echo " ERROR: Expected 2 tags after unconditional delete, got $REMAINING" - exit 1 -fi -echo " Unconditional delete verified ($REMAINING tags remaining)" - -# Delete tag with wrong value (should NOT delete) -echo " Attempting delete with wrong value (should be no-op)..." -$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=Environment,Value=production -ENV_STILL=$($AWS_EC2 describe-tags \ - --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Environment" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$ENV_STILL" -ne 1 ]; then - echo " ERROR: Value-conditional delete incorrectly removed tag" - exit 1 -fi -echo " Value-conditional mismatch preserved tag" - -# Delete tag with correct value -echo " Deleting Environment tag with correct value..." -$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=Environment,Value=testing -ENV_GONE=$($AWS_EC2 describe-tags \ - --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Environment" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$ENV_GONE" -ne 0 ]; then - echo " ERROR: Value-conditional delete failed to remove matching tag" - exit 1 -fi -echo " Value-conditional match deleted tag" - -# Verify only Name tag remains -FINAL_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$FINAL_COUNT" -ne 1 ]; then - echo " ERROR: Expected 1 tag remaining, got $FINAL_COUNT" - exit 1 -fi -echo " Instance tag tests passed" - -# Test 1d-ii: Tags on Volumes (multi-node) -echo "" -echo "Test 1d-ii: Tags on Volumes (Multi-Node)" -echo "----------------------------------------" - -# Get the root volume of the first instance -TAG_VOL_ID=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[0]}" \ - --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) -echo " Tagging volume $TAG_VOL_ID..." - -# Create tags on volume -$AWS_EC2 create-tags --resources "$TAG_VOL_ID" --tags Key=Name,Value=multinode-root-vol Key=Environment,Value=testing - -# Verify tags on volume -VOL_TAG_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_VOL_ID" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$VOL_TAG_COUNT" -ne 2 ]; then - echo " ERROR: Expected 2 tags on volume, got $VOL_TAG_COUNT" - exit 1 -fi -echo " Volume has $VOL_TAG_COUNT tags" - -# Filter by resource-type=volume -VOL_TYPE_TAGS=$($AWS_EC2 describe-tags --filters "Name=resource-type,Values=volume" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$VOL_TYPE_TAGS" -lt 2 ]; then - echo " ERROR: Expected at least 2 volume tags, got $VOL_TYPE_TAGS" - exit 1 -fi -echo " Volume resource-type filter returned $VOL_TYPE_TAGS tags" - -# Delete tags from volume -$AWS_EC2 delete-tags --resources "$TAG_VOL_ID" --tags Key=Name Key=Environment -VOL_TAG_AFTER=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_VOL_ID" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$VOL_TAG_AFTER" -ne 0 ]; then - echo " ERROR: Expected 0 tags after delete, got $VOL_TAG_AFTER" - exit 1 -fi -echo " Volume tag cleanup verified" - -echo " Tag management tests passed (instances + volumes)" - # Test 2: DescribeInstances Aggregation echo "" echo "Test 2: DescribeInstances Aggregation" @@ -986,1298 +476,6 @@ verify_nats_cluster 3 || { echo "WARNING: NATS cluster verification failed after operations" } -# Test 5: VM Crash Recovery (kill -9 โ†’ detect โ†’ auto-restart) -echo "" -echo "Test 5: VM Crash Recovery" -echo "----------------------------------------" -echo "Testing QEMU crash detection and auto-restart..." - -# Use the second instance for crash testing (first was used for stop/start in Test 3) -CRASH_INSTANCE="${INSTANCE_IDS[1]}" -echo " Crash test instance: $CRASH_INSTANCE" - -# Verify instance is running before crash -CRASH_STATE=$($AWS_EC2 describe-instances --instance-ids "$CRASH_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text) -if [ "$CRASH_STATE" != "running" ]; then - echo " ERROR: Instance not in running state before crash test (state: $CRASH_STATE)" - exit 1 -fi -echo " Instance is running" - -# Get the QEMU PID -QEMU_PID=$(get_qemu_pid "$CRASH_INSTANCE") -if [ -z "$QEMU_PID" ]; then - echo " ERROR: Could not find QEMU PID for $CRASH_INSTANCE" - exit 1 -fi -echo " QEMU PID: $QEMU_PID" - -# Kill QEMU with SIGKILL (simulates OOM kill) -echo " Killing QEMU process with SIGKILL (simulating OOM kill)..." -kill -9 "$QEMU_PID" - -# Brief pause for the daemon to detect the crash -sleep 3 - -# Verify the daemon detected the crash (state should be error or already recovering) -echo " Checking post-crash state..." -POST_CRASH_STATE=$($AWS_EC2 describe-instances --instance-ids "$CRASH_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text) -echo " Post-crash state: $POST_CRASH_STATE" - -if [ "$POST_CRASH_STATE" == "running" ]; then - # Might have recovered very quickly, check if it's a new PID - NEW_PID=$(get_qemu_pid "$CRASH_INSTANCE" || echo "") - if [ "$NEW_PID" == "$QEMU_PID" ]; then - echo " ERROR: Instance still has same PID after kill -9, crash not detected" - exit 1 - fi - echo " Instance already recovered with new PID: $NEW_PID (was: $QEMU_PID)" -else - # Wait for auto-restart (backoff starts at 5s) - echo " Instance in $POST_CRASH_STATE state, waiting for auto-restart..." - wait_for_instance_recovery "$CRASH_INSTANCE" 30 || { - echo " ERROR: Instance failed to recover from crash" - # Dump daemon logs for debugging - for i in 1 2 3; do - if [ -f "$HOME/node$i/logs/spinifex.log" ]; then - echo "" - echo " --- node$i daemon log (last 30 lines) ---" - tail -30 "$HOME/node$i/logs/spinifex.log" - fi - done - exit 1 - } -fi - -# Verify the instance is running with a new QEMU process -RECOVERED_PID=$(get_qemu_pid "$CRASH_INSTANCE") -if [ -z "$RECOVERED_PID" ]; then - echo " ERROR: No QEMU process found after recovery" - exit 1 -fi - -if [ "$RECOVERED_PID" == "$QEMU_PID" ]; then - echo " ERROR: QEMU PID unchanged after crash recovery (expected new process)" - exit 1 -fi -echo " New QEMU PID: $RECOVERED_PID (was: $QEMU_PID)" - -# Verify describe-instances shows running state -FINAL_STATE=$($AWS_EC2 describe-instances --instance-ids "$CRASH_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text) -if [ "$FINAL_STATE" != "running" ]; then - echo " ERROR: Instance not running after recovery (state: $FINAL_STATE)" - exit 1 -fi -echo " Instance is running after crash recovery" - -# Verify SSH works after recovery (VM rebooted with fresh OS from root volume) -echo " Verifying SSH after crash recovery..." -CRASH_SSH_PORT=$(get_ssh_port "$CRASH_INSTANCE" 30) -CRASH_SSH_HOST=$(get_ssh_host "$CRASH_INSTANCE") -if [ -n "$CRASH_SSH_PORT" ]; then - echo " SSH endpoint after recovery: $CRASH_SSH_HOST:$CRASH_SSH_PORT" - # Update SSH details for later termination verification - SSH_PORTS[1]="$CRASH_SSH_PORT" - SSH_HOSTS[1]="$CRASH_SSH_HOST" - wait_for_ssh "$CRASH_SSH_HOST" "$CRASH_SSH_PORT" "multinode-test-key.pem" 30 || { - echo " WARNING: SSH not ready after crash recovery (non-fatal, VM may still be booting)" - } -else - echo " WARNING: Could not determine SSH port after recovery (non-fatal)" -fi - -# Test 5b: Crash Loop Prevention (kill 4 times rapidly to exceed max restarts) -echo "" -echo "Test 5b: Crash Loop Prevention" -echo "----------------------------------------" -echo "Testing that crash loop is detected and restarts stop after max attempts..." - -# Use the third instance for crash loop testing -LOOP_INSTANCE="${INSTANCE_IDS[2]}" -echo " Crash loop test instance: $LOOP_INSTANCE" - -# Verify instance is running -LOOP_STATE=$($AWS_EC2 describe-instances --instance-ids "$LOOP_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text) -if [ "$LOOP_STATE" != "running" ]; then - echo " ERROR: Instance not running before crash loop test (state: $LOOP_STATE)" - exit 1 -fi - -# Kill QEMU repeatedly to exhaust restart attempts (max 3 in 10 min window) -for crash_num in 1 2 3 4; do - echo " Crash $crash_num/4: killing QEMU..." - LOOP_PID=$(get_qemu_pid "$LOOP_INSTANCE" || echo "") - if [ -z "$LOOP_PID" ]; then - echo " No QEMU process found (instance may be in error state)" - break - fi - - kill -9 "$LOOP_PID" - - if [ $crash_num -lt 4 ]; then - # Wait for restart (backoff increases: 5s, 10s, 20s) - # Give generous time for each restart cycle - local_max=$((30 + crash_num * 20)) - echo " Waiting up to ${local_max}s for restart or error state..." - attempt=0 - while [ $attempt -lt $local_max ]; do - state=$($AWS_EC2 describe-instances --instance-ids "$LOOP_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "unknown") - if [ "$state" == "running" ]; then - echo " Instance restarted (crash $crash_num)" - break - fi - if [ "$state" == "error" ] && [ $crash_num -ge 3 ]; then - echo " Instance in error state after crash $crash_num (restart limit may be reached)" - break - fi - sleep 1 - attempt=$((attempt + 2)) - done - fi -done - -# After 4 rapid crashes, the instance should be in error state (exceeded max 3 restarts) -sleep 5 -LOOP_FINAL_STATE=$($AWS_EC2 describe-instances --instance-ids "$LOOP_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text) -echo " Final state after crash loop: $LOOP_FINAL_STATE" - -if [ "$LOOP_FINAL_STATE" == "error" ]; then - echo " Crash loop prevention working: instance stayed in error state" -else - echo " WARNING: Instance in state '$LOOP_FINAL_STATE' (expected 'error' after exceeding max restarts)" - echo " This may be expected if timing allowed restarts to spread across the window" -fi - -echo " Crash recovery tests passed" - -# Phase 5c: IAM Accounts & Cross-Account Isolation -echo "" -echo "Phase 5c: IAM Accounts & Cross-Account Isolation" -echo "========================================" - -AWS_EP="--endpoint-url https://${NODE1_IP}:${AWSGW_PORT}" - -# Step 1: Create accounts -echo "" -echo "Step 1: Create Accounts" -echo "----------------------------------------" - -echo " Creating Team Alpha account..." -ALPHA_OUTPUT=$(./bin/spx admin account create --name "Team Alpha" --config "$HOME/node1/config/spinifex.toml" 2>&1) -echo "$ALPHA_OUTPUT" -ALPHA_ACCOUNT=$(echo "$ALPHA_OUTPUT" | grep "Account ID:" | awk '{print $NF}') -ALPHA_KEY_ID=$(echo "$ALPHA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}') -ALPHA_SECRET=$(echo "$ALPHA_OUTPUT" | grep "Secret Access Key:" | awk '{print $NF}') - -if [ -z "$ALPHA_ACCOUNT" ] || [ -z "$ALPHA_KEY_ID" ]; then - echo " ERROR: Failed to parse Team Alpha account output" - exit 1 -fi -echo " Team Alpha: account=$ALPHA_ACCOUNT key=$ALPHA_KEY_ID" - -# Configure alpha profile (in case auto-config used a different endpoint) -aws configure set aws_access_key_id "$ALPHA_KEY_ID" --profile spinifex-team-alpha -aws configure set aws_secret_access_key "$ALPHA_SECRET" --profile spinifex-team-alpha -aws configure set region us-east-1 --profile spinifex-team-alpha - -echo " Creating Team Beta account..." -BETA_OUTPUT=$(./bin/spx admin account create --name "Team Beta" --config "$HOME/node1/config/spinifex.toml" 2>&1) -echo "$BETA_OUTPUT" -BETA_ACCOUNT=$(echo "$BETA_OUTPUT" | grep "Account ID:" | awk '{print $NF}') -BETA_KEY_ID=$(echo "$BETA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}') -BETA_SECRET=$(echo "$BETA_OUTPUT" | grep "Secret Access Key:" | awk '{print $NF}') - -if [ -z "$BETA_ACCOUNT" ] || [ -z "$BETA_KEY_ID" ]; then - echo " ERROR: Failed to parse Team Beta account output" - exit 1 -fi -echo " Team Beta: account=$BETA_ACCOUNT key=$BETA_KEY_ID" - -aws configure set aws_access_key_id "$BETA_KEY_ID" --profile spinifex-team-beta -aws configure set aws_secret_access_key "$BETA_SECRET" --profile spinifex-team-beta -aws configure set region us-east-1 --profile spinifex-team-beta - -# Verify sequential IDs -echo " Verifying sequential account IDs..." -if [ "$ALPHA_ACCOUNT" == "000000000001" ] && [ "$BETA_ACCOUNT" == "000000000002" ]; then - echo " Sequential IDs correct: $ALPHA_ACCOUNT, $BETA_ACCOUNT" -else - echo " WARNING: Account IDs may not be sequential: $ALPHA_ACCOUNT, $BETA_ACCOUNT" -fi - -# List accounts -echo " Listing accounts..." -ACCOUNT_LIST=$(./bin/spx admin account list --config "$HOME/node1/config/spinifex.toml" 2>&1) -echo "$ACCOUNT_LIST" -if echo "$ACCOUNT_LIST" | grep -q "Team Alpha" && echo "$ACCOUNT_LIST" | grep -q "Team Beta"; then - echo " Account list verified" -else - echo " ERROR: Account list missing expected accounts" - exit 1 -fi - -# Step 2: Account admin auth -echo "" -echo "Step 2: Account Admin Auth" -echo "----------------------------------------" - -echo " Testing alpha admin auth..." -aws $AWS_EP ec2 describe-instances --profile spinifex-team-alpha > /dev/null -echo " Alpha admin: ec2 OK" -aws $AWS_EP iam list-users --profile spinifex-team-alpha > /dev/null -echo " Alpha admin: iam OK" - -echo " Testing beta admin auth..." -aws $AWS_EP ec2 describe-instances --profile spinifex-team-beta > /dev/null -echo " Beta admin: ec2 OK" -aws $AWS_EP iam list-users --profile spinifex-team-beta > /dev/null -echo " Beta admin: iam OK" - -# Step 3: Account-scoped users -echo "" -echo "Step 3: Account-Scoped Users" -echo "----------------------------------------" - -# Same username in different accounts -echo " Creating alice in both accounts..." -aws $AWS_EP iam create-user --user-name alice --profile spinifex-team-alpha > /dev/null -ALPHA_ALICE_ARN=$(aws $AWS_EP iam get-user --user-name alice --profile spinifex-team-alpha | jq -r '.User.Arn') -echo " Alpha alice: $ALPHA_ALICE_ARN" - -aws $AWS_EP iam create-user --user-name alice --profile spinifex-team-beta > /dev/null -BETA_ALICE_ARN=$(aws $AWS_EP iam get-user --user-name alice --profile spinifex-team-beta | jq -r '.User.Arn') -echo " Beta alice: $BETA_ALICE_ARN" - -# Additional users -aws $AWS_EP iam create-user --user-name team-member --profile spinifex-team-alpha > /dev/null -aws $AWS_EP iam create-user --user-name dev-user --profile spinifex-team-beta > /dev/null - -# List users โ€” scoped -echo " Verifying scoped user lists..." -ALPHA_USERS=$(aws $AWS_EP iam list-users --profile spinifex-team-alpha | jq -r '.Users[].UserName' | sort | tr '\n' ',') -echo " Alpha users: $ALPHA_USERS" -BETA_USERS=$(aws $AWS_EP iam list-users --profile spinifex-team-beta | jq -r '.Users[].UserName' | sort | tr '\n' ',') -echo " Beta users: $BETA_USERS" - -# Cross-account isolation -echo " Verifying cross-account isolation..." -set +e -CROSS_CHECK=$(aws $AWS_EP iam get-user --user-name dev-user --profile spinifex-team-alpha 2>&1) -set -e -if echo "$CROSS_CHECK" | grep -q "NoSuchEntity"; then - echo " Alpha cannot see Beta's dev-user โ€” isolation OK" -else - echo " ERROR: Alpha can see Beta's user (cross-account leak)" - exit 1 -fi - -set +e -CROSS_CHECK2=$(aws $AWS_EP iam get-user --user-name team-member --profile spinifex-team-beta 2>&1) -set -e -if echo "$CROSS_CHECK2" | grep -q "NoSuchEntity"; then - echo " Beta cannot see Alpha's team-member โ€” isolation OK" -else - echo " ERROR: Beta can see Alpha's user (cross-account leak)" - exit 1 -fi - -# Step 4: Account-scoped access keys -echo "" -echo "Step 4: Account-Scoped Access Keys" -echo "----------------------------------------" - -ALPHA_ALICE_KEY=$(aws $AWS_EP iam create-access-key --user-name alice --profile spinifex-team-alpha) -ALPHA_ALICE_KEY_ID=$(echo "$ALPHA_ALICE_KEY" | jq -r '.AccessKey.AccessKeyId') -ALPHA_ALICE_SECRET=$(echo "$ALPHA_ALICE_KEY" | jq -r '.AccessKey.SecretAccessKey') -echo " Alpha alice key: $ALPHA_ALICE_KEY_ID" - -aws configure set aws_access_key_id "$ALPHA_ALICE_KEY_ID" --profile spinifex-alpha-alice -aws configure set aws_secret_access_key "$ALPHA_ALICE_SECRET" --profile spinifex-alpha-alice -aws configure set region us-east-1 --profile spinifex-alpha-alice - -BETA_ALICE_KEY=$(aws $AWS_EP iam create-access-key --user-name alice --profile spinifex-team-beta) -BETA_ALICE_KEY_ID=$(echo "$BETA_ALICE_KEY" | jq -r '.AccessKey.AccessKeyId') -BETA_ALICE_SECRET=$(echo "$BETA_ALICE_KEY" | jq -r '.AccessKey.SecretAccessKey') -echo " Beta alice key: $BETA_ALICE_KEY_ID" - -aws configure set aws_access_key_id "$BETA_ALICE_KEY_ID" --profile spinifex-beta-alice -aws configure set aws_secret_access_key "$BETA_ALICE_SECRET" --profile spinifex-beta-alice -aws configure set region us-east-1 --profile spinifex-beta-alice - -# Step 5: Account-scoped policies & enforcement -echo "" -echo "Step 5: Account-Scoped Policies & Enforcement" -echo "----------------------------------------" - -# Alpha: narrow EC2 read-only -echo " Creating EC2ReadOnly in Alpha (narrow)..." -aws $AWS_EP iam create-policy \ - --policy-name EC2ReadOnly \ - --policy-document '{ - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": ["ec2:DescribeInstances", "ec2:DescribeVpcs"], - "Resource": "*" - }] - }' --profile spinifex-team-alpha > /dev/null - -# Beta: broad EC2 Describe* wildcard -echo " Creating EC2ReadOnly in Beta (broad Describe*)..." -aws $AWS_EP iam create-policy \ - --policy-name EC2ReadOnly \ - --policy-document '{ - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": "ec2:Describe*", - "Resource": "*" - }] - }' --profile spinifex-team-beta > /dev/null - -# Attach policies -echo " Attaching policies..." -aws $AWS_EP iam attach-user-policy --user-name alice \ - --policy-arn "arn:aws:iam::${ALPHA_ACCOUNT}:policy/EC2ReadOnly" --profile spinifex-team-alpha -aws $AWS_EP iam attach-user-policy --user-name alice \ - --policy-arn "arn:aws:iam::${BETA_ACCOUNT}:policy/EC2ReadOnly" --profile spinifex-team-beta - -# Verify scoped enforcement -echo " Verifying scoped enforcement..." - -# Alpha alice: narrow policy -aws $AWS_EP ec2 describe-instances --profile spinifex-alpha-alice > /dev/null -echo " Alpha alice: describe-instances โ€” allowed" -aws $AWS_EP ec2 describe-vpcs --profile spinifex-alpha-alice > /dev/null -echo " Alpha alice: describe-vpcs โ€” allowed" -expect_error "AccessDenied" aws $AWS_EP ec2 describe-key-pairs --profile spinifex-alpha-alice -echo " Alpha alice: describe-key-pairs โ€” denied (narrow policy)" - -# Beta alice: broad policy -aws $AWS_EP ec2 describe-instances --profile spinifex-beta-alice > /dev/null -echo " Beta alice: describe-instances โ€” allowed" -aws $AWS_EP ec2 describe-key-pairs --profile spinifex-beta-alice > /dev/null -echo " Beta alice: describe-key-pairs โ€” allowed (Describe* wildcard)" - -# Both denied non-Describe -expect_error "AccessDenied" aws $AWS_EP ec2 create-key-pair --key-name x --profile spinifex-alpha-alice -echo " Alpha alice: create-key-pair โ€” denied" -expect_error "AccessDenied" aws $AWS_EP ec2 create-key-pair --key-name x --profile spinifex-beta-alice -echo " Beta alice: create-key-pair โ€” denied" - -echo " Scoped enforcement verified" - -# Step 6: Cross-account delete isolation -echo "" -echo "Step 6: Cross-Account Delete Isolation" -echo "----------------------------------------" - -# Delete Alpha's alice -echo " Deleting Alpha's alice..." -aws $AWS_EP iam detach-user-policy --user-name alice \ - --policy-arn "arn:aws:iam::${ALPHA_ACCOUNT}:policy/EC2ReadOnly" --profile spinifex-team-alpha -aws $AWS_EP iam delete-access-key --user-name alice \ - --access-key-id "$ALPHA_ALICE_KEY_ID" --profile spinifex-team-alpha -aws $AWS_EP iam delete-user --user-name alice --profile spinifex-team-alpha - -# Verify Alpha alice gone -set +e -ALPHA_CHECK=$(aws $AWS_EP iam get-user --user-name alice --profile spinifex-team-alpha 2>&1) -set -e -if echo "$ALPHA_CHECK" | grep -q "NoSuchEntity"; then - echo " Alpha alice deleted" -else - echo " ERROR: Alpha alice still exists after delete" - exit 1 -fi - -# Verify Beta alice unaffected -aws $AWS_EP iam get-user --user-name alice --profile spinifex-team-beta > /dev/null -echo " Beta alice still exists (isolation OK)" - -aws $AWS_EP ec2 describe-instances --profile spinifex-beta-alice > /dev/null -echo " Beta alice auth still works (isolation OK)" - -# ========================================================================== -# Step 7: EC2 Resource Scoping -# ========================================================================== -# Tests that EC2 resources are properly isolated between accounts. -# Reuses Alpha/Beta accounts created in Step 1. -# Skips: Section 6 (CreateImage โ€” mulga-612), instance tags (mulga-613) -echo "" -echo "Step 7: EC2 Resource Scoping" -echo "========================================" - -# --- 7a: Instance Scoping --- -echo "" -echo "Step 7a: Instance Scoping" -echo "----------------------------------------" - -# Create per-account key pairs (key pairs are account-scoped, root's multinode-test-key is invisible) -$AWS_EC2 create-key-pair --key-name alpha-instance-key --profile spinifex-team-alpha > /dev/null -$AWS_EC2 create-key-pair --key-name beta-instance-key --profile spinifex-team-beta > /dev/null -echo " Created per-account key pairs for instance launches" - -echo " Alpha launching instance..." -ALPHA_INST_RUN=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name alpha-instance-key \ - --profile spinifex-team-alpha) -ALPHA_INST=$(echo "$ALPHA_INST_RUN" | jq -r '.Instances[0].InstanceId') -echo " Alpha instance: $ALPHA_INST" - -echo " Beta launching instance..." -BETA_INST_RUN=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name beta-instance-key \ - --profile spinifex-team-beta) -BETA_INST=$(echo "$BETA_INST_RUN" | jq -r '.Instances[0].InstanceId') -echo " Beta instance: $BETA_INST" - -# Wait for running (inline โ€” wait_for_instance_state uses root profile, can't see tenant instances) -echo " Waiting for instances to reach running state..." -COUNT=0 -while [ $COUNT -lt 30 ]; do - A_STATE=$($AWS_EC2 describe-instances --instance-ids "$ALPHA_INST" --profile spinifex-team-alpha \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending") - B_STATE=$($AWS_EC2 describe-instances --instance-ids "$BETA_INST" --profile spinifex-team-beta \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending") - echo " Alpha=$A_STATE, Beta=$B_STATE" - if [ "$A_STATE" == "running" ] && [ "$B_STATE" == "running" ]; then break; fi - sleep 1 - COUNT=$((COUNT + 1)) -done -if [ "$A_STATE" != "running" ] || [ "$B_STATE" != "running" ]; then - echo " ERROR: Instances failed to reach running state" - exit 1 -fi -echo " Both instances running" - -# Describe isolation -ALPHA_DESC=$($AWS_EC2 describe-instances --profile spinifex-team-alpha \ - --query 'Reservations[].Instances[].InstanceId' --output text) -if echo "$ALPHA_DESC" | grep -q "$BETA_INST"; then - echo " ERROR: Alpha can see Beta's instance" - exit 1 -fi -echo " Alpha sees only own instances" - -BETA_DESC=$($AWS_EC2 describe-instances --profile spinifex-team-beta \ - --query 'Reservations[].Instances[].InstanceId' --output text) -if echo "$BETA_DESC" | grep -q "$ALPHA_INST"; then - echo " ERROR: Beta can see Alpha's instance" - exit 1 -fi -echo " Beta sees only own instances" - -# OwnerId verification -ALPHA_OWNER=$($AWS_EC2 describe-instances --profile spinifex-team-alpha \ - --query 'Reservations[0].OwnerId' --output text) -if [ "$ALPHA_OWNER" != "$ALPHA_ACCOUNT" ]; then - echo " ERROR: Alpha OwnerId mismatch: expected $ALPHA_ACCOUNT, got $ALPHA_OWNER" - exit 1 -fi -echo " Alpha OwnerId correct: $ALPHA_OWNER" - -# Cross-account operations -expect_error "InvalidInstanceID.NotFound" \ - $AWS_EC2 stop-instances --instance-ids "$BETA_INST" --profile spinifex-team-alpha -echo " Alpha cannot stop Beta's instance" - -expect_error "InvalidInstanceID.NotFound" \ - $AWS_EC2 terminate-instances --instance-ids "$ALPHA_INST" --profile spinifex-team-beta -echo " Beta cannot terminate Alpha's instance" - -expect_error "InvalidInstanceID.NotFound" \ - $AWS_EC2 get-console-output --instance-id "$ALPHA_INST" --profile spinifex-team-beta -echo " Beta cannot get console output of Alpha's instance" - -echo " Instance scoping passed" - -# --- 7b: Volume Scoping --- -echo "" -echo "Step 7b: Volume Scoping" -echo "----------------------------------------" - -ALPHA_VOL=$($AWS_EC2 create-volume --availability-zone "$SPINIFEX_AZ" --size 10 \ - --volume-type gp3 --profile spinifex-team-alpha | jq -r '.VolumeId') -echo " Alpha volume: $ALPHA_VOL" - -BETA_VOL=$($AWS_EC2 create-volume --availability-zone "$SPINIFEX_AZ" --size 10 \ - --volume-type gp3 --profile spinifex-team-beta | jq -r '.VolumeId') -echo " Beta volume: $BETA_VOL" - -# Describe isolation -ALPHA_VOLS=$($AWS_EC2 describe-volumes --profile spinifex-team-alpha \ - --query 'Volumes[].VolumeId' --output text) -if echo "$ALPHA_VOLS" | grep -q "$BETA_VOL"; then - echo " ERROR: Alpha can see Beta's volume" - exit 1 -fi -echo " Alpha sees only own volumes" - -# Cross-account operations -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 describe-volumes --volume-ids "$BETA_VOL" --profile spinifex-team-alpha -echo " Alpha cannot describe Beta's volume by ID" - -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 delete-volume --volume-id "$ALPHA_VOL" --profile spinifex-team-beta -echo " Beta cannot delete Alpha's volume" - -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 attach-volume --volume-id "$ALPHA_VOL" \ - --instance-id "$BETA_INST" --device /dev/sdf --profile spinifex-team-beta -echo " Beta cannot attach Alpha's volume" - -# Attach Alpha's volume, then test cross-account detach -$AWS_EC2 attach-volume --volume-id "$ALPHA_VOL" \ - --instance-id "$ALPHA_INST" --device /dev/sdf --profile spinifex-team-alpha > /dev/null -sleep 2 - -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 detach-volume --volume-id "$ALPHA_VOL" --profile spinifex-team-beta -echo " Beta cannot detach Alpha's volume" - -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 modify-volume --volume-id "$ALPHA_VOL" --size 20 --profile spinifex-team-beta -echo " Beta cannot modify Alpha's volume" - -# Detach for later cleanup -$AWS_EC2 detach-volume --volume-id "$ALPHA_VOL" --profile spinifex-team-alpha > /dev/null -sleep 2 - -echo " Volume scoping passed" - -# --- 7c: Key Pair Scoping --- -echo "" -echo "Step 7c: Key Pair Scoping" -echo "----------------------------------------" - -$AWS_EC2 create-key-pair --key-name alpha-key --profile spinifex-team-alpha > /dev/null -ALPHA_KEYPAIR_ID=$($AWS_EC2 describe-key-pairs --key-names alpha-key \ - --profile spinifex-team-alpha --query 'KeyPairs[0].KeyPairId' --output text) -echo " Alpha key: alpha-key ($ALPHA_KEYPAIR_ID)" - -$AWS_EC2 create-key-pair --key-name beta-key --profile spinifex-team-beta > /dev/null -echo " Beta key: beta-key" - -# Describe isolation -ALPHA_KEYS=$($AWS_EC2 describe-key-pairs --profile spinifex-team-alpha \ - --query 'KeyPairs[].KeyName' --output text) -if echo "$ALPHA_KEYS" | grep -q "beta-key"; then - echo " ERROR: Alpha can see Beta's key" - exit 1 -fi -echo " Alpha sees only own keys" - -# Same name, different accounts -$AWS_EC2 create-key-pair --key-name shared-name --profile spinifex-team-alpha > /dev/null -$AWS_EC2 create-key-pair --key-name shared-name --profile spinifex-team-beta > /dev/null -ALPHA_SHARED_ID=$($AWS_EC2 describe-key-pairs --key-names shared-name \ - --profile spinifex-team-alpha --query 'KeyPairs[0].KeyPairId' --output text) -BETA_SHARED_ID=$($AWS_EC2 describe-key-pairs --key-names shared-name \ - --profile spinifex-team-beta --query 'KeyPairs[0].KeyPairId' --output text) -if [ "$ALPHA_SHARED_ID" == "$BETA_SHARED_ID" ]; then - echo " ERROR: Same KeyPairId for shared-name in both accounts" - exit 1 -fi -echo " Namespace isolation: alpha=$ALPHA_SHARED_ID, beta=$BETA_SHARED_ID" - -# Cross-account delete (idempotent, but shouldn't affect other account) -$AWS_EC2 delete-key-pair --key-name alpha-key --profile spinifex-team-beta -ALPHA_KEY_CHECK=$($AWS_EC2 describe-key-pairs --key-names alpha-key \ - --profile spinifex-team-alpha --query 'KeyPairs[0].KeyPairId' --output text) -if [ "$ALPHA_KEY_CHECK" != "$ALPHA_KEYPAIR_ID" ]; then - echo " ERROR: Beta's delete affected Alpha's key" - exit 1 -fi -echo " Cross-account delete had no effect on Alpha's key" - -# Import key pair โ€” account scoped -ssh-keygen -t ed25519 -f /tmp/test-import-key -N "" -q -$AWS_EC2 import-key-pair --key-name imported-key \ - --public-key-material fileb:///tmp/test-import-key.pub --profile spinifex-team-alpha > /dev/null -BETA_IMPORT_CHECK=$($AWS_EC2 describe-key-pairs --profile spinifex-team-beta \ - --query 'KeyPairs[].KeyName' --output text) -if echo "$BETA_IMPORT_CHECK" | grep -q "imported-key"; then - echo " ERROR: Beta can see Alpha's imported key" - exit 1 -fi -echo " Imported key invisible to Beta" -rm -f /tmp/test-import-key /tmp/test-import-key.pub - -echo " Key pair scoping passed" - -# --- 7d: Snapshot Scoping --- -echo "" -echo "Step 7d: Snapshot Scoping" -echo "----------------------------------------" - -ALPHA_SNAP=$($AWS_EC2 create-snapshot --volume-id "$ALPHA_VOL" \ - --description "Alpha snapshot" --profile spinifex-team-alpha | jq -r '.SnapshotId') -echo " Alpha snapshot: $ALPHA_SNAP" - -BETA_SNAP=$($AWS_EC2 create-snapshot --volume-id "$BETA_VOL" \ - --description "Beta snapshot" --profile spinifex-team-beta | jq -r '.SnapshotId') -echo " Beta snapshot: $BETA_SNAP" - -# Describe isolation -ALPHA_SNAPS=$($AWS_EC2 describe-snapshots --owner-ids self --profile spinifex-team-alpha \ - --query 'Snapshots[].SnapshotId' --output text) -if echo "$ALPHA_SNAPS" | grep -q "$BETA_SNAP"; then - echo " ERROR: Alpha can see Beta's snapshot" - exit 1 -fi -echo " Alpha sees only own snapshots" - -# OwnerId verification -ALPHA_SNAP_OWNER=$($AWS_EC2 describe-snapshots --owner-ids self --profile spinifex-team-alpha \ - --query 'Snapshots[0].OwnerId' --output text) -if [ "$ALPHA_SNAP_OWNER" != "$ALPHA_ACCOUNT" ]; then - echo " ERROR: Snapshot OwnerId mismatch: expected $ALPHA_ACCOUNT, got $ALPHA_SNAP_OWNER" - exit 1 -fi -echo " Alpha snapshot OwnerId correct" - -# Cross-account delete -expect_error "UnauthorizedOperation" \ - $AWS_EC2 delete-snapshot --snapshot-id "$ALPHA_SNAP" --profile spinifex-team-beta -echo " Beta cannot delete Alpha's snapshot" - -# Cross-account snapshot from other's volume -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 create-snapshot --volume-id "$ALPHA_VOL" \ - --description "stolen" --profile spinifex-team-beta -echo " Beta cannot snapshot Alpha's volume" - -echo " Snapshot scoping passed" - -# --- 7e: VPC/Subnet Scoping --- -echo "" -echo "Step 7e: VPC/Subnet Scoping" -echo "----------------------------------------" - -ALPHA_VPC=$($AWS_EC2 create-vpc --cidr-block 10.0.0.0/16 \ - --profile spinifex-team-alpha --query 'Vpc.VpcId' --output text) -echo " Alpha VPC: $ALPHA_VPC" - -BETA_VPC=$($AWS_EC2 create-vpc --cidr-block 10.0.0.0/16 \ - --profile spinifex-team-beta --query 'Vpc.VpcId' --output text) -echo " Beta VPC: $BETA_VPC (same CIDR โ€” no conflict)" - -# Describe isolation -ALPHA_VPCS=$($AWS_EC2 describe-vpcs --profile spinifex-team-alpha \ - --query 'Vpcs[].VpcId' --output text) -if echo "$ALPHA_VPCS" | grep -q "$BETA_VPC"; then - echo " ERROR: Alpha can see Beta's VPC" - exit 1 -fi -echo " VPC describe isolation OK" - -expect_error "InvalidVpcID.NotFound" \ - $AWS_EC2 describe-vpcs --vpc-ids "$BETA_VPC" --profile spinifex-team-alpha -echo " Alpha cannot describe Beta's VPC by ID" - -expect_error "InvalidVpcID.NotFound" \ - $AWS_EC2 delete-vpc --vpc-id "$ALPHA_VPC" --profile spinifex-team-beta -echo " Beta cannot delete Alpha's VPC" - -# Create subnets -ALPHA_SUBNET=$($AWS_EC2 create-subnet --vpc-id "$ALPHA_VPC" --cidr-block 10.0.1.0/24 \ - --profile spinifex-team-alpha --query 'Subnet.SubnetId' --output text) -echo " Alpha subnet: $ALPHA_SUBNET" - -BETA_SUBNET=$($AWS_EC2 create-subnet --vpc-id "$BETA_VPC" --cidr-block 10.0.1.0/24 \ - --profile spinifex-team-beta --query 'Subnet.SubnetId' --output text) -echo " Beta subnet: $BETA_SUBNET" - -# Subnet describe isolation -ALPHA_SUBNETS=$($AWS_EC2 describe-subnets --profile spinifex-team-alpha \ - --query 'Subnets[].SubnetId' --output text) -if echo "$ALPHA_SUBNETS" | grep -q "$BETA_SUBNET"; then - echo " ERROR: Alpha can see Beta's subnet" - exit 1 -fi -echo " Subnet describe isolation OK" - -expect_error "InvalidVpcID.NotFound" \ - $AWS_EC2 create-subnet --vpc-id "$ALPHA_VPC" --cidr-block 10.0.2.0/24 \ - --profile spinifex-team-beta -echo " Beta cannot create subnet in Alpha's VPC" - -expect_error "InvalidSubnetID.NotFound" \ - $AWS_EC2 delete-subnet --subnet-id "$ALPHA_SUBNET" --profile spinifex-team-beta -echo " Beta cannot delete Alpha's subnet" - -echo " VPC/Subnet scoping passed" - -# --- 7f: IGW + EIGW Scoping --- -echo "" -echo "Step 7f: IGW + EIGW Scoping" -echo "----------------------------------------" - -ALPHA_IGW=$($AWS_EC2 create-internet-gateway --profile spinifex-team-alpha \ - --query 'InternetGateway.InternetGatewayId' --output text) -echo " Alpha IGW: $ALPHA_IGW" - -BETA_IGW=$($AWS_EC2 create-internet-gateway --profile spinifex-team-beta \ - --query 'InternetGateway.InternetGatewayId' --output text) -echo " Beta IGW: $BETA_IGW" - -# IGW describe isolation -ALPHA_IGWS=$($AWS_EC2 describe-internet-gateways --profile spinifex-team-alpha \ - --query 'InternetGateways[].InternetGatewayId' --output text) -if echo "$ALPHA_IGWS" | grep -q "$BETA_IGW"; then - echo " ERROR: Alpha can see Beta's IGW" - exit 1 -fi -echo " IGW describe isolation OK" - -expect_error "InvalidInternetGatewayID.NotFound" \ - $AWS_EC2 describe-internet-gateways --internet-gateway-ids "$BETA_IGW" \ - --profile spinifex-team-alpha -echo " Alpha cannot describe Beta's IGW by ID" - -expect_error "InvalidInternetGatewayID.NotFound" \ - $AWS_EC2 delete-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ - --profile spinifex-team-beta -echo " Beta cannot delete Alpha's IGW" - -expect_error "InvalidInternetGatewayID.NotFound" \ - $AWS_EC2 attach-internet-gateway --internet-gateway-id "$BETA_IGW" \ - --vpc-id "$ALPHA_VPC" --profile spinifex-team-alpha -echo " Alpha cannot attach Beta's IGW to own VPC" - -# Attach Alpha's IGW, test cross-account detach -$AWS_EC2 attach-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ - --vpc-id "$ALPHA_VPC" --profile spinifex-team-alpha > /dev/null -expect_error "InvalidInternetGatewayID.NotFound" \ - $AWS_EC2 detach-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ - --vpc-id "$ALPHA_VPC" --profile spinifex-team-beta -echo " Beta cannot detach Alpha's IGW" - -# EIGW -ALPHA_EIGW=$($AWS_EC2 create-egress-only-internet-gateway --vpc-id "$ALPHA_VPC" \ - --profile spinifex-team-alpha \ - --query 'EgressOnlyInternetGateway.EgressOnlyInternetGatewayId' --output text) -echo " Alpha EIGW: $ALPHA_EIGW" - -BETA_EIGW=$($AWS_EC2 create-egress-only-internet-gateway --vpc-id "$BETA_VPC" \ - --profile spinifex-team-beta \ - --query 'EgressOnlyInternetGateway.EgressOnlyInternetGatewayId' --output text) -echo " Beta EIGW: $BETA_EIGW" - -# EIGW describe isolation -ALPHA_EIGWS=$($AWS_EC2 describe-egress-only-internet-gateways --profile spinifex-team-alpha \ - --query 'EgressOnlyInternetGateways[].EgressOnlyInternetGatewayId' --output text) -if echo "$ALPHA_EIGWS" | grep -q "$BETA_EIGW"; then - echo " ERROR: Alpha can see Beta's EIGW" - exit 1 -fi -echo " EIGW describe isolation OK" - -# Cross-account EIGW delete (verify Alpha's still exists after Beta's attempt) -set +e -$AWS_EC2 delete-egress-only-internet-gateway \ - --egress-only-internet-gateway-id "$ALPHA_EIGW" --profile spinifex-team-beta 2>/dev/null -set -e -ALPHA_EIGW_CHECK=$($AWS_EC2 describe-egress-only-internet-gateways --profile spinifex-team-alpha \ - --query 'EgressOnlyInternetGateways[].EgressOnlyInternetGatewayId' --output text) -if ! echo "$ALPHA_EIGW_CHECK" | grep -q "$ALPHA_EIGW"; then - echo " ERROR: Alpha's EIGW was deleted by Beta" - exit 1 -fi -echo " Beta cannot delete Alpha's EIGW" - -echo " IGW + EIGW scoping passed" - -# --- 7g: Account Settings --- -echo "" -echo "Step 7g: Account Settings" -echo "----------------------------------------" - -$AWS_EC2 enable-ebs-encryption-by-default --profile spinifex-team-alpha > /dev/null -BETA_ENC=$($AWS_EC2 get-ebs-encryption-by-default --profile spinifex-team-beta \ - --query 'EbsEncryptionByDefault' --output text) -if [ "$BETA_ENC" != "False" ]; then - echo " ERROR: Alpha's encryption setting leaked to Beta (got $BETA_ENC)" - exit 1 -fi -echo " Alpha enable did not affect Beta" - -$AWS_EC2 enable-ebs-encryption-by-default --profile spinifex-team-beta > /dev/null -$AWS_EC2 disable-ebs-encryption-by-default --profile spinifex-team-alpha > /dev/null -ALPHA_ENC=$($AWS_EC2 get-ebs-encryption-by-default --profile spinifex-team-alpha \ - --query 'EbsEncryptionByDefault' --output text) -BETA_ENC=$($AWS_EC2 get-ebs-encryption-by-default --profile spinifex-team-beta \ - --query 'EbsEncryptionByDefault' --output text) -if [ "$ALPHA_ENC" != "False" ] || [ "$BETA_ENC" != "True" ]; then - echo " ERROR: Independent settings failed: alpha=$ALPHA_ENC beta=$BETA_ENC" - exit 1 -fi -echo " Independent toggle verified: alpha=$ALPHA_ENC, beta=$BETA_ENC" -$AWS_EC2 disable-ebs-encryption-by-default --profile spinifex-team-beta > /dev/null - -echo " Account settings scoping passed" - -# --- 7h: Global Resources --- -echo "" -echo "Step 7h: Global Resources" -echo "----------------------------------------" - -ALPHA_REGIONS=$($AWS_EC2 describe-regions --profile spinifex-team-alpha \ - --query 'Regions[].RegionName' --output text) -BETA_REGIONS=$($AWS_EC2 describe-regions --profile spinifex-team-beta \ - --query 'Regions[].RegionName' --output text) -if [ "$ALPHA_REGIONS" != "$BETA_REGIONS" ]; then - echo " ERROR: Regions differ between accounts" - exit 1 -fi -echo " Regions identical" - -ALPHA_AZS=$($AWS_EC2 describe-availability-zones --profile spinifex-team-alpha \ - --query 'AvailabilityZones[].ZoneName' --output text) -BETA_AZS=$($AWS_EC2 describe-availability-zones --profile spinifex-team-beta \ - --query 'AvailabilityZones[].ZoneName' --output text) -if [ "$ALPHA_AZS" != "$BETA_AZS" ]; then - echo " ERROR: AZs differ between accounts" - exit 1 -fi -echo " Availability zones identical" - -ALPHA_TYPES=$($AWS_EC2 describe-instance-types --profile spinifex-team-alpha \ - --query 'InstanceTypes[].InstanceType' --output text | tr '\t' '\n' | sort) -BETA_TYPES=$($AWS_EC2 describe-instance-types --profile spinifex-team-beta \ - --query 'InstanceTypes[].InstanceType' --output text | tr '\t' '\n' | sort) -if [ "$ALPHA_TYPES" != "$BETA_TYPES" ]; then - echo " ERROR: Instance types differ between accounts" - exit 1 -fi -echo " Instance types identical" - -echo " Global resources passed" - -# --- Step 8: Edge Cases --- -echo "" -echo "Step 8: Edge Cases" -echo "----------------------------------------" - -# Empty account (Gamma) -echo " Creating empty Gamma account..." -GAMMA_OUTPUT=$(./bin/spx admin account create --name "Team Gamma" --config "$HOME/node1/config/spinifex.toml" 2>&1) -GAMMA_KEY_ID=$(echo "$GAMMA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}') -GAMMA_SECRET=$(echo "$GAMMA_OUTPUT" | grep "Secret Access Key:" | awk '{print $NF}') -aws configure set aws_access_key_id "$GAMMA_KEY_ID" --profile spinifex-team-gamma -aws configure set aws_secret_access_key "$GAMMA_SECRET" --profile spinifex-team-gamma -aws configure set region us-east-1 --profile spinifex-team-gamma - -GAMMA_INSTANCES=$($AWS_EC2 describe-instances --profile spinifex-team-gamma \ - --query 'Reservations' --output text) -if [ -n "$GAMMA_INSTANCES" ] && [ "$GAMMA_INSTANCES" != "None" ]; then - echo " ERROR: Gamma has instances" - exit 1 -fi -echo " Gamma: no instances" - -# Skip volume check: root-account volumes (empty TenantID) are visible to all accounts by design -echo " Gamma: volumes skipped (root legacy volumes visible to all)" - -GAMMA_KEYS=$($AWS_EC2 describe-key-pairs --profile spinifex-team-gamma \ - --query 'KeyPairs' --output text) -if [ -n "$GAMMA_KEYS" ] && [ "$GAMMA_KEYS" != "None" ]; then - echo " ERROR: Gamma has key pairs" - exit 1 -fi -echo " Gamma: no key pairs" - -# Root isolation from tenants -echo " Verifying root isolation from tenants..." -ROOT_INSTANCE_CHECK=$($AWS_EC2 describe-instances \ - --query 'Reservations[].Instances[].InstanceId' --output text) -if echo "$ROOT_INSTANCE_CHECK" | grep -q "$ALPHA_INST"; then - echo " ERROR: Root can see Alpha's instance" - exit 1 -fi -echo " Root cannot see tenant instances" - -# Non-existent resource IDs โ€” same error as cross-account -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 delete-volume --volume-id vol-00000000000000000 --profile spinifex-team-alpha -echo " Non-existent volume: same error as cross-account" - -expect_error "InvalidSnapshot.NotFound" \ - $AWS_EC2 delete-snapshot --snapshot-id snap-00000000000000000 --profile spinifex-team-alpha -echo " Non-existent snapshot: same error as cross-account" - -# Race condition: parallel key creation from both accounts -echo " Testing parallel key creation (race condition)..." -for i in $(seq 1 5); do - $AWS_EC2 create-key-pair --key-name "race-alpha-$i" --profile spinifex-team-alpha > /dev/null & - $AWS_EC2 create-key-pair --key-name "race-beta-$i" --profile spinifex-team-beta > /dev/null & -done -wait - -ALPHA_RACE_COUNT=$($AWS_EC2 describe-key-pairs --profile spinifex-team-alpha \ - --query 'KeyPairs[].KeyName' --output text | tr '\t' '\n' | grep -c "race-alpha" || true) -if [ "$ALPHA_RACE_COUNT" -ne 5 ]; then - echo " ERROR: Expected 5 race-alpha keys, got $ALPHA_RACE_COUNT" - exit 1 -fi -ALPHA_RACE_LEAK=$($AWS_EC2 describe-key-pairs --profile spinifex-team-alpha \ - --query 'KeyPairs[].KeyName' --output text | tr '\t' '\n' | grep -c "race-beta" || true) -if [ "$ALPHA_RACE_LEAK" -ne 0 ]; then - echo " ERROR: Alpha sees Beta's race keys (cross-contamination)" - exit 1 -fi -echo " Parallel key creation: no cross-contamination ($ALPHA_RACE_COUNT alpha keys, 0 beta leaks)" - -echo " Edge cases passed" - -# --- Step 9: EC2 + IAM Cleanup --- -echo "" -echo "Step 9: EC2 + IAM Cleanup" -echo "========================================" - -# Terminate instances -echo " Terminating account-scoped instances..." -$AWS_EC2 terminate-instances --instance-ids "$ALPHA_INST" --profile spinifex-team-alpha > /dev/null -$AWS_EC2 terminate-instances --instance-ids "$BETA_INST" --profile spinifex-team-beta > /dev/null - -COUNT=0 -while [ $COUNT -lt 30 ]; do - A_STATE=$($AWS_EC2 describe-instances --instance-ids "$ALPHA_INST" --profile spinifex-team-alpha \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "terminated") - B_STATE=$($AWS_EC2 describe-instances --instance-ids "$BETA_INST" --profile spinifex-team-beta \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "terminated") - if [ "$A_STATE" == "terminated" ] && [ "$B_STATE" == "terminated" ]; then - break - fi - sleep 1 - COUNT=$((COUNT + 1)) -done -echo " Instances terminated" - -# Delete snapshots -echo " Deleting snapshots..." -$AWS_EC2 delete-snapshot --snapshot-id "$ALPHA_SNAP" --profile spinifex-team-alpha 2>/dev/null || true -$AWS_EC2 delete-snapshot --snapshot-id "$BETA_SNAP" --profile spinifex-team-beta 2>/dev/null || true - -# Delete volumes -sleep 3 -echo " Deleting volumes..." -$AWS_EC2 delete-volume --volume-id "$ALPHA_VOL" --profile spinifex-team-alpha 2>/dev/null || true -$AWS_EC2 delete-volume --volume-id "$BETA_VOL" --profile spinifex-team-beta 2>/dev/null || true - -# Delete key pairs -echo " Deleting key pairs..." -for key in alpha-key alpha-instance-key shared-name imported-key; do - $AWS_EC2 delete-key-pair --key-name "$key" --profile spinifex-team-alpha 2>/dev/null || true -done -for key in beta-key beta-instance-key shared-name; do - $AWS_EC2 delete-key-pair --key-name "$key" --profile spinifex-team-beta 2>/dev/null || true -done -for i in $(seq 1 5); do - $AWS_EC2 delete-key-pair --key-name "race-alpha-$i" --profile spinifex-team-alpha 2>/dev/null || true - $AWS_EC2 delete-key-pair --key-name "race-beta-$i" --profile spinifex-team-beta 2>/dev/null || true -done - -# Delete EIGWs -echo " Deleting EIGWs..." -$AWS_EC2 delete-egress-only-internet-gateway \ - --egress-only-internet-gateway-id "$ALPHA_EIGW" --profile spinifex-team-alpha 2>/dev/null || true -$AWS_EC2 delete-egress-only-internet-gateway \ - --egress-only-internet-gateway-id "$BETA_EIGW" --profile spinifex-team-beta 2>/dev/null || true - -# Detach + delete IGWs -echo " Deleting IGWs..." -$AWS_EC2 detach-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ - --vpc-id "$ALPHA_VPC" --profile spinifex-team-alpha 2>/dev/null || true -$AWS_EC2 delete-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ - --profile spinifex-team-alpha 2>/dev/null || true -$AWS_EC2 delete-internet-gateway --internet-gateway-id "$BETA_IGW" \ - --profile spinifex-team-beta 2>/dev/null || true - -# Delete subnets -echo " Deleting subnets..." -$AWS_EC2 delete-subnet --subnet-id "$ALPHA_SUBNET" --profile spinifex-team-alpha 2>/dev/null || true -$AWS_EC2 delete-subnet --subnet-id "$BETA_SUBNET" --profile spinifex-team-beta 2>/dev/null || true - -# Delete VPCs -echo " Deleting VPCs..." -$AWS_EC2 delete-vpc --vpc-id "$ALPHA_VPC" --profile spinifex-team-alpha 2>/dev/null || true -$AWS_EC2 delete-vpc --vpc-id "$BETA_VPC" --profile spinifex-team-beta 2>/dev/null || true - -echo " EC2 resources cleaned up" - -# IAM cleanup (originally Step 7) -echo " Cleaning up IAM resources..." - -# Beta cleanup -aws $AWS_EP iam detach-user-policy --user-name alice \ - --policy-arn "arn:aws:iam::${BETA_ACCOUNT}:policy/EC2ReadOnly" --profile spinifex-team-beta -aws $AWS_EP iam delete-access-key --user-name alice \ - --access-key-id "$BETA_ALICE_KEY_ID" --profile spinifex-team-beta -aws $AWS_EP iam delete-user --user-name alice --profile spinifex-team-beta -aws $AWS_EP iam delete-user --user-name dev-user --profile spinifex-team-beta -aws $AWS_EP iam delete-policy \ - --policy-arn "arn:aws:iam::${BETA_ACCOUNT}:policy/EC2ReadOnly" --profile spinifex-team-beta - -# Alpha cleanup (alice already deleted in Step 6) -aws $AWS_EP iam delete-user --user-name team-member --profile spinifex-team-alpha -aws $AWS_EP iam delete-policy \ - --policy-arn "arn:aws:iam::${ALPHA_ACCOUNT}:policy/EC2ReadOnly" --profile spinifex-team-alpha - -# Clean up AWS CLI profiles -for p in spinifex-team-alpha spinifex-team-beta spinifex-alpha-alice spinifex-beta-alice spinifex-team-gamma; do - aws configure set aws_access_key_id "" --profile $p 2>/dev/null || true - aws configure set aws_secret_access_key "" --profile $p 2>/dev/null || true -done - -echo " IAM + EC2 account cleanup complete" -echo "" -echo " IAM Accounts & Cross-Account Isolation tests passed" - -# Phase 5d: VPC Networking -echo "" -echo "Phase 5d: VPC Networking" -echo "========================================" -echo "Testing VPC instance launch, PrivateIpAddress, and same-subnet connectivity..." - -# Step 1: Create VPC and Subnet -echo "" -echo "Step 1: Create VPC + Subnet" -echo "----------------------------------------" - -VPC_OUTPUT=$($AWS_EC2 create-vpc --cidr-block 10.100.0.0/16) -VPC_ID=$(echo "$VPC_OUTPUT" | jq -r '.Vpc.VpcId') -if [ -z "$VPC_ID" ] || [ "$VPC_ID" == "null" ]; then - echo " ERROR: Failed to create VPC" - echo " Output: $VPC_OUTPUT" - exit 1 -fi -echo " Created VPC: $VPC_ID (10.100.0.0/16)" - -SUBNET_OUTPUT=$($AWS_EC2 create-subnet --vpc-id "$VPC_ID" --cidr-block 10.100.1.0/24) -SUBNET_ID=$(echo "$SUBNET_OUTPUT" | jq -r '.Subnet.SubnetId') -if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" == "null" ]; then - echo " ERROR: Failed to create subnet" - echo " Output: $SUBNET_OUTPUT" - exit 1 -fi -echo " Created Subnet: $SUBNET_ID (10.100.1.0/24)" - -# Brief pause for OVN topology to be programmed (logical switch + router port + DHCP) -sleep 1 - -# Step 2: Launch 3 VPC instances -echo "" -echo "Step 2: Launch 3 VPC instances" -echo "----------------------------------------" - -VPC_INSTANCE_IDS=() -for i in 1 2 3; do - echo " Launching VPC instance $i with subnet $SUBNET_ID..." - RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --subnet-id "$SUBNET_ID") - - VPC_INST_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId') - VPC_INST_IP=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].PrivateIpAddress // empty') - - if [ -z "$VPC_INST_ID" ] || [ "$VPC_INST_ID" == "null" ]; then - echo " ERROR: Failed to launch VPC instance $i" - echo " Output: $RUN_OUTPUT" - exit 1 - fi - echo " Launched: $VPC_INST_ID (PrivateIpAddress: ${VPC_INST_IP:-not yet assigned})" - VPC_INSTANCE_IDS+=("$VPC_INST_ID") - - sleep 1 -done - -# Wait for all VPC instances to be running -echo "" -echo "Waiting for VPC instances to reach running state..." -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - wait_for_instance_state "$vpc_inst" "running" 30 || { - echo "ERROR: VPC instance $vpc_inst failed to start" - exit 1 - } -done - -# Step 3: Verify PrivateIpAddress in DescribeInstances -echo "" -echo "Step 3: Verify PrivateIpAddress in DescribeInstances" -echo "----------------------------------------" - -VPC_PRIVATE_IPS=() -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - DESCRIBE_OUT=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst") - PRIVATE_IP=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].PrivateIpAddress // empty') - INST_SUBNET=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].SubnetId // empty') - INST_VPC=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].VpcId // empty') - ENI_COUNT=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].NetworkInterfaces | length') - - if [ -z "$PRIVATE_IP" ]; then - echo " ERROR: $vpc_inst has no PrivateIpAddress" - echo " Describe output: $(echo "$DESCRIBE_OUT" | jq -c '.Reservations[0].Instances[0] | {PrivateIpAddress, SubnetId, VpcId, NetworkInterfaces}')" - exit 1 - fi - - echo " $vpc_inst: IP=$PRIVATE_IP, Subnet=$INST_SUBNET, VPC=$INST_VPC, ENIs=$ENI_COUNT" - - if [ "$INST_SUBNET" != "$SUBNET_ID" ]; then - echo " ERROR: SubnetId mismatch (expected $SUBNET_ID, got $INST_SUBNET)" - exit 1 - fi - if [ "$INST_VPC" != "$VPC_ID" ]; then - echo " ERROR: VpcId mismatch (expected $VPC_ID, got $INST_VPC)" - exit 1 - fi - if [ "$ENI_COUNT" -lt 1 ]; then - echo " ERROR: No NetworkInterfaces found" - exit 1 - fi - - VPC_PRIVATE_IPS+=("$PRIVATE_IP") -done - -# Verify all IPs are unique and in the subnet range (10.100.1.x) -echo "" -echo " Verifying IP uniqueness and subnet range..." -UNIQUE_IPS=$(printf '%s\n' "${VPC_PRIVATE_IPS[@]}" | sort -u | wc -l) -if [ "$UNIQUE_IPS" -ne "${#VPC_PRIVATE_IPS[@]}" ]; then - echo " ERROR: Duplicate IPs detected: ${VPC_PRIVATE_IPS[*]}" - exit 1 -fi -for ip in "${VPC_PRIVATE_IPS[@]}"; do - if ! echo "$ip" | grep -qE '^10\.100\.1\.[0-9]+$'; then - echo " ERROR: IP $ip not in expected subnet 10.100.1.0/24" - exit 1 - fi -done -echo " All IPs unique and in correct subnet: ${VPC_PRIVATE_IPS[*]}" - -# Step 4: SSH via DEV_NETWORKING hostfwd and test ping connectivity -# DISABLED: VPC instances use OVN DHCP which takes ~6 min per instance to get SSH ready. -# With 3 instances this adds ~18 min to the E2E run. The SSH+ping test is best-effort -# anyway (OVN overlay not fully programmed in Docker single-host mode). IP allocation -# and subnet correctness are already verified in Step 3. -echo "" -echo "Step 4: SSH + Ping Connectivity (SKIPPED โ€” OVN DHCP wait too slow in CI)" -echo "----------------------------------------" - -# Step 5: Stop/Start IP persistence -echo "" -echo "Step 5: Stop/Start IP Persistence" -echo "----------------------------------------" -echo "Verifying private IPs persist through stop/start cycle (AWS behavior)..." - -# Record IPs before stop -echo " IPs before stop: ${VPC_PRIVATE_IPS[*]}" - -# Stop all VPC instances -echo "" -echo " Stopping all VPC instances..." -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - echo " Stopping $vpc_inst..." - $AWS_EC2 stop-instances --instance-ids "$vpc_inst" > /dev/null -done - -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - wait_for_instance_state "$vpc_inst" "stopped" 30 || { - echo " ERROR: VPC instance $vpc_inst failed to stop" - exit 1 - } -done -echo " All VPC instances stopped" - -# Verify IPs are still present in DescribeInstances while stopped -echo "" -echo " Verifying IPs persist in stopped state..." -for idx in "${!VPC_INSTANCE_IDS[@]}"; do - vpc_inst="${VPC_INSTANCE_IDS[$idx]}" - expected_ip="${VPC_PRIVATE_IPS[$idx]}" - - STOPPED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \ - --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text) - - if [ "$STOPPED_IP" != "$expected_ip" ]; then - echo " ERROR: $vpc_inst IP changed while stopped (expected $expected_ip, got $STOPPED_IP)" - exit 1 - fi - echo " $vpc_inst: IP=$STOPPED_IP (unchanged)" -done - -# Start all VPC instances -echo "" -echo " Starting all VPC instances..." -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - echo " Starting $vpc_inst..." - $AWS_EC2 start-instances --instance-ids "$vpc_inst" > /dev/null -done - -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - wait_for_instance_state "$vpc_inst" "running" 30 || { - echo " ERROR: VPC instance $vpc_inst failed to restart" - exit 1 - } -done -echo " All VPC instances restarted" - -# Verify IPs are identical after restart -echo "" -echo " Verifying IPs persist after restart..." -IP_MISMATCHES=0 -for idx in "${!VPC_INSTANCE_IDS[@]}"; do - vpc_inst="${VPC_INSTANCE_IDS[$idx]}" - expected_ip="${VPC_PRIVATE_IPS[$idx]}" - - RESTARTED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \ - --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text) - - if [ "$RESTARTED_IP" == "$expected_ip" ]; then - echo " $vpc_inst: IP=$RESTARTED_IP (matches pre-stop)" - else - echo " ERROR: $vpc_inst IP changed after restart (expected $expected_ip, got $RESTARTED_IP)" - IP_MISMATCHES=$((IP_MISMATCHES + 1)) - fi -done - -if [ "$IP_MISMATCHES" -gt 0 ]; then - echo " ERROR: $IP_MISMATCHES instances had IP changes โ€” ENI not persisting through stop/start" - exit 1 -fi - -echo " Stop/start IP persistence verified โ€” all IPs match" - -# Step 6: Clean up VPC instances -echo "" -echo "Step 6: Clean up VPC resources" -echo "----------------------------------------" - -terminate_and_wait "${VPC_INSTANCE_IDS[@]}" || true - -# Clean up subnet and VPC -echo " Deleting subnet $SUBNET_ID..." -$AWS_EC2 delete-subnet --subnet-id "$SUBNET_ID" 2>/dev/null || echo " (subnet delete failed, may have ENIs)" - -echo " Deleting VPC $VPC_ID..." -$AWS_EC2 delete-vpc --vpc-id "$VPC_ID" 2>/dev/null || echo " (vpc delete failed, may have subnets)" - -echo " VPC networking tests passed" - # Phase 6: Cluster Shutdown + Restart echo "" echo "Phase 6: Cluster Shutdown + Restart" @@ -2373,11 +571,11 @@ echo "Test 6d: Instance Relaunch + Terminate" echo "----------------------------------------" echo "Waiting for instances to relaunch after cluster restart..." -# Two instances were running before shutdown โ€” the daemon will relaunch them. +# All 3 instances were running before shutdown โ€” the daemon will relaunch them. # Must wait for them to finish launching (pending โ†’ running) before terminate # will work, because the NATS per-instance subscription is only created after # QEMU starts. -for instance_id in "${INSTANCE_IDS[0]}" "${INSTANCE_IDS[1]}"; do +for instance_id in "${INSTANCE_IDS[@]}"; do echo " Waiting for $instance_id to finish relaunching..." COUNT=0 while [ $COUNT -lt 60 ]; do @@ -2395,16 +593,7 @@ for instance_id in "${INSTANCE_IDS[0]}" "${INSTANCE_IDS[1]}"; do fi done -# Verify crash-loop instance (INSTANCE_IDS[2]) stayed in error โ€” daemon should -# not relaunch an instance that exceeded its crash restart limit. -CRASH_LOOP_STATE=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[2]}" \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "unknown") -echo " Crash-loop instance ${INSTANCE_IDS[2]} state: $CRASH_LOOP_STATE" -if [ "$CRASH_LOOP_STATE" != "error" ]; then - echo " WARNING: Expected crash-loop instance to remain in error state (got: $CRASH_LOOP_STATE)" -fi - -# Terminate all instances (including the crash-loop one in error state) +# Terminate all instances echo "" echo "Terminating all instances..." if ! terminate_and_wait "${INSTANCE_IDS[@]}"; then