diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index da9d2f4d3..c25beec41 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -42,6 +42,10 @@ jobs: sparse-checkout: scripts/tofu-cluster path: mulga + - name: Clone and build + working-directory: mulga/scripts/tofu-cluster + run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/hive-artifacts.tar" + - name: Provision VM working-directory: mulga/scripts/tofu-cluster run: | @@ -54,7 +58,10 @@ jobs: - name: Bootstrap Cluster working-directory: mulga/scripts/tofu-cluster - run: ./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1 + run: | + HIVE_PREBUILT=true \ + HIVE_ARTIFACT_TAR="${RUNNER_TEMP}/hive-artifacts.tar" \ + ./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1 - name: Run E2E Tests working-directory: mulga/scripts/tofu-cluster @@ -96,9 +103,12 @@ jobs: repository: mulgadc/mulga token: ${{ steps.app-token.outputs.token }} sparse-checkout: scripts/tofu-cluster - path: mulga + - name: Clone and build + working-directory: mulga/scripts/tofu-cluster + run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/hive-artifacts.tar" + - name: Provision VM working-directory: mulga/scripts/tofu-cluster run: | @@ -111,7 +121,10 @@ jobs: - name: Bootstrap Cluster working-directory: mulga/scripts/tofu-cluster - run: ./bootstrap.sh env4 >> "$TOFU_LOG" 2>&1 + run: | + HIVE_PREBUILT=true \ + HIVE_ARTIFACT_TAR="${RUNNER_TEMP}/hive-artifacts.tar" \ + ./bootstrap.sh env4 >> "$TOFU_LOG" 2>&1 - name: Run E2E Tests working-directory: mulga/scripts/tofu-cluster @@ -155,6 +168,10 @@ jobs: sparse-checkout: scripts/tofu-cluster path: mulga + - name: Clone and build + working-directory: mulga/scripts/tofu-cluster + run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/hive-artifacts.tar" + - name: Provision VMs working-directory: mulga/scripts/tofu-cluster run: | @@ -167,7 +184,10 @@ jobs: - name: Bootstrap Cluster working-directory: mulga/scripts/tofu-cluster - run: ./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1 + run: | + HIVE_PREBUILT=true \ + HIVE_ARTIFACT_TAR="${RUNNER_TEMP}/hive-artifacts.tar" \ + ./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1 - name: Run E2E Tests working-directory: mulga/scripts/tofu-cluster diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index 8327ebfd2..522808942 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -115,7 +115,7 @@ start_service() { echo " PID: $pid, Log: $logfile" # Brief pause to let service start - sleep 2 + sleep 1 } # Function to start service in foreground (for final daemon) diff --git a/tests/e2e/lib/multinode-helpers.sh b/tests/e2e/lib/multinode-helpers.sh index 9e27cb474..09dfe232f 100755 --- a/tests/e2e/lib/multinode-helpers.sh +++ b/tests/e2e/lib/multinode-helpers.sh @@ -150,7 +150,7 @@ verify_nats_cluster() { wait_for_instance_state() { local instance_id="$1" local target_state="$2" - local max_attempts="${3:-10}" + local max_attempts="${3:-20}" local attempt=0 echo "Waiting for instance $instance_id to reach state: $target_state..." @@ -162,7 +162,7 @@ wait_for_instance_state() { --query 'Reservations[0].Instances[0].State.Name' \ --output text) || { echo " Attempt $((attempt + 1))/$max_attempts - Gateway request failed, retrying..." - sleep 2 + sleep 1 attempt=$((attempt + 1)) continue } @@ -179,7 +179,7 @@ wait_for_instance_state() { return 1 fi - sleep 2 + sleep 1 attempt=$((attempt + 1)) done @@ -204,7 +204,7 @@ terminate_and_wait() { local failed=0 for instance_id in "${ids[@]}"; do - if ! wait_for_instance_state "$instance_id" "terminated" 30; then + if ! wait_for_instance_state "$instance_id" "terminated" 60; then echo " WARNING: Failed to confirm termination of $instance_id" failed=1 fi @@ -217,7 +217,7 @@ terminate_and_wait() { # Usage: wait_for_gateway [host] [max_attempts] wait_for_gateway() { local host="${1:-localhost}" - local max_attempts="${2:-15}" + local max_attempts="${2:-30}" local attempt=0 echo "Waiting for AWS Gateway at $host:${AWSGW_PORT}..." @@ -229,7 +229,7 @@ wait_for_gateway() { fi echo " Waiting for gateway... ($((attempt + 1))/$max_attempts)" - sleep 2 + sleep 1 attempt=$((attempt + 1)) done @@ -242,7 +242,7 @@ wait_for_gateway() { # Usage: wait_for_daemon_ready [max_attempts] wait_for_daemon_ready() { local endpoint="$1" - local max_attempts="${2:-15}" + local max_attempts="${2:-30}" local attempt=0 echo "Waiting for daemon readiness (NATS subscriptions)..." @@ -256,7 +256,7 @@ wait_for_daemon_ready() { return 0 fi echo " Waiting... ($((attempt + 1))/$max_attempts)" - sleep 2 + sleep 1 attempt=$((attempt + 1)) done @@ -544,7 +544,7 @@ get_qemu_pid() { # Expects state transition: error → pending → running wait_for_instance_recovery() { local instance_id="$1" - local max_attempts="${2:-30}" + local max_attempts="${2:-60}" local attempt=0 local saw_error=false @@ -556,7 +556,7 @@ wait_for_instance_recovery() { --instance-ids "$instance_id" \ --query 'Reservations[0].Instances[0].State.Name' \ --output text 2>/dev/null) || { - sleep 2 + sleep 1 attempt=$((attempt + 1)) continue } @@ -576,7 +576,7 @@ wait_for_instance_recovery() { : fi - sleep 2 + sleep 1 attempt=$((attempt + 1)) done @@ -671,7 +671,7 @@ force_cleanup_all_nodes() { # Kill any remaining QEMU processes pkill -9 -x qemu-system-x86_64 2>/dev/null || true - sleep 2 + sleep 1 # Step 3: Remove stale badger LOCK files from predastore directories for i in 1 2 3; do diff --git a/tests/e2e/run-e2e.sh b/tests/e2e/run-e2e.sh index 0a6d05bf6..8cd59f377 100755 --- a/tests/e2e/run-e2e.sh +++ b/tests/e2e/run-e2e.sh @@ -122,12 +122,12 @@ fi # Wait for health checks on AWS Gateway echo "Waiting for AWS Gateway..." -MAX_RETRIES=15 +MAX_RETRIES=30 COUNT=0 until curl -sk "https://${GATEWAY_HOST}:9999" > /dev/null || [ $COUNT -eq $MAX_RETRIES ]; do echo "Waiting for gateway... ($COUNT/$MAX_RETRIES)" - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -346,18 +346,18 @@ echo "Launched Instance ID: $INSTANCE_ID" echo "Polling for instance running state..." COUNT=0 STATE="unknown" -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do # Capture full output to check if instance even exists in the response DESCRIBE_OUTPUT=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID") || { - echo "⚠️ Gateway request failed, retrying... ($COUNT/30)" - sleep 2 + echo "⚠️ Gateway request failed, retrying... ($COUNT/60)" + sleep 1 COUNT=$((COUNT + 1)) continue } if [ -z "$DESCRIBE_OUTPUT" ]; then echo "⚠️ Gateway returned empty response, retrying..." - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) continue fi @@ -375,7 +375,7 @@ while [ $COUNT -lt 30 ]; do exit 1 fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -537,7 +537,7 @@ aws ec2 modify-volume --volume-id "$TEST_VOLUME_ID" --size "$NEW_SIZE" # Verify resize echo "Verifying resize..." COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do VOLUME_SIZE=$(aws ec2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ --query 'Volumes[0].Size' --output text) @@ -546,7 +546,7 @@ while [ $COUNT -lt 30 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -562,7 +562,7 @@ aws ec2 attach-volume --volume-id "$TEST_VOLUME_ID" --instance-id "$INSTANCE_ID" # Verify attachment echo "Verifying volume attachment..." COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do ATTACH_STATE=$(aws ec2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ --query 'Volumes[0].Attachments[0].State' --output text) ATTACH_INSTANCE=$(aws ec2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ @@ -575,7 +575,7 @@ while [ $COUNT -lt 30 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -591,7 +591,7 @@ aws ec2 detach-volume --volume-id "$TEST_VOLUME_ID" # Verify detachment echo "Verifying volume detachment..." COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do VOL_STATE=$(aws ec2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ --query 'Volumes[0].State' --output text) @@ -600,7 +600,7 @@ while [ $COUNT -lt 30 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -616,7 +616,7 @@ aws ec2 delete-volume --volume-id "$TEST_VOLUME_ID" # Verify deletion echo "Verifying volume deletion..." COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do set +e VOLUME_CHECK=$(aws ec2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ --query 'Volumes[0].VolumeId' --output text 2>&1) @@ -628,11 +628,11 @@ while [ $COUNT -lt 30 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done -if [ $COUNT -ge 30 ]; then +if [ $COUNT -ge 60 ]; then echo "Volume deletion verification timed out" exit 1 fi @@ -694,7 +694,7 @@ echo "Snapshot create response verified (State=$SNAP_STATE, VolumeId=$SNAP_VOL_R # Poll until snapshot is completed (should be immediate in v1, but poll for forward-compat) echo "Waiting for snapshot to complete..." COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do SNAP_STATE=$(aws ec2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ --query 'Snapshots[0].State' --output text) @@ -703,7 +703,7 @@ while [ $COUNT -lt 30 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -777,7 +777,7 @@ aws ec2 delete-snapshot --snapshot-id "$SNAPSHOT_ID" # Verify original is gone, copy remains echo "Verifying snapshot deletion..." COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do set +e SNAP_CHECK=$(aws ec2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ --query 'Snapshots[0].SnapshotId' --output text 2>&1) @@ -789,11 +789,11 @@ while [ $COUNT -lt 30 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done -if [ $COUNT -ge 30 ]; then +if [ $COUNT -ge 60 ]; then echo "Snapshot deletion verification timed out" exit 1 fi @@ -991,13 +991,13 @@ echo "Phase 7: Instance State Transitions" echo "Stopping instance..." aws ec2 stop-instances --instance-ids "$INSTANCE_ID" COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do STATE=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text) echo "Instance state: $STATE" if [ "$STATE" == "stopped" ]; then break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -1060,14 +1060,14 @@ echo "Instance type updated to $MODIFIED_TYPE" echo "Starting instance with modified type..." aws ec2 start-instances --instance-ids "$INSTANCE_ID" COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do STATE=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" \ --query 'Reservations[0].Instances[0].State.Name' --output text) echo "Instance state: $STATE" if [ "$STATE" == "running" ]; then break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -1136,14 +1136,14 @@ aws ec2 reboot-instances --instance-ids "$INSTANCE_ID" # Verify state stays running (poll a few times to confirm no transient state change) COUNT=0 -while [ $COUNT -lt 5 ]; do +while [ $COUNT -lt 10 ]; do STATE=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" \ --query 'Reservations[0].Instances[0].State.Name' --output text) if [ "$STATE" != "running" ]; then echo "Instance unexpectedly left running state: $STATE" exit 1 fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done echo "Instance state remained running during reboot" @@ -1196,10 +1196,10 @@ echo "Launched 2 instances: $MULTI_ID_1, $MULTI_ID_2" for MID in "$MULTI_ID_1" "$MULTI_ID_2"; do echo "Waiting for $MID to reach running state..." COUNT=0 - while [ $COUNT -lt 30 ]; do + while [ $COUNT -lt 60 ]; do MSTATE=$(aws ec2 describe-instances --instance-ids "$MID" \ --query 'Reservations[0].Instances[0].State.Name' --output text) || { - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) continue } @@ -1207,7 +1207,7 @@ for MID in "$MULTI_ID_1" "$MULTI_ID_2"; do echo "Instance $MID is running" break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done if [ "$MSTATE" != "running" ]; then @@ -1221,13 +1221,13 @@ echo "Terminating multi-launch instances..." aws ec2 terminate-instances --instance-ids "$MULTI_ID_1" "$MULTI_ID_2" for MID in "$MULTI_ID_1" "$MULTI_ID_2"; do COUNT=0 - while [ $COUNT -lt 30 ]; do + while [ $COUNT -lt 60 ]; do MSTATE=$(aws ec2 describe-instances --instance-ids "$MID" \ --query 'Reservations[0].Instances[0].State.Name' --output text) if [ "$MSTATE" == "terminated" ] || [ "$MSTATE" == "None" ]; then break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done done @@ -1933,14 +1933,14 @@ echo " Beta instance: $BETA_INST" # Wait for running echo " Waiting for instances to reach running state..." COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do A_STATE=$(aws ec2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \ --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending") B_STATE=$(aws ec2 describe-instances --instance-ids "$BETA_INST" --profile hive-team-beta \ --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending") echo " Alpha=$A_STATE, Beta=$B_STATE" if [ "$A_STATE" == "running" ] && [ "$B_STATE" == "running" ]; then break; fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done if [ "$A_STATE" != "running" ] || [ "$B_STATE" != "running" ]; then @@ -1991,11 +1991,11 @@ echo " Alpha cannot reboot Beta's instance" # Stop Alpha's instance for cross-account start/modify/console tests aws ec2 stop-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha > /dev/null COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do A_STATE=$(aws ec2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \ --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null) if [ "$A_STATE" == "stopped" ]; then break; fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -2015,11 +2015,11 @@ echo " Beta cannot get console output of Alpha's instance" # Restart Alpha's instance for later tests aws ec2 start-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha > /dev/null COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do A_STATE=$(aws ec2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \ --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null) if [ "$A_STATE" == "running" ]; then break; fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -2066,7 +2066,7 @@ echo " Beta cannot attach Alpha's volume" # Attach Alpha's volume to Alpha's instance, then test cross-account detach aws ec2 attach-volume --volume-id "$ALPHA_VOL" \ --instance-id "$ALPHA_INST" --device /dev/sdf --profile hive-team-alpha > /dev/null -sleep 2 +sleep 1 expect_error "InvalidVolume.NotFound" \ aws ec2 detach-volume --volume-id "$ALPHA_VOL" --profile hive-team-beta @@ -2079,7 +2079,7 @@ echo " Beta cannot modify Alpha's volume" # Detach for cleanup later aws ec2 detach-volume --volume-id "$ALPHA_VOL" --profile hive-team-alpha > /dev/null -sleep 2 +sleep 1 echo " Volume scoping passed" @@ -2492,7 +2492,7 @@ aws ec2 terminate-instances --instance-ids "$BETA_INST" --profile hive-team-beta # Wait for termination COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do A_STATE=$(aws ec2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \ --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "terminated") B_STATE=$(aws ec2 describe-instances --instance-ids "$BETA_INST" --profile hive-team-beta \ @@ -2500,7 +2500,7 @@ while [ $COUNT -lt 30 ]; do if [ "$A_STATE" == "terminated" ] && [ "$B_STATE" == "terminated" ]; then break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done echo " Instances terminated" @@ -2580,13 +2580,13 @@ fi echo "Terminating instance..." aws ec2 terminate-instances --instance-ids "$INSTANCE_ID" COUNT=0 -while [ $COUNT -lt 30 ]; do +while [ $COUNT -lt 60 ]; do STATE=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text) echo "Instance state: $STATE" if [ "$STATE" == "terminated" ] || [ "$STATE" == "None" ]; then break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done diff --git a/tests/e2e/run-multinode-e2e.sh b/tests/e2e/run-multinode-e2e.sh index 27c402a04..f48269822 100755 --- a/tests/e2e/run-multinode-e2e.sh +++ b/tests/e2e/run-multinode-e2e.sh @@ -88,7 +88,7 @@ aws_via_retry() { } attempt=$((attempt + 1)) echo " Retry $attempt/$max_attempts..." >&2 - sleep 2 + sleep 1 done return 1 } @@ -131,7 +131,7 @@ dump_all_node_logs() { wait_for_instance_state() { local instance_id="$1" local target_state="$2" - local max_attempts="${3:-30}" + local max_attempts="${3:-60}" local gw_ip="${4:-$LOCAL_IP}" local attempt=0 @@ -143,7 +143,7 @@ wait_for_instance_state() { --instance-ids "$instance_id" \ --query 'Reservations[0].Instances[0].State.Name' \ --output text 2>/dev/null) || { - sleep 2 + sleep 1 attempt=$((attempt + 1)) continue } @@ -158,7 +158,7 @@ wait_for_instance_state() { return 1 fi - sleep 2 + sleep 1 attempt=$((attempt + 1)) done @@ -264,7 +264,7 @@ terminate_and_wait() { local failed=0 for instance_id in "${ids[@]}"; do - if ! wait_for_instance_state "$instance_id" "terminated" 30; then + if ! wait_for_instance_state "$instance_id" "terminated" 60; then echo " WARNING: Failed to confirm termination of $instance_id" failed=1 fi @@ -384,15 +384,15 @@ pass_test "All gateways" # Daemon readiness: describe-instance-types must return results echo "Checking daemon readiness..." ATTEMPT=0 -while [ $ATTEMPT -lt 15 ]; do +while [ $ATTEMPT -lt 30 ]; do TYPES=$($AWS_EC2 describe-instance-types \ --query 'InstanceTypes[*].InstanceType' --output text 2>/dev/null || true) if [ -n "$TYPES" ] && [ "$TYPES" != "None" ]; then echo " Daemon ready (instance types: $TYPES)" break fi - echo " Waiting for daemon... ($((ATTEMPT + 1))/15)" - sleep 2 + echo " Waiting for daemon... ($((ATTEMPT + 1))/30)" + sleep 1 ATTEMPT=$((ATTEMPT + 1)) done if [ -z "$TYPES" ] || [ "$TYPES" == "None" ]; then @@ -469,7 +469,7 @@ done echo "" echo "Waiting for instances to reach running state..." for instance_id in "${INSTANCE_IDS[@]}"; do - wait_for_instance_state "$instance_id" "running" 30 || { + wait_for_instance_state "$instance_id" "running" 60 || { echo "ERROR: Instance $instance_id failed to start" exit 1 } @@ -626,7 +626,7 @@ else # Wait for attachment COUNT=0 - while [ $COUNT -lt 15 ]; do + while [ $COUNT -lt 30 ]; do ATTACH_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ --query 'Volumes[0].Attachments[0].State' --output text) VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ @@ -635,7 +635,7 @@ else echo " Volume attached" break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -650,14 +650,14 @@ else $AWS_EC2 detach-volume --volume-id "$TEST_VOLUME_ID" > /dev/null COUNT=0 - while [ $COUNT -lt 15 ]; do + while [ $COUNT -lt 30 ]; do VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ --query 'Volumes[0].State' --output text) if [ "$VOL_STATE" == "available" ]; then echo " Volume detached" break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -672,7 +672,7 @@ else $AWS_EC2 delete-volume --volume-id "$TEST_VOLUME_ID" COUNT=0 - while [ $COUNT -lt 15 ]; do + while [ $COUNT -lt 30 ]; do set +e VOL_CHECK=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ --query 'Volumes[0].VolumeId' --output text 2>&1) @@ -682,11 +682,11 @@ else echo " Volume deleted" break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done - if [ $COUNT -lt 15 ]; then + if [ $COUNT -lt 30 ]; then pass_test "Volume delete" else fail_test "Volume delete" @@ -746,7 +746,7 @@ echo " Will operate via gateway on $OTHER_GW" # Stop via other gateway echo " Stopping instance via $OTHER_GW..." aws_via "$OTHER_GW" ec2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null -wait_for_instance_state "$TEST_INSTANCE" "stopped" 30 "$OTHER_GW" +wait_for_instance_state "$TEST_INSTANCE" "stopped" 60 "$OTHER_GW" # Pick yet another gateway for start (or same other if only 2 choices) THIRD_GW="" @@ -759,7 +759,7 @@ done THIRD_GW="${THIRD_GW:-$OTHER_GW}" echo " Starting instance via $THIRD_GW..." aws_via "$THIRD_GW" ec2 start-instances --instance-ids "$TEST_INSTANCE" > /dev/null -wait_for_instance_state "$TEST_INSTANCE" "running" 30 "$THIRD_GW" +wait_for_instance_state "$TEST_INSTANCE" "running" 60 "$THIRD_GW" pass_test "Cross-node stop/start" @@ -844,7 +844,7 @@ peer_ssh "$NODE2_IP" "cd ~/Development/mulga/spinifex && ./scripts/start-dev.sh" echo " Waiting for NATS cluster to reform..." ATTEMPT=0 REFORMED=false -while [ $ATTEMPT -lt 30 ]; do +while [ $ATTEMPT -lt 60 ]; do NATS_RECOVER=$(curl -s "http://${LOCAL_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null) RECOVER_PEERS=$(echo "$NATS_RECOVER" | jq -r '[.routes[].remote_name] | unique | length' 2>/dev/null || echo "0") if [ "$RECOVER_PEERS" -ge 2 ]; then @@ -852,8 +852,8 @@ while [ $ATTEMPT -lt 30 ]; do REFORMED=true break fi - echo " Waiting for NATS reform... ($((ATTEMPT + 1))/30, peers: $RECOVER_PEERS)" - sleep 2 + echo " Waiting for NATS reform... ($((ATTEMPT + 1))/60, peers: $RECOVER_PEERS)" + sleep 1 ATTEMPT=$((ATTEMPT + 1)) done @@ -868,13 +868,13 @@ fi echo " Waiting for node2 gateway..." ATTEMPT=0 GW_BACK=false -while [ $ATTEMPT -lt 15 ]; do +while [ $ATTEMPT -lt 30 ]; do if curl -k -s "https://${NODE2_IP}:${AWSGW_PORT}" > /dev/null 2>&1; then echo " Node2 gateway is back" GW_BACK=true break fi - sleep 2 + sleep 1 ATTEMPT=$((ATTEMPT + 1)) done diff --git a/tests/e2e/run-pseudo-multinode-e2e.sh b/tests/e2e/run-pseudo-multinode-e2e.sh index 471aaa67b..ffaad9188 100755 --- a/tests/e2e/run-pseudo-multinode-e2e.sh +++ b/tests/e2e/run-pseudo-multinode-e2e.sh @@ -344,7 +344,7 @@ for i in 1 2 3; do INSTANCE_IDS+=("$INSTANCE_ID") # Small delay between launches to encourage distribution - sleep 2 + sleep 1 done # Wait for all instances to be running @@ -489,7 +489,7 @@ $AWS_EC2 modify-volume --volume-id "$TEST_VOLUME_ID" --size "$NEW_SIZE" # Verify resize echo " Verifying resize..." COUNT=0 -while [ $COUNT -lt 15 ]; do +while [ $COUNT -lt 30 ]; do VOLUME_SIZE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ --query 'Volumes[0].Size' --output text) @@ -498,7 +498,7 @@ while [ $COUNT -lt 15 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -514,7 +514,7 @@ $AWS_EC2 attach-volume --volume-id "$TEST_VOLUME_ID" --instance-id "${INSTANCE_I # Verify attachment echo " Verifying volume attachment..." COUNT=0 -while [ $COUNT -lt 15 ]; do +while [ $COUNT -lt 30 ]; do ATTACH_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ --query 'Volumes[0].Attachments[0].State' --output text) ATTACH_INSTANCE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ @@ -527,7 +527,7 @@ while [ $COUNT -lt 15 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -543,7 +543,7 @@ $AWS_EC2 detach-volume --volume-id "$TEST_VOLUME_ID" # Verify detachment echo " Verifying volume detachment..." COUNT=0 -while [ $COUNT -lt 15 ]; do +while [ $COUNT -lt 30 ]; do VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ --query 'Volumes[0].State' --output text) @@ -552,7 +552,7 @@ while [ $COUNT -lt 15 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -568,7 +568,7 @@ $AWS_EC2 delete-volume --volume-id "$TEST_VOLUME_ID" # Verify deletion echo " Verifying volume deletion..." COUNT=0 -while [ $COUNT -lt 15 ]; do +while [ $COUNT -lt 30 ]; do set +e VOLUME_CHECK=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ --query 'Volumes[0].VolumeId' --output text 2>&1) @@ -580,11 +580,11 @@ while [ $COUNT -lt 15 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done -if [ $COUNT -ge 15 ]; then +if [ $COUNT -ge 30 ]; then echo " ERROR: Volume deletion verification timed out" exit 1 fi @@ -635,7 +635,7 @@ echo " Create response verified (State=$SNAP_STATE, VolumeId=$SNAP_VOL_REF, Siz # Poll until completed echo " Waiting for snapshot to complete..." COUNT=0 -while [ $COUNT -lt 15 ]; do +while [ $COUNT -lt 30 ]; do SNAP_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ --query 'Snapshots[0].State' --output text) @@ -644,7 +644,7 @@ while [ $COUNT -lt 15 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done @@ -717,7 +717,7 @@ $AWS_EC2 delete-snapshot --snapshot-id "$SNAPSHOT_ID" # Verify original gone, copy remains echo " Verifying snapshot deletion..." COUNT=0 -while [ $COUNT -lt 15 ]; do +while [ $COUNT -lt 30 ]; do set +e SNAP_CHECK=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ --query 'Snapshots[0].SnapshotId' --output text 2>&1) @@ -729,11 +729,11 @@ while [ $COUNT -lt 15 ]; do break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done -if [ $COUNT -ge 15 ]; then +if [ $COUNT -ge 30 ]; then echo " ERROR: Snapshot deletion verification timed out" exit 1 fi @@ -1121,7 +1121,7 @@ for crash_num in 1 2 3 4; do if [ $crash_num -lt 4 ]; then # Wait for restart (backoff increases: 5s, 10s, 20s) # Give generous time for each restart cycle - local_max=$((15 + crash_num * 10)) + local_max=$((30 + crash_num * 20)) echo " Waiting up to ${local_max}s for restart or error state..." attempt=0 while [ $attempt -lt $local_max ]; do @@ -1135,7 +1135,7 @@ for crash_num in 1 2 3 4; do echo " Instance in error state after crash $crash_num (restart limit may be reached)" break fi - sleep 2 + sleep 1 attempt=$((attempt + 2)) done fi @@ -1452,7 +1452,7 @@ while [ $COUNT -lt 30 ]; do --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending") echo " Alpha=$A_STATE, Beta=$B_STATE" if [ "$A_STATE" == "running" ] && [ "$B_STATE" == "running" ]; then break; fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done if [ "$A_STATE" != "running" ] || [ "$B_STATE" != "running" ]; then @@ -1961,7 +1961,7 @@ while [ $COUNT -lt 30 ]; do if [ "$A_STATE" == "terminated" ] && [ "$B_STATE" == "terminated" ]; then break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done echo " Instances terminated" @@ -2076,7 +2076,7 @@ fi echo " Created Subnet: $SUBNET_ID (10.100.1.0/24)" # Brief pause for OVN topology to be programmed (logical switch + router port + DHCP) -sleep 2 +sleep 1 # Step 2: Launch 3 VPC instances echo "" @@ -2103,7 +2103,7 @@ for i in 1 2 3; do echo " Launched: $VPC_INST_ID (PrivateIpAddress: ${VPC_INST_IP:-not yet assigned})" VPC_INSTANCE_IDS+=("$VPC_INST_ID") - sleep 2 + sleep 1 done # Wait for all VPC instances to be running @@ -2316,7 +2316,7 @@ CLUSTER_SERVICES_STARTED="false" # Verify all services are down echo " Waiting for services to stop..." -sleep 2 +sleep 1 if ! verify_all_services_down; then echo " Some services still running, force-cleaning..." force_cleanup_all_nodes @@ -2380,17 +2380,17 @@ echo "Waiting for instances to relaunch after cluster restart..." for instance_id in "${INSTANCE_IDS[0]}" "${INSTANCE_IDS[1]}"; do echo " Waiting for $instance_id to finish relaunching..." COUNT=0 - while [ $COUNT -lt 30 ]; do + while [ $COUNT -lt 60 ]; do STATE=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "unknown") if [ "$STATE" = "running" ]; then echo " $instance_id relaunched successfully: $STATE" break fi - sleep 2 + sleep 1 COUNT=$((COUNT + 1)) done - if [ $COUNT -ge 30 ]; then + if [ $COUNT -ge 60 ]; then echo " WARNING: $instance_id still in $STATE after 60s" fi done