gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: e2e pipeline speedup (#78)

Browse Source
This commit is contained in:
Julian Sommer
2026-03-18 12:50:45 +10:00
committed by GitHub
parent 454f381826
commit ba0d8b67ff
6 changed files with 129 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/e2e.yml
+24 -4
View File
@@ -42,6 +42,10 @@ jobs:
sparse-checkout: scripts/tofu-cluster
path: mulga
- name: Clone and build
working-directory: mulga/scripts/tofu-cluster
run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/hive-artifacts.tar"
- name: Provision VM
working-directory: mulga/scripts/tofu-cluster
run: |
@@ -54,7 +58,10 @@ jobs:
- name: Bootstrap Cluster
working-directory: mulga/scripts/tofu-cluster
run: ./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1
run: |
HIVE_PREBUILT=true \
HIVE_ARTIFACT_TAR="${RUNNER_TEMP}/hive-artifacts.tar" \
./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1
- name: Run E2E Tests
working-directory: mulga/scripts/tofu-cluster
@@ -96,9 +103,12 @@ jobs:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- name: Clone and build
working-directory: mulga/scripts/tofu-cluster
run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/hive-artifacts.tar"
- name: Provision VM
working-directory: mulga/scripts/tofu-cluster
run: |
@@ -111,7 +121,10 @@ jobs:
- name: Bootstrap Cluster
working-directory: mulga/scripts/tofu-cluster
run: ./bootstrap.sh env4 >> "$TOFU_LOG" 2>&1
run: |
HIVE_PREBUILT=true \
HIVE_ARTIFACT_TAR="${RUNNER_TEMP}/hive-artifacts.tar" \
./bootstrap.sh env4 >> "$TOFU_LOG" 2>&1
- name: Run E2E Tests
working-directory: mulga/scripts/tofu-cluster
@@ -155,6 +168,10 @@ jobs:
sparse-checkout: scripts/tofu-cluster
path: mulga
- name: Clone and build
working-directory: mulga/scripts/tofu-cluster
run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/hive-artifacts.tar"
- name: Provision VMs
working-directory: mulga/scripts/tofu-cluster
run: |
@@ -167,7 +184,10 @@ jobs:
- name: Bootstrap Cluster
working-directory: mulga/scripts/tofu-cluster
run: ./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1
run: |
HIVE_PREBUILT=true \
HIVE_ARTIFACT_TAR="${RUNNER_TEMP}/hive-artifacts.tar" \
./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1
- name: Run E2E Tests
working-directory: mulga/scripts/tofu-cluster
scripts/start-dev.sh
+1 -1
View File
@@ -115,7 +115,7 @@ start_service() {
echo " PID: $pid, Log: $logfile"
# Brief pause to let service start
sleep 2
sleep 1
}
# Function to start service in foreground (for final daemon)
tests/e2e/lib/multinode-helpers.sh
+12 -12
View File
@@ -150,7 +150,7 @@ verify_nats_cluster() {
wait_for_instance_state() {
local instance_id="$1"
local target_state="$2"
local max_attempts="${3:-10}"
local max_attempts="${3:-20}"
local attempt=0
echo "Waiting for instance $instance_id to reach state: $target_state..."
@@ -162,7 +162,7 @@ wait_for_instance_state() {
--query 'Reservations[0].Instances[0].State.Name' \
--output text) || {
echo " Attempt $((attempt + 1))/$max_attempts - Gateway request failed, retrying..."
sleep 2
sleep 1
attempt=$((attempt + 1))
continue
}
@@ -179,7 +179,7 @@ wait_for_instance_state() {
return 1
fi
sleep 2
sleep 1
attempt=$((attempt + 1))
done
@@ -204,7 +204,7 @@ terminate_and_wait() {
local failed=0
for instance_id in "${ids[@]}"; do
if ! wait_for_instance_state "$instance_id" "terminated" 30; then
if ! wait_for_instance_state "$instance_id" "terminated" 60; then
echo " WARNING: Failed to confirm termination of $instance_id"
failed=1
fi
@@ -217,7 +217,7 @@ terminate_and_wait() {
# Usage: wait_for_gateway [host] [max_attempts]
wait_for_gateway() {
local host="${1:-localhost}"
local max_attempts="${2:-15}"
local max_attempts="${2:-30}"
local attempt=0
echo "Waiting for AWS Gateway at $host:${AWSGW_PORT}..."
@@ -229,7 +229,7 @@ wait_for_gateway() {
fi
echo " Waiting for gateway... ($((attempt + 1))/$max_attempts)"
sleep 2
sleep 1
attempt=$((attempt + 1))
done
@@ -242,7 +242,7 @@ wait_for_gateway() {
# Usage: wait_for_daemon_ready <gateway_endpoint> [max_attempts]
wait_for_daemon_ready() {
local endpoint="$1"
local max_attempts="${2:-15}"
local max_attempts="${2:-30}"
local attempt=0
echo "Waiting for daemon readiness (NATS subscriptions)..."
@@ -256,7 +256,7 @@ wait_for_daemon_ready() {
return 0
fi
echo " Waiting... ($((attempt + 1))/$max_attempts)"
sleep 2
sleep 1
attempt=$((attempt + 1))
done
@@ -544,7 +544,7 @@ get_qemu_pid() {
# Expects state transition: error → pending → running
wait_for_instance_recovery() {
local instance_id="$1"
local max_attempts="${2:-30}"
local max_attempts="${2:-60}"
local attempt=0
local saw_error=false
@@ -556,7 +556,7 @@ wait_for_instance_recovery() {
--instance-ids "$instance_id" \
--query 'Reservations[0].Instances[0].State.Name' \
--output text 2>/dev/null) || {
sleep 2
sleep 1
attempt=$((attempt + 1))
continue
}
@@ -576,7 +576,7 @@ wait_for_instance_recovery() {
:
fi
sleep 2
sleep 1
attempt=$((attempt + 1))
done
@@ -671,7 +671,7 @@ force_cleanup_all_nodes() {
# Kill any remaining QEMU processes
pkill -9 -x qemu-system-x86_64 2>/dev/null || true
sleep 2
sleep 1
# Step 3: Remove stale badger LOCK files from predastore directories
for i in 1 2 3; do
tests/e2e/run-e2e.sh
+44 -44
View File
@@ -122,12 +122,12 @@ fi
# Wait for health checks on AWS Gateway
echo "Waiting for AWS Gateway..."
MAX_RETRIES=15
MAX_RETRIES=30
COUNT=0
until curl -sk "https://${GATEWAY_HOST}:9999" > /dev/null || [ $COUNT -eq $MAX_RETRIES ]; do
echo "Waiting for gateway... ($COUNT/$MAX_RETRIES)"
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -346,18 +346,18 @@ echo "Launched Instance ID: $INSTANCE_ID"
echo "Polling for instance running state..."
COUNT=0
STATE="unknown"
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
# Capture full output to check if instance even exists in the response
DESCRIBE_OUTPUT=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID") || {
echo "⚠️ Gateway request failed, retrying... ($COUNT/30)"
sleep 2
echo "⚠️ Gateway request failed, retrying... ($COUNT/60)"
sleep 1
COUNT=$((COUNT + 1))
continue
}
if [ -z "$DESCRIBE_OUTPUT" ]; then
echo "⚠️ Gateway returned empty response, retrying..."
sleep 2
sleep 1
COUNT=$((COUNT + 1))
continue
fi
@@ -375,7 +375,7 @@ while [ $COUNT -lt 30 ]; do
exit 1
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -537,7 +537,7 @@ aws ec2 modify-volume --volume-id "$TEST_VOLUME_ID" --size "$NEW_SIZE"
# Verify resize
echo "Verifying resize..."
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
VOLUME_SIZE=$(aws ec2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
--query 'Volumes[0].Size' --output text)
@@ -546,7 +546,7 @@ while [ $COUNT -lt 30 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -562,7 +562,7 @@ aws ec2 attach-volume --volume-id "$TEST_VOLUME_ID" --instance-id "$INSTANCE_ID"
# Verify attachment
echo "Verifying volume attachment..."
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
ATTACH_STATE=$(aws ec2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
--query 'Volumes[0].Attachments[0].State' --output text)
ATTACH_INSTANCE=$(aws ec2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
@@ -575,7 +575,7 @@ while [ $COUNT -lt 30 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -591,7 +591,7 @@ aws ec2 detach-volume --volume-id "$TEST_VOLUME_ID"
# Verify detachment
echo "Verifying volume detachment..."
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
VOL_STATE=$(aws ec2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
--query 'Volumes[0].State' --output text)
@@ -600,7 +600,7 @@ while [ $COUNT -lt 30 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -616,7 +616,7 @@ aws ec2 delete-volume --volume-id "$TEST_VOLUME_ID"
# Verify deletion
echo "Verifying volume deletion..."
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
set +e
VOLUME_CHECK=$(aws ec2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
--query 'Volumes[0].VolumeId' --output text 2>&1)
@@ -628,11 +628,11 @@ while [ $COUNT -lt 30 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
if [ $COUNT -ge 30 ]; then
if [ $COUNT -ge 60 ]; then
echo "Volume deletion verification timed out"
exit 1
fi
@@ -694,7 +694,7 @@ echo "Snapshot create response verified (State=$SNAP_STATE, VolumeId=$SNAP_VOL_R
# Poll until snapshot is completed (should be immediate in v1, but poll for forward-compat)
echo "Waiting for snapshot to complete..."
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
SNAP_STATE=$(aws ec2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \
--query 'Snapshots[0].State' --output text)
@@ -703,7 +703,7 @@ while [ $COUNT -lt 30 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -777,7 +777,7 @@ aws ec2 delete-snapshot --snapshot-id "$SNAPSHOT_ID"
# Verify original is gone, copy remains
echo "Verifying snapshot deletion..."
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
set +e
SNAP_CHECK=$(aws ec2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \
--query 'Snapshots[0].SnapshotId' --output text 2>&1)
@@ -789,11 +789,11 @@ while [ $COUNT -lt 30 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
if [ $COUNT -ge 30 ]; then
if [ $COUNT -ge 60 ]; then
echo "Snapshot deletion verification timed out"
exit 1
fi
@@ -991,13 +991,13 @@ echo "Phase 7: Instance State Transitions"
echo "Stopping instance..."
aws ec2 stop-instances --instance-ids "$INSTANCE_ID"
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
STATE=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text)
echo "Instance state: $STATE"
if [ "$STATE" == "stopped" ]; then
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -1060,14 +1060,14 @@ echo "Instance type updated to $MODIFIED_TYPE"
echo "Starting instance with modified type..."
aws ec2 start-instances --instance-ids "$INSTANCE_ID"
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
STATE=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" \
--query 'Reservations[0].Instances[0].State.Name' --output text)
echo "Instance state: $STATE"
if [ "$STATE" == "running" ]; then
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -1136,14 +1136,14 @@ aws ec2 reboot-instances --instance-ids "$INSTANCE_ID"
# Verify state stays running (poll a few times to confirm no transient state change)
COUNT=0
while [ $COUNT -lt 5 ]; do
while [ $COUNT -lt 10 ]; do
STATE=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" \
--query 'Reservations[0].Instances[0].State.Name' --output text)
if [ "$STATE" != "running" ]; then
echo "Instance unexpectedly left running state: $STATE"
exit 1
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
echo "Instance state remained running during reboot"
@@ -1196,10 +1196,10 @@ echo "Launched 2 instances: $MULTI_ID_1, $MULTI_ID_2"
for MID in "$MULTI_ID_1" "$MULTI_ID_2"; do
echo "Waiting for $MID to reach running state..."
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
MSTATE=$(aws ec2 describe-instances --instance-ids "$MID" \
--query 'Reservations[0].Instances[0].State.Name' --output text) || {
sleep 2
sleep 1
COUNT=$((COUNT + 1))
continue
}
@@ -1207,7 +1207,7 @@ for MID in "$MULTI_ID_1" "$MULTI_ID_2"; do
echo "Instance $MID is running"
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
if [ "$MSTATE" != "running" ]; then
@@ -1221,13 +1221,13 @@ echo "Terminating multi-launch instances..."
aws ec2 terminate-instances --instance-ids "$MULTI_ID_1" "$MULTI_ID_2"
for MID in "$MULTI_ID_1" "$MULTI_ID_2"; do
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
MSTATE=$(aws ec2 describe-instances --instance-ids "$MID" \
--query 'Reservations[0].Instances[0].State.Name' --output text)
if [ "$MSTATE" == "terminated" ] || [ "$MSTATE" == "None" ]; then
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
done
@@ -1933,14 +1933,14 @@ echo " Beta instance: $BETA_INST"
# Wait for running
echo " Waiting for instances to reach running state..."
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
A_STATE=$(aws ec2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \
--query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending")
B_STATE=$(aws ec2 describe-instances --instance-ids "$BETA_INST" --profile hive-team-beta \
--query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending")
echo " Alpha=$A_STATE, Beta=$B_STATE"
if [ "$A_STATE" == "running" ] && [ "$B_STATE" == "running" ]; then break; fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
if [ "$A_STATE" != "running" ] || [ "$B_STATE" != "running" ]; then
@@ -1991,11 +1991,11 @@ echo " Alpha cannot reboot Beta's instance"
# Stop Alpha's instance for cross-account start/modify/console tests
aws ec2 stop-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha > /dev/null
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
A_STATE=$(aws ec2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \
--query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null)
if [ "$A_STATE" == "stopped" ]; then break; fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -2015,11 +2015,11 @@ echo " Beta cannot get console output of Alpha's instance"
# Restart Alpha's instance for later tests
aws ec2 start-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha > /dev/null
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
A_STATE=$(aws ec2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \
--query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null)
if [ "$A_STATE" == "running" ]; then break; fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -2066,7 +2066,7 @@ echo " Beta cannot attach Alpha's volume"
# Attach Alpha's volume to Alpha's instance, then test cross-account detach
aws ec2 attach-volume --volume-id "$ALPHA_VOL" \
--instance-id "$ALPHA_INST" --device /dev/sdf --profile hive-team-alpha > /dev/null
sleep 2
sleep 1
expect_error "InvalidVolume.NotFound" \
aws ec2 detach-volume --volume-id "$ALPHA_VOL" --profile hive-team-beta
@@ -2079,7 +2079,7 @@ echo " Beta cannot modify Alpha's volume"
# Detach for cleanup later
aws ec2 detach-volume --volume-id "$ALPHA_VOL" --profile hive-team-alpha > /dev/null
sleep 2
sleep 1
echo " Volume scoping passed"
@@ -2492,7 +2492,7 @@ aws ec2 terminate-instances --instance-ids "$BETA_INST" --profile hive-team-beta
# Wait for termination
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
A_STATE=$(aws ec2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \
--query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "terminated")
B_STATE=$(aws ec2 describe-instances --instance-ids "$BETA_INST" --profile hive-team-beta \
@@ -2500,7 +2500,7 @@ while [ $COUNT -lt 30 ]; do
if [ "$A_STATE" == "terminated" ] && [ "$B_STATE" == "terminated" ]; then
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
echo " Instances terminated"
@@ -2580,13 +2580,13 @@ fi
echo "Terminating instance..."
aws ec2 terminate-instances --instance-ids "$INSTANCE_ID"
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
STATE=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text)
echo "Instance state: $STATE"
if [ "$STATE" == "terminated" ] || [ "$STATE" == "None" ]; then
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
tests/e2e/run-multinode-e2e.sh
+23 -23
View File
@@ -88,7 +88,7 @@ aws_via_retry() {
}
attempt=$((attempt + 1))
echo " Retry $attempt/$max_attempts..." >&2
sleep 2
sleep 1
done
return 1
}
@@ -131,7 +131,7 @@ dump_all_node_logs() {
wait_for_instance_state() {
local instance_id="$1"
local target_state="$2"
local max_attempts="${3:-30}"
local max_attempts="${3:-60}"
local gw_ip="${4:-$LOCAL_IP}"
local attempt=0
@@ -143,7 +143,7 @@ wait_for_instance_state() {
--instance-ids "$instance_id" \
--query 'Reservations[0].Instances[0].State.Name' \
--output text 2>/dev/null) || {
sleep 2
sleep 1
attempt=$((attempt + 1))
continue
}
@@ -158,7 +158,7 @@ wait_for_instance_state() {
return 1
fi
sleep 2
sleep 1
attempt=$((attempt + 1))
done
@@ -264,7 +264,7 @@ terminate_and_wait() {
local failed=0
for instance_id in "${ids[@]}"; do
if ! wait_for_instance_state "$instance_id" "terminated" 30; then
if ! wait_for_instance_state "$instance_id" "terminated" 60; then
echo " WARNING: Failed to confirm termination of $instance_id"
failed=1
fi
@@ -384,15 +384,15 @@ pass_test "All gateways"
# Daemon readiness: describe-instance-types must return results
echo "Checking daemon readiness..."
ATTEMPT=0
while [ $ATTEMPT -lt 15 ]; do
while [ $ATTEMPT -lt 30 ]; do
TYPES=$($AWS_EC2 describe-instance-types \
--query 'InstanceTypes[*].InstanceType' --output text 2>/dev/null || true)
if [ -n "$TYPES" ] && [ "$TYPES" != "None" ]; then
echo " Daemon ready (instance types: $TYPES)"
break
fi
echo " Waiting for daemon... ($((ATTEMPT + 1))/15)"
sleep 2
echo " Waiting for daemon... ($((ATTEMPT + 1))/30)"
sleep 1
ATTEMPT=$((ATTEMPT + 1))
done
if [ -z "$TYPES" ] || [ "$TYPES" == "None" ]; then
@@ -469,7 +469,7 @@ done
echo ""
echo "Waiting for instances to reach running state..."
for instance_id in "${INSTANCE_IDS[@]}"; do
wait_for_instance_state "$instance_id" "running" 30 || {
wait_for_instance_state "$instance_id" "running" 60 || {
echo "ERROR: Instance $instance_id failed to start"
exit 1
}
@@ -626,7 +626,7 @@ else
# Wait for attachment
COUNT=0
while [ $COUNT -lt 15 ]; do
while [ $COUNT -lt 30 ]; do
ATTACH_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
--query 'Volumes[0].Attachments[0].State' --output text)
VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
@@ -635,7 +635,7 @@ else
echo " Volume attached"
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -650,14 +650,14 @@ else
$AWS_EC2 detach-volume --volume-id "$TEST_VOLUME_ID" > /dev/null
COUNT=0
while [ $COUNT -lt 15 ]; do
while [ $COUNT -lt 30 ]; do
VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
--query 'Volumes[0].State' --output text)
if [ "$VOL_STATE" == "available" ]; then
echo " Volume detached"
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -672,7 +672,7 @@ else
$AWS_EC2 delete-volume --volume-id "$TEST_VOLUME_ID"
COUNT=0
while [ $COUNT -lt 15 ]; do
while [ $COUNT -lt 30 ]; do
set +e
VOL_CHECK=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
--query 'Volumes[0].VolumeId' --output text 2>&1)
@@ -682,11 +682,11 @@ else
echo " Volume deleted"
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
if [ $COUNT -lt 15 ]; then
if [ $COUNT -lt 30 ]; then
pass_test "Volume delete"
else
fail_test "Volume delete"
@@ -746,7 +746,7 @@ echo " Will operate via gateway on $OTHER_GW"
# Stop via other gateway
echo " Stopping instance via $OTHER_GW..."
aws_via "$OTHER_GW" ec2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null
wait_for_instance_state "$TEST_INSTANCE" "stopped" 30 "$OTHER_GW"
wait_for_instance_state "$TEST_INSTANCE" "stopped" 60 "$OTHER_GW"
# Pick yet another gateway for start (or same other if only 2 choices)
THIRD_GW=""
@@ -759,7 +759,7 @@ done
THIRD_GW="${THIRD_GW:-$OTHER_GW}"
echo " Starting instance via $THIRD_GW..."
aws_via "$THIRD_GW" ec2 start-instances --instance-ids "$TEST_INSTANCE" > /dev/null
wait_for_instance_state "$TEST_INSTANCE" "running" 30 "$THIRD_GW"
wait_for_instance_state "$TEST_INSTANCE" "running" 60 "$THIRD_GW"
pass_test "Cross-node stop/start"
@@ -844,7 +844,7 @@ peer_ssh "$NODE2_IP" "cd ~/Development/mulga/spinifex && ./scripts/start-dev.sh"
echo " Waiting for NATS cluster to reform..."
ATTEMPT=0
REFORMED=false
while [ $ATTEMPT -lt 30 ]; do
while [ $ATTEMPT -lt 60 ]; do
NATS_RECOVER=$(curl -s "http://${LOCAL_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null)
RECOVER_PEERS=$(echo "$NATS_RECOVER" | jq -r '[.routes[].remote_name] | unique | length' 2>/dev/null || echo "0")
if [ "$RECOVER_PEERS" -ge 2 ]; then
@@ -852,8 +852,8 @@ while [ $ATTEMPT -lt 30 ]; do
REFORMED=true
break
fi
echo " Waiting for NATS reform... ($((ATTEMPT + 1))/30, peers: $RECOVER_PEERS)"
sleep 2
echo " Waiting for NATS reform... ($((ATTEMPT + 1))/60, peers: $RECOVER_PEERS)"
sleep 1
ATTEMPT=$((ATTEMPT + 1))
done
@@ -868,13 +868,13 @@ fi
echo " Waiting for node2 gateway..."
ATTEMPT=0
GW_BACK=false
while [ $ATTEMPT -lt 15 ]; do
while [ $ATTEMPT -lt 30 ]; do
if curl -k -s "https://${NODE2_IP}:${AWSGW_PORT}" > /dev/null 2>&1; then
echo " Node2 gateway is back"
GW_BACK=true
break
fi
sleep 2
sleep 1
ATTEMPT=$((ATTEMPT + 1))
done
tests/e2e/run-pseudo-multinode-e2e.sh
+25 -25
View File
@@ -344,7 +344,7 @@ for i in 1 2 3; do
INSTANCE_IDS+=("$INSTANCE_ID")
# Small delay between launches to encourage distribution
sleep 2
sleep 1
done
# Wait for all instances to be running
@@ -489,7 +489,7 @@ $AWS_EC2 modify-volume --volume-id "$TEST_VOLUME_ID" --size "$NEW_SIZE"
# Verify resize
echo " Verifying resize..."
COUNT=0
while [ $COUNT -lt 15 ]; do
while [ $COUNT -lt 30 ]; do
VOLUME_SIZE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
--query 'Volumes[0].Size' --output text)
@@ -498,7 +498,7 @@ while [ $COUNT -lt 15 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -514,7 +514,7 @@ $AWS_EC2 attach-volume --volume-id "$TEST_VOLUME_ID" --instance-id "${INSTANCE_I
# Verify attachment
echo " Verifying volume attachment..."
COUNT=0
while [ $COUNT -lt 15 ]; do
while [ $COUNT -lt 30 ]; do
ATTACH_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
--query 'Volumes[0].Attachments[0].State' --output text)
ATTACH_INSTANCE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
@@ -527,7 +527,7 @@ while [ $COUNT -lt 15 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -543,7 +543,7 @@ $AWS_EC2 detach-volume --volume-id "$TEST_VOLUME_ID"
# Verify detachment
echo " Verifying volume detachment..."
COUNT=0
while [ $COUNT -lt 15 ]; do
while [ $COUNT -lt 30 ]; do
VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
--query 'Volumes[0].State' --output text)
@@ -552,7 +552,7 @@ while [ $COUNT -lt 15 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -568,7 +568,7 @@ $AWS_EC2 delete-volume --volume-id "$TEST_VOLUME_ID"
# Verify deletion
echo " Verifying volume deletion..."
COUNT=0
while [ $COUNT -lt 15 ]; do
while [ $COUNT -lt 30 ]; do
set +e
VOLUME_CHECK=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \
--query 'Volumes[0].VolumeId' --output text 2>&1)
@@ -580,11 +580,11 @@ while [ $COUNT -lt 15 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
if [ $COUNT -ge 15 ]; then
if [ $COUNT -ge 30 ]; then
echo " ERROR: Volume deletion verification timed out"
exit 1
fi
@@ -635,7 +635,7 @@ echo " Create response verified (State=$SNAP_STATE, VolumeId=$SNAP_VOL_REF, Siz
# Poll until completed
echo " Waiting for snapshot to complete..."
COUNT=0
while [ $COUNT -lt 15 ]; do
while [ $COUNT -lt 30 ]; do
SNAP_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \
--query 'Snapshots[0].State' --output text)
@@ -644,7 +644,7 @@ while [ $COUNT -lt 15 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
@@ -717,7 +717,7 @@ $AWS_EC2 delete-snapshot --snapshot-id "$SNAPSHOT_ID"
# Verify original gone, copy remains
echo " Verifying snapshot deletion..."
COUNT=0
while [ $COUNT -lt 15 ]; do
while [ $COUNT -lt 30 ]; do
set +e
SNAP_CHECK=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \
--query 'Snapshots[0].SnapshotId' --output text 2>&1)
@@ -729,11 +729,11 @@ while [ $COUNT -lt 15 ]; do
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
if [ $COUNT -ge 15 ]; then
if [ $COUNT -ge 30 ]; then
echo " ERROR: Snapshot deletion verification timed out"
exit 1
fi
@@ -1121,7 +1121,7 @@ for crash_num in 1 2 3 4; do
if [ $crash_num -lt 4 ]; then
# Wait for restart (backoff increases: 5s, 10s, 20s)
# Give generous time for each restart cycle
local_max=$((15 + crash_num * 10))
local_max=$((30 + crash_num * 20))
echo " Waiting up to ${local_max}s for restart or error state..."
attempt=0
while [ $attempt -lt $local_max ]; do
@@ -1135,7 +1135,7 @@ for crash_num in 1 2 3 4; do
echo " Instance in error state after crash $crash_num (restart limit may be reached)"
break
fi
sleep 2
sleep 1
attempt=$((attempt + 2))
done
fi
@@ -1452,7 +1452,7 @@ while [ $COUNT -lt 30 ]; do
--query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending")
echo " Alpha=$A_STATE, Beta=$B_STATE"
if [ "$A_STATE" == "running" ] && [ "$B_STATE" == "running" ]; then break; fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
if [ "$A_STATE" != "running" ] || [ "$B_STATE" != "running" ]; then
@@ -1961,7 +1961,7 @@ while [ $COUNT -lt 30 ]; do
if [ "$A_STATE" == "terminated" ] && [ "$B_STATE" == "terminated" ]; then
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
echo " Instances terminated"
@@ -2076,7 +2076,7 @@ fi
echo " Created Subnet: $SUBNET_ID (10.100.1.0/24)"
# Brief pause for OVN topology to be programmed (logical switch + router port + DHCP)
sleep 2
sleep 1
# Step 2: Launch 3 VPC instances
echo ""
@@ -2103,7 +2103,7 @@ for i in 1 2 3; do
echo " Launched: $VPC_INST_ID (PrivateIpAddress: ${VPC_INST_IP:-not yet assigned})"
VPC_INSTANCE_IDS+=("$VPC_INST_ID")
sleep 2
sleep 1
done
# Wait for all VPC instances to be running
@@ -2316,7 +2316,7 @@ CLUSTER_SERVICES_STARTED="false"
# Verify all services are down
echo " Waiting for services to stop..."
sleep 2
sleep 1
if ! verify_all_services_down; then
echo " Some services still running, force-cleaning..."
force_cleanup_all_nodes
@@ -2380,17 +2380,17 @@ echo "Waiting for instances to relaunch after cluster restart..."
for instance_id in "${INSTANCE_IDS[0]}" "${INSTANCE_IDS[1]}"; do
echo " Waiting for $instance_id to finish relaunching..."
COUNT=0
while [ $COUNT -lt 30 ]; do
while [ $COUNT -lt 60 ]; do
STATE=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \
--query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "unknown")
if [ "$STATE" = "running" ]; then
echo " $instance_id relaunched successfully: $STATE"
break
fi
sleep 2
sleep 1
COUNT=$((COUNT + 1))
done
if [ $COUNT -ge 30 ]; then
if [ $COUNT -ge 60 ]; then
echo " WARNING: $instance_id still in $STATE after 60s"
fi
done