diff --git a/hive/handlers/ec2/instance/service_impl.go b/hive/handlers/ec2/instance/service_impl.go index 10b713d0c..bc7ec1912 100644 --- a/hive/handlers/ec2/instance/service_impl.go +++ b/hive/handlers/ec2/instance/service_impl.go @@ -164,7 +164,7 @@ func (s *InstanceServiceImpl) GenerateVolumes(input *ec2.RunInstancesInput, inst } if bdm.Ebs != nil { if bdm.Ebs.VolumeSize != nil { - size = int(*bdm.Ebs.VolumeSize) + size = int(*bdm.Ebs.VolumeSize) * 1024 * 1024 * 1024 // AWS API sends GiB, convert to bytes } if bdm.Ebs.VolumeType != nil { volumeType = *bdm.Ebs.VolumeType diff --git a/hive/handlers/ec2/snapshot/service_impl.go b/hive/handlers/ec2/snapshot/service_impl.go index 842b443e8..8209f9346 100644 --- a/hive/handlers/ec2/snapshot/service_impl.go +++ b/hive/handlers/ec2/snapshot/service_impl.go @@ -167,11 +167,17 @@ func (s *SnapshotServiceImpl) CreateSnapshot(input *ec2.CreateSnapshotInput) (*e } defer volumeResult.Body.Close() - var volumeConfig viperblock.VolumeConfig - if err := json.NewDecoder(volumeResult.Body).Decode(&volumeConfig); err != nil { + var volumeState viperblock.VBState + if err := json.NewDecoder(volumeResult.Body).Decode(&volumeState); err != nil { slog.Error("CreateSnapshot failed to decode volume config", "volumeId", volumeID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } + volumeConfig := volumeState.VolumeConfig + + if volumeConfig.VolumeMetadata.SizeGiB == 0 { + slog.Error("CreateSnapshot: source volume has zero size in config", "volumeId", volumeID) + return nil, errors.New(awserrors.ErrorServerInternal) + } now := time.Now() diff --git a/hive/handlers/ec2/snapshot/service_impl_test.go b/hive/handlers/ec2/snapshot/service_impl_test.go index c58303d7c..202facb26 100644 --- a/hive/handlers/ec2/snapshot/service_impl_test.go +++ b/hive/handlers/ec2/snapshot/service_impl_test.go @@ -31,15 +31,18 @@ func setupTestSnapshotService(t *testing.T) (*SnapshotServiceImpl, *objectstore. } // createTestVolume creates a test volume in the mock store +// The real S3 stores VBState (which wraps VolumeConfig), so we match that format. func createTestVolume(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string, sizeGiB int) { - volumeConfig := viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - SizeGiB: uint64(sizeGiB), - IsEncrypted: false, - AvailabilityZone: "us-east-1a", + volumeState := viperblock.VBState{ + VolumeConfig: viperblock.VolumeConfig{ + VolumeMetadata: viperblock.VolumeMetadata{ + SizeGiB: uint64(sizeGiB), + IsEncrypted: false, + AvailabilityZone: "us-east-1a", + }, }, } - data, err := json.Marshal(volumeConfig) + data, err := json.Marshal(volumeState) require.NoError(t, err) _, err = store.PutObject(&s3.PutObjectInput{ @@ -97,6 +100,35 @@ func TestCreateSnapshot_MissingVolumeId(t *testing.T) { assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameterValue) } +// TestCreateSnapshot_VolumeZeroSize tests that creating a snapshot from a volume with zero SizeGiB fails +func TestCreateSnapshot_VolumeZeroSize(t *testing.T) { + svc, store := setupTestSnapshotService(t) + + // Store a volume config with SizeGiB == 0 + volumeState := viperblock.VBState{ + VolumeConfig: viperblock.VolumeConfig{ + VolumeMetadata: viperblock.VolumeMetadata{ + SizeGiB: 0, + }, + }, + } + data, err := json.Marshal(volumeState) + require.NoError(t, err) + + _, err = store.PutObject(&s3.PutObjectInput{ + Bucket: aws.String("test-bucket"), + Key: aws.String("vol-zerosize/config.json"), + Body: strings.NewReader(string(data)), + }) + require.NoError(t, err) + + _, err = svc.CreateSnapshot(&ec2.CreateSnapshotInput{ + VolumeId: aws.String("vol-zerosize"), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), awserrors.ErrorServerInternal) +} + // TestCreateSnapshot_VolumeNotFound tests creating a snapshot from non-existent volume func TestCreateSnapshot_VolumeNotFound(t *testing.T) { svc, _ := setupTestSnapshotService(t) diff --git a/hive/handlers/ec2/volume/service_impl.go b/hive/handlers/ec2/volume/service_impl.go index 6c623e46f..5f2c64e3e 100644 --- a/hive/handlers/ec2/volume/service_impl.go +++ b/hive/handlers/ec2/volume/service_impl.go @@ -383,6 +383,11 @@ func (s *VolumeServiceImpl) getVolumeByID(volumeID string) (*ec2.Volume, error) return nil, errors.New("volume ID is empty") } + if volMeta.SizeGiB == 0 { + slog.Error("Volume has zero size in config", "volumeId", volumeID) + return nil, fmt.Errorf("volume %s has zero size in config", volumeID) + } + state := volMeta.State if state == "" { state = "available" diff --git a/tests/e2e/TEST_COVERAGE.md b/tests/e2e/TEST_COVERAGE.md new file mode 100644 index 000000000..8fdcf90a8 --- /dev/null +++ b/tests/e2e/TEST_COVERAGE.md @@ -0,0 +1,142 @@ +# E2E Test Coverage + +## Single-Node (`run-e2e.sh`) + +### Phase 1: Environment Setup +- KVM support check +- `hive admin init` (region/az/node config) +- CA certificate trust +- Start all services (`start-dev.sh`) +- Wait for AWS gateway on `localhost:9999` + +### Phase 2: Discovery & Metadata +- `describe-regions` +- `describe-instance-types` (discover available types) +- Select nano instance type and detect architecture + +### Phase 3: SSH Key Management +- `create-key-pair` (test-key-1, verify private key material) +- `import-key-pair` (test-key-2, from local RSA key) +- `describe-key-pairs` (verify both keys exist) +- `delete-key-pair` (test-key-2, verify only test-key-1 remains) + +### Phase 4: Image Management +- `hive admin images import` (file-based, architecture-aware) +- `describe-images` (verify AMI by ID) + +### Phase 5: Instance Lifecycle +- `run-instances` (launch VM with key pair) +- `describe-instances` (poll pending -> running) +- `describe-volumes` (verify root volume attached) + +### Phase 5b: Volume Lifecycle +- `create-volume` (10GB, ap-southeast-2a) +- `modify-volume` (resize to 20GB, poll to verify) +- `attach-volume` (to running instance, /dev/sdf) +- `describe-volumes` (verify in-use + attached state) +- `detach-volume` (verify available state) +- `delete-volume` (verify gone) + +### Phase 5c: Snapshot Lifecycle +- `create-snapshot` (from volume, with description) +- Verify create response fields (VolumeId, VolumeSize, State, Progress) +- `describe-snapshots` (by ID, verify VolumeId/Size/Description) +- `copy-snapshot` (with new description, verify distinct ID) +- `describe-snapshots` (verify both original + copy visible) +- `delete-snapshot` (original, verify gone while copy survives) +- `delete-snapshot` (copy, cleanup) + +### Phase 6: Tag Management +- `create-tags` (3 tags on instance) +- `describe-tags` (filter by resource-id) +- `create-tags` (2 tags on volume) +- `describe-tags` (filter by key) +- `describe-tags` (filter by resource-type) +- `create-tags` (overwrite existing tag value) +- `delete-tags` (unconditional by key) +- `delete-tags` (with wrong value — should be no-op) +- `delete-tags` (with correct value) +- Verify final tag count + +### Phase 7: Instance State Transitions +- `stop-instances` (poll -> stopped) +- `start-instances` (poll -> running) +- `terminate-instances` (poll -> terminated) + +--- + +## Multi-Node (`run-multinode-e2e.sh`) + +### Phase 1: Environment Setup +- KVM support check +- Simulated network IPs (10.11.12.{1,2,3} on loopback) +- Create ramdisk mount + +### Phase 2: Cluster Initialization +- `hive admin init` (leader node1) +- CA certificate trust +- Start node1 services +- `hive admin join` (node2, node3) +- Start node2 + node3 services + +### Phase 3: Cluster Health Verification +- Verify NATS cluster (3 nodes) +- Verify Predastore cluster (3 nodes) +- Wait for gateway on node1 +- `describe-regions` (gateway connectivity check) + +### Phase 4: Image and Key Setup +- `describe-instance-types` (discover + select nano) +- `create-key-pair` +- `hive admin images import` (with node1 config paths) +- `describe-images` (verify AMI) + +### Phase 5: Multi-Node Instance Tests + +#### Test 1: Instance Distribution +- `run-instances` x3 (distribute across nodes) +- Poll all instances to running state +- Check instance distribution across nodes + +#### Test 1b: Volume Lifecycle +- `create-volume` (10GB) +- `modify-volume` (resize to 20GB) +- `attach-volume` (to first instance) +- `detach-volume` +- `delete-volume` + +#### Test 1c: Snapshot Lifecycle +- `create-snapshot` (from volume, with description) +- Verify create response fields (VolumeId, VolumeSize, State) +- `describe-snapshots` (by ID, verify fields) +- `copy-snapshot` (with new description) +- `describe-snapshots` (verify both exist) +- `delete-snapshot` (original, verify copy survives) +- `delete-snapshot` (copy, cleanup) + +#### Test 1d: Tag Management +- `create-tags` (3 tags on instance) +- `describe-tags` (filter by resource-id) +- `describe-tags` (filter by key) +- `describe-tags` (filter by resource-type) +- `create-tags` (overwrite tag value) +- `delete-tags` (unconditional by key) +- `delete-tags` (wrong value — no-op) +- `delete-tags` (correct value) +- Verify final tag count + +#### Test 2: DescribeInstances Aggregation +- `describe-instances` (fan-out across all nodes, verify count) + +#### Test 3: Cross-Node Operations +- `stop-instances` (poll -> stopped) +- `start-instances` (poll -> running) + +#### Test 4: NATS Cluster Health (Post-Operations) +- Verify NATS cluster still healthy after all operations + +### Cleanup +- `terminate-instances` (all 3 instances) +- Poll all to terminated state +- Stop all node services +- Remove simulated IPs diff --git a/tests/e2e/run-e2e.sh b/tests/e2e/run-e2e.sh index dfc35411e..6ccdabcf6 100755 --- a/tests/e2e/run-e2e.sh +++ b/tests/e2e/run-e2e.sh @@ -377,6 +377,175 @@ fi echo "Volume lifecycle test passed (create -> resize -> attach -> detach -> delete)" +# Phase 5c: Snapshot Lifecycle +echo "Phase 5c: Snapshot Lifecycle" +echo "Testing snapshot create -> describe -> copy -> delete..." + +# Create a volume to snapshot (snapshots need a source volume) +echo "Creating source volume for snapshot tests..." +SNAP_VOL_OUTPUT=$($AWS_EC2 create-volume --size 10 --availability-zone ap-southeast-2a) +SNAP_VOL_ID=$(echo "$SNAP_VOL_OUTPUT" | jq -r '.VolumeId') + +if [ -z "$SNAP_VOL_ID" ] || [ "$SNAP_VOL_ID" == "null" ]; then + echo "Failed to create source volume for snapshot tests" + exit 1 +fi +echo "Created source volume: $SNAP_VOL_ID" + +# Create a snapshot from the volume +echo "Creating snapshot from volume $SNAP_VOL_ID..." +SNAP_OUTPUT=$($AWS_EC2 create-snapshot --volume-id "$SNAP_VOL_ID" --description "e2e-test-snapshot") +SNAPSHOT_ID=$(echo "$SNAP_OUTPUT" | jq -r '.SnapshotId') + +if [ -z "$SNAPSHOT_ID" ] || [ "$SNAPSHOT_ID" == "null" ]; then + echo "Failed to create snapshot" + echo "Output: $SNAP_OUTPUT" + exit 1 +fi +echo "Created snapshot: $SNAPSHOT_ID" + +# Verify snapshot fields from create response +SNAP_STATE=$(echo "$SNAP_OUTPUT" | jq -r '.State') +SNAP_VOL_REF=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeId') +SNAP_SIZE=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeSize') +SNAP_PROGRESS=$(echo "$SNAP_OUTPUT" | jq -r '.Progress') + +if [ "$SNAP_VOL_REF" != "$SNAP_VOL_ID" ]; then + echo "Snapshot VolumeId mismatch: expected $SNAP_VOL_ID, got $SNAP_VOL_REF" + exit 1 +fi +if [ "$SNAP_SIZE" -ne 10 ]; then + echo "Snapshot VolumeSize mismatch: expected 10, got $SNAP_SIZE" + exit 1 +fi +echo "Snapshot create response verified (State=$SNAP_STATE, VolumeId=$SNAP_VOL_REF, Size=$SNAP_SIZE, Progress=$SNAP_PROGRESS)" + +# Poll until snapshot is completed (should be immediate in v1, but poll for forward-compat) +echo "Waiting for snapshot to complete..." +COUNT=0 +while [ $COUNT -lt 30 ]; do + SNAP_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ + --query 'Snapshots[0].State' --output text) + + if [ "$SNAP_STATE" == "completed" ]; then + echo "Snapshot completed" + break + fi + + sleep 2 + COUNT=$((COUNT + 1)) +done + +if [ "$SNAP_STATE" != "completed" ]; then + echo "Snapshot failed to reach completed state (State=$SNAP_STATE)" + exit 1 +fi + +# Describe snapshot by ID and verify fields +echo "Verifying snapshot via describe-snapshots..." +DESCRIBE_SNAP=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID") +DESC_VOL_ID=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeId') +DESC_SIZE=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeSize') +DESC_DESC=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].Description') + +if [ "$DESC_VOL_ID" != "$SNAP_VOL_ID" ]; then + echo "Describe snapshot VolumeId mismatch: expected $SNAP_VOL_ID, got $DESC_VOL_ID" + exit 1 +fi +if [ "$DESC_SIZE" -ne 10 ]; then + echo "Describe snapshot VolumeSize mismatch: expected 10, got $DESC_SIZE" + exit 1 +fi +if [ "$DESC_DESC" != "e2e-test-snapshot" ]; then + echo "Describe snapshot Description mismatch: expected 'e2e-test-snapshot', got '$DESC_DESC'" + exit 1 +fi +echo "Describe snapshot verified (VolumeId=$DESC_VOL_ID, Size=$DESC_SIZE, Description=$DESC_DESC)" + +# Copy the snapshot +echo "Copying snapshot $SNAPSHOT_ID..." +COPY_OUTPUT=$($AWS_EC2 copy-snapshot --source-snapshot-id "$SNAPSHOT_ID" --source-region ap-southeast-2 --description "e2e-copy") +COPY_SNAPSHOT_ID=$(echo "$COPY_OUTPUT" | jq -r '.SnapshotId') + +if [ -z "$COPY_SNAPSHOT_ID" ] || [ "$COPY_SNAPSHOT_ID" == "null" ]; then + echo "Failed to copy snapshot" + echo "Output: $COPY_OUTPUT" + exit 1 +fi +echo "Copied snapshot: $COPY_SNAPSHOT_ID" + +# Verify the copy is a distinct snapshot +if [ "$COPY_SNAPSHOT_ID" == "$SNAPSHOT_ID" ]; then + echo "Copy snapshot ID should differ from original" + exit 1 +fi + +# Describe all snapshots — should see both original and copy +TOTAL_SNAPS=$($AWS_EC2 describe-snapshots \ + --snapshot-ids "$SNAPSHOT_ID" "$COPY_SNAPSHOT_ID" \ + --query 'length(Snapshots)' --output text) + +if [ "$TOTAL_SNAPS" -ne 2 ]; then + echo "Expected 2 snapshots, got $TOTAL_SNAPS" + exit 1 +fi +echo "Both snapshots visible via describe-snapshots" + +# Verify copy has correct description +COPY_DESC=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \ + --query 'Snapshots[0].Description' --output text) +if [ "$COPY_DESC" != "e2e-copy" ]; then + echo "Copy description mismatch: expected 'e2e-copy', got '$COPY_DESC'" + exit 1 +fi + +# Delete the original snapshot +echo "Deleting original snapshot $SNAPSHOT_ID..." +$AWS_EC2 delete-snapshot --snapshot-id "$SNAPSHOT_ID" + +# Verify original is gone, copy remains +echo "Verifying snapshot deletion..." +COUNT=0 +while [ $COUNT -lt 30 ]; do + set +e + SNAP_CHECK=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ + --query 'Snapshots[0].SnapshotId' --output text 2>&1) + SNAP_EXIT=$? + set -e + + if [ $SNAP_EXIT -ne 0 ] || [ "$SNAP_CHECK" == "None" ] || [ -z "$SNAP_CHECK" ]; then + echo "Original snapshot deleted successfully" + break + fi + + sleep 2 + COUNT=$((COUNT + 1)) +done + +if [ $COUNT -ge 30 ]; then + echo "Snapshot deletion verification timed out" + exit 1 +fi + +# Verify copy still exists +COPY_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \ + --query 'Snapshots[0].State' --output text) +if [ "$COPY_STATE" != "completed" ]; then + echo "Copy snapshot should still exist (State=$COPY_STATE)" + exit 1 +fi +echo "Copy snapshot still intact after original deletion" + +# Delete the copy +echo "Deleting copy snapshot $COPY_SNAPSHOT_ID..." +$AWS_EC2 delete-snapshot --snapshot-id "$COPY_SNAPSHOT_ID" + +# Clean up the source volume +echo "Deleting source volume $SNAP_VOL_ID..." +$AWS_EC2 delete-volume --volume-id "$SNAP_VOL_ID" + +echo "Snapshot lifecycle test passed (create -> describe -> copy -> delete)" + # Phase 6: Tag Management echo "Phase 6: Tag Management" diff --git a/tests/e2e/run-multinode-e2e.sh b/tests/e2e/run-multinode-e2e.sh index 5f1cdd268..222121c96 100755 --- a/tests/e2e/run-multinode-e2e.sh +++ b/tests/e2e/run-multinode-e2e.sh @@ -382,6 +382,273 @@ fi echo " Volume lifecycle test passed (create -> resize -> attach -> detach -> delete)" +# Test 1c: Snapshot Lifecycle +echo "" +echo "Test 1c: Snapshot Lifecycle" +echo "----------------------------------------" +echo "Testing snapshot create -> describe -> copy -> delete..." + +# Create a volume to snapshot +echo " Creating source volume for snapshot tests..." +SNAP_VOL_OUTPUT=$($AWS_EC2 create-volume --size 10 --availability-zone ap-southeast-2a) +SNAP_VOL_ID=$(echo "$SNAP_VOL_OUTPUT" | jq -r '.VolumeId') + +if [ -z "$SNAP_VOL_ID" ] || [ "$SNAP_VOL_ID" == "null" ]; then + echo " ERROR: Failed to create source volume for snapshot tests" + exit 1 +fi +echo " Created source volume: $SNAP_VOL_ID" + +# Create a snapshot +echo " Creating snapshot from volume $SNAP_VOL_ID..." +SNAP_OUTPUT=$($AWS_EC2 create-snapshot --volume-id "$SNAP_VOL_ID" --description "multinode-e2e-snapshot") +SNAPSHOT_ID=$(echo "$SNAP_OUTPUT" | jq -r '.SnapshotId') + +if [ -z "$SNAPSHOT_ID" ] || [ "$SNAPSHOT_ID" == "null" ]; then + echo " ERROR: Failed to create snapshot" + echo " Output: $SNAP_OUTPUT" + exit 1 +fi +echo " Created snapshot: $SNAPSHOT_ID" + +# Verify create response fields +SNAP_STATE=$(echo "$SNAP_OUTPUT" | jq -r '.State') +SNAP_VOL_REF=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeId') +SNAP_SIZE=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeSize') + +if [ "$SNAP_VOL_REF" != "$SNAP_VOL_ID" ]; then + echo " ERROR: Snapshot VolumeId mismatch: expected $SNAP_VOL_ID, got $SNAP_VOL_REF" + exit 1 +fi +if [ "$SNAP_SIZE" -ne 10 ]; then + echo " ERROR: Snapshot VolumeSize mismatch: expected 10, got $SNAP_SIZE" + exit 1 +fi +echo " Create response verified (State=$SNAP_STATE, VolumeId=$SNAP_VOL_REF, Size=$SNAP_SIZE)" + +# Poll until completed +echo " Waiting for snapshot to complete..." +COUNT=0 +while [ $COUNT -lt 30 ]; do + SNAP_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ + --query 'Snapshots[0].State' --output text) + + if [ "$SNAP_STATE" == "completed" ]; then + echo " Snapshot completed" + break + fi + + sleep 2 + COUNT=$((COUNT + 1)) +done + +if [ "$SNAP_STATE" != "completed" ]; then + echo " ERROR: Snapshot failed to reach completed state (State=$SNAP_STATE)" + exit 1 +fi + +# Describe by ID and verify +echo " Verifying snapshot via describe-snapshots..." +DESCRIBE_SNAP=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID") +DESC_VOL_ID=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeId') +DESC_SIZE=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeSize') +DESC_DESC=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].Description') + +if [ "$DESC_VOL_ID" != "$SNAP_VOL_ID" ]; then + echo " ERROR: Describe VolumeId mismatch: expected $SNAP_VOL_ID, got $DESC_VOL_ID" + exit 1 +fi +if [ "$DESC_SIZE" -ne 10 ]; then + echo " ERROR: Describe VolumeSize mismatch: expected 10, got $DESC_SIZE" + exit 1 +fi +if [ "$DESC_DESC" != "multinode-e2e-snapshot" ]; then + echo " ERROR: Describe Description mismatch: expected 'multinode-e2e-snapshot', got '$DESC_DESC'" + exit 1 +fi +echo " Describe verified (VolumeId=$DESC_VOL_ID, Size=$DESC_SIZE, Description=$DESC_DESC)" + +# Copy the snapshot +echo " Copying snapshot $SNAPSHOT_ID..." +COPY_OUTPUT=$($AWS_EC2 copy-snapshot --source-snapshot-id "$SNAPSHOT_ID" --source-region ap-southeast-2 --description "multinode-e2e-copy") +COPY_SNAPSHOT_ID=$(echo "$COPY_OUTPUT" | jq -r '.SnapshotId') + +if [ -z "$COPY_SNAPSHOT_ID" ] || [ "$COPY_SNAPSHOT_ID" == "null" ]; then + echo " ERROR: Failed to copy snapshot" + echo " Output: $COPY_OUTPUT" + exit 1 +fi +echo " Copied snapshot: $COPY_SNAPSHOT_ID" + +if [ "$COPY_SNAPSHOT_ID" == "$SNAPSHOT_ID" ]; then + echo " ERROR: Copy snapshot ID should differ from original" + exit 1 +fi + +# Verify both exist +TOTAL_SNAPS=$($AWS_EC2 describe-snapshots \ + --snapshot-ids "$SNAPSHOT_ID" "$COPY_SNAPSHOT_ID" \ + --query 'length(Snapshots)' --output text) + +if [ "$TOTAL_SNAPS" -ne 2 ]; then + echo " ERROR: Expected 2 snapshots, got $TOTAL_SNAPS" + exit 1 +fi +echo " Both snapshots visible via describe-snapshots" + +# Verify copy description +COPY_DESC=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \ + --query 'Snapshots[0].Description' --output text) +if [ "$COPY_DESC" != "multinode-e2e-copy" ]; then + echo " ERROR: Copy description mismatch: expected 'multinode-e2e-copy', got '$COPY_DESC'" + exit 1 +fi + +# Delete original +echo " Deleting original snapshot $SNAPSHOT_ID..." +$AWS_EC2 delete-snapshot --snapshot-id "$SNAPSHOT_ID" + +# Verify original gone, copy remains +echo " Verifying snapshot deletion..." +COUNT=0 +while [ $COUNT -lt 30 ]; do + set +e + SNAP_CHECK=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ + --query 'Snapshots[0].SnapshotId' --output text 2>&1) + SNAP_EXIT=$? + set -e + + if [ $SNAP_EXIT -ne 0 ] || [ "$SNAP_CHECK" == "None" ] || [ -z "$SNAP_CHECK" ]; then + echo " Original snapshot deleted successfully" + break + fi + + sleep 2 + COUNT=$((COUNT + 1)) +done + +if [ $COUNT -ge 30 ]; then + echo " ERROR: Snapshot deletion verification timed out" + exit 1 +fi + +# Verify copy still exists +COPY_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \ + --query 'Snapshots[0].State' --output text) +if [ "$COPY_STATE" != "completed" ]; then + echo " ERROR: Copy snapshot should still exist (State=$COPY_STATE)" + exit 1 +fi +echo " Copy snapshot intact after original deletion" + +# Delete copy and source volume +echo " Deleting copy snapshot $COPY_SNAPSHOT_ID..." +$AWS_EC2 delete-snapshot --snapshot-id "$COPY_SNAPSHOT_ID" + +echo " Deleting source volume $SNAP_VOL_ID..." +$AWS_EC2 delete-volume --volume-id "$SNAP_VOL_ID" + +echo " Snapshot lifecycle test passed (create -> describe -> copy -> delete)" + +# Test 1d: Tag Management +echo "" +echo "Test 1d: Tag Management" +echo "----------------------------------------" +echo "Testing create-tags -> describe-tags -> delete-tags..." + +# Use the first instance for tag tests +TAG_INSTANCE="${INSTANCE_IDS[0]}" + +# Create tags on instance +echo " Creating tags on instance $TAG_INSTANCE..." +$AWS_EC2 create-tags --resources "$TAG_INSTANCE" --tags Key=Name,Value=multinode-test Key=Environment,Value=testing Key=DeleteMe,Value=please + +# Verify tags with describe-tags (resource-id filter) +echo " Verifying tags on instance..." +TAG_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$TAG_COUNT" -ne 3 ]; then + echo " ERROR: Expected 3 tags on instance, got $TAG_COUNT" + exit 1 +fi +echo " Instance has $TAG_COUNT tags" + +# Filter by key +echo " Testing key filter..." +ENV_TAGS=$($AWS_EC2 describe-tags --filters "Name=key,Values=Environment" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$ENV_TAGS" -lt 1 ]; then + echo " ERROR: Expected at least 1 'Environment' tag, got $ENV_TAGS" + exit 1 +fi +echo " Key filter returned $ENV_TAGS tags" + +# Filter by resource-type +echo " Testing resource-type filter..." +INSTANCE_TAGS=$($AWS_EC2 describe-tags --filters "Name=resource-type,Values=instance" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$INSTANCE_TAGS" -lt 3 ]; then + echo " ERROR: Expected at least 3 instance tags, got $INSTANCE_TAGS" + exit 1 +fi +echo " Resource-type filter returned $INSTANCE_TAGS instance tags" + +# Overwrite a tag value +echo " Overwriting Name tag..." +$AWS_EC2 create-tags --resources "$TAG_INSTANCE" --tags Key=Name,Value=multinode-updated +UPDATED_NAME=$($AWS_EC2 describe-tags \ + --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Name" \ + --query 'Tags[0].Value' --output text) +if [ "$UPDATED_NAME" != "multinode-updated" ]; then + echo " ERROR: Tag overwrite failed: expected 'multinode-updated', got '$UPDATED_NAME'" + exit 1 +fi +echo " Tag overwrite verified" + +# Delete tag by key (unconditional) +echo " Deleting DeleteMe tag unconditionally..." +$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=DeleteMe +REMAINING=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$REMAINING" -ne 2 ]; then + echo " ERROR: Expected 2 tags after unconditional delete, got $REMAINING" + exit 1 +fi +echo " Unconditional delete verified ($REMAINING tags remaining)" + +# Delete tag with wrong value (should NOT delete) +echo " Attempting delete with wrong value (should be no-op)..." +$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=Environment,Value=production +ENV_STILL=$($AWS_EC2 describe-tags \ + --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Environment" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$ENV_STILL" -ne 1 ]; then + echo " ERROR: Value-conditional delete incorrectly removed tag" + exit 1 +fi +echo " Value-conditional mismatch preserved tag" + +# Delete tag with correct value +echo " Deleting Environment tag with correct value..." +$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=Environment,Value=testing +ENV_GONE=$($AWS_EC2 describe-tags \ + --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Environment" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$ENV_GONE" -ne 0 ]; then + echo " ERROR: Value-conditional delete failed to remove matching tag" + exit 1 +fi +echo " Value-conditional match deleted tag" + +# Verify only Name tag remains +FINAL_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$FINAL_COUNT" -ne 1 ]; then + echo " ERROR: Expected 1 tag remaining, got $FINAL_COUNT" + exit 1 +fi +echo " Tag management tests passed" + # Test 2: DescribeInstances Aggregation echo "" echo "Test 2: DescribeInstances Aggregation"