feat: snapshot e2e (#31)

hive/handlers/ec2/instance/service_impl.go

@@ -164,7 +164,7 @@ func (s *InstanceServiceImpl) GenerateVolumes(input *ec2.RunInstancesInput, inst
 		}
 		if bdm.Ebs != nil {
 			if bdm.Ebs.VolumeSize != nil {
-				size = int(*bdm.Ebs.VolumeSize)
+				size = int(*bdm.Ebs.VolumeSize) * 1024 * 1024 * 1024 // AWS API sends GiB, convert to bytes
 			}
 			if bdm.Ebs.VolumeType != nil {
 				volumeType = *bdm.Ebs.VolumeType

hive/handlers/ec2/snapshot/service_impl.go

@@ -167,11 +167,17 @@ func (s *SnapshotServiceImpl) CreateSnapshot(input *ec2.CreateSnapshotInput) (*e
 	}
 	defer volumeResult.Body.Close()
 
-	var volumeConfig viperblock.VolumeConfig
-	if err := json.NewDecoder(volumeResult.Body).Decode(&volumeConfig); err != nil {
+	var volumeState viperblock.VBState
+	if err := json.NewDecoder(volumeResult.Body).Decode(&volumeState); err != nil {
 		slog.Error("CreateSnapshot failed to decode volume config", "volumeId", volumeID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
+	volumeConfig := volumeState.VolumeConfig
+
+	if volumeConfig.VolumeMetadata.SizeGiB == 0 {
+		slog.Error("CreateSnapshot: source volume has zero size in config", "volumeId", volumeID)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
 
 	now := time.Now()
 

hive/handlers/ec2/snapshot/service_impl_test.go

@@ -31,15 +31,18 @@ func setupTestSnapshotService(t *testing.T) (*SnapshotServiceImpl, *objectstore.
 }
 
 // createTestVolume creates a test volume in the mock store
+// The real S3 stores VBState (which wraps VolumeConfig), so we match that format.
 func createTestVolume(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string, sizeGiB int) {
-	volumeConfig := viperblock.VolumeConfig{
-		VolumeMetadata: viperblock.VolumeMetadata{
-			SizeGiB:          uint64(sizeGiB),
-			IsEncrypted:      false,
-			AvailabilityZone: "us-east-1a",
+	volumeState := viperblock.VBState{
+		VolumeConfig: viperblock.VolumeConfig{
+			VolumeMetadata: viperblock.VolumeMetadata{
+				SizeGiB:          uint64(sizeGiB),
+				IsEncrypted:      false,
+				AvailabilityZone: "us-east-1a",
+			},
 		},
 	}
-	data, err := json.Marshal(volumeConfig)
+	data, err := json.Marshal(volumeState)
 	require.NoError(t, err)
 
 	_, err = store.PutObject(&s3.PutObjectInput{
@@ -97,6 +100,35 @@ func TestCreateSnapshot_MissingVolumeId(t *testing.T) {
 	assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameterValue)
 }
 
+// TestCreateSnapshot_VolumeZeroSize tests that creating a snapshot from a volume with zero SizeGiB fails
+func TestCreateSnapshot_VolumeZeroSize(t *testing.T) {
+	svc, store := setupTestSnapshotService(t)
+
+	// Store a volume config with SizeGiB == 0
+	volumeState := viperblock.VBState{
+		VolumeConfig: viperblock.VolumeConfig{
+			VolumeMetadata: viperblock.VolumeMetadata{
+				SizeGiB: 0,
+			},
+		},
+	}
+	data, err := json.Marshal(volumeState)
+	require.NoError(t, err)
+
+	_, err = store.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String("test-bucket"),
+		Key:    aws.String("vol-zerosize/config.json"),
+		Body:   strings.NewReader(string(data)),
+	})
+	require.NoError(t, err)
+
+	_, err = svc.CreateSnapshot(&ec2.CreateSnapshotInput{
+		VolumeId: aws.String("vol-zerosize"),
+	})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), awserrors.ErrorServerInternal)
+}
+
 // TestCreateSnapshot_VolumeNotFound tests creating a snapshot from non-existent volume
 func TestCreateSnapshot_VolumeNotFound(t *testing.T) {
 	svc, _ := setupTestSnapshotService(t)

hive/handlers/ec2/volume/service_impl.go

@@ -383,6 +383,11 @@ func (s *VolumeServiceImpl) getVolumeByID(volumeID string) (*ec2.Volume, error)
 		return nil, errors.New("volume ID is empty")
 	}
 
+	if volMeta.SizeGiB == 0 {
+		slog.Error("Volume has zero size in config", "volumeId", volumeID)
+		return nil, fmt.Errorf("volume %s has zero size in config", volumeID)
+	}
+
 	state := volMeta.State
 	if state == "" {
 		state = "available"

tests/e2e/TEST_COVERAGE.md

@@ -0,0 +1,142 @@
+# E2E Test Coverage
+
+## Single-Node (`run-e2e.sh`)
+
+### Phase 1: Environment Setup
+- KVM support check
+- `hive admin init` (region/az/node config)
+- CA certificate trust
+- Start all services (`start-dev.sh`)
+- Wait for AWS gateway on `localhost:9999`
+
+### Phase 2: Discovery & Metadata
+- `describe-regions`
+- `describe-instance-types` (discover available types)
+- Select nano instance type and detect architecture
+
+### Phase 3: SSH Key Management
+- `create-key-pair` (test-key-1, verify private key material)
+- `import-key-pair` (test-key-2, from local RSA key)
+- `describe-key-pairs` (verify both keys exist)
+- `delete-key-pair` (test-key-2, verify only test-key-1 remains)
+
+### Phase 4: Image Management
+- `hive admin images import` (file-based, architecture-aware)
+- `describe-images` (verify AMI by ID)
+
+### Phase 5: Instance Lifecycle
+- `run-instances` (launch VM with key pair)
+- `describe-instances` (poll pending -> running)
+- `describe-volumes` (verify root volume attached)
+
+### Phase 5b: Volume Lifecycle
+- `create-volume` (10GB, ap-southeast-2a)
+- `modify-volume` (resize to 20GB, poll to verify)
+- `attach-volume` (to running instance, /dev/sdf)
+- `describe-volumes` (verify in-use + attached state)
+- `detach-volume` (verify available state)
+- `delete-volume` (verify gone)
+
+### Phase 5c: Snapshot Lifecycle
+- `create-snapshot` (from volume, with description)
+- Verify create response fields (VolumeId, VolumeSize, State, Progress)
+- `describe-snapshots` (by ID, verify VolumeId/Size/Description)
+- `copy-snapshot` (with new description, verify distinct ID)
+- `describe-snapshots` (verify both original + copy visible)
+- `delete-snapshot` (original, verify gone while copy survives)
+- `delete-snapshot` (copy, cleanup)
+
+### Phase 6: Tag Management
+- `create-tags` (3 tags on instance)
+- `describe-tags` (filter by resource-id)
+- `create-tags` (2 tags on volume)
+- `describe-tags` (filter by key)
+- `describe-tags` (filter by resource-type)
+- `create-tags` (overwrite existing tag value)
+- `delete-tags` (unconditional by key)
+- `delete-tags` (with wrong value — should be no-op)
+- `delete-tags` (with correct value)
+- Verify final tag count
+
+### Phase 7: Instance State Transitions
+- `stop-instances` (poll -> stopped)
+- `start-instances` (poll -> running)
+- `terminate-instances` (poll -> terminated)
+
+---
+
+## Multi-Node (`run-multinode-e2e.sh`)
+
+### Phase 1: Environment Setup
+- KVM support check
+- Simulated network IPs (10.11.12.{1,2,3} on loopback)
+- Create ramdisk mount
+
+### Phase 2: Cluster Initialization
+- `hive admin init` (leader node1)
+- CA certificate trust
+- Start node1 services
+- `hive admin join` (node2, node3)
+- Start node2 + node3 services
+
+### Phase 3: Cluster Health Verification
+- Verify NATS cluster (3 nodes)
+- Verify Predastore cluster (3 nodes)
+- Wait for gateway on node1
+- `describe-regions` (gateway connectivity check)
+
+### Phase 4: Image and Key Setup
+- `describe-instance-types` (discover + select nano)
+- `create-key-pair`
+- `hive admin images import` (with node1 config paths)
+- `describe-images` (verify AMI)
+
+### Phase 5: Multi-Node Instance Tests
+
+#### Test 1: Instance Distribution
+- `run-instances` x3 (distribute across nodes)
+- Poll all instances to running state
+- Check instance distribution across nodes
+
+#### Test 1b: Volume Lifecycle
+- `create-volume` (10GB)
+- `modify-volume` (resize to 20GB)
+- `attach-volume` (to first instance)
+- `detach-volume`
+- `delete-volume`
+
+#### Test 1c: Snapshot Lifecycle
+- `create-snapshot` (from volume, with description)
+- Verify create response fields (VolumeId, VolumeSize, State)
+- `describe-snapshots` (by ID, verify fields)
+- `copy-snapshot` (with new description)
+- `describe-snapshots` (verify both exist)
+- `delete-snapshot` (original, verify copy survives)
+- `delete-snapshot` (copy, cleanup)
+
+#### Test 1d: Tag Management
+- `create-tags` (3 tags on instance)
+- `describe-tags` (filter by resource-id)
+- `describe-tags` (filter by key)
+- `describe-tags` (filter by resource-type)
+- `create-tags` (overwrite tag value)
+- `delete-tags` (unconditional by key)
+- `delete-tags` (wrong value — no-op)
+- `delete-tags` (correct value)
+- Verify final tag count
+
+#### Test 2: DescribeInstances Aggregation
+- `describe-instances` (fan-out across all nodes, verify count)
+
+#### Test 3: Cross-Node Operations
+- `stop-instances` (poll -> stopped)
+- `start-instances` (poll -> running)
+
+#### Test 4: NATS Cluster Health (Post-Operations)
+- Verify NATS cluster still healthy after all operations
+
+### Cleanup
+- `terminate-instances` (all 3 instances)
+- Poll all to terminated state
+- Stop all node services
+- Remove simulated IPs

tests/e2e/run-e2e.sh

@@ -377,6 +377,175 @@ fi
 
 echo "Volume lifecycle test passed (create -> resize -> attach -> detach -> delete)"
 
+# Phase 5c: Snapshot Lifecycle
+echo "Phase 5c: Snapshot Lifecycle"
+echo "Testing snapshot create -> describe -> copy -> delete..."
+
+# Create a volume to snapshot (snapshots need a source volume)
+echo "Creating source volume for snapshot tests..."
+SNAP_VOL_OUTPUT=$($AWS_EC2 create-volume --size 10 --availability-zone ap-southeast-2a)
+SNAP_VOL_ID=$(echo "$SNAP_VOL_OUTPUT" | jq -r '.VolumeId')
+
+if [ -z "$SNAP_VOL_ID" ] || [ "$SNAP_VOL_ID" == "null" ]; then
+    echo "Failed to create source volume for snapshot tests"
+    exit 1
+fi
+echo "Created source volume: $SNAP_VOL_ID"
+
+# Create a snapshot from the volume
+echo "Creating snapshot from volume $SNAP_VOL_ID..."
+SNAP_OUTPUT=$($AWS_EC2 create-snapshot --volume-id "$SNAP_VOL_ID" --description "e2e-test-snapshot")
+SNAPSHOT_ID=$(echo "$SNAP_OUTPUT" | jq -r '.SnapshotId')
+
+if [ -z "$SNAPSHOT_ID" ] || [ "$SNAPSHOT_ID" == "null" ]; then
+    echo "Failed to create snapshot"
+    echo "Output: $SNAP_OUTPUT"
+    exit 1
+fi
+echo "Created snapshot: $SNAPSHOT_ID"
+
+# Verify snapshot fields from create response
+SNAP_STATE=$(echo "$SNAP_OUTPUT" | jq -r '.State')
+SNAP_VOL_REF=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeId')
+SNAP_SIZE=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeSize')
+SNAP_PROGRESS=$(echo "$SNAP_OUTPUT" | jq -r '.Progress')
+
+if [ "$SNAP_VOL_REF" != "$SNAP_VOL_ID" ]; then
+    echo "Snapshot VolumeId mismatch: expected $SNAP_VOL_ID, got $SNAP_VOL_REF"
+    exit 1
+fi
+if [ "$SNAP_SIZE" -ne 10 ]; then
+    echo "Snapshot VolumeSize mismatch: expected 10, got $SNAP_SIZE"
+    exit 1
+fi
+echo "Snapshot create response verified (State=$SNAP_STATE, VolumeId=$SNAP_VOL_REF, Size=$SNAP_SIZE, Progress=$SNAP_PROGRESS)"
+
+# Poll until snapshot is completed (should be immediate in v1, but poll for forward-compat)
+echo "Waiting for snapshot to complete..."
+COUNT=0
+while [ $COUNT -lt 30 ]; do
+    SNAP_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \
+        --query 'Snapshots[0].State' --output text)
+
+    if [ "$SNAP_STATE" == "completed" ]; then
+        echo "Snapshot completed"
+        break
+    fi
+
+    sleep 2
+    COUNT=$((COUNT + 1))
+done
+
+if [ "$SNAP_STATE" != "completed" ]; then
+    echo "Snapshot failed to reach completed state (State=$SNAP_STATE)"
+    exit 1
+fi
+
+# Describe snapshot by ID and verify fields
+echo "Verifying snapshot via describe-snapshots..."
+DESCRIBE_SNAP=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID")
+DESC_VOL_ID=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeId')
+DESC_SIZE=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeSize')
+DESC_DESC=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].Description')
+
+if [ "$DESC_VOL_ID" != "$SNAP_VOL_ID" ]; then
+    echo "Describe snapshot VolumeId mismatch: expected $SNAP_VOL_ID, got $DESC_VOL_ID"
+    exit 1
+fi
+if [ "$DESC_SIZE" -ne 10 ]; then
+    echo "Describe snapshot VolumeSize mismatch: expected 10, got $DESC_SIZE"
+    exit 1
+fi
+if [ "$DESC_DESC" != "e2e-test-snapshot" ]; then
+    echo "Describe snapshot Description mismatch: expected 'e2e-test-snapshot', got '$DESC_DESC'"
+    exit 1
+fi
+echo "Describe snapshot verified (VolumeId=$DESC_VOL_ID, Size=$DESC_SIZE, Description=$DESC_DESC)"
+
+# Copy the snapshot
+echo "Copying snapshot $SNAPSHOT_ID..."
+COPY_OUTPUT=$($AWS_EC2 copy-snapshot --source-snapshot-id "$SNAPSHOT_ID" --source-region ap-southeast-2 --description "e2e-copy")
+COPY_SNAPSHOT_ID=$(echo "$COPY_OUTPUT" | jq -r '.SnapshotId')
+
+if [ -z "$COPY_SNAPSHOT_ID" ] || [ "$COPY_SNAPSHOT_ID" == "null" ]; then
+    echo "Failed to copy snapshot"
+    echo "Output: $COPY_OUTPUT"
+    exit 1
+fi
+echo "Copied snapshot: $COPY_SNAPSHOT_ID"
+
+# Verify the copy is a distinct snapshot
+if [ "$COPY_SNAPSHOT_ID" == "$SNAPSHOT_ID" ]; then
+    echo "Copy snapshot ID should differ from original"
+    exit 1
+fi
+
+# Describe all snapshots — should see both original and copy
+TOTAL_SNAPS=$($AWS_EC2 describe-snapshots \
+    --snapshot-ids "$SNAPSHOT_ID" "$COPY_SNAPSHOT_ID" \
+    --query 'length(Snapshots)' --output text)
+
+if [ "$TOTAL_SNAPS" -ne 2 ]; then
+    echo "Expected 2 snapshots, got $TOTAL_SNAPS"
+    exit 1
+fi
+echo "Both snapshots visible via describe-snapshots"
+
+# Verify copy has correct description
+COPY_DESC=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \
+    --query 'Snapshots[0].Description' --output text)
+if [ "$COPY_DESC" != "e2e-copy" ]; then
+    echo "Copy description mismatch: expected 'e2e-copy', got '$COPY_DESC'"
+    exit 1
+fi
+
+# Delete the original snapshot
+echo "Deleting original snapshot $SNAPSHOT_ID..."
+$AWS_EC2 delete-snapshot --snapshot-id "$SNAPSHOT_ID"
+
+# Verify original is gone, copy remains
+echo "Verifying snapshot deletion..."
+COUNT=0
+while [ $COUNT -lt 30 ]; do
+    set +e
+    SNAP_CHECK=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \
+        --query 'Snapshots[0].SnapshotId' --output text 2>&1)
+    SNAP_EXIT=$?
+    set -e
+
+    if [ $SNAP_EXIT -ne 0 ] || [ "$SNAP_CHECK" == "None" ] || [ -z "$SNAP_CHECK" ]; then
+        echo "Original snapshot deleted successfully"
+        break
+    fi
+
+    sleep 2
+    COUNT=$((COUNT + 1))
+done
+
+if [ $COUNT -ge 30 ]; then
+    echo "Snapshot deletion verification timed out"
+    exit 1
+fi
+
+# Verify copy still exists
+COPY_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \
+    --query 'Snapshots[0].State' --output text)
+if [ "$COPY_STATE" != "completed" ]; then
+    echo "Copy snapshot should still exist (State=$COPY_STATE)"
+    exit 1
+fi
+echo "Copy snapshot still intact after original deletion"
+
+# Delete the copy
+echo "Deleting copy snapshot $COPY_SNAPSHOT_ID..."
+$AWS_EC2 delete-snapshot --snapshot-id "$COPY_SNAPSHOT_ID"
+
+# Clean up the source volume
+echo "Deleting source volume $SNAP_VOL_ID..."
+$AWS_EC2 delete-volume --volume-id "$SNAP_VOL_ID"
+
+echo "Snapshot lifecycle test passed (create -> describe -> copy -> delete)"
+
 # Phase 6: Tag Management
 echo "Phase 6: Tag Management"
 

tests/e2e/run-multinode-e2e.sh

@@ -382,6 +382,273 @@ fi
 
 echo "  Volume lifecycle test passed (create -> resize -> attach -> detach -> delete)"
 
+# Test 1c: Snapshot Lifecycle
+echo ""
+echo "Test 1c: Snapshot Lifecycle"
+echo "----------------------------------------"
+echo "Testing snapshot create -> describe -> copy -> delete..."
+
+# Create a volume to snapshot
+echo "  Creating source volume for snapshot tests..."
+SNAP_VOL_OUTPUT=$($AWS_EC2 create-volume --size 10 --availability-zone ap-southeast-2a)
+SNAP_VOL_ID=$(echo "$SNAP_VOL_OUTPUT" | jq -r '.VolumeId')
+
+if [ -z "$SNAP_VOL_ID" ] || [ "$SNAP_VOL_ID" == "null" ]; then
+    echo "  ERROR: Failed to create source volume for snapshot tests"
+    exit 1
+fi
+echo "  Created source volume: $SNAP_VOL_ID"
+
+# Create a snapshot
+echo "  Creating snapshot from volume $SNAP_VOL_ID..."
+SNAP_OUTPUT=$($AWS_EC2 create-snapshot --volume-id "$SNAP_VOL_ID" --description "multinode-e2e-snapshot")
+SNAPSHOT_ID=$(echo "$SNAP_OUTPUT" | jq -r '.SnapshotId')
+
+if [ -z "$SNAPSHOT_ID" ] || [ "$SNAPSHOT_ID" == "null" ]; then
+    echo "  ERROR: Failed to create snapshot"
+    echo "  Output: $SNAP_OUTPUT"
+    exit 1
+fi
+echo "  Created snapshot: $SNAPSHOT_ID"
+
+# Verify create response fields
+SNAP_STATE=$(echo "$SNAP_OUTPUT" | jq -r '.State')
+SNAP_VOL_REF=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeId')
+SNAP_SIZE=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeSize')
+
+if [ "$SNAP_VOL_REF" != "$SNAP_VOL_ID" ]; then
+    echo "  ERROR: Snapshot VolumeId mismatch: expected $SNAP_VOL_ID, got $SNAP_VOL_REF"
+    exit 1
+fi
+if [ "$SNAP_SIZE" -ne 10 ]; then
+    echo "  ERROR: Snapshot VolumeSize mismatch: expected 10, got $SNAP_SIZE"
+    exit 1
+fi
+echo "  Create response verified (State=$SNAP_STATE, VolumeId=$SNAP_VOL_REF, Size=$SNAP_SIZE)"
+
+# Poll until completed
+echo "  Waiting for snapshot to complete..."
+COUNT=0
+while [ $COUNT -lt 30 ]; do
+    SNAP_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \
+        --query 'Snapshots[0].State' --output text)
+
+    if [ "$SNAP_STATE" == "completed" ]; then
+        echo "  Snapshot completed"
+        break
+    fi
+
+    sleep 2
+    COUNT=$((COUNT + 1))
+done
+
+if [ "$SNAP_STATE" != "completed" ]; then
+    echo "  ERROR: Snapshot failed to reach completed state (State=$SNAP_STATE)"
+    exit 1
+fi
+
+# Describe by ID and verify
+echo "  Verifying snapshot via describe-snapshots..."
+DESCRIBE_SNAP=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID")
+DESC_VOL_ID=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeId')
+DESC_SIZE=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeSize')
+DESC_DESC=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].Description')
+
+if [ "$DESC_VOL_ID" != "$SNAP_VOL_ID" ]; then
+    echo "  ERROR: Describe VolumeId mismatch: expected $SNAP_VOL_ID, got $DESC_VOL_ID"
+    exit 1
+fi
+if [ "$DESC_SIZE" -ne 10 ]; then
+    echo "  ERROR: Describe VolumeSize mismatch: expected 10, got $DESC_SIZE"
+    exit 1
+fi
+if [ "$DESC_DESC" != "multinode-e2e-snapshot" ]; then
+    echo "  ERROR: Describe Description mismatch: expected 'multinode-e2e-snapshot', got '$DESC_DESC'"
+    exit 1
+fi
+echo "  Describe verified (VolumeId=$DESC_VOL_ID, Size=$DESC_SIZE, Description=$DESC_DESC)"
+
+# Copy the snapshot
+echo "  Copying snapshot $SNAPSHOT_ID..."
+COPY_OUTPUT=$($AWS_EC2 copy-snapshot --source-snapshot-id "$SNAPSHOT_ID" --source-region ap-southeast-2 --description "multinode-e2e-copy")
+COPY_SNAPSHOT_ID=$(echo "$COPY_OUTPUT" | jq -r '.SnapshotId')
+
+if [ -z "$COPY_SNAPSHOT_ID" ] || [ "$COPY_SNAPSHOT_ID" == "null" ]; then
+    echo "  ERROR: Failed to copy snapshot"
+    echo "  Output: $COPY_OUTPUT"
+    exit 1
+fi
+echo "  Copied snapshot: $COPY_SNAPSHOT_ID"
+
+if [ "$COPY_SNAPSHOT_ID" == "$SNAPSHOT_ID" ]; then
+    echo "  ERROR: Copy snapshot ID should differ from original"
+    exit 1
+fi
+
+# Verify both exist
+TOTAL_SNAPS=$($AWS_EC2 describe-snapshots \
+    --snapshot-ids "$SNAPSHOT_ID" "$COPY_SNAPSHOT_ID" \
+    --query 'length(Snapshots)' --output text)
+
+if [ "$TOTAL_SNAPS" -ne 2 ]; then
+    echo "  ERROR: Expected 2 snapshots, got $TOTAL_SNAPS"
+    exit 1
+fi
+echo "  Both snapshots visible via describe-snapshots"
+
+# Verify copy description
+COPY_DESC=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \
+    --query 'Snapshots[0].Description' --output text)
+if [ "$COPY_DESC" != "multinode-e2e-copy" ]; then
+    echo "  ERROR: Copy description mismatch: expected 'multinode-e2e-copy', got '$COPY_DESC'"
+    exit 1
+fi
+
+# Delete original
+echo "  Deleting original snapshot $SNAPSHOT_ID..."
+$AWS_EC2 delete-snapshot --snapshot-id "$SNAPSHOT_ID"
+
+# Verify original gone, copy remains
+echo "  Verifying snapshot deletion..."
+COUNT=0
+while [ $COUNT -lt 30 ]; do
+    set +e
+    SNAP_CHECK=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \
+        --query 'Snapshots[0].SnapshotId' --output text 2>&1)
+    SNAP_EXIT=$?
+    set -e
+
+    if [ $SNAP_EXIT -ne 0 ] || [ "$SNAP_CHECK" == "None" ] || [ -z "$SNAP_CHECK" ]; then
+        echo "  Original snapshot deleted successfully"
+        break
+    fi
+
+    sleep 2
+    COUNT=$((COUNT + 1))
+done
+
+if [ $COUNT -ge 30 ]; then
+    echo "  ERROR: Snapshot deletion verification timed out"
+    exit 1
+fi
+
+# Verify copy still exists
+COPY_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \
+    --query 'Snapshots[0].State' --output text)
+if [ "$COPY_STATE" != "completed" ]; then
+    echo "  ERROR: Copy snapshot should still exist (State=$COPY_STATE)"
+    exit 1
+fi
+echo "  Copy snapshot intact after original deletion"
+
+# Delete copy and source volume
+echo "  Deleting copy snapshot $COPY_SNAPSHOT_ID..."
+$AWS_EC2 delete-snapshot --snapshot-id "$COPY_SNAPSHOT_ID"
+
+echo "  Deleting source volume $SNAP_VOL_ID..."
+$AWS_EC2 delete-volume --volume-id "$SNAP_VOL_ID"
+
+echo "  Snapshot lifecycle test passed (create -> describe -> copy -> delete)"
+
+# Test 1d: Tag Management
+echo ""
+echo "Test 1d: Tag Management"
+echo "----------------------------------------"
+echo "Testing create-tags -> describe-tags -> delete-tags..."
+
+# Use the first instance for tag tests
+TAG_INSTANCE="${INSTANCE_IDS[0]}"
+
+# Create tags on instance
+echo "  Creating tags on instance $TAG_INSTANCE..."
+$AWS_EC2 create-tags --resources "$TAG_INSTANCE" --tags Key=Name,Value=multinode-test Key=Environment,Value=testing Key=DeleteMe,Value=please
+
+# Verify tags with describe-tags (resource-id filter)
+echo "  Verifying tags on instance..."
+TAG_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \
+    --query 'length(Tags || `[]`)' --output text)
+if [ "$TAG_COUNT" -ne 3 ]; then
+    echo "  ERROR: Expected 3 tags on instance, got $TAG_COUNT"
+    exit 1
+fi
+echo "  Instance has $TAG_COUNT tags"
+
+# Filter by key
+echo "  Testing key filter..."
+ENV_TAGS=$($AWS_EC2 describe-tags --filters "Name=key,Values=Environment" \
+    --query 'length(Tags || `[]`)' --output text)
+if [ "$ENV_TAGS" -lt 1 ]; then
+    echo "  ERROR: Expected at least 1 'Environment' tag, got $ENV_TAGS"
+    exit 1
+fi
+echo "  Key filter returned $ENV_TAGS tags"
+
+# Filter by resource-type
+echo "  Testing resource-type filter..."
+INSTANCE_TAGS=$($AWS_EC2 describe-tags --filters "Name=resource-type,Values=instance" \
+    --query 'length(Tags || `[]`)' --output text)
+if [ "$INSTANCE_TAGS" -lt 3 ]; then
+    echo "  ERROR: Expected at least 3 instance tags, got $INSTANCE_TAGS"
+    exit 1
+fi
+echo "  Resource-type filter returned $INSTANCE_TAGS instance tags"
+
+# Overwrite a tag value
+echo "  Overwriting Name tag..."
+$AWS_EC2 create-tags --resources "$TAG_INSTANCE" --tags Key=Name,Value=multinode-updated
+UPDATED_NAME=$($AWS_EC2 describe-tags \
+    --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Name" \
+    --query 'Tags[0].Value' --output text)
+if [ "$UPDATED_NAME" != "multinode-updated" ]; then
+    echo "  ERROR: Tag overwrite failed: expected 'multinode-updated', got '$UPDATED_NAME'"
+    exit 1
+fi
+echo "  Tag overwrite verified"
+
+# Delete tag by key (unconditional)
+echo "  Deleting DeleteMe tag unconditionally..."
+$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=DeleteMe
+REMAINING=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \
+    --query 'length(Tags || `[]`)' --output text)
+if [ "$REMAINING" -ne 2 ]; then
+    echo "  ERROR: Expected 2 tags after unconditional delete, got $REMAINING"
+    exit 1
+fi
+echo "  Unconditional delete verified ($REMAINING tags remaining)"
+
+# Delete tag with wrong value (should NOT delete)
+echo "  Attempting delete with wrong value (should be no-op)..."
+$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=Environment,Value=production
+ENV_STILL=$($AWS_EC2 describe-tags \
+    --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Environment" \
+    --query 'length(Tags || `[]`)' --output text)
+if [ "$ENV_STILL" -ne 1 ]; then
+    echo "  ERROR: Value-conditional delete incorrectly removed tag"
+    exit 1
+fi
+echo "  Value-conditional mismatch preserved tag"
+
+# Delete tag with correct value
+echo "  Deleting Environment tag with correct value..."
+$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=Environment,Value=testing
+ENV_GONE=$($AWS_EC2 describe-tags \
+    --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Environment" \
+    --query 'length(Tags || `[]`)' --output text)
+if [ "$ENV_GONE" -ne 0 ]; then
+    echo "  ERROR: Value-conditional delete failed to remove matching tag"
+    exit 1
+fi
+echo "  Value-conditional match deleted tag"
+
+# Verify only Name tag remains
+FINAL_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \
+    --query 'length(Tags || `[]`)' --output text)
+if [ "$FINAL_COUNT" -ne 1 ]; then
+    echo "  ERROR: Expected 1 tag remaining, got $FINAL_COUNT"
+    exit 1
+fi
+echo "  Tag management tests passed"
+
 # Test 2: DescribeInstances Aggregation
 echo ""
 echo "Test 2: DescribeInstances Aggregation"