diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 779acd4ef..725f13b1c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,4 +1,4 @@ -name: Hive E2E Test +name: Hive E2E Tests on: push: @@ -11,70 +11,186 @@ on: - go.sum - Makefile - tests/e2e/** - - scripts/** - .github/workflows/e2e.yml workflow_dispatch: +env: + GIT_BRANCH: ${{ github.ref_name }} + jobs: - build_e2e_image: - name: Build E2E Image - runs-on: self-hosted - + e2e_single: + name: Single-Node E2E + runs-on: [self-hosted, ci-single] + timeout-minutes: 25 + env: + TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log steps: - - name: Checkout Code - uses: actions/checkout@v6 + - name: Clean workspace + run: rm -rf "$GITHUB_WORKSPACE"/* + + - uses: actions/create-github-app-token@v2 + id: app-token with: - path: hive + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: mulga - - name: Clone Dependencies - run: ./hive/scripts/clone-deps.sh + - uses: actions/checkout@v6 + with: + repository: mulgadc/mulga + token: ${{ steps.app-token.outputs.token }} + sparse-checkout: scripts/tofu-cluster + path: mulga - - name: Build E2E Image + - name: Provision VM + working-directory: mulga/scripts/tofu-cluster run: | - docker build \ - -t hive-e2e:latest \ - -f hive/tests/e2e/Dockerfile.e2e \ - . + export TF_VAR_git_branch="${GIT_BRANCH}" + tofu init -input=false >> "$TOFU_LOG" 2>&1 + tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1 + tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true + ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true + tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 - hive_e2e_test: - name: Hive Single-Node E2E Test - runs-on: self-hosted - needs: build_e2e_image + - name: Bootstrap Cluster + working-directory: mulga/scripts/tofu-cluster + run: ./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1 - steps: - name: Run E2E Tests + working-directory: mulga/scripts/tofu-cluster run: | - docker run --privileged --rm \ - -v /dev/kvm:/dev/kvm \ - --name hive-e2e-test \ - hive-e2e:latest + INVENTORY=$(tofu output -json inventory) + WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \ + 'cd ~/Development/mulga/hive && ./tests/e2e/run-e2e.sh' - hive_multinode_e2e_test: - name: Hive Multi-Node E2E Test - runs-on: self-hosted - needs: build_e2e_image + - name: Teardown + if: always() + working-directory: mulga/scripts/tofu-cluster + run: | + tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1 + tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 + ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true + e2e_pseudo_multi: + name: Pseudo Multi-Node E2E + runs-on: [self-hosted, ci-pseudo] + timeout-minutes: 25 + env: + TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log steps: - - name: Run Multi-Node E2E Tests + - name: Clean workspace + run: rm -rf "$GITHUB_WORKSPACE"/* + + - uses: actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: mulga + + - uses: actions/checkout@v6 + with: + repository: mulgadc/mulga + token: ${{ steps.app-token.outputs.token }} + sparse-checkout: scripts/tofu-cluster + + path: mulga + + - name: Provision VM + working-directory: mulga/scripts/tofu-cluster run: | - docker run --privileged --rm \ - -v /dev/kvm:/dev/kvm \ - --cap-add=NET_ADMIN \ - --name hive-multinode-e2e-test \ - hive-e2e:latest \ - ./tests/e2e/run-multinode-e2e.sh + export TF_VAR_git_branch="${GIT_BRANCH}" + tofu init -input=false >> "$TOFU_LOG" 2>&1 + tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1 + tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true + ./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true + tofu apply -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 - cleanup: - name: Cleanup - runs-on: self-hosted - needs: [hive_e2e_test, hive_multinode_e2e_test] - if: always() + - name: Bootstrap Cluster + working-directory: mulga/scripts/tofu-cluster + run: ./bootstrap.sh env4 >> "$TOFU_LOG" 2>&1 + - name: Run E2E Tests + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory) + WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \ + 'cd ~/Development/mulga/hive && ./tests/e2e/run-pseudo-multinode-e2e.sh' + + - name: Teardown + if: always() + working-directory: mulga/scripts/tofu-cluster + run: | + tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1 + tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 + ./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true + + e2e_real_multi: + name: Real Multi-Node E2E + runs-on: [self-hosted, ci-multi] + timeout-minutes: 25 + env: + TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log steps: - - name: Remove Stale Containers - run: | - docker rm -f hive-e2e-test 2>/dev/null || true - docker rm -f hive-multinode-e2e-test 2>/dev/null || true + - name: Clean workspace + run: rm -rf "$GITHUB_WORKSPACE"/* - - name: Prune Docker Images - run: docker image prune -f + - uses: actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: mulga + + - uses: actions/checkout@v6 + with: + repository: mulgadc/mulga + token: ${{ steps.app-token.outputs.token }} + sparse-checkout: scripts/tofu-cluster + path: mulga + + - name: Provision VMs + working-directory: mulga/scripts/tofu-cluster + run: | + export TF_VAR_git_branch="${GIT_BRANCH}" + tofu init -input=false >> "$TOFU_LOG" 2>&1 + tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1 + tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true + ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true + tofu apply -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 + + - name: Bootstrap Cluster + working-directory: mulga/scripts/tofu-cluster + run: ./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1 + + - name: Run E2E Tests + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory) + NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + + # Build node IP list + NODE_IPS="" + NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length') + for i in $(seq 0 $((NODE_COUNT - 1))); do + NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")" + done + + ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \ + "cd ~/Development/mulga/hive && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS}" + + - name: Teardown + if: always() + working-directory: mulga/scripts/tofu-cluster + run: | + tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1 + tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 + ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true diff --git a/Makefile b/Makefile index 9ea632fae..d14f54d47 100644 --- a/Makefile +++ b/Makefile @@ -193,39 +193,6 @@ security-check: $(_Q)set -o pipefail && go tool staticcheck -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-SA1019,-SA9005" ./... $(_SECQ) tests/staticcheck-report.txt $(if $(QUIET),|| { cat tests/staticcheck-report.txt; exit 1; }) @echo " staticcheck ok" -# Docker E2E tests (mirrors GitHub Actions e2e.yml) -# Usage: make test-docker # both suites -# make test-docker-single # single-node only -# make test-docker-multi # multi-node only -PARENT_DIR := $(shell cd .. && pwd) -E2E_IMAGE := hive-e2e:latest - -test-docker-build: - @echo -e "\n....Building E2E Docker image...." - @for dep in viperblock predastore; do \ - if [ ! -d "$(PARENT_DIR)/$$dep" ]; then \ - echo "Missing sibling repo $$dep — running clone-deps.sh"; \ - ./scripts/clone-deps.sh; \ - break; \ - fi; \ - done - docker build -t $(E2E_IMAGE) -f tests/e2e/Dockerfile.e2e $(PARENT_DIR) - -test-docker-single: test-docker-build - @echo -e "\n....Running Single-Node E2E Tests...." - docker run --privileged --rm -v /dev/kvm:/dev/kvm $(E2E_IMAGE) - -test-docker-multi: test-docker-build - @echo -e "\n....Running Multi-Node E2E Tests...." - docker run --privileged --rm -v /dev/kvm:/dev/kvm --cap-add=NET_ADMIN $(E2E_IMAGE) ./tests/e2e/run-multinode-e2e.sh - -test-docker: test-docker-build - @echo -e "\n....Running Single-Node E2E Tests...." - docker run --privileged --rm -v /dev/kvm:/dev/kvm $(E2E_IMAGE) - @echo -e "\n....Running Multi-Node E2E Tests...." - docker run --privileged --rm -v /dev/kvm:/dev/kvm --cap-add=NET_ADMIN $(E2E_IMAGE) ./tests/e2e/run-multinode-e2e.sh - .PHONY: build build-ui go_build go_run preflight test test-cover test-race diff-coverage bench run clean \ install-system install-go install-aws quickinstall \ - format check-format modernize check-modernize vet security-check \ - test-docker-build test-docker-single test-docker-multi test-docker + format check-format modernize check-modernize vet security-check diff --git a/hive/services/hiveui/frontend/.nvmrc b/hive/services/hiveui/frontend/.nvmrc index 248216ad5..d845d9d88 100644 --- a/hive/services/hiveui/frontend/.nvmrc +++ b/hive/services/hiveui/frontend/.nvmrc @@ -1 +1 @@ -24.12.0 +24.14.0 diff --git a/hive/services/hiveui/frontend/package.json b/hive/services/hiveui/frontend/package.json index 004922767..635eb693f 100644 --- a/hive/services/hiveui/frontend/package.json +++ b/hive/services/hiveui/frontend/package.json @@ -48,7 +48,7 @@ "@vitejs/plugin-react": "^5.1.4", "@vitest/coverage-v8": "^4.0.18", "babel-plugin-react-compiler": "^1.0.0", - "jsdom": "^28.1.0", + "happy-dom": "^20.8.3", "typescript": "^5.9.3", "ultracite": "^7.2.5", "vite": "^7.3.1", diff --git a/hive/services/hiveui/frontend/pnpm-lock.yaml b/hive/services/hiveui/frontend/pnpm-lock.yaml index 9080603a5..507c74bf9 100644 --- a/hive/services/hiveui/frontend/pnpm-lock.yaml +++ b/hive/services/hiveui/frontend/pnpm-lock.yaml @@ -104,13 +104,13 @@ importers: version: 5.1.4(vite@7.3.1(@types/node@24.12.0)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)) '@vitest/coverage-v8': specifier: ^4.0.18 - version: 4.0.18(vitest@4.0.18(@types/node@24.12.0)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0)) + version: 4.0.18(vitest@4.0.18(@types/node@24.12.0)(happy-dom@20.8.3)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0)) babel-plugin-react-compiler: specifier: ^1.0.0 version: 1.0.0 - jsdom: - specifier: ^28.1.0 - version: 28.1.0 + happy-dom: + specifier: ^20.8.3 + version: 20.8.3 typescript: specifier: ^5.9.3 version: 5.9.3 @@ -122,7 +122,7 @@ importers: version: 7.3.1(@types/node@24.12.0)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0) vitest: specifier: ^4.0.18 - version: 4.0.18(@types/node@24.12.0)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0) + version: 4.0.18(@types/node@24.12.0)(happy-dom@20.8.3)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0) packages: @@ -1338,6 +1338,12 @@ packages: '@types/react@19.2.14': resolution: {integrity: sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==} + '@types/whatwg-mimetype@3.0.2': + resolution: {integrity: sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==} + + '@types/ws@8.18.1': + resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260307.1': resolution: {integrity: sha512-VpnrMP4iDLSTT9Hg/KrHwuIHLZr5dxYPMFErfv3ZDA0tv48u2H1lBhHVVMMopCuskuX3C35EOJbxLkxCJd6zDw==} cpu: [arm64] @@ -1608,6 +1614,10 @@ packages: resolution: {integrity: sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==} engines: {node: '>=0.12'} + entities@7.0.1: + resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==} + engines: {node: '>=0.12'} + es-module-lexer@1.7.0: resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} @@ -1679,6 +1689,10 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + happy-dom@20.8.3: + resolution: {integrity: sha512-lMHQRRwIPyJ70HV0kkFT7jH/gXzSI7yDkQFe07E2flwmNDFoWUTRMKpW2sglsnpeA7b6S2TJPp98EbQxai8eaQ==} + engines: {node: '>=20.0.0'} + has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} @@ -2262,6 +2276,10 @@ packages: webpack-virtual-modules@0.6.2: resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==} + whatwg-mimetype@3.0.0: + resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==} + engines: {node: '>=12'} + whatwg-mimetype@5.0.0: resolution: {integrity: sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==} engines: {node: '>=20'} @@ -2275,6 +2293,18 @@ packages: engines: {node: '>=8'} hasBin: true + ws@8.19.0: + resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + xml-name-validator@5.0.0: resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} engines: {node: '>=18'} @@ -2293,7 +2323,8 @@ packages: snapshots: - '@acemir/cssom@0.9.31': {} + '@acemir/cssom@0.9.31': + optional: true '@adobe/css-tools@4.4.4': {} @@ -2304,6 +2335,7 @@ snapshots: '@csstools/css-parser-algorithms': 4.0.0(@csstools/css-tokenizer@4.0.0) '@csstools/css-tokenizer': 4.0.0 lru-cache: 11.2.6 + optional: true '@asamuzakjp/dom-selector@6.8.1': dependencies: @@ -2312,8 +2344,10 @@ snapshots: css-tree: 3.2.1 is-potential-custom-element-name: 1.0.1 lru-cache: 11.2.6 + optional: true - '@asamuzakjp/nwsapi@2.3.9': {} + '@asamuzakjp/nwsapi@2.3.9': + optional: true '@aws-crypto/crc32@5.2.0': dependencies: @@ -3066,6 +3100,7 @@ snapshots: '@bramus/specificity@2.4.2': dependencies: css-tree: 3.2.1 + optional: true '@clack/core@1.1.0': dependencies: @@ -3076,12 +3111,14 @@ snapshots: '@clack/core': 1.1.0 sisteransi: 1.0.5 - '@csstools/color-helpers@6.0.2': {} + '@csstools/color-helpers@6.0.2': + optional: true '@csstools/css-calc@3.1.1(@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.0))(@csstools/css-tokenizer@4.0.0)': dependencies: '@csstools/css-parser-algorithms': 4.0.0(@csstools/css-tokenizer@4.0.0) '@csstools/css-tokenizer': 4.0.0 + optional: true '@csstools/css-color-parser@4.0.2(@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.0))(@csstools/css-tokenizer@4.0.0)': dependencies: @@ -3089,14 +3126,18 @@ snapshots: '@csstools/css-calc': 3.1.1(@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.0))(@csstools/css-tokenizer@4.0.0) '@csstools/css-parser-algorithms': 4.0.0(@csstools/css-tokenizer@4.0.0) '@csstools/css-tokenizer': 4.0.0 + optional: true '@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.0)': dependencies: '@csstools/css-tokenizer': 4.0.0 + optional: true - '@csstools/css-syntax-patches-for-csstree@1.1.0': {} + '@csstools/css-syntax-patches-for-csstree@1.1.0': + optional: true - '@csstools/css-tokenizer@4.0.0': {} + '@csstools/css-tokenizer@4.0.0': + optional: true '@esbuild/aix-ppc64@0.27.3': optional: true @@ -3176,7 +3217,8 @@ snapshots: '@esbuild/win32-x64@0.27.3': optional: true - '@exodus/bytes@1.15.0': {} + '@exodus/bytes@1.15.0': + optional: true '@floating-ui/core@1.7.5': dependencies: @@ -3877,6 +3919,12 @@ snapshots: dependencies: csstype: 3.2.3 + '@types/whatwg-mimetype@3.0.2': {} + + '@types/ws@8.18.1': + dependencies: + '@types/node': 24.12.0 + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260307.1': optional: true @@ -3924,7 +3972,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitest/coverage-v8@4.0.18(vitest@4.0.18(@types/node@24.12.0)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0))': + '@vitest/coverage-v8@4.0.18(vitest@4.0.18(@types/node@24.12.0)(happy-dom@20.8.3)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0))': dependencies: '@bcoe/v8-coverage': 1.0.2 '@vitest/utils': 4.0.18 @@ -3936,7 +3984,7 @@ snapshots: obug: 2.1.1 std-env: 3.10.0 tinyrainbow: 3.0.3 - vitest: 4.0.18(@types/node@24.12.0)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0) + vitest: 4.0.18(@types/node@24.12.0)(happy-dom@20.8.3)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0) '@vitest/expect@4.0.18': dependencies: @@ -3979,7 +4027,8 @@ snapshots: acorn@8.16.0: {} - agent-base@7.1.4: {} + agent-base@7.1.4: + optional: true ansi-regex@5.0.1: {} @@ -4032,6 +4081,7 @@ snapshots: bidi-js@1.0.3: dependencies: require-from-string: 2.0.2 + optional: true binary-extensions@2.3.0: {} @@ -4092,6 +4142,7 @@ snapshots: dependencies: mdn-data: 2.27.1 source-map-js: 1.2.1 + optional: true css.escape@1.5.1: {} @@ -4101,6 +4152,7 @@ snapshots: '@csstools/css-syntax-patches-for-csstree': 1.1.0 css-tree: 3.2.1 lru-cache: 11.2.6 + optional: true csstype@3.2.3: {} @@ -4110,12 +4162,14 @@ snapshots: whatwg-url: 16.0.1 transitivePeerDependencies: - '@noble/hashes' + optional: true debug@4.4.3: dependencies: ms: 2.1.3 - decimal.js@10.6.0: {} + decimal.js@10.6.0: + optional: true deepmerge@4.3.1: {} @@ -4136,7 +4190,10 @@ snapshots: graceful-fs: 4.2.11 tapable: 2.3.0 - entities@6.0.1: {} + entities@6.0.1: + optional: true + + entities@7.0.1: {} es-module-lexer@1.7.0: {} @@ -4217,6 +4274,18 @@ snapshots: graceful-fs@4.2.11: {} + happy-dom@20.8.3: + dependencies: + '@types/node': 24.12.0 + '@types/whatwg-mimetype': 3.0.2 + '@types/ws': 8.18.1 + entities: 7.0.1 + whatwg-mimetype: 3.0.0 + ws: 8.19.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + has-flag@4.0.0: {} html-encoding-sniffer@6.0.0: @@ -4224,6 +4293,7 @@ snapshots: '@exodus/bytes': 1.15.0 transitivePeerDependencies: - '@noble/hashes' + optional: true html-escaper@2.0.2: {} @@ -4233,6 +4303,7 @@ snapshots: debug: 4.4.3 transitivePeerDependencies: - supports-color + optional: true https-proxy-agent@7.0.6: dependencies: @@ -4240,6 +4311,7 @@ snapshots: debug: 4.4.3 transitivePeerDependencies: - supports-color + optional: true ieee754@1.2.1: {} @@ -4259,7 +4331,8 @@ snapshots: is-number@7.0.0: {} - is-potential-custom-element-name@1.0.1: {} + is-potential-custom-element-name@1.0.1: + optional: true isbot@5.1.35: {} @@ -4308,6 +4381,7 @@ snapshots: transitivePeerDependencies: - '@noble/hashes' - supports-color + optional: true jsesc@3.1.0: {} @@ -4390,7 +4464,8 @@ snapshots: dependencies: semver: 7.7.4 - mdn-data@2.27.1: {} + mdn-data@2.27.1: + optional: true min-indent@1.0.1: {} @@ -4419,6 +4494,7 @@ snapshots: parse5@8.0.0: dependencies: entities: 6.0.1 + optional: true path-scurry@2.0.2: dependencies: @@ -4447,7 +4523,8 @@ snapshots: ansi-styles: 5.2.0 react-is: 17.0.2 - punycode@2.3.1: {} + punycode@2.3.1: + optional: true react-dom@19.2.4(react@19.2.4): dependencies: @@ -4487,7 +4564,8 @@ snapshots: indent-string: 4.0.0 strip-indent: 3.0.0 - require-from-string@2.0.2: {} + require-from-string@2.0.2: + optional: true reselect@5.1.1: {} @@ -4529,6 +4607,7 @@ snapshots: saxes@6.0.0: dependencies: xmlchars: 2.2.0 + optional: true scheduler@0.27.0: {} @@ -4575,7 +4654,8 @@ snapshots: dependencies: has-flag: 4.0.0 - symbol-tree@3.2.4: {} + symbol-tree@3.2.4: + optional: true tabbable@6.4.0: {} @@ -4600,11 +4680,13 @@ snapshots: tinyrainbow@3.0.3: {} - tldts-core@7.0.25: {} + tldts-core@7.0.25: + optional: true tldts@7.0.25: dependencies: tldts-core: 7.0.25 + optional: true to-regex-range@5.0.1: dependencies: @@ -4613,10 +4695,12 @@ snapshots: tough-cookie@6.0.0: dependencies: tldts: 7.0.25 + optional: true tr46@6.0.0: dependencies: punycode: 2.3.1 + optional: true tslib@2.8.1: {} @@ -4640,7 +4724,8 @@ snapshots: undici-types@7.16.0: {} - undici@7.22.0: {} + undici@7.22.0: + optional: true unplugin@2.3.11: dependencies: @@ -4676,7 +4761,7 @@ snapshots: lightningcss: 1.31.1 tsx: 4.21.0 - vitest@4.0.18(@types/node@24.12.0)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0): + vitest@4.0.18(@types/node@24.12.0)(happy-dom@20.8.3)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0): dependencies: '@vitest/expect': 4.0.18 '@vitest/mocker': 4.0.18(vite@7.3.1(@types/node@24.12.0)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)) @@ -4700,6 +4785,7 @@ snapshots: why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 24.12.0 + happy-dom: 20.8.3 jsdom: 28.1.0 transitivePeerDependencies: - jiti @@ -4717,12 +4803,17 @@ snapshots: w3c-xmlserializer@5.0.0: dependencies: xml-name-validator: 5.0.0 + optional: true - webidl-conversions@8.0.1: {} + webidl-conversions@8.0.1: + optional: true webpack-virtual-modules@0.6.2: {} - whatwg-mimetype@5.0.0: {} + whatwg-mimetype@3.0.0: {} + + whatwg-mimetype@5.0.0: + optional: true whatwg-url@16.0.1: dependencies: @@ -4731,15 +4822,20 @@ snapshots: webidl-conversions: 8.0.1 transitivePeerDependencies: - '@noble/hashes' + optional: true why-is-node-running@2.3.0: dependencies: siginfo: 2.0.0 stackback: 0.0.2 - xml-name-validator@5.0.0: {} + ws@8.19.0: {} - xmlchars@2.2.0: {} + xml-name-validator@5.0.0: + optional: true + + xmlchars@2.2.0: + optional: true yallist@3.1.1: {} diff --git a/hive/services/hiveui/frontend/src/hooks/use-copy-to-clipboard.test.tsx b/hive/services/hiveui/frontend/src/hooks/use-copy-to-clipboard.test.tsx index 792cd92ac..24d0eaa4f 100644 --- a/hive/services/hiveui/frontend/src/hooks/use-copy-to-clipboard.test.tsx +++ b/hive/services/hiveui/frontend/src/hooks/use-copy-to-clipboard.test.tsx @@ -5,8 +5,10 @@ import { useCopyToClipboard } from "./use-copy-to-clipboard" beforeEach(() => { vi.useFakeTimers() - Object.assign(navigator, { - clipboard: { writeText: vi.fn().mockResolvedValue(undefined) }, + Object.defineProperty(navigator, "clipboard", { + value: { writeText: vi.fn().mockResolvedValue(undefined) }, + writable: true, + configurable: true, }) }) diff --git a/hive/services/hiveui/frontend/vite.config.ts b/hive/services/hiveui/frontend/vite.config.ts index bd61757f8..df256f0f4 100644 --- a/hive/services/hiveui/frontend/vite.config.ts +++ b/hive/services/hiveui/frontend/vite.config.ts @@ -45,7 +45,7 @@ export default defineConfig({ }, test: { globals: true, - environment: "jsdom", + environment: "happy-dom", setupFiles: "./src/test/setup.ts", coverage: { all: true, diff --git a/scripts/run-e2e-local.sh b/scripts/run-e2e-local.sh deleted file mode 100755 index 25317672a..000000000 --- a/scripts/run-e2e-local.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash -# Run E2E tests locally using Docker, matching the CI process (e2e.yml). -# Requires: Docker, /dev/kvm -# -# Usage: -# ./scripts/run-e2e-local.sh # Run both suites -# ./scripts/run-e2e-local.sh single # Single-node only -# ./scripts/run-e2e-local.sh multi # Multi-node only - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -PARENT_DIR="$(cd "$PROJECT_ROOT/.." && pwd)" - -IMAGE_NAME="hive-e2e:latest" - -# Determine which suites to run -SUITE="${1:-both}" -case "$SUITE" in - single|multi|both) ;; - *) - echo "Usage: $0 [single|multi|both]" - exit 1 - ;; -esac - -# Verify /dev/kvm exists -if [ ! -e /dev/kvm ]; then - echo "ERROR: /dev/kvm not found. KVM is required for E2E tests." - exit 1 -fi - -# Verify sibling repos exist, clone if missing -for dep in viperblock predastore; do - if [ ! -d "$PARENT_DIR/$dep" ]; then - echo "Sibling repo '$dep' not found at $PARENT_DIR/$dep" - echo "Running clone-deps.sh..." - "$PROJECT_ROOT/scripts/clone-deps.sh" - break - fi -done - -# Build Docker image (same context as CI — parent directory) -echo "Building E2E Docker image..." -docker build \ - -t "$IMAGE_NAME" \ - -f "$PROJECT_ROOT/tests/e2e/Dockerfile.e2e" \ - "$PARENT_DIR" - -SINGLE_PASS="" -MULTI_PASS="" - -# Run single-node E2E -if [ "$SUITE" = "single" ] || [ "$SUITE" = "both" ]; then - echo "" - echo "========================================" - echo "Running Single-Node E2E Tests" - echo "========================================" - if docker run --privileged --rm \ - -v /dev/kvm:/dev/kvm \ - --name hive-e2e-local \ - "$IMAGE_NAME"; then - SINGLE_PASS="PASS" - echo "Single-node E2E: PASS" - else - SINGLE_PASS="FAIL" - echo "Single-node E2E: FAIL" - fi -fi - -# Run multi-node E2E -if [ "$SUITE" = "multi" ] || [ "$SUITE" = "both" ]; then - echo "" - echo "========================================" - echo "Running Multi-Node E2E Tests" - echo "========================================" - if docker run --privileged --rm \ - -v /dev/kvm:/dev/kvm \ - --cap-add=NET_ADMIN \ - --name hive-multinode-e2e-local \ - "$IMAGE_NAME" \ - ./tests/e2e/run-multinode-e2e.sh; then - MULTI_PASS="PASS" - echo "Multi-node E2E: PASS" - else - MULTI_PASS="FAIL" - echo "Multi-node E2E: FAIL" - fi -fi - -# Summary -echo "" -echo "========================================" -echo "E2E Results" -echo "========================================" -[ -n "$SINGLE_PASS" ] && echo " Single-node: $SINGLE_PASS" -[ -n "$MULTI_PASS" ] && echo " Multi-node: $MULTI_PASS" - -# Exit with failure if any suite failed -if [ "$SINGLE_PASS" = "FAIL" ] || [ "$MULTI_PASS" = "FAIL" ]; then - exit 1 -fi diff --git a/tests/e2e/Dockerfile.e2e b/tests/e2e/Dockerfile.e2e deleted file mode 100644 index c51dba238..000000000 --- a/tests/e2e/Dockerfile.e2e +++ /dev/null @@ -1,76 +0,0 @@ -FROM ubuntu:24.04 - -# Avoid prompts -ENV DEBIAN_FRONTEND=noninteractive - -# Install make and iproute2 (for multi-node IP management) -RUN apt-get update && apt-get install -y make sudo iproute2 && rm -rf /var/lib/apt/lists/* - -# Set up working directory for Makefile targets -WORKDIR /tmp/hive-setup - -# Copy only the Makefile for dependency installation -COPY hive/Makefile . - -# Install system dependencies in separate layers for better caching -RUN make install-system -RUN make install-go -RUN make install-aws - -# Ensure Go is in the PATH -ENV PATH="/usr/local/go/bin:/root/go/bin:${PATH}" - -# Set up working directory structure to match development layout -WORKDIR /root/Development/mulga - -# Pre-download Ubuntu cloud image for e2e tests (cached in Docker layer) -# Placed before dependency copy so go.mod changes don't invalidate this layer -RUN mkdir -p /root/images && \ - ARCH=$(dpkg --print-architecture) && \ - if [ "$ARCH" = "amd64" ]; then \ - curl -L -o /root/images/ubuntu-24.04.img \ - https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img; \ - elif [ "$ARCH" = "arm64" ]; then \ - curl -L -o /root/images/ubuntu-24.04.img \ - https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-arm64.img; \ - fi - -# 1. Pre-copy dependency manifests for cache layer -# go.work is gitignored (generated by init-workspace.sh), so create it inline -RUN printf 'go 1.26.1\n\nuse (\n\t./hive\n\t./predastore\n\t./viperblock\n)\n' > go.work -COPY hive/go.mod hive/go.sum ./hive/ -COPY viperblock/go.mod viperblock/go.sum ./viperblock/ -COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/ -COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/ -COPY predastore/go.mod predastore/go.sum ./predastore/ - -# 2. Download dependencies (using cache mount) -RUN --mount=type=cache,target=/root/go/pkg/mod \ - (cd hive && go mod download) && \ - (cd viperblock && go mod download) && \ - (cd viperblock/nbd && go mod download) && \ - (cd viperblock/nbd/libguestfs.org/nbdkit && go mod download) && \ - (cd predastore && go mod download) - -# 3. Copy the rest of the source code -COPY hive/ ./hive/ -COPY viperblock/ ./viperblock/ -COPY predastore/ ./predastore/ - -# Set working directory to hive -WORKDIR /root/Development/mulga/hive - -# Ensure scripts are executable -RUN chmod +x ./scripts/*.sh ./tests/e2e/*.sh ./tests/e2e/lib/*.sh - -# 4. Build everything (using cache mounts) -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/root/go/pkg/mod \ - ./scripts/dev-setup.sh - -# Set environment variables for runtime -ENV PATH="/root/Development/mulga/hive/bin:${PATH}" -ENV HOME="/root" - -# Run the E2E test runner -CMD ["./tests/e2e/run-e2e.sh"] diff --git a/tests/e2e/TEST_COVERAGE.md b/tests/e2e/TEST_COVERAGE.md index f176e7817..0d9489bba 100644 --- a/tests/e2e/TEST_COVERAGE.md +++ b/tests/e2e/TEST_COVERAGE.md @@ -298,12 +298,12 @@ Tests that EC2 resources are properly isolated between tenant accounts (Alpha, B --- -## Multi-Node (`run-multinode-e2e.sh`) +## Pseudo Multi-Node (`run-pseudo-multinode-e2e.sh`) ### Phase 1: Environment Setup - KVM support check - Simulated network IPs (10.11.12.{1,2,3} on loopback) -- Create ramdisk mount +- Simulated network IPs (no ramdisk — start-dev.sh uses disk-backed WAL/VB in CI) ### Phase 2: Cluster Initialization - `hive admin init` (leader node1) @@ -550,6 +550,91 @@ Tests that EC2 resources are properly isolated between the Alpha/Beta accounts. --- +## Real Multi-Node (`run-multinode-e2e.sh`) + +Runs on a real 3-node libvirt cluster provisioned by OpenTofu (`scripts/tofu-cluster/`). Each node is a separate VM with its own OVN, NATS, Predastore, and Hive daemon. Bootstrap (`bootstrap.sh`) handles all provisioning before the test script runs. + +### Bootstrap (pre-test) + +1. OpenTofu provisions 3 libvirt VMs (bottlebrush, ironbark, casuarina) with cloud-init +2. Wait for all nodes to be SSH-reachable +3. Copy SSH key to node1 for peer access +4. `git fetch` + checkout test branch on all nodes (all 3 repos) +5. `make build` on all nodes (parallel) +6. `setup-ovn.sh` — primary gets `--management`, secondaries connect to primary's OVN central +7. `hive admin init` on primary, `hive admin join` on secondaries (formation server handshake) +8. `start-dev.sh` on all nodes +9. Wait for `/health` (awsgw=ok) on all nodes +10. Install Hive CA certificate on all nodes +11. `import-key-pair` (hive-key) + `hive admin images import` (Ubuntu 24.04) +12. `create-vpc` (10.200.0.0/16) + `create-subnet` (10.200.1.0/24) + +### Phase 1: Pre-flight Validation +- KVM support check (`/dev/kvm` writable) +- SSH connectivity to all peer nodes + +### Phase 2: Cluster Health +- NATS cluster: verify 2 unique route peers (3-node quorum) +- Predastore reachable on all 3 nodes +- AWS gateway reachable on all 3 nodes +- Daemon readiness (`describe-instance-types` returns results) +- `hive get nodes` — verify all 3 nodes show Ready +- `hive get vms` — verify empty (no instances yet) + +### Phase 3: Instance Lifecycle + Distribution +- Discover nano instance type and AMI (from bootstrap import) +- `run-instances` x3 with staggered launches +- Poll all instances to running state +- Check instance distribution across physical nodes (QEMU process check) +- Verify at least 2 different hosting nodes (non-deterministic, non-fatal if all on 1) +- `hive get vms` — verify all 3 instances visible + +### Phase 4: SSH into Guest VMs +- For each instance across all nodes: + - Find hosting node via QEMU process scan + - Extract SSH hostfwd port from QEMU command line on remote node + - Wait for SSH readiness (up to 60s) + - Verify SSH connectivity (`id` returns ec2-user) + - Verify block device (`lsblk`) + +### Phase 5: Volume Lifecycle +- `create-volume` (10GB) +- `attach-volume` (to first instance, /dev/sdf) +- Poll for in-use + attached state +- `detach-volume` (poll for available) +- `delete-volume` (verify gone) + +### Phase 6: Cross-Node Gateway Access +- `describe-instances` via node1 gateway (baseline count) +- `describe-instances` via node2 and node3 gateways +- Verify all gateways return the same instance count (fan-out aggregation) + +### Phase 7: Cross-Node Operations +- Find which node hosts instance 0 +- Pick a different node's gateway for stop +- `stop-instances` via remote gateway (poll -> stopped) +- Pick a third node's gateway for start +- `start-instances` via another remote gateway (poll -> running) + +### Phase 8: Node Failure +- `stop-dev.sh` on node2 (simulate node failure) +- Verify node1 still serves requests (`describe-instance-types`) +- Verify node3 still serves requests +- Check NATS degraded state (1 peer instead of 2) +- `describe-instances` still works from surviving nodes + +### Phase 9: Node Recovery +- `start-dev.sh` on node2 (restart failed node) +- Wait for NATS cluster to reform (2 peers again) +- Verify node2 gateway is back +- `hive get nodes` — verify all 3 nodes Ready again +- Verify node2 serves requests after recovery + +### Phase 10: Cleanup +- `terminate-instances` (all 3 instances, poll -> terminated) + +--- + ## VPC (`run-vpc-e2e.sh`) Standalone VPC networking test suite. Runs against a running Hive endpoint (configurable via `ENDPOINT` env var). OVN topology tests are skipped when OVN is unavailable. diff --git a/tests/e2e/lib/multinode-helpers.sh b/tests/e2e/lib/multinode-helpers.sh index 0b827f46a..b9f0318d8 100755 --- a/tests/e2e/lib/multinode-helpers.sh +++ b/tests/e2e/lib/multinode-helpers.sh @@ -5,117 +5,6 @@ # and verifying NATS cluster health. -# Bootstrap OVN/OVS services inside a Docker container (no systemd). -# Starts ovsdb-server, ovs-vswitchd, OVN central (NB/SB DBs + northd), -# creates br-int, and starts ovn-controller. -# All 3 simulated nodes share one br-int and one ovn-controller. -bootstrap_ovn_docker() { - echo "Bootstrapping OVN/OVS for Docker (no systemd)..." - - # Start ovsdb-server - mkdir -p /var/run/openvswitch - if [ ! -f /etc/openvswitch/conf.db ]; then - ovsdb-tool create /etc/openvswitch/conf.db /usr/share/openvswitch/vswitch.ovsschema - fi - ovsdb-server --remote=punix:/var/run/openvswitch/db.sock \ - --remote=db:Open_vSwitch,Open_vSwitch,manager_options \ - --pidfile --detach --log-file=/var/log/openvswitch/ovsdb-server.log \ - /etc/openvswitch/conf.db - ovs-vsctl --no-wait init - - # Start ovs-vswitchd - ovs-vswitchd --pidfile --detach --log-file=/var/log/openvswitch/ovs-vswitchd.log - - # Create br-int with secure fail-mode - ovs-vsctl --may-exist add-br br-int - ovs-vsctl set Bridge br-int fail-mode=secure - ovs-vsctl set Bridge br-int other-config:disable-in-band=true - ip link set br-int up - - # Start OVN central (NB + SB DBs + northd) - mkdir -p /var/run/ovn /var/lib/ovn /var/log/ovn - - # NB database - if [ ! -f /var/lib/ovn/ovnnb_db.db ]; then - ovsdb-tool create /var/lib/ovn/ovnnb_db.db /usr/share/ovn/ovn-nb.ovsschema - fi - ovsdb-server --remote=punix:/var/run/ovn/ovnnb_db.sock \ - --remote=ptcp:6641 \ - --pidfile=/var/run/ovn/ovnnb_db.pid \ - --detach --log-file=/var/log/ovn/ovnnb_db.log \ - /var/lib/ovn/ovnnb_db.db - - # SB database - if [ ! -f /var/lib/ovn/ovnsb_db.db ]; then - ovsdb-tool create /var/lib/ovn/ovnsb_db.db /usr/share/ovn/ovn-sb.ovsschema - fi - ovsdb-server --remote=punix:/var/run/ovn/ovnsb_db.sock \ - --remote=ptcp:6642 \ - --pidfile=/var/run/ovn/ovnsb_db.pid \ - --detach --log-file=/var/log/ovn/ovnsb_db.log \ - /var/lib/ovn/ovnsb_db.db - - # Start ovn-northd - ovn-northd --pidfile=/var/run/ovn/ovn-northd.pid \ - --detach --log-file=/var/log/ovn/ovn-northd.log \ - --ovnnb-db=unix:/var/run/ovn/ovnnb_db.sock \ - --ovnsb-db=unix:/var/run/ovn/ovnsb_db.sock - - # Configure OVS external_ids for OVN - local chassis_id="chassis-docker" - ovs-vsctl set Open_vSwitch . \ - external_ids:system-id="$chassis_id" \ - external_ids:ovn-remote="tcp:127.0.0.1:6642" \ - external_ids:ovn-encap-ip="127.0.0.1" \ - external_ids:ovn-encap-type="geneve" - - # Start ovn-controller with pidfile in OVS rundir - ovn-controller --pidfile=/var/run/openvswitch/ovn-controller.pid \ - --detach --log-file=/var/log/ovn/ovn-controller.log - - # Apply sysctl for overlay networking - sysctl -w net.ipv4.ip_forward=1 >/dev/null 2>&1 || true - sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null 2>&1 || true - sysctl -w net.ipv4.conf.default.rp_filter=0 >/dev/null 2>&1 || true - - # Load geneve kernel module - modprobe geneve 2>/dev/null || true - - # Wait for ovn-controller to become responsive. - # ovn-controller creates its ctl socket at /var/run/ovn/ovn-controller.PID.ctl - # but ovs-appctl -t ovn-controller looks in /var/run/openvswitch/, so we - # symlink it once the socket appears. - echo " Waiting for ovn-controller to start..." - local ovn_pid - ovn_pid=$(cat /var/run/openvswitch/ovn-controller.pid 2>/dev/null || true) - local attempt=0 - while [ $attempt -lt 15 ]; do - # Create symlink once the ctl socket appears - if [ -n "$ovn_pid" ] && [ -S "/var/run/ovn/ovn-controller.${ovn_pid}.ctl" ] \ - && [ ! -e "/var/run/openvswitch/ovn-controller.${ovn_pid}.ctl" ]; then - ln -sf "/var/run/ovn/ovn-controller.${ovn_pid}.ctl" \ - "/var/run/openvswitch/ovn-controller.${ovn_pid}.ctl" - fi - if ovs-appctl -t ovn-controller version >/dev/null 2>&1; then - break - fi - sleep 1 - attempt=$((attempt + 1)) - done - - # Verify - if ovs-vsctl br-exists br-int && ovs-appctl -t ovn-controller version >/dev/null 2>&1; then - echo " OVN/OVS bootstrap complete (br-int up, ovn-controller running)" - else - echo " ERROR: OVN bootstrap failed" - ovs-vsctl br-exists br-int && echo " br-int: OK" || echo " br-int: MISSING" - ovs-appctl -t ovn-controller version >/dev/null 2>&1 && echo " ovn-controller: OK" || echo " ovn-controller: NOT RUNNING" - # Dump ovn-controller log for debugging - echo " --- ovn-controller.log ---" - cat /var/log/ovn/ovn-controller.log 2>/dev/null | tail -20 || true - return 1 - fi -} # Network configuration SIMULATED_NETWORK="10.11.12" @@ -139,7 +28,7 @@ add_simulated_ips() { for i in 1 2 3; do local ip="${SIMULATED_NETWORK}.$i" if ! ip addr show lo | grep -q "$ip"; then - ip addr add "${ip}/24" dev lo + sudo ip addr add "${ip}/24" dev lo echo " Added $ip to lo" else echo " $ip already exists on lo" @@ -168,7 +57,7 @@ remove_simulated_ips() { for i in 1 2 3; do local ip="${SIMULATED_NETWORK}.$i" if ip addr show lo | grep -q "$ip"; then - ip addr del "${ip}/24" dev lo 2>/dev/null || true + sudo ip addr del "${ip}/24" dev lo 2>/dev/null || true echo " Removed $ip from lo" fi done diff --git a/tests/e2e/run-e2e.sh b/tests/e2e/run-e2e.sh index cec865ebb..e890ac94a 100755 --- a/tests/e2e/run-e2e.sh +++ b/tests/e2e/run-e2e.sh @@ -1,7 +1,13 @@ #!/bin/bash set -e -# Source helper functions for SSH testing +# Ensure Go is on PATH (SSH non-interactive shells don't source .bashrc) +export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" + +# Ensure we are in the project root +cd "$(dirname "$0")/../.." + +# Source helper functions source ./tests/e2e/lib/multinode-helpers.sh # Ensure services are stopped on exit and print logs on failure @@ -34,7 +40,10 @@ cleanup() { tail -200 ~/hive/logs/awsgw.log 2>/dev/null fi fi - ./scripts/stop-dev.sh + # Only stop services if we started them (not when bootstrapped) + if [ "$BOOTSTRAPPED" != "true" ]; then + ./scripts/stop-dev.sh + fi exit $EXIT_CODE } trap cleanup EXIT @@ -42,6 +51,33 @@ trap cleanup EXIT # Use Hive profile (hive admin init sets endpoint_url, ca_bundle, region in ~/.aws/config) export AWS_PROFILE=hive +# Detect bootstrapped environment from hive.toml config +BOOTSTRAPPED="false" +GATEWAY_HOST="localhost" +PREDASTORE_HOST="localhost" +if [ -f ~/hive/config/hive.toml ]; then + # Resolve gateway host from AWS profile endpoint_url (set by hive admin init) + ENDPOINT_URL=$(aws configure get endpoint_url 2>/dev/null || true) + if [ -n "$ENDPOINT_URL" ]; then + DETECTED_GW_HOST=$(echo "$ENDPOINT_URL" | sed 's|https\?://||;s|:.*||') + if [ -n "$DETECTED_GW_HOST" ]; then + GATEWAY_HOST="$DETECTED_GW_HOST" + fi + fi + # Resolve predastore host from hive.toml + DETECTED_PS_HOST=$(awk -F'"' '/\[nodes\..*\.predastore\]/{found=1} found && /^host/{print $2; exit}' ~/hive/config/hive.toml) + if [ -n "$DETECTED_PS_HOST" ]; then + PREDASTORE_HOST="${DETECTED_PS_HOST%%:*}" + fi + # Check if gateway is actually responding + if curl -sk "https://${GATEWAY_HOST}:9999" > /dev/null 2>&1; then + BOOTSTRAPPED="true" + echo "Detected bootstrapped environment — skipping cluster setup" + echo " Gateway host: $GATEWAY_HOST" + echo " Predastore host: $PREDASTORE_HOST" + fi +fi + # Helper: set up an AWS CLI profile with credentials + endpoint/CA config from the hive profile setup_test_profile() { local profile="$1" key_id="$2" secret="$3" @@ -52,9 +88,6 @@ setup_test_profile() { aws configure set ca_bundle "$(aws configure get ca_bundle)" --profile "$profile" } -# Ensure we are in the project root -cd "$(dirname "$0")/../.." - # Phase 1: Environment Setup echo "Phase 1: Environment Setup" @@ -73,29 +106,26 @@ else exit 1 fi -# Bootstrap OVN/OVS (required — start-dev.sh will block without it) -echo "Bootstrapping OVN/OVS networking..." -bootstrap_ovn_docker +if [ "$BOOTSTRAPPED" != "true" ]; then + ./bin/hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1 -./bin/hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1 + # Trust the Hive CA certificate for AWS CLI SSL verification + echo "Adding Hive CA certificate to system trust store..." + sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt + sudo update-ca-certificates -# Trust the Hive CA certificate for AWS CLI SSL verification -echo "Adding Hive CA certificate to system trust store..." -sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt -sudo update-ca-certificates + # Start all services + # Ensure logs directory exists for start-dev.sh + mkdir -p ~/hive/logs + ./scripts/start-dev.sh +fi -# Start all services -# Ensure logs directory exists for start-dev.sh -mkdir -p ~/hive/logs -mkdir -p /mnt/ramdisk -./scripts/start-dev.sh - -# Wait for health checks on https://localhost:9999 (AWS Gateway) +# Wait for health checks on AWS Gateway echo "Waiting for AWS Gateway..." MAX_RETRIES=15 COUNT=0 -until curl -s https://localhost:9999 > /dev/null || [ $COUNT -eq $MAX_RETRIES ]; do +until curl -sk "https://${GATEWAY_HOST}:9999" > /dev/null || [ $COUNT -eq $MAX_RETRIES ]; do echo "Waiting for gateway... ($COUNT/$MAX_RETRIES)" sleep 2 COUNT=$((COUNT + 1)) @@ -107,7 +137,12 @@ if [ $COUNT -eq $MAX_RETRIES ]; then fi # Wait for daemon NATS subscriptions to be active -wait_for_daemon_ready "https://localhost:9999" +wait_for_daemon_ready "https://${GATEWAY_HOST}:9999" + +# Discover the cluster's availability zone and region dynamically +HIVE_AZ=$(aws ec2 describe-availability-zones --query 'AvailabilityZones[0].ZoneName' --output text) +HIVE_REGION=$(aws ec2 describe-availability-zones --query 'AvailabilityZones[0].RegionName' --output text) +echo "Discovered AZ: $HIVE_AZ, Region: $HIVE_REGION" # No need for AWS_EC2/AWS_IAM vars — endpoint_url and ca_bundle are in the profile config @@ -260,17 +295,23 @@ fi echo "Using image: $IMAGE_NAME" # Import the pre-downloaded Ubuntu image using file-based import -echo "Importing pre-cached Ubuntu image..." -IMPORT_LOG=$(./bin/hive admin images import \ - --file /root/images/ubuntu-24.04.img \ - --arch "$ARCH" \ - --distro ubuntu \ - --version 24.04 \ - --force 2>/dev/null) -AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+') +# When bootstrapped, the image is already imported — discover the existing AMI +if [ "$BOOTSTRAPPED" = "true" ]; then + echo "Discovering existing AMI (imported by bootstrap)..." + AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text) +else + echo "Importing pre-cached Ubuntu image..." + IMPORT_LOG=$(./bin/hive admin images import \ + --file ~/images/ubuntu-24.04.img \ + --arch "$ARCH" \ + --distro ubuntu \ + --version 24.04 \ + --force 2>/dev/null) + AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+') +fi -if [ -z "$AMI_ID" ]; then - echo "Failed to capture AMI ID from import command" +if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then + echo "Failed to capture AMI ID" exit 1 fi echo "Captured AMI ID: $AMI_ID" @@ -477,8 +518,8 @@ echo "Phase 5b: Volume Lifecycle (Attach/Detach)" echo "Testing volume create -> resize -> attach -> detach -> delete..." # Create a test volume -echo "Creating 10GB volume in ap-southeast-2a..." -CREATE_OUTPUT=$(aws ec2 create-volume --size 10 --availability-zone ap-southeast-2a) +echo "Creating 10GB volume in ${HIVE_AZ}..." +CREATE_OUTPUT=$(aws ec2 create-volume --size 10 --availability-zone "$HIVE_AZ") TEST_VOLUME_ID=$(echo "$CREATE_OUTPUT" | jq -r '.VolumeId') if [ -z "$TEST_VOLUME_ID" ] || [ "$TEST_VOLUME_ID" == "null" ]; then @@ -694,7 +735,7 @@ echo "Describe snapshot verified (VolumeId=$DESC_VOL_ID, Size=$DESC_SIZE, Descri # Copy the snapshot echo "Copying snapshot $SNAPSHOT_ID..." -COPY_OUTPUT=$(aws ec2 copy-snapshot --source-snapshot-id "$SNAPSHOT_ID" --source-region ap-southeast-2 --description "e2e-copy") +COPY_OUTPUT=$(aws ec2 copy-snapshot --source-snapshot-id "$SNAPSHOT_ID" --source-region "$HIVE_REGION" --description "e2e-copy") COPY_SNAPSHOT_ID=$(echo "$COPY_OUTPUT" | jq -r '.SnapshotId') if [ -z "$COPY_SNAPSHOT_ID" ] || [ "$COPY_SNAPSHOT_ID" == "null" ]; then @@ -777,7 +818,7 @@ echo "Phase 5d: Verify Snapshot-Backed Instance Launch" echo "All run-instances calls go through cloneAMIToVolume() -> OpenFromSnapshot()," echo "so the Phase 5 instance is already snapshot-backed. Verify its volume config." -AWS_S3="aws --endpoint-url https://localhost:8443 s3" +AWS_S3="aws --endpoint-url https://${PREDASTORE_HOST}:8443 s3" # Verify the AMI snapshot exists in Predastore echo "Checking AMI snapshot in Predastore..." @@ -973,7 +1014,7 @@ echo "Reboot-stopped correctly rejected" # Phase 7a: Attach volume to stopped instance (should fail) echo "Phase 7a: Attach Volume to Stopped Instance (Error Path)" echo "Creating a volume to test attach-to-stopped..." -STOPPED_VOL_OUTPUT=$(aws ec2 create-volume --size 10 --availability-zone ap-southeast-2a) +STOPPED_VOL_OUTPUT=$(aws ec2 create-volume --size 10 --availability-zone "$HIVE_AZ") STOPPED_VOL_ID=$(echo "$STOPPED_VOL_OUTPUT" | jq -r '.VolumeId') echo "Created volume: $STOPPED_VOL_ID" @@ -990,8 +1031,9 @@ echo "Cleaned up test volume $STOPPED_VOL_ID" echo "Phase 7b: ModifyInstanceAttribute" echo "Instance is stopped — modifying instance type to verify changes take effect on restart" -# Derive an upsized type in the same family: nano → xlarge (4 vCPUs instead of 2) -MODIFY_TYPE="${INSTANCE_TYPE%.nano}.xlarge" +# Derive an upsized type in the same family: nano → small (same vCPUs, more RAM) +# Note: xlarge needs 16GB RAM which exceeds the 8GB CI VM — use small (2GB) instead +MODIFY_TYPE="${INSTANCE_TYPE%.nano}.small" echo "Changing instance type from $INSTANCE_TYPE to $MODIFY_TYPE..." # Get expected vCPU and memory for the new type @@ -1222,7 +1264,7 @@ expect_error "InvalidSnapshot.NotFound" aws ec2 delete-snapshot \ # 8f: Call an unsupported Action (use raw curl to send an invalid Action) echo "8f: Unsupported Action..." set +e -UNSUPPORTED_OUTPUT=$(curl -s -k -X POST https://localhost:9999/ \ +UNSUPPORTED_OUTPUT=$(curl -s -k -X POST "https://${GATEWAY_HOST}:9999/" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "Action=DescribeFakeThings&Version=2016-11-15" 2>&1) set -e diff --git a/tests/e2e/run-multinode-e2e.sh b/tests/e2e/run-multinode-e2e.sh index fc65cd39d..245572d24 100755 --- a/tests/e2e/run-multinode-e2e.sh +++ b/tests/e2e/run-multinode-e2e.sh @@ -1,337 +1,434 @@ #!/bin/bash -set -e +set -euo pipefail -# Multi-node E2E test runner -# This script sets up a 3-node Hive cluster using simulated IPs on the loopback interface -# and runs distributed instance tests. +# Multi-Node E2E Test Suite (real 3-node cluster) +# Runs inside node1 of a 3-node cluster provisioned by tofu (env2). +# Bootstrap has already: built hive, set up OVN, run init/join, started services, +# installed CA certs, imported SSH key + AMI, created VPC + subnet. +# +# Usage: run-multinode-e2e.sh -# Ensure we are in the project root +# Ensure Go is on PATH (SSH non-interactive shells don't source .bashrc) +export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" + +# Ensure we are in the hive project root cd "$(dirname "$0")/../.." -# Source helper functions -source ./tests/e2e/lib/multinode-helpers.sh +# ========================================================================== +# Argument parsing +# ========================================================================== +if [ $# -lt 3 ]; then + echo "Usage: $0 " + echo " All 3 node WAN IPs are required." + exit 1 +fi -# Cleanup function - ensure resources are cleaned up on exit -cleanup() { - local exit_code=$? +NODE_IPS=("$@") +NODE_COUNT=${#NODE_IPS[@]} +LOCAL_IP="${NODE_IPS[0]}" +NODE2_IP="${NODE_IPS[1]}" +NODE3_IP="${NODE_IPS[2]}" - echo "" - echo "Cleanup triggered (exit code: $exit_code)..." - - if [ $exit_code -ne 0 ]; then - dump_all_node_logs - fi - - # Kill any lingering formation background processes - [ -n "$LEADER_INIT_PID" ] && kill "$LEADER_INIT_PID" 2>/dev/null || true - [ -n "$JOIN2_PID" ] && kill "$JOIN2_PID" 2>/dev/null || true - [ -n "$JOIN3_PID" ] && kill "$JOIN3_PID" 2>/dev/null || true - - # Try coordinated shutdown first (only if NATS is likely still up) - if [ "$CLUSTER_SERVICES_STARTED" = "true" ]; then - echo "Attempting coordinated cluster shutdown..." - if timeout 60 ./bin/hive admin cluster shutdown --force --timeout 30s --config "$HOME/node1/config/hive.toml" 2>/dev/null; then - echo "Coordinated shutdown succeeded" - else - echo "Coordinated shutdown failed, falling back to per-node stop..." - stop_all_nodes || true - fi - else - stop_all_nodes || true - fi - - # Force-kill anything that survived and clean up stale locks - force_cleanup_all_nodes || true - - # Remove simulated IPs - remove_simulated_ips || true - - echo "Cleanup complete" -} -trap cleanup EXIT - -# PIDs for background formation processes (used in cleanup) -JOIN2_PID="" -JOIN3_PID="" - -# Track whether cluster services have been started (for cleanup trap) -CLUSTER_SERVICES_STARTED="false" +# ========================================================================== +# Constants +# ========================================================================== +NATS_MONITOR_PORT=8222 +PREDASTORE_PORT=8443 +AWSGW_PORT=9999 +SSH_KEY_PATH="$HOME/.ssh/tf-user-ap-southeast-2" +HIVE_DATA_DIR="$HOME/hive" +HIVE_CONFIG="$HIVE_DATA_DIR/config/hive.toml" +HIVE_BIN="./bin/hive" # Use Hive profile for AWS CLI export AWS_PROFILE=hive -# Trust Hive CA for all profiles (AWS CLI v2 bundles its own Python/certifi, ignores system CA store) -export AWS_CA_BUNDLE="$HOME/node1/config/ca.pem" +# Trust Hive CA for AWS CLI v2 (bundles its own Python/certifi, ignores system CA store) +export AWS_CA_BUNDLE="$HIVE_DATA_DIR/config/ca.pem" +# Track test results +TESTS_PASSED=0 +TESTS_FAILED=0 +FAILED_TESTS=() + +# ========================================================================== +# Helper functions +# ========================================================================== + +# SSH to a peer node +peer_ssh() { + local ip="$1"; shift + ssh -i "$SSH_KEY_PATH" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "tf-user@${ip}" "$@" +} + +# Run AWS CLI against a specific node's gateway +aws_via() { + local node_ip="$1"; shift + aws --endpoint-url "https://${node_ip}:${AWSGW_PORT}" "$@" +} + +# Default AWS CLI shorthand (via local node) +AWS_EC2="aws --endpoint-url https://${LOCAL_IP}:${AWSGW_PORT} ec2" + +# Dump logs from ALL nodes on failure +dump_all_node_logs() { + echo "" + echo "==========================================" + echo "DUMPING LOGS FROM ALL NODES" + echo "==========================================" + for i in $(seq 0 $((NODE_COUNT - 1))); do + local ip="${NODE_IPS[$i]}" + echo "" + echo "=== Node $((i+1)) ($ip) ===" + if [ "$ip" = "$LOCAL_IP" ]; then + for f in "$HIVE_DATA_DIR/logs/"*.log; do + [ -f "$f" ] || continue + echo "--- $(basename "$f") (last 50 lines) ---" + tail -50 "$f" 2>/dev/null || echo "(not found)" + done + else + peer_ssh "$ip" 'for f in ~/hive/logs/*.log; do + [ -f "$f" ] || continue + echo "--- $(basename "$f") (last 50 lines) ---" + tail -50 "$f" 2>/dev/null || echo "(not found)" + done' || echo "(node unreachable)" + fi + done + echo "" + echo "==========================================" + echo "END OF LOG DUMP" + echo "==========================================" +} + +# Wait for a specific instance state +# Usage: wait_for_instance_state [max_attempts] [gateway_ip] +wait_for_instance_state() { + local instance_id="$1" + local target_state="$2" + local max_attempts="${3:-30}" + local gw_ip="${4:-$LOCAL_IP}" + local attempt=0 + + echo " Waiting for $instance_id to reach state: $target_state..." + + while [ $attempt -lt $max_attempts ]; do + local state + state=$(aws_via "$gw_ip" ec2 describe-instances \ + --instance-ids "$instance_id" \ + --query 'Reservations[0].Instances[0].State.Name' \ + --output text 2>/dev/null) || { + sleep 2 + attempt=$((attempt + 1)) + continue + } + + if [ "$state" == "$target_state" ]; then + echo " Instance reached state: $target_state" + return 0 + fi + + if [ "$state" == "terminated" ] && [ "$target_state" != "terminated" ]; then + echo " ERROR: Instance terminated unexpectedly" + return 1 + fi + + sleep 2 + attempt=$((attempt + 1)) + done + + echo " ERROR: Instance did not reach $target_state within $max_attempts attempts" + return 1 +} + +# Find which node runs a QEMU instance (by checking ps on each node) +# Usage: find_instance_node +# Returns: the WAN IP of the hosting node +find_instance_node() { + local instance_id="$1" + + for ip in "${NODE_IPS[@]}"; do + local found + if [ "$ip" = "$LOCAL_IP" ]; then + found=$(ps auxw | grep "$instance_id" | grep qemu-system | grep -v grep || true) + else + found=$(peer_ssh "$ip" "ps auxw | grep '$instance_id' | grep qemu-system | grep -v grep" 2>/dev/null || true) + fi + if [ -n "$found" ]; then + echo "$ip" + return 0 + fi + done + + return 1 +} + +# Extract the SSH hostfwd port for an instance from the QEMU process on a remote node +# Usage: get_remote_ssh_port [max_attempts] +get_remote_ssh_port() { + local node_ip="$1" + local instance_id="$2" + local max_attempts="${3:-30}" + local attempt=0 + + while [ $attempt -lt $max_attempts ]; do + local qemu_cmd + if [ "$node_ip" = "$LOCAL_IP" ]; then + qemu_cmd=$(ps auxw | grep "$instance_id" | grep qemu-system | grep -v grep || true) + else + qemu_cmd=$(peer_ssh "$node_ip" "ps auxw | grep '$instance_id' | grep qemu-system | grep -v grep" 2>/dev/null || true) + fi + + if [ -n "$qemu_cmd" ]; then + local ssh_port + ssh_port=$(echo "$qemu_cmd" | sed -n 's/.*hostfwd=tcp:[^:]*:\([0-9]*\)-:22.*/\1/p') + if [ -n "$ssh_port" ]; then + echo "$ssh_port" + return 0 + fi + fi + + attempt=$((attempt + 1)) + [ $attempt -lt $max_attempts ] && sleep 1 + done + + return 1 +} + +# Expect an AWS CLI command to fail with a specific error code +# Usage: expect_error "ErrorCode" aws ec2 some-command --args... +expect_error() { + local expected_error="$1" + shift + + set +e + local output + output=$("$@" 2>&1) + local exit_code=$? + set -e + + if [ $exit_code -eq 0 ]; then + echo " FAIL: Expected error '$expected_error' but command succeeded" + echo " Output: $output" + return 1 + fi + + if echo "$output" | grep -q "$expected_error"; then + echo " Got expected error: $expected_error" + return 0 + else + echo " FAIL: Expected error '$expected_error' but got different error" + echo " Output: $output" + return 1 + fi +} + +# Terminate instances and wait for terminated state +terminate_and_wait() { + local ids=("$@") + + for instance_id in "${ids[@]}"; do + local state + state=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ + --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "unknown") + if [ "$state" != "terminated" ] && [ "$state" != "unknown" ]; then + echo " Terminating $instance_id (state: $state)..." + $AWS_EC2 terminate-instances --instance-ids "$instance_id" > /dev/null 2>&1 || true + fi + done + + local failed=0 + for instance_id in "${ids[@]}"; do + if ! wait_for_instance_state "$instance_id" "terminated" 30; then + echo " WARNING: Failed to confirm termination of $instance_id" + failed=1 + fi + done + + return $failed +} + +# Record test result +pass_test() { + local name="$1" + echo " $name PASSED" + TESTS_PASSED=$((TESTS_PASSED + 1)) +} + +fail_test() { + local name="$1" + echo " $name FAILED" + TESTS_FAILED=$((TESTS_FAILED + 1)) + FAILED_TESTS+=("$name") +} + +# ========================================================================== +# EXIT trap — dump logs on failure +# ========================================================================== +trap 'EXIT_CODE=$?; if [ $EXIT_CODE -ne 0 ]; then dump_all_node_logs; fi; exit $EXIT_CODE' EXIT + +# Track instance IDs for cleanup +INSTANCE_IDS=() echo "========================================" -echo "Multi-Node E2E Test Suite" +echo "Real Multi-Node E2E Test Suite" echo "========================================" +echo "Nodes: ${NODE_IPS[*]}" +echo "Local: $LOCAL_IP" echo "" -# Phase 1: Environment Setup -echo "Phase 1: Environment Setup" +# ========================================================================== +# Phase 1: Pre-flight Validation +# ========================================================================== +echo "Phase 1: Pre-flight Validation" echo "========================================" -# Check for KVM support -echo "Checking for KVM support..." -if [ -e /dev/kvm ]; then - echo " /dev/kvm exists" - if [ -w /dev/kvm ]; then - echo " /dev/kvm is writable" +# Check KVM support +echo "Checking KVM support..." +if [ -e /dev/kvm ] && [ -w /dev/kvm ]; then + echo " /dev/kvm exists and is writable" +else + echo " ERROR: /dev/kvm missing or not writable" + exit 1 +fi + +# Verify SSH to peer nodes +echo "Verifying SSH connectivity to peer nodes..." +for i in 1 2; do + local_idx=$((i)) + ip="${NODE_IPS[$local_idx]}" + echo -n " SSH to node$((local_idx + 1)) ($ip)..." + if peer_ssh "$ip" "hostname" > /dev/null 2>&1; then + echo " OK" else - echo " ERROR: /dev/kvm is NOT writable" + echo " FAILED" + echo " ERROR: Cannot SSH to $ip" exit 1 fi +done + +echo "" + +# ========================================================================== +# Phase 2: Cluster Health +# ========================================================================== +echo "Phase 2: Cluster Health" +echo "========================================" + +# NATS cluster: verify 2 unique peers from node1 +echo "Checking NATS cluster..." +NATS_INFO=$(curl -s "http://${LOCAL_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null) || { + echo " ERROR: Cannot reach NATS monitoring endpoint" + exit 1 +} +UNIQUE_PEERS=$(echo "$NATS_INFO" | jq -r '[.routes[].remote_name] | unique | length') +echo " NATS unique peers: $UNIQUE_PEERS (expected: 2)" +if [ "$UNIQUE_PEERS" -ge 2 ]; then + pass_test "NATS quorum" else - echo " ERROR: /dev/kvm does NOT exist" + echo " ERROR: NATS cluster not fully formed" exit 1 fi -# Check for ip command (iproute2) -if ! command -v ip &> /dev/null; then - echo " ERROR: 'ip' command not found. Install iproute2." +# Predastore: check each node +echo "Checking Predastore on all nodes..." +for i in $(seq 0 $((NODE_COUNT - 1))); do + ip="${NODE_IPS[$i]}" + if curl -k -s "https://${ip}:${PREDASTORE_PORT}" > /dev/null 2>&1; then + echo " Node$((i+1)) ($ip): Predastore reachable" + else + echo " ERROR: Predastore not reachable on node$((i+1)) ($ip)" + exit 1 + fi +done +pass_test "Predastore cluster" + +# Gateway: check each node +echo "Checking gateway on all nodes..." +for i in $(seq 0 $((NODE_COUNT - 1))); do + ip="${NODE_IPS[$i]}" + if curl -k -s "https://${ip}:${AWSGW_PORT}" > /dev/null 2>&1; then + echo " Node$((i+1)) ($ip): Gateway reachable" + else + echo " ERROR: Gateway not reachable on node$((i+1)) ($ip)" + exit 1 + fi +done +pass_test "All gateways" + +# Daemon readiness: describe-instance-types must return results +echo "Checking daemon readiness..." +ATTEMPT=0 +while [ $ATTEMPT -lt 15 ]; do + TYPES=$($AWS_EC2 describe-instance-types \ + --query 'InstanceTypes[*].InstanceType' --output text 2>/dev/null || true) + if [ -n "$TYPES" ] && [ "$TYPES" != "None" ]; then + echo " Daemon ready (instance types: $TYPES)" + break + fi + echo " Waiting for daemon... ($((ATTEMPT + 1))/15)" + sleep 2 + ATTEMPT=$((ATTEMPT + 1)) +done +if [ -z "$TYPES" ] || [ "$TYPES" == "None" ]; then + echo " ERROR: Daemon not ready" exit 1 fi +pass_test "Daemon readiness" -# Bootstrap OVN/OVS (required — start-dev.sh will block without it) -echo "" -echo "Bootstrapping OVN/OVS networking..." -bootstrap_ovn_docker - -# Setup simulated network -echo "" -echo "Setting up simulated network..." -add_simulated_ips - -# Create ramdisk mount point -mkdir -p /mnt/ramdisk - -echo "" - -# Phase 2: Cluster Initialization -echo "Phase 2: Cluster Initialization" -echo "========================================" - -# Background init — starts formation server, generates certs first -echo "" -init_leader_node - -# Trust CA cert (exists before formation completes — cert generation is the first step) -echo "" -echo "Adding Hive CA certificate to system trust store..." -sudo cp ~/node1/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt -sudo update-ca-certificates - -# Background BOTH joins — they poll /formation/status until all 3 nodes have joined. -# Must be concurrent: each join blocks until formation is complete. -echo "" -echo "Joining follower nodes concurrently..." -join_follower_node 2 & -JOIN2_PID=$! -join_follower_node 3 & -JOIN3_PID=$! - -# Wait for formation to complete (all processes generate their configs) -echo "Waiting for cluster formation to complete..." -wait $JOIN2_PID || { echo "ERROR: Node 2 join failed"; exit 1; } -wait $JOIN3_PID || { echo "ERROR: Node 3 join failed"; exit 1; } -wait $LEADER_INIT_PID || { echo "ERROR: Leader init failed"; exit 1; } -echo "Cluster formation complete — all configs generated" - -# Now start services (configs exist for all nodes) -echo "" -echo "Starting node services..." -start_node_services 1 "$HOME/node1" -start_node_services 2 "$HOME/node2" -start_node_services 3 "$HOME/node3" -CLUSTER_SERVICES_STARTED="true" - -# Wait for all services to stabilize -echo "" -echo "Waiting for cluster to stabilize..." -sleep 5 - -# Phase 3: Cluster Health Verification -echo "" -echo "Phase 3: Cluster Health Verification" -echo "========================================" - -# Verify NATS cluster -echo "" -verify_nats_cluster 3 || { - echo "WARNING: NATS cluster verification failed, continuing anyway..." -} - -# Verify Predastore cluster -echo "" -verify_predastore_cluster 3 || { - echo "ERROR: Predastore cluster verification failed" - dump_all_node_logs - exit 1 -} - -# Wait for gateway on node1 (primary gateway) -echo "" -wait_for_gateway "${NODE1_IP}" 15 - -# Wait for daemon NATS subscriptions to be active -wait_for_daemon_ready "https://${NODE1_IP}:${AWSGW_PORT}" - -# Define AWS CLI args pointing to node1's gateway -AWS_EC2="aws --endpoint-url https://${NODE1_IP}:${AWSGW_PORT} ec2" -AWS_IAM="aws --endpoint-url https://${NODE1_IP}:${AWSGW_PORT} iam" - -# Phase 3b: Cluster Stats CLI (Multi-Node) -echo "" -echo "Phase 3b: Cluster Stats CLI (Multi-Node)" -echo "========================================" - -# Test hive get nodes — should show all 3 nodes as Ready -echo "Testing hive get nodes..." -GET_NODES_OUTPUT=$(./bin/hive get nodes --config "$HOME/node1/config/hive.toml" --timeout 5s 2>/dev/null) +# Hive CLI: get nodes +echo "Checking hive get nodes..." +GET_NODES_OUTPUT=$($HIVE_BIN get nodes --config "$HIVE_CONFIG" --timeout 5s 2>/dev/null) echo "$GET_NODES_OUTPUT" READY_COUNT=$(echo "$GET_NODES_OUTPUT" | grep -c "Ready" || true) -if [ "$READY_COUNT" -lt 3 ]; then - echo "WARNING: hive get nodes shows $READY_COUNT Ready nodes (expected 3)" +if [ "$READY_COUNT" -ge 3 ]; then + pass_test "hive get nodes ($READY_COUNT Ready)" +else + echo " WARNING: hive get nodes shows $READY_COUNT Ready nodes (expected 3)" + fail_test "hive get nodes" fi -echo "hive get nodes passed ($READY_COUNT Ready nodes)" -# Test hive top nodes — should show resource stats for all nodes -echo "Testing hive top nodes..." -TOP_NODES_OUTPUT=$(./bin/hive top nodes --config "$HOME/node1/config/hive.toml" --timeout 5s 2>/dev/null) -echo "$TOP_NODES_OUTPUT" -if ! echo "$TOP_NODES_OUTPUT" | grep -q "INSTANCE TYPE"; then - echo "WARNING: hive top nodes did not show instance type capacity table" -fi -echo "hive top nodes passed" - -# Test hive get vms — should show no VMs yet -echo "Testing hive get vms (empty)..." -GET_VMS_OUTPUT=$(./bin/hive get vms --config "$HOME/node1/config/hive.toml" --timeout 5s 2>/dev/null) +# Hive CLI: get vms (should be empty) +echo "Checking hive get vms (empty)..." +GET_VMS_OUTPUT=$($HIVE_BIN get vms --config "$HIVE_CONFIG" --timeout 5s 2>/dev/null) echo "$GET_VMS_OUTPUT" -echo "hive get vms (empty) passed" +pass_test "hive get vms (empty)" -# Verify gateway responds echo "" -echo "Testing gateway connectivity..." -$AWS_EC2 describe-regions | jq -e '.Regions | length > 0' || { - echo "ERROR: Gateway not responding correctly" - exit 1 -} -echo " Gateway is responding" -# Phase 4: Image and Key Setup -echo "" -echo "Phase 4: Image and Key Setup" +# ========================================================================== +# Phase 3: Instance Lifecycle + Distribution +# ========================================================================== +echo "Phase 3: Instance Lifecycle + Distribution" echo "========================================" -# Discover instance types -echo "Discovering available instance types..." -AVAILABLE_TYPES=$($AWS_EC2 describe-instance-types --query 'InstanceTypes[*].InstanceType' --output text) -echo " Available: $AVAILABLE_TYPES" - -# Pick nano instance type -INSTANCE_TYPE=$(echo $AVAILABLE_TYPES | tr ' ' '\n' | grep -m1 'nano') -if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" == "None" ]; then - echo "ERROR: No instance types found" +# Discover instance type +INSTANCE_TYPE=$(echo $TYPES | tr ' ' '\n' | grep -m1 'nano') +if [ -z "$INSTANCE_TYPE" ]; then + echo "ERROR: No nano instance type found" exit 1 fi -echo " Selected: $INSTANCE_TYPE" +echo "Using instance type: $INSTANCE_TYPE" -# Get architecture -ARCH=$($AWS_EC2 describe-instance-types --instance-types "$INSTANCE_TYPE" \ - --query 'InstanceTypes[0].ProcessorInfo.SupportedArchitectures[0]' --output text) -echo " Architecture: $ARCH" - -# Create test key -echo "" -echo "Creating test key pair..." -KEY_MATERIAL=$($AWS_EC2 create-key-pair --key-name multinode-test-key --query 'KeyMaterial' --output text) -echo "$KEY_MATERIAL" > multinode-test-key.pem -chmod 600 multinode-test-key.pem -echo " Key created: multinode-test-key" - -# Import Ubuntu image (use node1's config and hive-dir) -echo "" -echo "Importing Ubuntu image..." -IMPORT_LOG=$(./bin/hive admin images import \ - --file /root/images/ubuntu-24.04.img \ - --arch "$ARCH" \ - --distro ubuntu \ - --version 24.04 \ - --config "$HOME/node1/config/hive.toml" \ - --hive-dir "$HOME/node1/" \ - --force) -echo "Import output: $IMPORT_LOG" -AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+') - -if [ -z "$AMI_ID" ]; then - echo "ERROR: Failed to capture AMI ID" +# Discover AMI +AMI_ID=$($AWS_EC2 describe-images --query 'Images[0].ImageId' --output text) +if [ -z "$AMI_ID" ] || [ "$AMI_ID" == "None" ]; then + echo "ERROR: No AMI found (bootstrap should have imported one)" exit 1 fi -echo " AMI ID: $AMI_ID" +echo "Using AMI: $AMI_ID" -# Verify AMI -$AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.ImageId==\"$AMI_ID\")" > /dev/null -echo " AMI verified" - -# Phase 4b: Multi-Node Key Pair Operations -echo "" -echo "Phase 4b: Multi-Node Key Pair Operations" -echo "========================================" -echo "Testing key pair CRUD across the cluster..." - -# Import a key pair (goes to a random node via queue group) -echo " Generating local RSA key for import..." -ssh-keygen -t rsa -b 2048 -f multinode-test-key-2-local -N "" -$AWS_EC2 import-key-pair --key-name multinode-test-key-2 --public-key-material "fileb://multinode-test-key-2-local.pub" -echo " Imported key: multinode-test-key-2" - -# Describe key pairs — verify both keys are visible -echo " Verifying both keys via describe-key-pairs..." -KEY_LIST=$($AWS_EC2 describe-key-pairs --query 'KeyPairs[*].KeyName' --output text) -echo " Keys found: $KEY_LIST" -echo "$KEY_LIST" | grep -q "multinode-test-key" || { - echo " ERROR: multinode-test-key not found in describe-key-pairs" - exit 1 -} -echo "$KEY_LIST" | grep -q "multinode-test-key-2" || { - echo " ERROR: multinode-test-key-2 not found in describe-key-pairs" - exit 1 -} -echo " Both keys visible" - -# Delete the imported key -echo " Deleting multinode-test-key-2..." -$AWS_EC2 delete-key-pair --key-name multinode-test-key-2 - -# Verify deletion -REMAINING_KEYS=$($AWS_EC2 describe-key-pairs --query 'KeyPairs[*].KeyName' --output text) -if echo "$REMAINING_KEYS" | grep -q "multinode-test-key-2"; then - echo " ERROR: multinode-test-key-2 was not deleted" - exit 1 -fi -echo " Key deletion verified" -echo " Multi-node key pair operations passed" - -# Phase 5: Multi-Node Instance Tests -echo "" -echo "Phase 5: Multi-Node Instance Tests" -echo "========================================" - -# Test 1: Instance Distribution -echo "" -echo "Test 1: Instance Distribution" -echo "----------------------------------------" -echo "Launching 3 instances to test distribution across nodes..." - -INSTANCE_IDS=() +# Launch 3 instances with stagger to encourage distribution +echo "Launching 3 instances..." for i in 1 2 3; do echo " Launching instance $i..." RUN_OUTPUT=$($AWS_EC2 run-instances \ --image-id "$AMI_ID" \ --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key) + --key-name hive-key) INSTANCE_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId') if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "null" ]; then @@ -342,8 +439,8 @@ for i in 1 2 3; do echo " Launched: $INSTANCE_ID" INSTANCE_IDS+=("$INSTANCE_ID") - # Small delay between launches to encourage distribution - sleep 2 + # Stagger launches to encourage distribution across nodes + [ $i -lt 3 ] && sleep 2 done # Wait for all instances to be running @@ -356,2067 +453,474 @@ for instance_id in "${INSTANCE_IDS[@]}"; do } done -# Check distribution +# Check distribution via hive get vms or QEMU process check echo "" -check_instance_distribution +echo "Checking instance distribution across nodes..." +declare -A NODE_INSTANCE_COUNT +HOSTING_NODES=() +for instance_id in "${INSTANCE_IDS[@]}"; do + HOST_IP=$(find_instance_node "$instance_id" || echo "unknown") + echo " $instance_id -> $HOST_IP" + HOSTING_NODES+=("$HOST_IP") + NODE_INSTANCE_COUNT[$HOST_IP]=$(( ${NODE_INSTANCE_COUNT[$HOST_IP]:-0} + 1 )) +done -# Verify hive get vms shows all running instances +# Count unique hosting nodes +UNIQUE_HOSTS=$(printf '%s\n' "${HOSTING_NODES[@]}" | sort -u | wc -l) +echo " Instances on $UNIQUE_HOSTS different nodes" +if [ "$UNIQUE_HOSTS" -ge 2 ]; then + pass_test "Instance distribution (>= 2 nodes)" +else + echo " WARNING: All instances on same node (distribution not guaranteed, non-fatal)" + pass_test "Instance distribution (non-deterministic)" +fi + +# Verify hive get vms shows all instances echo "" -echo "Verifying hive get vms (with running VMs)..." -GET_VMS_OUTPUT=$(./bin/hive get vms --config "$HOME/node1/config/hive.toml" --timeout 5s 2>/dev/null) +echo "Verifying hive get vms..." +GET_VMS_OUTPUT=$($HIVE_BIN get vms --config "$HIVE_CONFIG" --timeout 5s 2>/dev/null) echo "$GET_VMS_OUTPUT" for instance_id in "${INSTANCE_IDS[@]}"; do if ! echo "$GET_VMS_OUTPUT" | grep -q "$instance_id"; then - echo "WARNING: hive get vms did not show instance $instance_id" + echo " WARNING: hive get vms did not show $instance_id" fi done -echo "hive get vms shows launched instances" +pass_test "hive get vms (with instances)" -# Test 1a-ii: SSH Connectivity & Volume Verification echo "" -echo "Test 1a-ii: SSH Connectivity & Volume Verification" -echo "----------------------------------------" -echo "Testing SSH into all 3 instances..." -# Arrays to store SSH details for post-termination verification -SSH_PORTS=() -SSH_HOSTS=() +# ========================================================================== +# Phase 4: SSH into Guest VMs +# ========================================================================== +echo "Phase 4: SSH into Guest VMs" +echo "========================================" for idx in "${!INSTANCE_IDS[@]}"; do instance_id="${INSTANCE_IDS[$idx]}" + host_ip="${HOSTING_NODES[$idx]}" echo "" - echo " Instance $((idx + 1)): $instance_id" + echo " Instance $((idx + 1)): $instance_id (on $host_ip)" - # Get SSH connection details from QEMU process - echo " Getting SSH port..." - SSH_PORT=$(get_ssh_port "$instance_id") + # Get SSH port from QEMU process on the hosting node + SSH_PORT=$(get_remote_ssh_port "$host_ip" "$instance_id" 10) if [ -z "$SSH_PORT" ]; then - echo " ERROR: Failed to get SSH port for instance $instance_id" - exit 1 + echo " ERROR: Failed to get SSH port for $instance_id on $host_ip" + fail_test "Guest SSH ($instance_id)" + continue fi - SSH_HOST=$(get_ssh_host "$instance_id") - echo " SSH endpoint: $SSH_HOST:$SSH_PORT" + echo " SSH endpoint: $host_ip:$SSH_PORT" - SSH_PORTS+=("$SSH_PORT") - SSH_HOSTS+=("$SSH_HOST") + # Wait for SSH to be ready (VM boot + cloud-init) + echo " Waiting for SSH to be ready..." + ATTEMPT=0 + SSH_READY=false + while [ $ATTEMPT -lt 60 ]; do + if ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=2 \ + -o BatchMode=yes \ + -o LogLevel=ERROR \ + -p "$SSH_PORT" \ + -i "$HOME/.ssh/hive-key" \ + ec2-user@"$host_ip" 'echo ready' > /dev/null 2>&1; then + SSH_READY=true + break + fi + ATTEMPT=$((ATTEMPT + 1)) + [ $((ATTEMPT % 10)) -eq 0 ] && echo " Waiting for SSH... ($ATTEMPT/60)" + sleep 1 + done - # Wait for SSH to become ready (VM boot + cloud-init) - wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 30 + if [ "$SSH_READY" = false ]; then + echo " ERROR: SSH not ready after 60 attempts" + fail_test "Guest SSH ($instance_id)" + continue + fi - # Test basic SSH connectivity - test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" - - # Check root volume size via lsblk - echo " Verifying root volume size from inside the VM..." - ROOT_VOL_ID_SSH=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ - --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) - ROOT_VOL_SIZE_API=$($AWS_EC2 describe-volumes --volume-ids "$ROOT_VOL_ID_SSH" \ - --query 'Volumes[0].Size' --output text) - # Find the disk backing the root filesystem (avoids picking up floppy/cdrom devices) - # 1. findmnt gets the source device for / (e.g. /dev/vda1) - # 2. lsblk PKNAME resolves to parent disk name (e.g. vda) - # 3. lsblk -b -d gets that disk's byte size - ROOT_DISK_BYTES=$(ssh -o StrictHostKeyChecking=no \ + # Test SSH connectivity + ID_OUTPUT=$(ssh -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 \ + -o BatchMode=yes \ -o LogLevel=ERROR \ - -o ConnectTimeout=5 \ - -o BatchMode=yes \ -p "$SSH_PORT" \ - -i "multinode-test-key.pem" \ - ec2-user@"$SSH_HOST" 'SRC=$(findmnt -n -o SOURCE /); PKN=$(lsblk -n -o PKNAME "$SRC" 2>/dev/null | head -1); DEV=${PKN:-$(basename "$SRC")}; lsblk -b -d -n -o SIZE "/dev/$DEV"' | tr -d '[:space:]') - if [ -z "$ROOT_DISK_BYTES" ] || [ "$ROOT_DISK_BYTES" = "0" ]; then - echo " ERROR: Failed to get root disk size from VM (got: '$ROOT_DISK_BYTES')" - echo " lsblk debug output:" - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ - -o ConnectTimeout=5 -o BatchMode=yes -p "$SSH_PORT" -i "multinode-test-key.pem" \ - ec2-user@"$SSH_HOST" 'lsblk -b -d; echo "---"; findmnt -n -o SOURCE /; cat /proc/partitions' || true - exit 1 - fi - ROOT_DISK_GIB=$((ROOT_DISK_BYTES / 1073741824)) - echo " Root disk size from VM: ${ROOT_DISK_GIB}GiB (API reports: ${ROOT_VOL_SIZE_API}GiB)" - if [ "$ROOT_DISK_GIB" -ne "$ROOT_VOL_SIZE_API" ]; then - echo " ERROR: Root volume size mismatch: VM reports ${ROOT_DISK_GIB}GiB, API reports ${ROOT_VOL_SIZE_API}GiB" - exit 1 - fi - echo " Root volume size verified" + -i "$HOME/.ssh/hive-key" \ + ec2-user@"$host_ip" 'id' 2>&1) || { + echo " ERROR: SSH 'id' command failed" + fail_test "Guest SSH ($instance_id)" + continue + } - # Verify hostname contains instance ID - echo " Verifying hostname inside the VM..." - VM_HOSTNAME=$(ssh -o StrictHostKeyChecking=no \ + echo " SSH 'id' output: $ID_OUTPUT" + if echo "$ID_OUTPUT" | grep -q "ec2-user"; then + echo " ec2-user confirmed" + else + echo " ERROR: Expected 'ec2-user' in id output" + fail_test "Guest SSH ($instance_id)" + continue + fi + + # Verify block device + LSBLK_OUTPUT=$(ssh -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -o ConnectTimeout=5 \ -o BatchMode=yes \ + -o LogLevel=ERROR \ -p "$SSH_PORT" \ - -i "multinode-test-key.pem" \ - ec2-user@"$SSH_HOST" 'hostname' 2>/dev/null) - echo " VM hostname: $VM_HOSTNAME" - # Hostname uses truncated ID: hive-vm- - SHORT_ID=$(echo "$instance_id" | sed 's/^i-//' | cut -c1-8) - if echo "$VM_HOSTNAME" | grep -q "$SHORT_ID"; then - echo " Hostname contains instance ID prefix ($SHORT_ID)" - else - echo " WARNING: Hostname '$VM_HOSTNAME' does not contain instance ID prefix '$SHORT_ID' (non-fatal)" - fi + -i "$HOME/.ssh/hive-key" \ + ec2-user@"$host_ip" 'lsblk' 2>&1) || true + echo " lsblk: $(echo "$LSBLK_OUTPUT" | head -5)" + + pass_test "Guest SSH ($instance_id)" done echo "" -echo " SSH connectivity and volume verification passed for all instances" -# Test 1b: Volume Lifecycle (Attach/Detach) -echo "" -echo "Test 1b: Volume Lifecycle (Attach/Detach)" -echo "----------------------------------------" -echo "Testing volume create -> resize -> attach -> detach -> delete..." +# ========================================================================== +# Phase 5: Volume Lifecycle +# ========================================================================== +echo "Phase 5: Volume Lifecycle" +echo "========================================" -# Create a test volume -echo " Creating 10GB volume in ap-southeast-2a..." -CREATE_OUTPUT=$($AWS_EC2 create-volume --size 10 --availability-zone ap-southeast-2a) +# Discover AZ +HIVE_AZ=$($AWS_EC2 describe-availability-zones --query 'AvailabilityZones[0].ZoneName' --output text) +echo "AZ: $HIVE_AZ" + +# Create volume +echo "Creating 10GB test volume..." +CREATE_OUTPUT=$($AWS_EC2 create-volume --size 10 --availability-zone "$HIVE_AZ") TEST_VOLUME_ID=$(echo "$CREATE_OUTPUT" | jq -r '.VolumeId') - if [ -z "$TEST_VOLUME_ID" ] || [ "$TEST_VOLUME_ID" == "null" ]; then - echo " ERROR: Failed to create test volume" - echo " Output: $CREATE_OUTPUT" - exit 1 -fi -echo " Created volume: $TEST_VOLUME_ID" - -# Resize to 20GB -NEW_SIZE=20 -echo " Modifying volume to ${NEW_SIZE}GB..." -$AWS_EC2 modify-volume --volume-id "$TEST_VOLUME_ID" --size "$NEW_SIZE" - -# Verify resize -echo " Verifying resize..." -COUNT=0 -while [ $COUNT -lt 15 ]; do - VOLUME_SIZE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].Size' --output text) - - if [ "$VOLUME_SIZE" -eq "$NEW_SIZE" ]; then - echo " Volume resized successfully to ${NEW_SIZE}GB" - break - fi - - sleep 2 - COUNT=$((COUNT + 1)) -done - -if [ "$VOLUME_SIZE" -ne "$NEW_SIZE" ]; then - echo " ERROR: Volume failed to resize to ${NEW_SIZE}GB (current: ${VOLUME_SIZE}GB)" - exit 1 -fi - -# Attach volume to the first running instance -echo " Attaching volume $TEST_VOLUME_ID to instance ${INSTANCE_IDS[0]}..." -$AWS_EC2 attach-volume --volume-id "$TEST_VOLUME_ID" --instance-id "${INSTANCE_IDS[0]}" --device /dev/sdf - -# Verify attachment -echo " Verifying volume attachment..." -COUNT=0 -while [ $COUNT -lt 15 ]; do - ATTACH_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].Attachments[0].State' --output text) - ATTACH_INSTANCE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].Attachments[0].InstanceId' --output text) - VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].State' --output text) - - if [ "$VOL_STATE" == "in-use" ] && [ "$ATTACH_STATE" == "attached" ] && [ "$ATTACH_INSTANCE" == "${INSTANCE_IDS[0]}" ]; then - echo " Volume attached successfully (State=$VOL_STATE, AttachState=$ATTACH_STATE, Instance=$ATTACH_INSTANCE)" - break - fi - - sleep 2 - COUNT=$((COUNT + 1)) -done - -if [ "$ATTACH_STATE" != "attached" ] || [ "$ATTACH_INSTANCE" != "${INSTANCE_IDS[0]}" ]; then - echo " ERROR: Volume attachment verification failed (AttachState=$ATTACH_STATE, Instance=$ATTACH_INSTANCE)" - exit 1 -fi - -# Detach volume (without --instance-id to test gateway resolution path) -echo " Detaching volume $TEST_VOLUME_ID..." -$AWS_EC2 detach-volume --volume-id "$TEST_VOLUME_ID" - -# Verify detachment -echo " Verifying volume detachment..." -COUNT=0 -while [ $COUNT -lt 15 ]; do - VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].State' --output text) - - if [ "$VOL_STATE" == "available" ]; then - echo " Volume detached successfully (State=$VOL_STATE)" - break - fi - - sleep 2 - COUNT=$((COUNT + 1)) -done - -if [ "$VOL_STATE" != "available" ]; then - echo " ERROR: Volume detachment verification failed (State=$VOL_STATE)" - exit 1 -fi - -# Delete the test volume -echo " Deleting test volume $TEST_VOLUME_ID..." -$AWS_EC2 delete-volume --volume-id "$TEST_VOLUME_ID" - -# Verify deletion -echo " Verifying volume deletion..." -COUNT=0 -while [ $COUNT -lt 15 ]; do - set +e - VOLUME_CHECK=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ - --query 'Volumes[0].VolumeId' --output text 2>&1) - DESCRIBE_EXIT=$? - set -e - - if [ $DESCRIBE_EXIT -ne 0 ] || [ "$VOLUME_CHECK" == "None" ] || [ -z "$VOLUME_CHECK" ]; then - echo " Volume deleted successfully" - break - fi - - sleep 2 - COUNT=$((COUNT + 1)) -done - -if [ $COUNT -ge 15 ]; then - echo " ERROR: Volume deletion verification timed out" - exit 1 -fi - -echo " Volume lifecycle test passed (create -> resize -> attach -> detach -> delete)" - -# Test 1c: Snapshot Lifecycle -echo "" -echo "Test 1c: Snapshot Lifecycle" -echo "----------------------------------------" -echo "Testing snapshot create -> describe -> copy -> delete..." - -# Use the root volume of the first instance — it's already attached and mounted -# in viperblockd, which is required for create-snapshot. -SNAP_VOL_ID=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[0]}" \ - --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) -echo " Using root volume $SNAP_VOL_ID (attached to ${INSTANCE_IDS[0]})" -SNAP_VOL_SIZE=$($AWS_EC2 describe-volumes --volume-ids "$SNAP_VOL_ID" \ - --query 'Volumes[0].Size' --output text) - -# Create a snapshot -echo " Creating snapshot from volume $SNAP_VOL_ID..." -SNAP_OUTPUT=$($AWS_EC2 create-snapshot --volume-id "$SNAP_VOL_ID" --description "multinode-e2e-snapshot") -SNAPSHOT_ID=$(echo "$SNAP_OUTPUT" | jq -r '.SnapshotId') - -if [ -z "$SNAPSHOT_ID" ] || [ "$SNAPSHOT_ID" == "null" ]; then - echo " ERROR: Failed to create snapshot" - echo " Output: $SNAP_OUTPUT" - exit 1 -fi -echo " Created snapshot: $SNAPSHOT_ID" - -# Verify create response fields -SNAP_STATE=$(echo "$SNAP_OUTPUT" | jq -r '.State') -SNAP_VOL_REF=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeId') -SNAP_SIZE=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeSize') - -if [ "$SNAP_VOL_REF" != "$SNAP_VOL_ID" ]; then - echo " ERROR: Snapshot VolumeId mismatch: expected $SNAP_VOL_ID, got $SNAP_VOL_REF" - exit 1 -fi -if [ "$SNAP_SIZE" -ne "$SNAP_VOL_SIZE" ]; then - echo " ERROR: Snapshot VolumeSize mismatch: expected $SNAP_VOL_SIZE, got $SNAP_SIZE" - exit 1 -fi -echo " Create response verified (State=$SNAP_STATE, VolumeId=$SNAP_VOL_REF, Size=$SNAP_SIZE)" - -# Poll until completed -echo " Waiting for snapshot to complete..." -COUNT=0 -while [ $COUNT -lt 15 ]; do - SNAP_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ - --query 'Snapshots[0].State' --output text) - - if [ "$SNAP_STATE" == "completed" ]; then - echo " Snapshot completed" - break - fi - - sleep 2 - COUNT=$((COUNT + 1)) -done - -if [ "$SNAP_STATE" != "completed" ]; then - echo " ERROR: Snapshot failed to reach completed state (State=$SNAP_STATE)" - exit 1 -fi - -# Describe by ID and verify -echo " Verifying snapshot via describe-snapshots..." -DESCRIBE_SNAP=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID") -DESC_VOL_ID=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeId') -DESC_SIZE=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeSize') -DESC_DESC=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].Description') - -if [ "$DESC_VOL_ID" != "$SNAP_VOL_ID" ]; then - echo " ERROR: Describe VolumeId mismatch: expected $SNAP_VOL_ID, got $DESC_VOL_ID" - exit 1 -fi -if [ "$DESC_SIZE" -ne "$SNAP_VOL_SIZE" ]; then - echo " ERROR: Describe VolumeSize mismatch: expected $SNAP_VOL_SIZE, got $DESC_SIZE" - exit 1 -fi -if [ "$DESC_DESC" != "multinode-e2e-snapshot" ]; then - echo " ERROR: Describe Description mismatch: expected 'multinode-e2e-snapshot', got '$DESC_DESC'" - exit 1 -fi -echo " Describe verified (VolumeId=$DESC_VOL_ID, Size=$DESC_SIZE, Description=$DESC_DESC)" - -# Copy the snapshot -echo " Copying snapshot $SNAPSHOT_ID..." -COPY_OUTPUT=$($AWS_EC2 copy-snapshot --source-snapshot-id "$SNAPSHOT_ID" --source-region ap-southeast-2 --description "multinode-e2e-copy") -COPY_SNAPSHOT_ID=$(echo "$COPY_OUTPUT" | jq -r '.SnapshotId') - -if [ -z "$COPY_SNAPSHOT_ID" ] || [ "$COPY_SNAPSHOT_ID" == "null" ]; then - echo " ERROR: Failed to copy snapshot" - echo " Output: $COPY_OUTPUT" - exit 1 -fi -echo " Copied snapshot: $COPY_SNAPSHOT_ID" - -if [ "$COPY_SNAPSHOT_ID" == "$SNAPSHOT_ID" ]; then - echo " ERROR: Copy snapshot ID should differ from original" - exit 1 -fi - -# Verify both exist -TOTAL_SNAPS=$($AWS_EC2 describe-snapshots \ - --snapshot-ids "$SNAPSHOT_ID" "$COPY_SNAPSHOT_ID" \ - --query 'length(Snapshots)' --output text) - -if [ "$TOTAL_SNAPS" -ne 2 ]; then - echo " ERROR: Expected 2 snapshots, got $TOTAL_SNAPS" - exit 1 -fi -echo " Both snapshots visible via describe-snapshots" - -# Verify copy description -COPY_DESC=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \ - --query 'Snapshots[0].Description' --output text) -if [ "$COPY_DESC" != "multinode-e2e-copy" ]; then - echo " ERROR: Copy description mismatch: expected 'multinode-e2e-copy', got '$COPY_DESC'" - exit 1 -fi - -# Delete original -echo " Deleting original snapshot $SNAPSHOT_ID..." -$AWS_EC2 delete-snapshot --snapshot-id "$SNAPSHOT_ID" - -# Verify original gone, copy remains -echo " Verifying snapshot deletion..." -COUNT=0 -while [ $COUNT -lt 15 ]; do - set +e - SNAP_CHECK=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ - --query 'Snapshots[0].SnapshotId' --output text 2>&1) - SNAP_EXIT=$? - set -e - - if [ $SNAP_EXIT -ne 0 ] || [ "$SNAP_CHECK" == "None" ] || [ -z "$SNAP_CHECK" ]; then - echo " Original snapshot deleted successfully" - break - fi - - sleep 2 - COUNT=$((COUNT + 1)) -done - -if [ $COUNT -ge 15 ]; then - echo " ERROR: Snapshot deletion verification timed out" - exit 1 -fi - -# Verify copy still exists -COPY_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \ - --query 'Snapshots[0].State' --output text) -if [ "$COPY_STATE" != "completed" ]; then - echo " ERROR: Copy snapshot should still exist (State=$COPY_STATE)" - exit 1 -fi -echo " Copy snapshot intact after original deletion" - -# Delete copy -echo " Deleting copy snapshot $COPY_SNAPSHOT_ID..." -$AWS_EC2 delete-snapshot --snapshot-id "$COPY_SNAPSHOT_ID" - -echo " Snapshot lifecycle test passed (create -> describe -> copy -> delete)" - -# Test 1c-ii: Verify Snapshot-Backed Instance Launch -echo "" -echo "Test 1c-ii: Verify Snapshot-Backed Instance Launch" -echo "----------------------------------------" -echo "All run-instances calls go through cloneAMIToVolume() -> OpenFromSnapshot()," -echo "so the Test 1 instances are already snapshot-backed. Verify their volume configs." - -AWS_S3="aws --endpoint-url https://${NODE1_IP}:${PREDASTORE_PORT} s3" - -# Verify the AMI snapshot exists in Predastore -echo " Checking AMI snapshot in Predastore..." -SNAP_PREFIX="snap-$AMI_ID" -SNAP_FILES=$($AWS_S3 ls "s3://predastore/$SNAP_PREFIX/" 2>&1 || echo "") -if echo "$SNAP_FILES" | grep -q "config.json"; then - echo " AMI snapshot config found at $SNAP_PREFIX/" + echo " ERROR: Failed to create volume" + fail_test "Volume create" else - echo " ERROR: AMI snapshot config not found at $SNAP_PREFIX/" - exit 1 -fi + echo " Created: $TEST_VOLUME_ID" + pass_test "Volume create" -# Verify the first instance's root volume has SnapshotID and SourceVolumeName -SNAP_ROOT_VOL=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[0]}" \ - --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) -echo " Verifying root volume $SNAP_ROOT_VOL is snapshot-backed via Predastore config..." -VOL_CONFIG=$($AWS_S3 cp "s3://predastore/$SNAP_ROOT_VOL/config.json" - 2>/dev/null || echo "{}") -VOL_SNAPSHOT_ID=$(echo "$VOL_CONFIG" | jq -r '.SnapshotID // empty') -VOL_SOURCE_NAME=$(echo "$VOL_CONFIG" | jq -r '.SourceVolumeName // empty') + # Attach to first instance + echo " Attaching to ${INSTANCE_IDS[0]}..." + $AWS_EC2 attach-volume --volume-id "$TEST_VOLUME_ID" \ + --instance-id "${INSTANCE_IDS[0]}" --device /dev/sdf > /dev/null -if [ -z "$VOL_SNAPSHOT_ID" ]; then - echo " ERROR: Volume config missing SnapshotID — launch was NOT snapshot-backed" - exit 1 -fi -if [ -z "$VOL_SOURCE_NAME" ]; then - echo " ERROR: Volume config missing SourceVolumeName — launch was NOT snapshot-backed" - exit 1 -fi -echo " Volume is snapshot-backed (SnapshotID=$VOL_SNAPSHOT_ID, SourceVolumeName=$VOL_SOURCE_NAME)" - -echo " Snapshot-backed instance launch verified" - -# Test 1d: Tag Management -echo "" -echo "Test 1d: Tag Management" -echo "----------------------------------------" -echo "Testing create-tags -> describe-tags -> delete-tags..." - -# Use the first instance for tag tests -TAG_INSTANCE="${INSTANCE_IDS[0]}" - -# Create tags on instance -echo " Creating tags on instance $TAG_INSTANCE..." -$AWS_EC2 create-tags --resources "$TAG_INSTANCE" --tags Key=Name,Value=multinode-test Key=Environment,Value=testing Key=DeleteMe,Value=please - -# Verify tags with describe-tags (resource-id filter) -echo " Verifying tags on instance..." -TAG_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$TAG_COUNT" -ne 3 ]; then - echo " ERROR: Expected 3 tags on instance, got $TAG_COUNT" - exit 1 -fi -echo " Instance has $TAG_COUNT tags" - -# Filter by key -echo " Testing key filter..." -ENV_TAGS=$($AWS_EC2 describe-tags --filters "Name=key,Values=Environment" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$ENV_TAGS" -lt 1 ]; then - echo " ERROR: Expected at least 1 'Environment' tag, got $ENV_TAGS" - exit 1 -fi -echo " Key filter returned $ENV_TAGS tags" - -# Filter by resource-type -echo " Testing resource-type filter..." -INSTANCE_TAGS=$($AWS_EC2 describe-tags --filters "Name=resource-type,Values=instance" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$INSTANCE_TAGS" -lt 3 ]; then - echo " ERROR: Expected at least 3 instance tags, got $INSTANCE_TAGS" - exit 1 -fi -echo " Resource-type filter returned $INSTANCE_TAGS instance tags" - -# Overwrite a tag value -echo " Overwriting Name tag..." -$AWS_EC2 create-tags --resources "$TAG_INSTANCE" --tags Key=Name,Value=multinode-updated -UPDATED_NAME=$($AWS_EC2 describe-tags \ - --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Name" \ - --query 'Tags[0].Value' --output text) -if [ "$UPDATED_NAME" != "multinode-updated" ]; then - echo " ERROR: Tag overwrite failed: expected 'multinode-updated', got '$UPDATED_NAME'" - exit 1 -fi -echo " Tag overwrite verified" - -# Delete tag by key (unconditional) -echo " Deleting DeleteMe tag unconditionally..." -$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=DeleteMe -REMAINING=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$REMAINING" -ne 2 ]; then - echo " ERROR: Expected 2 tags after unconditional delete, got $REMAINING" - exit 1 -fi -echo " Unconditional delete verified ($REMAINING tags remaining)" - -# Delete tag with wrong value (should NOT delete) -echo " Attempting delete with wrong value (should be no-op)..." -$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=Environment,Value=production -ENV_STILL=$($AWS_EC2 describe-tags \ - --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Environment" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$ENV_STILL" -ne 1 ]; then - echo " ERROR: Value-conditional delete incorrectly removed tag" - exit 1 -fi -echo " Value-conditional mismatch preserved tag" - -# Delete tag with correct value -echo " Deleting Environment tag with correct value..." -$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=Environment,Value=testing -ENV_GONE=$($AWS_EC2 describe-tags \ - --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Environment" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$ENV_GONE" -ne 0 ]; then - echo " ERROR: Value-conditional delete failed to remove matching tag" - exit 1 -fi -echo " Value-conditional match deleted tag" - -# Verify only Name tag remains -FINAL_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$FINAL_COUNT" -ne 1 ]; then - echo " ERROR: Expected 1 tag remaining, got $FINAL_COUNT" - exit 1 -fi -echo " Instance tag tests passed" - -# Test 1d-ii: Tags on Volumes (multi-node) -echo "" -echo "Test 1d-ii: Tags on Volumes (Multi-Node)" -echo "----------------------------------------" - -# Get the root volume of the first instance -TAG_VOL_ID=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[0]}" \ - --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) -echo " Tagging volume $TAG_VOL_ID..." - -# Create tags on volume -$AWS_EC2 create-tags --resources "$TAG_VOL_ID" --tags Key=Name,Value=multinode-root-vol Key=Environment,Value=testing - -# Verify tags on volume -VOL_TAG_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_VOL_ID" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$VOL_TAG_COUNT" -ne 2 ]; then - echo " ERROR: Expected 2 tags on volume, got $VOL_TAG_COUNT" - exit 1 -fi -echo " Volume has $VOL_TAG_COUNT tags" - -# Filter by resource-type=volume -VOL_TYPE_TAGS=$($AWS_EC2 describe-tags --filters "Name=resource-type,Values=volume" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$VOL_TYPE_TAGS" -lt 2 ]; then - echo " ERROR: Expected at least 2 volume tags, got $VOL_TYPE_TAGS" - exit 1 -fi -echo " Volume resource-type filter returned $VOL_TYPE_TAGS tags" - -# Delete tags from volume -$AWS_EC2 delete-tags --resources "$TAG_VOL_ID" --tags Key=Name Key=Environment -VOL_TAG_AFTER=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_VOL_ID" \ - --query 'length(Tags || `[]`)' --output text) -if [ "$VOL_TAG_AFTER" -ne 0 ]; then - echo " ERROR: Expected 0 tags after delete, got $VOL_TAG_AFTER" - exit 1 -fi -echo " Volume tag cleanup verified" - -echo " Tag management tests passed (instances + volumes)" - -# Test 2: DescribeInstances Aggregation -echo "" -echo "Test 2: DescribeInstances Aggregation" -echo "----------------------------------------" -echo "Verifying all instances are returned via fan-out query..." - -DESCRIBE_OUTPUT=$($AWS_EC2 describe-instances --query 'Reservations[*].Instances[*].InstanceId' --output text) -DESCRIBED_COUNT=$(echo "$DESCRIBE_OUTPUT" | wc -w) - -echo " Launched: ${#INSTANCE_IDS[@]} instances" -echo " Described: $DESCRIBED_COUNT instances" - -if [ "$DESCRIBED_COUNT" -lt "${#INSTANCE_IDS[@]}" ]; then - echo "ERROR: DescribeInstances did not return all instances" - echo " Expected: ${#INSTANCE_IDS[@]}, Got: $DESCRIBED_COUNT" - exit 1 -fi -echo " Aggregation test passed" - -# Test 3: Cross-Node Operations -echo "" -echo "Test 3: Cross-Node Operations" -echo "----------------------------------------" -echo "Testing stop/start/terminate via gateway regardless of instance location..." - -# Pick first instance for cross-node operations -TEST_INSTANCE="${INSTANCE_IDS[0]}" -echo " Test instance: $TEST_INSTANCE" - -# Stop instance -echo " Stopping instance..." -$AWS_EC2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null -wait_for_instance_state "$TEST_INSTANCE" "stopped" 30 - -# Start instance -echo " Starting instance..." -$AWS_EC2 start-instances --instance-ids "$TEST_INSTANCE" > /dev/null -wait_for_instance_state "$TEST_INSTANCE" "running" 30 - -echo " Cross-node operations test passed" - -# Test 4: NATS Cluster Health (Post-Operations) -echo "" -echo "Test 4: NATS Cluster Health (Post-Operations)" -echo "----------------------------------------" -echo "Verifying NATS cluster is still healthy after operations..." - -verify_nats_cluster 3 || { - echo "WARNING: NATS cluster verification failed after operations" -} - -# Test 5: VM Crash Recovery (kill -9 → detect → auto-restart) -echo "" -echo "Test 5: VM Crash Recovery" -echo "----------------------------------------" -echo "Testing QEMU crash detection and auto-restart..." - -# Use the second instance for crash testing (first was used for stop/start in Test 3) -CRASH_INSTANCE="${INSTANCE_IDS[1]}" -echo " Crash test instance: $CRASH_INSTANCE" - -# Verify instance is running before crash -CRASH_STATE=$($AWS_EC2 describe-instances --instance-ids "$CRASH_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text) -if [ "$CRASH_STATE" != "running" ]; then - echo " ERROR: Instance not in running state before crash test (state: $CRASH_STATE)" - exit 1 -fi -echo " Instance is running" - -# Get the QEMU PID -QEMU_PID=$(get_qemu_pid "$CRASH_INSTANCE") -if [ -z "$QEMU_PID" ]; then - echo " ERROR: Could not find QEMU PID for $CRASH_INSTANCE" - exit 1 -fi -echo " QEMU PID: $QEMU_PID" - -# Kill QEMU with SIGKILL (simulates OOM kill) -echo " Killing QEMU process with SIGKILL (simulating OOM kill)..." -kill -9 "$QEMU_PID" - -# Brief pause for the daemon to detect the crash -sleep 3 - -# Verify the daemon detected the crash (state should be error or already recovering) -echo " Checking post-crash state..." -POST_CRASH_STATE=$($AWS_EC2 describe-instances --instance-ids "$CRASH_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text) -echo " Post-crash state: $POST_CRASH_STATE" - -if [ "$POST_CRASH_STATE" == "running" ]; then - # Might have recovered very quickly, check if it's a new PID - NEW_PID=$(get_qemu_pid "$CRASH_INSTANCE" || echo "") - if [ "$NEW_PID" == "$QEMU_PID" ]; then - echo " ERROR: Instance still has same PID after kill -9, crash not detected" - exit 1 - fi - echo " Instance already recovered with new PID: $NEW_PID (was: $QEMU_PID)" -else - # Wait for auto-restart (backoff starts at 5s) - echo " Instance in $POST_CRASH_STATE state, waiting for auto-restart..." - wait_for_instance_recovery "$CRASH_INSTANCE" 30 || { - echo " ERROR: Instance failed to recover from crash" - # Dump daemon logs for debugging - for i in 1 2 3; do - if [ -f "$HOME/node$i/logs/hive.log" ]; then - echo "" - echo " --- node$i daemon log (last 30 lines) ---" - tail -30 "$HOME/node$i/logs/hive.log" - fi - done - exit 1 - } -fi - -# Verify the instance is running with a new QEMU process -RECOVERED_PID=$(get_qemu_pid "$CRASH_INSTANCE") -if [ -z "$RECOVERED_PID" ]; then - echo " ERROR: No QEMU process found after recovery" - exit 1 -fi - -if [ "$RECOVERED_PID" == "$QEMU_PID" ]; then - echo " ERROR: QEMU PID unchanged after crash recovery (expected new process)" - exit 1 -fi -echo " New QEMU PID: $RECOVERED_PID (was: $QEMU_PID)" - -# Verify describe-instances shows running state -FINAL_STATE=$($AWS_EC2 describe-instances --instance-ids "$CRASH_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text) -if [ "$FINAL_STATE" != "running" ]; then - echo " ERROR: Instance not running after recovery (state: $FINAL_STATE)" - exit 1 -fi -echo " Instance is running after crash recovery" - -# Verify SSH works after recovery (VM rebooted with fresh OS from root volume) -echo " Verifying SSH after crash recovery..." -CRASH_SSH_PORT=$(get_ssh_port "$CRASH_INSTANCE" 30) -CRASH_SSH_HOST=$(get_ssh_host "$CRASH_INSTANCE") -if [ -n "$CRASH_SSH_PORT" ]; then - echo " SSH endpoint after recovery: $CRASH_SSH_HOST:$CRASH_SSH_PORT" - # Update SSH details for later termination verification - SSH_PORTS[1]="$CRASH_SSH_PORT" - SSH_HOSTS[1]="$CRASH_SSH_HOST" - wait_for_ssh "$CRASH_SSH_HOST" "$CRASH_SSH_PORT" "multinode-test-key.pem" 30 || { - echo " WARNING: SSH not ready after crash recovery (non-fatal, VM may still be booting)" - } -else - echo " WARNING: Could not determine SSH port after recovery (non-fatal)" -fi - -# Test 5b: Crash Loop Prevention (kill 4 times rapidly to exceed max restarts) -echo "" -echo "Test 5b: Crash Loop Prevention" -echo "----------------------------------------" -echo "Testing that crash loop is detected and restarts stop after max attempts..." - -# Use the third instance for crash loop testing -LOOP_INSTANCE="${INSTANCE_IDS[2]}" -echo " Crash loop test instance: $LOOP_INSTANCE" - -# Verify instance is running -LOOP_STATE=$($AWS_EC2 describe-instances --instance-ids "$LOOP_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text) -if [ "$LOOP_STATE" != "running" ]; then - echo " ERROR: Instance not running before crash loop test (state: $LOOP_STATE)" - exit 1 -fi - -# Kill QEMU repeatedly to exhaust restart attempts (max 3 in 10 min window) -for crash_num in 1 2 3 4; do - echo " Crash $crash_num/4: killing QEMU..." - LOOP_PID=$(get_qemu_pid "$LOOP_INSTANCE" || echo "") - if [ -z "$LOOP_PID" ]; then - echo " No QEMU process found (instance may be in error state)" - break - fi - - kill -9 "$LOOP_PID" - - if [ $crash_num -lt 4 ]; then - # Wait for restart (backoff increases: 5s, 10s, 20s) - # Give generous time for each restart cycle - local_max=$((15 + crash_num * 10)) - echo " Waiting up to ${local_max}s for restart or error state..." - attempt=0 - while [ $attempt -lt $local_max ]; do - state=$($AWS_EC2 describe-instances --instance-ids "$LOOP_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "unknown") - if [ "$state" == "running" ]; then - echo " Instance restarted (crash $crash_num)" - break - fi - if [ "$state" == "error" ] && [ $crash_num -ge 3 ]; then - echo " Instance in error state after crash $crash_num (restart limit may be reached)" - break - fi - sleep 2 - attempt=$((attempt + 2)) - done - fi -done - -# After 4 rapid crashes, the instance should be in error state (exceeded max 3 restarts) -sleep 5 -LOOP_FINAL_STATE=$($AWS_EC2 describe-instances --instance-ids "$LOOP_INSTANCE" \ - --query 'Reservations[0].Instances[0].State.Name' --output text) -echo " Final state after crash loop: $LOOP_FINAL_STATE" - -if [ "$LOOP_FINAL_STATE" == "error" ]; then - echo " Crash loop prevention working: instance stayed in error state" -else - echo " WARNING: Instance in state '$LOOP_FINAL_STATE' (expected 'error' after exceeding max restarts)" - echo " This may be expected if timing allowed restarts to spread across the window" -fi - -echo " Crash recovery tests passed" - -# Phase 5c: IAM Accounts & Cross-Account Isolation -echo "" -echo "Phase 5c: IAM Accounts & Cross-Account Isolation" -echo "========================================" - -AWS_EP="--endpoint-url https://${NODE1_IP}:${AWSGW_PORT}" - -# Step 1: Create accounts -echo "" -echo "Step 1: Create Accounts" -echo "----------------------------------------" - -echo " Creating Team Alpha account..." -ALPHA_OUTPUT=$(./bin/hive admin account create --name "Team Alpha" --config "$HOME/node1/config/hive.toml" 2>&1) -echo "$ALPHA_OUTPUT" -ALPHA_ACCOUNT=$(echo "$ALPHA_OUTPUT" | grep "Account ID:" | awk '{print $NF}') -ALPHA_KEY_ID=$(echo "$ALPHA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}') -ALPHA_SECRET=$(echo "$ALPHA_OUTPUT" | grep "Secret Access Key:" | awk '{print $NF}') - -if [ -z "$ALPHA_ACCOUNT" ] || [ -z "$ALPHA_KEY_ID" ]; then - echo " ERROR: Failed to parse Team Alpha account output" - exit 1 -fi -echo " Team Alpha: account=$ALPHA_ACCOUNT key=$ALPHA_KEY_ID" - -# Configure alpha profile (in case auto-config used a different endpoint) -aws configure set aws_access_key_id "$ALPHA_KEY_ID" --profile hive-team-alpha -aws configure set aws_secret_access_key "$ALPHA_SECRET" --profile hive-team-alpha -aws configure set region us-east-1 --profile hive-team-alpha - -echo " Creating Team Beta account..." -BETA_OUTPUT=$(./bin/hive admin account create --name "Team Beta" --config "$HOME/node1/config/hive.toml" 2>&1) -echo "$BETA_OUTPUT" -BETA_ACCOUNT=$(echo "$BETA_OUTPUT" | grep "Account ID:" | awk '{print $NF}') -BETA_KEY_ID=$(echo "$BETA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}') -BETA_SECRET=$(echo "$BETA_OUTPUT" | grep "Secret Access Key:" | awk '{print $NF}') - -if [ -z "$BETA_ACCOUNT" ] || [ -z "$BETA_KEY_ID" ]; then - echo " ERROR: Failed to parse Team Beta account output" - exit 1 -fi -echo " Team Beta: account=$BETA_ACCOUNT key=$BETA_KEY_ID" - -aws configure set aws_access_key_id "$BETA_KEY_ID" --profile hive-team-beta -aws configure set aws_secret_access_key "$BETA_SECRET" --profile hive-team-beta -aws configure set region us-east-1 --profile hive-team-beta - -# Verify sequential IDs -echo " Verifying sequential account IDs..." -if [ "$ALPHA_ACCOUNT" == "000000000001" ] && [ "$BETA_ACCOUNT" == "000000000002" ]; then - echo " Sequential IDs correct: $ALPHA_ACCOUNT, $BETA_ACCOUNT" -else - echo " WARNING: Account IDs may not be sequential: $ALPHA_ACCOUNT, $BETA_ACCOUNT" -fi - -# List accounts -echo " Listing accounts..." -ACCOUNT_LIST=$(./bin/hive admin account list --config "$HOME/node1/config/hive.toml" 2>&1) -echo "$ACCOUNT_LIST" -if echo "$ACCOUNT_LIST" | grep -q "Team Alpha" && echo "$ACCOUNT_LIST" | grep -q "Team Beta"; then - echo " Account list verified" -else - echo " ERROR: Account list missing expected accounts" - exit 1 -fi - -# Step 2: Account admin auth -echo "" -echo "Step 2: Account Admin Auth" -echo "----------------------------------------" - -echo " Testing alpha admin auth..." -aws $AWS_EP ec2 describe-instances --profile hive-team-alpha > /dev/null -echo " Alpha admin: ec2 OK" -aws $AWS_EP iam list-users --profile hive-team-alpha > /dev/null -echo " Alpha admin: iam OK" - -echo " Testing beta admin auth..." -aws $AWS_EP ec2 describe-instances --profile hive-team-beta > /dev/null -echo " Beta admin: ec2 OK" -aws $AWS_EP iam list-users --profile hive-team-beta > /dev/null -echo " Beta admin: iam OK" - -# Step 3: Account-scoped users -echo "" -echo "Step 3: Account-Scoped Users" -echo "----------------------------------------" - -# Same username in different accounts -echo " Creating alice in both accounts..." -aws $AWS_EP iam create-user --user-name alice --profile hive-team-alpha > /dev/null -ALPHA_ALICE_ARN=$(aws $AWS_EP iam get-user --user-name alice --profile hive-team-alpha | jq -r '.User.Arn') -echo " Alpha alice: $ALPHA_ALICE_ARN" - -aws $AWS_EP iam create-user --user-name alice --profile hive-team-beta > /dev/null -BETA_ALICE_ARN=$(aws $AWS_EP iam get-user --user-name alice --profile hive-team-beta | jq -r '.User.Arn') -echo " Beta alice: $BETA_ALICE_ARN" - -# Additional users -aws $AWS_EP iam create-user --user-name team-member --profile hive-team-alpha > /dev/null -aws $AWS_EP iam create-user --user-name dev-user --profile hive-team-beta > /dev/null - -# List users — scoped -echo " Verifying scoped user lists..." -ALPHA_USERS=$(aws $AWS_EP iam list-users --profile hive-team-alpha | jq -r '.Users[].UserName' | sort | tr '\n' ',') -echo " Alpha users: $ALPHA_USERS" -BETA_USERS=$(aws $AWS_EP iam list-users --profile hive-team-beta | jq -r '.Users[].UserName' | sort | tr '\n' ',') -echo " Beta users: $BETA_USERS" - -# Cross-account isolation -echo " Verifying cross-account isolation..." -set +e -CROSS_CHECK=$(aws $AWS_EP iam get-user --user-name dev-user --profile hive-team-alpha 2>&1) -set -e -if echo "$CROSS_CHECK" | grep -q "NoSuchEntity"; then - echo " Alpha cannot see Beta's dev-user — isolation OK" -else - echo " ERROR: Alpha can see Beta's user (cross-account leak)" - exit 1 -fi - -set +e -CROSS_CHECK2=$(aws $AWS_EP iam get-user --user-name team-member --profile hive-team-beta 2>&1) -set -e -if echo "$CROSS_CHECK2" | grep -q "NoSuchEntity"; then - echo " Beta cannot see Alpha's team-member — isolation OK" -else - echo " ERROR: Beta can see Alpha's user (cross-account leak)" - exit 1 -fi - -# Step 4: Account-scoped access keys -echo "" -echo "Step 4: Account-Scoped Access Keys" -echo "----------------------------------------" - -ALPHA_ALICE_KEY=$(aws $AWS_EP iam create-access-key --user-name alice --profile hive-team-alpha) -ALPHA_ALICE_KEY_ID=$(echo "$ALPHA_ALICE_KEY" | jq -r '.AccessKey.AccessKeyId') -ALPHA_ALICE_SECRET=$(echo "$ALPHA_ALICE_KEY" | jq -r '.AccessKey.SecretAccessKey') -echo " Alpha alice key: $ALPHA_ALICE_KEY_ID" - -aws configure set aws_access_key_id "$ALPHA_ALICE_KEY_ID" --profile hive-alpha-alice -aws configure set aws_secret_access_key "$ALPHA_ALICE_SECRET" --profile hive-alpha-alice -aws configure set region us-east-1 --profile hive-alpha-alice - -BETA_ALICE_KEY=$(aws $AWS_EP iam create-access-key --user-name alice --profile hive-team-beta) -BETA_ALICE_KEY_ID=$(echo "$BETA_ALICE_KEY" | jq -r '.AccessKey.AccessKeyId') -BETA_ALICE_SECRET=$(echo "$BETA_ALICE_KEY" | jq -r '.AccessKey.SecretAccessKey') -echo " Beta alice key: $BETA_ALICE_KEY_ID" - -aws configure set aws_access_key_id "$BETA_ALICE_KEY_ID" --profile hive-beta-alice -aws configure set aws_secret_access_key "$BETA_ALICE_SECRET" --profile hive-beta-alice -aws configure set region us-east-1 --profile hive-beta-alice - -# Step 5: Account-scoped policies & enforcement -echo "" -echo "Step 5: Account-Scoped Policies & Enforcement" -echo "----------------------------------------" - -# Alpha: narrow EC2 read-only -echo " Creating EC2ReadOnly in Alpha (narrow)..." -aws $AWS_EP iam create-policy \ - --policy-name EC2ReadOnly \ - --policy-document '{ - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": ["ec2:DescribeInstances", "ec2:DescribeVpcs"], - "Resource": "*" - }] - }' --profile hive-team-alpha > /dev/null - -# Beta: broad EC2 Describe* wildcard -echo " Creating EC2ReadOnly in Beta (broad Describe*)..." -aws $AWS_EP iam create-policy \ - --policy-name EC2ReadOnly \ - --policy-document '{ - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": "ec2:Describe*", - "Resource": "*" - }] - }' --profile hive-team-beta > /dev/null - -# Attach policies -echo " Attaching policies..." -aws $AWS_EP iam attach-user-policy --user-name alice \ - --policy-arn "arn:aws:iam::${ALPHA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-alpha -aws $AWS_EP iam attach-user-policy --user-name alice \ - --policy-arn "arn:aws:iam::${BETA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-beta - -# Verify scoped enforcement -echo " Verifying scoped enforcement..." - -# Alpha alice: narrow policy -aws $AWS_EP ec2 describe-instances --profile hive-alpha-alice > /dev/null -echo " Alpha alice: describe-instances — allowed" -aws $AWS_EP ec2 describe-vpcs --profile hive-alpha-alice > /dev/null -echo " Alpha alice: describe-vpcs — allowed" -expect_error "AccessDenied" aws $AWS_EP ec2 describe-key-pairs --profile hive-alpha-alice -echo " Alpha alice: describe-key-pairs — denied (narrow policy)" - -# Beta alice: broad policy -aws $AWS_EP ec2 describe-instances --profile hive-beta-alice > /dev/null -echo " Beta alice: describe-instances — allowed" -aws $AWS_EP ec2 describe-key-pairs --profile hive-beta-alice > /dev/null -echo " Beta alice: describe-key-pairs — allowed (Describe* wildcard)" - -# Both denied non-Describe -expect_error "AccessDenied" aws $AWS_EP ec2 create-key-pair --key-name x --profile hive-alpha-alice -echo " Alpha alice: create-key-pair — denied" -expect_error "AccessDenied" aws $AWS_EP ec2 create-key-pair --key-name x --profile hive-beta-alice -echo " Beta alice: create-key-pair — denied" - -echo " Scoped enforcement verified" - -# Step 6: Cross-account delete isolation -echo "" -echo "Step 6: Cross-Account Delete Isolation" -echo "----------------------------------------" - -# Delete Alpha's alice -echo " Deleting Alpha's alice..." -aws $AWS_EP iam detach-user-policy --user-name alice \ - --policy-arn "arn:aws:iam::${ALPHA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-alpha -aws $AWS_EP iam delete-access-key --user-name alice \ - --access-key-id "$ALPHA_ALICE_KEY_ID" --profile hive-team-alpha -aws $AWS_EP iam delete-user --user-name alice --profile hive-team-alpha - -# Verify Alpha alice gone -set +e -ALPHA_CHECK=$(aws $AWS_EP iam get-user --user-name alice --profile hive-team-alpha 2>&1) -set -e -if echo "$ALPHA_CHECK" | grep -q "NoSuchEntity"; then - echo " Alpha alice deleted" -else - echo " ERROR: Alpha alice still exists after delete" - exit 1 -fi - -# Verify Beta alice unaffected -aws $AWS_EP iam get-user --user-name alice --profile hive-team-beta > /dev/null -echo " Beta alice still exists (isolation OK)" - -aws $AWS_EP ec2 describe-instances --profile hive-beta-alice > /dev/null -echo " Beta alice auth still works (isolation OK)" - -# ========================================================================== -# Step 7: EC2 Resource Scoping -# ========================================================================== -# Tests that EC2 resources are properly isolated between accounts. -# Reuses Alpha/Beta accounts created in Step 1. -# Skips: Section 6 (CreateImage — mulga-612), instance tags (mulga-613) -echo "" -echo "Step 7: EC2 Resource Scoping" -echo "========================================" - -# --- 7a: Instance Scoping --- -echo "" -echo "Step 7a: Instance Scoping" -echo "----------------------------------------" - -# Create per-account key pairs (key pairs are account-scoped, root's multinode-test-key is invisible) -$AWS_EC2 create-key-pair --key-name alpha-instance-key --profile hive-team-alpha > /dev/null -$AWS_EC2 create-key-pair --key-name beta-instance-key --profile hive-team-beta > /dev/null -echo " Created per-account key pairs for instance launches" - -echo " Alpha launching instance..." -ALPHA_INST_RUN=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name alpha-instance-key \ - --profile hive-team-alpha) -ALPHA_INST=$(echo "$ALPHA_INST_RUN" | jq -r '.Instances[0].InstanceId') -echo " Alpha instance: $ALPHA_INST" - -echo " Beta launching instance..." -BETA_INST_RUN=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name beta-instance-key \ - --profile hive-team-beta) -BETA_INST=$(echo "$BETA_INST_RUN" | jq -r '.Instances[0].InstanceId') -echo " Beta instance: $BETA_INST" - -# Wait for running (inline — wait_for_instance_state uses root profile, can't see tenant instances) -echo " Waiting for instances to reach running state..." -COUNT=0 -while [ $COUNT -lt 30 ]; do - A_STATE=$($AWS_EC2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending") - B_STATE=$($AWS_EC2 describe-instances --instance-ids "$BETA_INST" --profile hive-team-beta \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending") - echo " Alpha=$A_STATE, Beta=$B_STATE" - if [ "$A_STATE" == "running" ] && [ "$B_STATE" == "running" ]; then break; fi - sleep 2 - COUNT=$((COUNT + 1)) -done -if [ "$A_STATE" != "running" ] || [ "$B_STATE" != "running" ]; then - echo " ERROR: Instances failed to reach running state" - exit 1 -fi -echo " Both instances running" - -# Describe isolation -ALPHA_DESC=$($AWS_EC2 describe-instances --profile hive-team-alpha \ - --query 'Reservations[].Instances[].InstanceId' --output text) -if echo "$ALPHA_DESC" | grep -q "$BETA_INST"; then - echo " ERROR: Alpha can see Beta's instance" - exit 1 -fi -echo " Alpha sees only own instances" - -BETA_DESC=$($AWS_EC2 describe-instances --profile hive-team-beta \ - --query 'Reservations[].Instances[].InstanceId' --output text) -if echo "$BETA_DESC" | grep -q "$ALPHA_INST"; then - echo " ERROR: Beta can see Alpha's instance" - exit 1 -fi -echo " Beta sees only own instances" - -# OwnerId verification -ALPHA_OWNER=$($AWS_EC2 describe-instances --profile hive-team-alpha \ - --query 'Reservations[0].OwnerId' --output text) -if [ "$ALPHA_OWNER" != "$ALPHA_ACCOUNT" ]; then - echo " ERROR: Alpha OwnerId mismatch: expected $ALPHA_ACCOUNT, got $ALPHA_OWNER" - exit 1 -fi -echo " Alpha OwnerId correct: $ALPHA_OWNER" - -# Cross-account operations -expect_error "InvalidInstanceID.NotFound" \ - $AWS_EC2 stop-instances --instance-ids "$BETA_INST" --profile hive-team-alpha -echo " Alpha cannot stop Beta's instance" - -expect_error "InvalidInstanceID.NotFound" \ - $AWS_EC2 terminate-instances --instance-ids "$ALPHA_INST" --profile hive-team-beta -echo " Beta cannot terminate Alpha's instance" - -expect_error "InvalidInstanceID.NotFound" \ - $AWS_EC2 get-console-output --instance-id "$ALPHA_INST" --profile hive-team-beta -echo " Beta cannot get console output of Alpha's instance" - -echo " Instance scoping passed" - -# --- 7b: Volume Scoping --- -echo "" -echo "Step 7b: Volume Scoping" -echo "----------------------------------------" - -ALPHA_VOL=$($AWS_EC2 create-volume --availability-zone ap-southeast-2a --size 10 \ - --volume-type gp3 --profile hive-team-alpha | jq -r '.VolumeId') -echo " Alpha volume: $ALPHA_VOL" - -BETA_VOL=$($AWS_EC2 create-volume --availability-zone ap-southeast-2a --size 10 \ - --volume-type gp3 --profile hive-team-beta | jq -r '.VolumeId') -echo " Beta volume: $BETA_VOL" - -# Describe isolation -ALPHA_VOLS=$($AWS_EC2 describe-volumes --profile hive-team-alpha \ - --query 'Volumes[].VolumeId' --output text) -if echo "$ALPHA_VOLS" | grep -q "$BETA_VOL"; then - echo " ERROR: Alpha can see Beta's volume" - exit 1 -fi -echo " Alpha sees only own volumes" - -# Cross-account operations -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 describe-volumes --volume-ids "$BETA_VOL" --profile hive-team-alpha -echo " Alpha cannot describe Beta's volume by ID" - -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 delete-volume --volume-id "$ALPHA_VOL" --profile hive-team-beta -echo " Beta cannot delete Alpha's volume" - -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 attach-volume --volume-id "$ALPHA_VOL" \ - --instance-id "$BETA_INST" --device /dev/sdf --profile hive-team-beta -echo " Beta cannot attach Alpha's volume" - -# Attach Alpha's volume, then test cross-account detach -$AWS_EC2 attach-volume --volume-id "$ALPHA_VOL" \ - --instance-id "$ALPHA_INST" --device /dev/sdf --profile hive-team-alpha > /dev/null -sleep 2 - -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 detach-volume --volume-id "$ALPHA_VOL" --profile hive-team-beta -echo " Beta cannot detach Alpha's volume" - -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 modify-volume --volume-id "$ALPHA_VOL" --size 20 --profile hive-team-beta -echo " Beta cannot modify Alpha's volume" - -# Detach for later cleanup -$AWS_EC2 detach-volume --volume-id "$ALPHA_VOL" --profile hive-team-alpha > /dev/null -sleep 2 - -echo " Volume scoping passed" - -# --- 7c: Key Pair Scoping --- -echo "" -echo "Step 7c: Key Pair Scoping" -echo "----------------------------------------" - -$AWS_EC2 create-key-pair --key-name alpha-key --profile hive-team-alpha > /dev/null -ALPHA_KEYPAIR_ID=$($AWS_EC2 describe-key-pairs --key-names alpha-key \ - --profile hive-team-alpha --query 'KeyPairs[0].KeyPairId' --output text) -echo " Alpha key: alpha-key ($ALPHA_KEYPAIR_ID)" - -$AWS_EC2 create-key-pair --key-name beta-key --profile hive-team-beta > /dev/null -echo " Beta key: beta-key" - -# Describe isolation -ALPHA_KEYS=$($AWS_EC2 describe-key-pairs --profile hive-team-alpha \ - --query 'KeyPairs[].KeyName' --output text) -if echo "$ALPHA_KEYS" | grep -q "beta-key"; then - echo " ERROR: Alpha can see Beta's key" - exit 1 -fi -echo " Alpha sees only own keys" - -# Same name, different accounts -$AWS_EC2 create-key-pair --key-name shared-name --profile hive-team-alpha > /dev/null -$AWS_EC2 create-key-pair --key-name shared-name --profile hive-team-beta > /dev/null -ALPHA_SHARED_ID=$($AWS_EC2 describe-key-pairs --key-names shared-name \ - --profile hive-team-alpha --query 'KeyPairs[0].KeyPairId' --output text) -BETA_SHARED_ID=$($AWS_EC2 describe-key-pairs --key-names shared-name \ - --profile hive-team-beta --query 'KeyPairs[0].KeyPairId' --output text) -if [ "$ALPHA_SHARED_ID" == "$BETA_SHARED_ID" ]; then - echo " ERROR: Same KeyPairId for shared-name in both accounts" - exit 1 -fi -echo " Namespace isolation: alpha=$ALPHA_SHARED_ID, beta=$BETA_SHARED_ID" - -# Cross-account delete (idempotent, but shouldn't affect other account) -$AWS_EC2 delete-key-pair --key-name alpha-key --profile hive-team-beta -ALPHA_KEY_CHECK=$($AWS_EC2 describe-key-pairs --key-names alpha-key \ - --profile hive-team-alpha --query 'KeyPairs[0].KeyPairId' --output text) -if [ "$ALPHA_KEY_CHECK" != "$ALPHA_KEYPAIR_ID" ]; then - echo " ERROR: Beta's delete affected Alpha's key" - exit 1 -fi -echo " Cross-account delete had no effect on Alpha's key" - -# Import key pair — account scoped -ssh-keygen -t ed25519 -f /tmp/test-import-key -N "" -q -$AWS_EC2 import-key-pair --key-name imported-key \ - --public-key-material fileb:///tmp/test-import-key.pub --profile hive-team-alpha > /dev/null -BETA_IMPORT_CHECK=$($AWS_EC2 describe-key-pairs --profile hive-team-beta \ - --query 'KeyPairs[].KeyName' --output text) -if echo "$BETA_IMPORT_CHECK" | grep -q "imported-key"; then - echo " ERROR: Beta can see Alpha's imported key" - exit 1 -fi -echo " Imported key invisible to Beta" -rm -f /tmp/test-import-key /tmp/test-import-key.pub - -echo " Key pair scoping passed" - -# --- 7d: Snapshot Scoping --- -echo "" -echo "Step 7d: Snapshot Scoping" -echo "----------------------------------------" - -ALPHA_SNAP=$($AWS_EC2 create-snapshot --volume-id "$ALPHA_VOL" \ - --description "Alpha snapshot" --profile hive-team-alpha | jq -r '.SnapshotId') -echo " Alpha snapshot: $ALPHA_SNAP" - -BETA_SNAP=$($AWS_EC2 create-snapshot --volume-id "$BETA_VOL" \ - --description "Beta snapshot" --profile hive-team-beta | jq -r '.SnapshotId') -echo " Beta snapshot: $BETA_SNAP" - -# Describe isolation -ALPHA_SNAPS=$($AWS_EC2 describe-snapshots --owner-ids self --profile hive-team-alpha \ - --query 'Snapshots[].SnapshotId' --output text) -if echo "$ALPHA_SNAPS" | grep -q "$BETA_SNAP"; then - echo " ERROR: Alpha can see Beta's snapshot" - exit 1 -fi -echo " Alpha sees only own snapshots" - -# OwnerId verification -ALPHA_SNAP_OWNER=$($AWS_EC2 describe-snapshots --owner-ids self --profile hive-team-alpha \ - --query 'Snapshots[0].OwnerId' --output text) -if [ "$ALPHA_SNAP_OWNER" != "$ALPHA_ACCOUNT" ]; then - echo " ERROR: Snapshot OwnerId mismatch: expected $ALPHA_ACCOUNT, got $ALPHA_SNAP_OWNER" - exit 1 -fi -echo " Alpha snapshot OwnerId correct" - -# Cross-account delete -expect_error "UnauthorizedOperation" \ - $AWS_EC2 delete-snapshot --snapshot-id "$ALPHA_SNAP" --profile hive-team-beta -echo " Beta cannot delete Alpha's snapshot" - -# Cross-account snapshot from other's volume -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 create-snapshot --volume-id "$ALPHA_VOL" \ - --description "stolen" --profile hive-team-beta -echo " Beta cannot snapshot Alpha's volume" - -echo " Snapshot scoping passed" - -# --- 7e: VPC/Subnet Scoping --- -echo "" -echo "Step 7e: VPC/Subnet Scoping" -echo "----------------------------------------" - -ALPHA_VPC=$($AWS_EC2 create-vpc --cidr-block 10.0.0.0/16 \ - --profile hive-team-alpha --query 'Vpc.VpcId' --output text) -echo " Alpha VPC: $ALPHA_VPC" - -BETA_VPC=$($AWS_EC2 create-vpc --cidr-block 10.0.0.0/16 \ - --profile hive-team-beta --query 'Vpc.VpcId' --output text) -echo " Beta VPC: $BETA_VPC (same CIDR — no conflict)" - -# Describe isolation -ALPHA_VPCS=$($AWS_EC2 describe-vpcs --profile hive-team-alpha \ - --query 'Vpcs[].VpcId' --output text) -if echo "$ALPHA_VPCS" | grep -q "$BETA_VPC"; then - echo " ERROR: Alpha can see Beta's VPC" - exit 1 -fi -echo " VPC describe isolation OK" - -expect_error "InvalidVpcID.NotFound" \ - $AWS_EC2 describe-vpcs --vpc-ids "$BETA_VPC" --profile hive-team-alpha -echo " Alpha cannot describe Beta's VPC by ID" - -expect_error "InvalidVpcID.NotFound" \ - $AWS_EC2 delete-vpc --vpc-id "$ALPHA_VPC" --profile hive-team-beta -echo " Beta cannot delete Alpha's VPC" - -# Create subnets -ALPHA_SUBNET=$($AWS_EC2 create-subnet --vpc-id "$ALPHA_VPC" --cidr-block 10.0.1.0/24 \ - --profile hive-team-alpha --query 'Subnet.SubnetId' --output text) -echo " Alpha subnet: $ALPHA_SUBNET" - -BETA_SUBNET=$($AWS_EC2 create-subnet --vpc-id "$BETA_VPC" --cidr-block 10.0.1.0/24 \ - --profile hive-team-beta --query 'Subnet.SubnetId' --output text) -echo " Beta subnet: $BETA_SUBNET" - -# Subnet describe isolation -ALPHA_SUBNETS=$($AWS_EC2 describe-subnets --profile hive-team-alpha \ - --query 'Subnets[].SubnetId' --output text) -if echo "$ALPHA_SUBNETS" | grep -q "$BETA_SUBNET"; then - echo " ERROR: Alpha can see Beta's subnet" - exit 1 -fi -echo " Subnet describe isolation OK" - -expect_error "InvalidVpcID.NotFound" \ - $AWS_EC2 create-subnet --vpc-id "$ALPHA_VPC" --cidr-block 10.0.2.0/24 \ - --profile hive-team-beta -echo " Beta cannot create subnet in Alpha's VPC" - -expect_error "InvalidSubnetID.NotFound" \ - $AWS_EC2 delete-subnet --subnet-id "$ALPHA_SUBNET" --profile hive-team-beta -echo " Beta cannot delete Alpha's subnet" - -echo " VPC/Subnet scoping passed" - -# --- 7f: IGW + EIGW Scoping --- -echo "" -echo "Step 7f: IGW + EIGW Scoping" -echo "----------------------------------------" - -ALPHA_IGW=$($AWS_EC2 create-internet-gateway --profile hive-team-alpha \ - --query 'InternetGateway.InternetGatewayId' --output text) -echo " Alpha IGW: $ALPHA_IGW" - -BETA_IGW=$($AWS_EC2 create-internet-gateway --profile hive-team-beta \ - --query 'InternetGateway.InternetGatewayId' --output text) -echo " Beta IGW: $BETA_IGW" - -# IGW describe isolation -ALPHA_IGWS=$($AWS_EC2 describe-internet-gateways --profile hive-team-alpha \ - --query 'InternetGateways[].InternetGatewayId' --output text) -if echo "$ALPHA_IGWS" | grep -q "$BETA_IGW"; then - echo " ERROR: Alpha can see Beta's IGW" - exit 1 -fi -echo " IGW describe isolation OK" - -expect_error "InvalidInternetGatewayID.NotFound" \ - $AWS_EC2 describe-internet-gateways --internet-gateway-ids "$BETA_IGW" \ - --profile hive-team-alpha -echo " Alpha cannot describe Beta's IGW by ID" - -expect_error "InvalidInternetGatewayID.NotFound" \ - $AWS_EC2 delete-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ - --profile hive-team-beta -echo " Beta cannot delete Alpha's IGW" - -expect_error "InvalidInternetGatewayID.NotFound" \ - $AWS_EC2 attach-internet-gateway --internet-gateway-id "$BETA_IGW" \ - --vpc-id "$ALPHA_VPC" --profile hive-team-alpha -echo " Alpha cannot attach Beta's IGW to own VPC" - -# Attach Alpha's IGW, test cross-account detach -$AWS_EC2 attach-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ - --vpc-id "$ALPHA_VPC" --profile hive-team-alpha > /dev/null -expect_error "InvalidInternetGatewayID.NotFound" \ - $AWS_EC2 detach-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ - --vpc-id "$ALPHA_VPC" --profile hive-team-beta -echo " Beta cannot detach Alpha's IGW" - -# EIGW -ALPHA_EIGW=$($AWS_EC2 create-egress-only-internet-gateway --vpc-id "$ALPHA_VPC" \ - --profile hive-team-alpha \ - --query 'EgressOnlyInternetGateway.EgressOnlyInternetGatewayId' --output text) -echo " Alpha EIGW: $ALPHA_EIGW" - -BETA_EIGW=$($AWS_EC2 create-egress-only-internet-gateway --vpc-id "$BETA_VPC" \ - --profile hive-team-beta \ - --query 'EgressOnlyInternetGateway.EgressOnlyInternetGatewayId' --output text) -echo " Beta EIGW: $BETA_EIGW" - -# EIGW describe isolation -ALPHA_EIGWS=$($AWS_EC2 describe-egress-only-internet-gateways --profile hive-team-alpha \ - --query 'EgressOnlyInternetGateways[].EgressOnlyInternetGatewayId' --output text) -if echo "$ALPHA_EIGWS" | grep -q "$BETA_EIGW"; then - echo " ERROR: Alpha can see Beta's EIGW" - exit 1 -fi -echo " EIGW describe isolation OK" - -# Cross-account EIGW delete (verify Alpha's still exists after Beta's attempt) -set +e -$AWS_EC2 delete-egress-only-internet-gateway \ - --egress-only-internet-gateway-id "$ALPHA_EIGW" --profile hive-team-beta 2>/dev/null -set -e -ALPHA_EIGW_CHECK=$($AWS_EC2 describe-egress-only-internet-gateways --profile hive-team-alpha \ - --query 'EgressOnlyInternetGateways[].EgressOnlyInternetGatewayId' --output text) -if ! echo "$ALPHA_EIGW_CHECK" | grep -q "$ALPHA_EIGW"; then - echo " ERROR: Alpha's EIGW was deleted by Beta" - exit 1 -fi -echo " Beta cannot delete Alpha's EIGW" - -echo " IGW + EIGW scoping passed" - -# --- 7g: Account Settings --- -echo "" -echo "Step 7g: Account Settings" -echo "----------------------------------------" - -$AWS_EC2 enable-ebs-encryption-by-default --profile hive-team-alpha > /dev/null -BETA_ENC=$($AWS_EC2 get-ebs-encryption-by-default --profile hive-team-beta \ - --query 'EbsEncryptionByDefault' --output text) -if [ "$BETA_ENC" != "False" ]; then - echo " ERROR: Alpha's encryption setting leaked to Beta (got $BETA_ENC)" - exit 1 -fi -echo " Alpha enable did not affect Beta" - -$AWS_EC2 enable-ebs-encryption-by-default --profile hive-team-beta > /dev/null -$AWS_EC2 disable-ebs-encryption-by-default --profile hive-team-alpha > /dev/null -ALPHA_ENC=$($AWS_EC2 get-ebs-encryption-by-default --profile hive-team-alpha \ - --query 'EbsEncryptionByDefault' --output text) -BETA_ENC=$($AWS_EC2 get-ebs-encryption-by-default --profile hive-team-beta \ - --query 'EbsEncryptionByDefault' --output text) -if [ "$ALPHA_ENC" != "False" ] || [ "$BETA_ENC" != "True" ]; then - echo " ERROR: Independent settings failed: alpha=$ALPHA_ENC beta=$BETA_ENC" - exit 1 -fi -echo " Independent toggle verified: alpha=$ALPHA_ENC, beta=$BETA_ENC" -$AWS_EC2 disable-ebs-encryption-by-default --profile hive-team-beta > /dev/null - -echo " Account settings scoping passed" - -# --- 7h: Global Resources --- -echo "" -echo "Step 7h: Global Resources" -echo "----------------------------------------" - -ALPHA_REGIONS=$($AWS_EC2 describe-regions --profile hive-team-alpha \ - --query 'Regions[].RegionName' --output text) -BETA_REGIONS=$($AWS_EC2 describe-regions --profile hive-team-beta \ - --query 'Regions[].RegionName' --output text) -if [ "$ALPHA_REGIONS" != "$BETA_REGIONS" ]; then - echo " ERROR: Regions differ between accounts" - exit 1 -fi -echo " Regions identical" - -ALPHA_AZS=$($AWS_EC2 describe-availability-zones --profile hive-team-alpha \ - --query 'AvailabilityZones[].ZoneName' --output text) -BETA_AZS=$($AWS_EC2 describe-availability-zones --profile hive-team-beta \ - --query 'AvailabilityZones[].ZoneName' --output text) -if [ "$ALPHA_AZS" != "$BETA_AZS" ]; then - echo " ERROR: AZs differ between accounts" - exit 1 -fi -echo " Availability zones identical" - -ALPHA_TYPES=$($AWS_EC2 describe-instance-types --profile hive-team-alpha \ - --query 'InstanceTypes[].InstanceType' --output text | tr '\t' '\n' | sort) -BETA_TYPES=$($AWS_EC2 describe-instance-types --profile hive-team-beta \ - --query 'InstanceTypes[].InstanceType' --output text | tr '\t' '\n' | sort) -if [ "$ALPHA_TYPES" != "$BETA_TYPES" ]; then - echo " ERROR: Instance types differ between accounts" - exit 1 -fi -echo " Instance types identical" - -echo " Global resources passed" - -# --- Step 8: Edge Cases --- -echo "" -echo "Step 8: Edge Cases" -echo "----------------------------------------" - -# Empty account (Gamma) -echo " Creating empty Gamma account..." -GAMMA_OUTPUT=$(./bin/hive admin account create --name "Team Gamma" --config "$HOME/node1/config/hive.toml" 2>&1) -GAMMA_KEY_ID=$(echo "$GAMMA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}') -GAMMA_SECRET=$(echo "$GAMMA_OUTPUT" | grep "Secret Access Key:" | awk '{print $NF}') -aws configure set aws_access_key_id "$GAMMA_KEY_ID" --profile hive-team-gamma -aws configure set aws_secret_access_key "$GAMMA_SECRET" --profile hive-team-gamma -aws configure set region us-east-1 --profile hive-team-gamma - -GAMMA_INSTANCES=$($AWS_EC2 describe-instances --profile hive-team-gamma \ - --query 'Reservations' --output text) -if [ -n "$GAMMA_INSTANCES" ] && [ "$GAMMA_INSTANCES" != "None" ]; then - echo " ERROR: Gamma has instances" - exit 1 -fi -echo " Gamma: no instances" - -# Skip volume check: root-account volumes (empty TenantID) are visible to all accounts by design -echo " Gamma: volumes skipped (root legacy volumes visible to all)" - -GAMMA_KEYS=$($AWS_EC2 describe-key-pairs --profile hive-team-gamma \ - --query 'KeyPairs' --output text) -if [ -n "$GAMMA_KEYS" ] && [ "$GAMMA_KEYS" != "None" ]; then - echo " ERROR: Gamma has key pairs" - exit 1 -fi -echo " Gamma: no key pairs" - -# Root isolation from tenants -echo " Verifying root isolation from tenants..." -ROOT_INSTANCE_CHECK=$($AWS_EC2 describe-instances \ - --query 'Reservations[].Instances[].InstanceId' --output text) -if echo "$ROOT_INSTANCE_CHECK" | grep -q "$ALPHA_INST"; then - echo " ERROR: Root can see Alpha's instance" - exit 1 -fi -echo " Root cannot see tenant instances" - -# Non-existent resource IDs — same error as cross-account -expect_error "InvalidVolume.NotFound" \ - $AWS_EC2 delete-volume --volume-id vol-00000000000000000 --profile hive-team-alpha -echo " Non-existent volume: same error as cross-account" - -expect_error "InvalidSnapshot.NotFound" \ - $AWS_EC2 delete-snapshot --snapshot-id snap-00000000000000000 --profile hive-team-alpha -echo " Non-existent snapshot: same error as cross-account" - -# Race condition: parallel key creation from both accounts -echo " Testing parallel key creation (race condition)..." -for i in $(seq 1 5); do - $AWS_EC2 create-key-pair --key-name "race-alpha-$i" --profile hive-team-alpha > /dev/null & - $AWS_EC2 create-key-pair --key-name "race-beta-$i" --profile hive-team-beta > /dev/null & -done -wait - -ALPHA_RACE_COUNT=$($AWS_EC2 describe-key-pairs --profile hive-team-alpha \ - --query 'KeyPairs[].KeyName' --output text | tr '\t' '\n' | grep -c "race-alpha" || true) -if [ "$ALPHA_RACE_COUNT" -ne 5 ]; then - echo " ERROR: Expected 5 race-alpha keys, got $ALPHA_RACE_COUNT" - exit 1 -fi -ALPHA_RACE_LEAK=$($AWS_EC2 describe-key-pairs --profile hive-team-alpha \ - --query 'KeyPairs[].KeyName' --output text | tr '\t' '\n' | grep -c "race-beta" || true) -if [ "$ALPHA_RACE_LEAK" -ne 0 ]; then - echo " ERROR: Alpha sees Beta's race keys (cross-contamination)" - exit 1 -fi -echo " Parallel key creation: no cross-contamination ($ALPHA_RACE_COUNT alpha keys, 0 beta leaks)" - -echo " Edge cases passed" - -# --- Step 9: EC2 + IAM Cleanup --- -echo "" -echo "Step 9: EC2 + IAM Cleanup" -echo "========================================" - -# Terminate instances -echo " Terminating account-scoped instances..." -$AWS_EC2 terminate-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha > /dev/null -$AWS_EC2 terminate-instances --instance-ids "$BETA_INST" --profile hive-team-beta > /dev/null - -COUNT=0 -while [ $COUNT -lt 30 ]; do - A_STATE=$($AWS_EC2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "terminated") - B_STATE=$($AWS_EC2 describe-instances --instance-ids "$BETA_INST" --profile hive-team-beta \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "terminated") - if [ "$A_STATE" == "terminated" ] && [ "$B_STATE" == "terminated" ]; then - break - fi - sleep 2 - COUNT=$((COUNT + 1)) -done -echo " Instances terminated" - -# Delete snapshots -echo " Deleting snapshots..." -$AWS_EC2 delete-snapshot --snapshot-id "$ALPHA_SNAP" --profile hive-team-alpha 2>/dev/null || true -$AWS_EC2 delete-snapshot --snapshot-id "$BETA_SNAP" --profile hive-team-beta 2>/dev/null || true - -# Delete volumes -sleep 3 -echo " Deleting volumes..." -$AWS_EC2 delete-volume --volume-id "$ALPHA_VOL" --profile hive-team-alpha 2>/dev/null || true -$AWS_EC2 delete-volume --volume-id "$BETA_VOL" --profile hive-team-beta 2>/dev/null || true - -# Delete key pairs -echo " Deleting key pairs..." -for key in alpha-key alpha-instance-key shared-name imported-key; do - $AWS_EC2 delete-key-pair --key-name "$key" --profile hive-team-alpha 2>/dev/null || true -done -for key in beta-key beta-instance-key shared-name; do - $AWS_EC2 delete-key-pair --key-name "$key" --profile hive-team-beta 2>/dev/null || true -done -for i in $(seq 1 5); do - $AWS_EC2 delete-key-pair --key-name "race-alpha-$i" --profile hive-team-alpha 2>/dev/null || true - $AWS_EC2 delete-key-pair --key-name "race-beta-$i" --profile hive-team-beta 2>/dev/null || true -done - -# Delete EIGWs -echo " Deleting EIGWs..." -$AWS_EC2 delete-egress-only-internet-gateway \ - --egress-only-internet-gateway-id "$ALPHA_EIGW" --profile hive-team-alpha 2>/dev/null || true -$AWS_EC2 delete-egress-only-internet-gateway \ - --egress-only-internet-gateway-id "$BETA_EIGW" --profile hive-team-beta 2>/dev/null || true - -# Detach + delete IGWs -echo " Deleting IGWs..." -$AWS_EC2 detach-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ - --vpc-id "$ALPHA_VPC" --profile hive-team-alpha 2>/dev/null || true -$AWS_EC2 delete-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ - --profile hive-team-alpha 2>/dev/null || true -$AWS_EC2 delete-internet-gateway --internet-gateway-id "$BETA_IGW" \ - --profile hive-team-beta 2>/dev/null || true - -# Delete subnets -echo " Deleting subnets..." -$AWS_EC2 delete-subnet --subnet-id "$ALPHA_SUBNET" --profile hive-team-alpha 2>/dev/null || true -$AWS_EC2 delete-subnet --subnet-id "$BETA_SUBNET" --profile hive-team-beta 2>/dev/null || true - -# Delete VPCs -echo " Deleting VPCs..." -$AWS_EC2 delete-vpc --vpc-id "$ALPHA_VPC" --profile hive-team-alpha 2>/dev/null || true -$AWS_EC2 delete-vpc --vpc-id "$BETA_VPC" --profile hive-team-beta 2>/dev/null || true - -echo " EC2 resources cleaned up" - -# IAM cleanup (originally Step 7) -echo " Cleaning up IAM resources..." - -# Beta cleanup -aws $AWS_EP iam detach-user-policy --user-name alice \ - --policy-arn "arn:aws:iam::${BETA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-beta -aws $AWS_EP iam delete-access-key --user-name alice \ - --access-key-id "$BETA_ALICE_KEY_ID" --profile hive-team-beta -aws $AWS_EP iam delete-user --user-name alice --profile hive-team-beta -aws $AWS_EP iam delete-user --user-name dev-user --profile hive-team-beta -aws $AWS_EP iam delete-policy \ - --policy-arn "arn:aws:iam::${BETA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-beta - -# Alpha cleanup (alice already deleted in Step 6) -aws $AWS_EP iam delete-user --user-name team-member --profile hive-team-alpha -aws $AWS_EP iam delete-policy \ - --policy-arn "arn:aws:iam::${ALPHA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-alpha - -# Clean up AWS CLI profiles -for p in hive-team-alpha hive-team-beta hive-alpha-alice hive-beta-alice hive-team-gamma; do - aws configure set aws_access_key_id "" --profile $p 2>/dev/null || true - aws configure set aws_secret_access_key "" --profile $p 2>/dev/null || true -done - -echo " IAM + EC2 account cleanup complete" -echo "" -echo " IAM Accounts & Cross-Account Isolation tests passed" - -# Phase 5d: VPC Networking -echo "" -echo "Phase 5d: VPC Networking" -echo "========================================" -echo "Testing VPC instance launch, PrivateIpAddress, and same-subnet connectivity..." - -# Step 1: Create VPC and Subnet -echo "" -echo "Step 1: Create VPC + Subnet" -echo "----------------------------------------" - -VPC_OUTPUT=$($AWS_EC2 create-vpc --cidr-block 10.100.0.0/16) -VPC_ID=$(echo "$VPC_OUTPUT" | jq -r '.Vpc.VpcId') -if [ -z "$VPC_ID" ] || [ "$VPC_ID" == "null" ]; then - echo " ERROR: Failed to create VPC" - echo " Output: $VPC_OUTPUT" - exit 1 -fi -echo " Created VPC: $VPC_ID (10.100.0.0/16)" - -SUBNET_OUTPUT=$($AWS_EC2 create-subnet --vpc-id "$VPC_ID" --cidr-block 10.100.1.0/24) -SUBNET_ID=$(echo "$SUBNET_OUTPUT" | jq -r '.Subnet.SubnetId') -if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" == "null" ]; then - echo " ERROR: Failed to create subnet" - echo " Output: $SUBNET_OUTPUT" - exit 1 -fi -echo " Created Subnet: $SUBNET_ID (10.100.1.0/24)" - -# Brief pause for OVN topology to be programmed (logical switch + router port + DHCP) -sleep 2 - -# Step 2: Launch 3 VPC instances -echo "" -echo "Step 2: Launch 3 VPC instances" -echo "----------------------------------------" - -VPC_INSTANCE_IDS=() -for i in 1 2 3; do - echo " Launching VPC instance $i with subnet $SUBNET_ID..." - RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --subnet-id "$SUBNET_ID") - - VPC_INST_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId') - VPC_INST_IP=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].PrivateIpAddress // empty') - - if [ -z "$VPC_INST_ID" ] || [ "$VPC_INST_ID" == "null" ]; then - echo " ERROR: Failed to launch VPC instance $i" - echo " Output: $RUN_OUTPUT" - exit 1 - fi - echo " Launched: $VPC_INST_ID (PrivateIpAddress: ${VPC_INST_IP:-not yet assigned})" - VPC_INSTANCE_IDS+=("$VPC_INST_ID") - - sleep 2 -done - -# Wait for all VPC instances to be running -echo "" -echo "Waiting for VPC instances to reach running state..." -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - wait_for_instance_state "$vpc_inst" "running" 30 || { - echo "ERROR: VPC instance $vpc_inst failed to start" - exit 1 - } -done - -# Step 3: Verify PrivateIpAddress in DescribeInstances -echo "" -echo "Step 3: Verify PrivateIpAddress in DescribeInstances" -echo "----------------------------------------" - -VPC_PRIVATE_IPS=() -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - DESCRIBE_OUT=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst") - PRIVATE_IP=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].PrivateIpAddress // empty') - INST_SUBNET=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].SubnetId // empty') - INST_VPC=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].VpcId // empty') - ENI_COUNT=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].NetworkInterfaces | length') - - if [ -z "$PRIVATE_IP" ]; then - echo " ERROR: $vpc_inst has no PrivateIpAddress" - echo " Describe output: $(echo "$DESCRIBE_OUT" | jq -c '.Reservations[0].Instances[0] | {PrivateIpAddress, SubnetId, VpcId, NetworkInterfaces}')" - exit 1 - fi - - echo " $vpc_inst: IP=$PRIVATE_IP, Subnet=$INST_SUBNET, VPC=$INST_VPC, ENIs=$ENI_COUNT" - - if [ "$INST_SUBNET" != "$SUBNET_ID" ]; then - echo " ERROR: SubnetId mismatch (expected $SUBNET_ID, got $INST_SUBNET)" - exit 1 - fi - if [ "$INST_VPC" != "$VPC_ID" ]; then - echo " ERROR: VpcId mismatch (expected $VPC_ID, got $INST_VPC)" - exit 1 - fi - if [ "$ENI_COUNT" -lt 1 ]; then - echo " ERROR: No NetworkInterfaces found" - exit 1 - fi - - VPC_PRIVATE_IPS+=("$PRIVATE_IP") -done - -# Verify all IPs are unique and in the subnet range (10.100.1.x) -echo "" -echo " Verifying IP uniqueness and subnet range..." -UNIQUE_IPS=$(printf '%s\n' "${VPC_PRIVATE_IPS[@]}" | sort -u | wc -l) -if [ "$UNIQUE_IPS" -ne "${#VPC_PRIVATE_IPS[@]}" ]; then - echo " ERROR: Duplicate IPs detected: ${VPC_PRIVATE_IPS[*]}" - exit 1 -fi -for ip in "${VPC_PRIVATE_IPS[@]}"; do - if ! echo "$ip" | grep -qE '^10\.100\.1\.[0-9]+$'; then - echo " ERROR: IP $ip not in expected subnet 10.100.1.0/24" - exit 1 - fi -done -echo " All IPs unique and in correct subnet: ${VPC_PRIVATE_IPS[*]}" - -# Step 4: SSH via DEV_NETWORKING hostfwd and test ping connectivity -# DISABLED: VPC instances use OVN DHCP which takes ~6 min per instance to get SSH ready. -# With 3 instances this adds ~18 min to the E2E run. The SSH+ping test is best-effort -# anyway (OVN overlay not fully programmed in Docker single-host mode). IP allocation -# and subnet correctness are already verified in Step 3. -echo "" -echo "Step 4: SSH + Ping Connectivity (SKIPPED — OVN DHCP wait too slow in CI)" -echo "----------------------------------------" - -# Step 5: Stop/Start IP persistence -echo "" -echo "Step 5: Stop/Start IP Persistence" -echo "----------------------------------------" -echo "Verifying private IPs persist through stop/start cycle (AWS behavior)..." - -# Record IPs before stop -echo " IPs before stop: ${VPC_PRIVATE_IPS[*]}" - -# Stop all VPC instances -echo "" -echo " Stopping all VPC instances..." -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - echo " Stopping $vpc_inst..." - $AWS_EC2 stop-instances --instance-ids "$vpc_inst" > /dev/null -done - -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - wait_for_instance_state "$vpc_inst" "stopped" 30 || { - echo " ERROR: VPC instance $vpc_inst failed to stop" - exit 1 - } -done -echo " All VPC instances stopped" - -# Verify IPs are still present in DescribeInstances while stopped -echo "" -echo " Verifying IPs persist in stopped state..." -for idx in "${!VPC_INSTANCE_IDS[@]}"; do - vpc_inst="${VPC_INSTANCE_IDS[$idx]}" - expected_ip="${VPC_PRIVATE_IPS[$idx]}" - - STOPPED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \ - --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text) - - if [ "$STOPPED_IP" != "$expected_ip" ]; then - echo " ERROR: $vpc_inst IP changed while stopped (expected $expected_ip, got $STOPPED_IP)" - exit 1 - fi - echo " $vpc_inst: IP=$STOPPED_IP (unchanged)" -done - -# Start all VPC instances -echo "" -echo " Starting all VPC instances..." -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - echo " Starting $vpc_inst..." - $AWS_EC2 start-instances --instance-ids "$vpc_inst" > /dev/null -done - -for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do - wait_for_instance_state "$vpc_inst" "running" 30 || { - echo " ERROR: VPC instance $vpc_inst failed to restart" - exit 1 - } -done -echo " All VPC instances restarted" - -# Verify IPs are identical after restart -echo "" -echo " Verifying IPs persist after restart..." -IP_MISMATCHES=0 -for idx in "${!VPC_INSTANCE_IDS[@]}"; do - vpc_inst="${VPC_INSTANCE_IDS[$idx]}" - expected_ip="${VPC_PRIVATE_IPS[$idx]}" - - RESTARTED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \ - --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text) - - if [ "$RESTARTED_IP" == "$expected_ip" ]; then - echo " $vpc_inst: IP=$RESTARTED_IP (matches pre-stop)" - else - echo " ERROR: $vpc_inst IP changed after restart (expected $expected_ip, got $RESTARTED_IP)" - IP_MISMATCHES=$((IP_MISMATCHES + 1)) - fi -done - -if [ "$IP_MISMATCHES" -gt 0 ]; then - echo " ERROR: $IP_MISMATCHES instances had IP changes — ENI not persisting through stop/start" - exit 1 -fi - -echo " Stop/start IP persistence verified — all IPs match" - -# Step 6: Clean up VPC instances -echo "" -echo "Step 6: Clean up VPC resources" -echo "----------------------------------------" - -terminate_and_wait "${VPC_INSTANCE_IDS[@]}" || true - -# Clean up subnet and VPC -echo " Deleting subnet $SUBNET_ID..." -$AWS_EC2 delete-subnet --subnet-id "$SUBNET_ID" 2>/dev/null || echo " (subnet delete failed, may have ENIs)" - -echo " Deleting VPC $VPC_ID..." -$AWS_EC2 delete-vpc --vpc-id "$VPC_ID" 2>/dev/null || echo " (vpc delete failed, may have subnets)" - -echo " VPC networking tests passed" - -# Phase 6: Cluster Shutdown + Restart -echo "" -echo "Phase 6: Cluster Shutdown + Restart" -echo "========================================" -echo "Testing hive admin cluster shutdown command..." - -# Test 6a: Dry-run shutdown -echo "" -echo "Test 6a: Dry-Run Shutdown" -echo "----------------------------------------" -echo "Running cluster shutdown in dry-run mode..." - -DRY_RUN_OUTPUT=$(./bin/hive admin cluster shutdown --dry-run --config "$HOME/node1/config/hive.toml" 2>&1) -echo "$DRY_RUN_OUTPUT" - -# Validate dry-run output contains expected phases -for phase in GATE DRAIN STORAGE PERSIST INFRA; do - if echo "$DRY_RUN_OUTPUT" | grep -qi "$phase"; then - echo " Phase $phase found in shutdown plan" - else - echo " WARNING: Phase $phase not found in dry-run output" - fi -done -echo " Dry-run shutdown test passed" - -# Test 6b: Real coordinated shutdown -echo "" -echo "Test 6b: Coordinated Cluster Shutdown" -echo "----------------------------------------" -echo "Running cluster shutdown..." - -./bin/hive admin cluster shutdown --force --timeout 30s --config "$HOME/node1/config/hive.toml" 2>&1 || { - echo " WARNING: Cluster shutdown command returned non-zero exit code" -} -CLUSTER_SERVICES_STARTED="false" - -# Verify all services are down -echo " Waiting for services to stop..." -sleep 2 -if ! verify_all_services_down; then - echo " Some services still running, force-cleaning..." - force_cleanup_all_nodes -fi - -# Test 6c: Restart and recovery -echo "" -echo "Test 6c: Cluster Restart + Recovery" -echo "----------------------------------------" -echo "Restarting all node services concurrently..." - -# Cluster restart requires concurrent startup: NATS needs route peers to form, -# Predastore needs Raft quorum (2/3), and the daemon needs JetStream. -# Sequential start would leave node1 waiting for quorum that never arrives. -start_node_services 1 "$HOME/node1" & -start_node_services 2 "$HOME/node2" & -start_node_services 3 "$HOME/node3" & -wait -CLUSTER_SERVICES_STARTED="true" - -echo "" -echo "Waiting for cluster to stabilize..." -sleep 5 - -# Verify NATS cluster reformed -echo "" -verify_nats_cluster 3 || { - echo "WARNING: NATS cluster verification failed after restart" -} - -# Wait for gateway -echo "" -wait_for_gateway "${NODE1_IP}" 15 - -# Wait for daemon readiness -wait_for_daemon_ready "https://${NODE1_IP}:${AWSGW_PORT}" - -# Smoke test: describe-instance-types -echo "" -echo "Running post-restart smoke test..." -SMOKE_OUTPUT=$($AWS_EC2 describe-instance-types --query 'InstanceTypes[*].InstanceType' --output text 2>/dev/null) -if [ -n "$SMOKE_OUTPUT" ] && [ "$SMOKE_OUTPUT" != "None" ]; then - echo " Smoke test passed: describe-instance-types returned: $SMOKE_OUTPUT" -else - echo " ERROR: Smoke test failed: describe-instance-types returned empty/None" - exit 1 -fi - -echo " Cluster shutdown + restart test passed" - -# Test 6d: Instance relaunch and terminate after restart -echo "" -echo "Test 6d: Instance Relaunch + Terminate" -echo "----------------------------------------" -echo "Waiting for instances to relaunch after cluster restart..." - -# Two instances were running before shutdown — the daemon will relaunch them. -# Must wait for them to finish launching (pending → running) before terminate -# will work, because the NATS per-instance subscription is only created after -# QEMU starts. -for instance_id in "${INSTANCE_IDS[0]}" "${INSTANCE_IDS[1]}"; do - echo " Waiting for $instance_id to finish relaunching..." + # Wait for attachment COUNT=0 - while [ $COUNT -lt 30 ]; do - STATE=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "unknown") - if [ "$STATE" = "running" ]; then - echo " $instance_id relaunched successfully: $STATE" + while [ $COUNT -lt 15 ]; do + ATTACH_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ + --query 'Volumes[0].Attachments[0].State' --output text) + VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ + --query 'Volumes[0].State' --output text) + if [ "$VOL_STATE" == "in-use" ] && [ "$ATTACH_STATE" == "attached" ]; then + echo " Volume attached" break fi sleep 2 COUNT=$((COUNT + 1)) done - if [ $COUNT -ge 30 ]; then - echo " WARNING: $instance_id still in $STATE after 60s" + + if [ "$ATTACH_STATE" == "attached" ]; then + pass_test "Volume attach" + else + fail_test "Volume attach" + fi + + # Detach + echo " Detaching volume..." + $AWS_EC2 detach-volume --volume-id "$TEST_VOLUME_ID" > /dev/null + + COUNT=0 + while [ $COUNT -lt 15 ]; do + VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ + --query 'Volumes[0].State' --output text) + if [ "$VOL_STATE" == "available" ]; then + echo " Volume detached" + break + fi + sleep 2 + COUNT=$((COUNT + 1)) + done + + if [ "$VOL_STATE" == "available" ]; then + pass_test "Volume detach" + else + fail_test "Volume detach" + fi + + # Delete + echo " Deleting volume..." + $AWS_EC2 delete-volume --volume-id "$TEST_VOLUME_ID" + + COUNT=0 + while [ $COUNT -lt 15 ]; do + set +e + VOL_CHECK=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ + --query 'Volumes[0].VolumeId' --output text 2>&1) + DESCRIBE_EXIT=$? + set -e + if [ $DESCRIBE_EXIT -ne 0 ] || [ "$VOL_CHECK" == "None" ] || [ -z "$VOL_CHECK" ]; then + echo " Volume deleted" + break + fi + sleep 2 + COUNT=$((COUNT + 1)) + done + + if [ $COUNT -lt 15 ]; then + pass_test "Volume delete" + else + fail_test "Volume delete" + fi +fi + +echo "" + +# ========================================================================== +# Phase 6: Cross-Node Gateway Access +# ========================================================================== +echo "Phase 6: Cross-Node Gateway Access" +echo "========================================" +echo "Verifying describe-instances returns same results via each node's gateway..." + +BASELINE_COUNT=$($AWS_EC2 describe-instances \ + --query 'length(Reservations[*].Instances[*][])' --output text) +echo " Baseline (node1): $BASELINE_COUNT instances" + +for i in 1 2; do + ip="${NODE_IPS[$i]}" + COUNT=$(aws_via "$ip" ec2 describe-instances \ + --query 'length(Reservations[*].Instances[*][])' --output text 2>/dev/null || echo "0") + echo " Node$((i+1)) ($ip): $COUNT instances" + if [ "$COUNT" -eq "$BASELINE_COUNT" ]; then + pass_test "Cross-node gateway (node$((i+1)))" + else + echo " WARNING: Instance count mismatch (expected $BASELINE_COUNT, got $COUNT)" + fail_test "Cross-node gateway (node$((i+1)))" fi done -# Verify crash-loop instance (INSTANCE_IDS[2]) stayed in error — daemon should -# not relaunch an instance that exceeded its crash restart limit. -CRASH_LOOP_STATE=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[2]}" \ - --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "unknown") -echo " Crash-loop instance ${INSTANCE_IDS[2]} state: $CRASH_LOOP_STATE" -if [ "$CRASH_LOOP_STATE" != "error" ]; then - echo " WARNING: Expected crash-loop instance to remain in error state (got: $CRASH_LOOP_STATE)" +echo "" + +# ========================================================================== +# Phase 7: Cross-Node Operations +# ========================================================================== +echo "Phase 7: Cross-Node Operations" +echo "========================================" +echo "Testing stop/start via a gateway on a DIFFERENT node than the hosting node..." + +# Pick first instance and find its host +TEST_INSTANCE="${INSTANCE_IDS[0]}" +INSTANCE_HOST=$(find_instance_node "$TEST_INSTANCE" || echo "$LOCAL_IP") +echo " Instance $TEST_INSTANCE is on $INSTANCE_HOST" + +# Pick a different node's gateway for the operation +OTHER_GW="" +for ip in "${NODE_IPS[@]}"; do + if [ "$ip" != "$INSTANCE_HOST" ]; then + OTHER_GW="$ip" + break + fi +done +echo " Will operate via gateway on $OTHER_GW" + +# Stop via other gateway +echo " Stopping instance via $OTHER_GW..." +aws_via "$OTHER_GW" ec2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null +wait_for_instance_state "$TEST_INSTANCE" "stopped" 30 "$OTHER_GW" + +# Pick yet another gateway for start (or same other if only 2 choices) +THIRD_GW="" +for ip in "${NODE_IPS[@]}"; do + if [ "$ip" != "$INSTANCE_HOST" ] && [ "$ip" != "$OTHER_GW" ]; then + THIRD_GW="$ip" + break + fi +done +THIRD_GW="${THIRD_GW:-$OTHER_GW}" +echo " Starting instance via $THIRD_GW..." +aws_via "$THIRD_GW" ec2 start-instances --instance-ids "$TEST_INSTANCE" > /dev/null +wait_for_instance_state "$TEST_INSTANCE" "running" 30 "$THIRD_GW" + +pass_test "Cross-node stop/start" + +echo "" + +# ========================================================================== +# Phase 8: Node Failure +# ========================================================================== +echo "Phase 8: Node Failure" +echo "========================================" +echo "Stopping services on node2 ($NODE2_IP) to simulate node failure..." + +# Stop services on node2 only — HIVE_FORCE_LOCAL_STOP prevents coordinated +# cluster shutdown which would kill all nodes via NATS +peer_ssh "$NODE2_IP" "cd ~/Development/mulga/hive && HIVE_FORCE_LOCAL_STOP=1 ./scripts/stop-dev.sh" || { + echo " WARNING: stop-dev.sh returned non-zero (may be expected)" +} + +# Wait a moment for cluster to detect the failure +sleep 5 + +# Verify node1 and node3 still serve requests +echo " Verifying node1 still serves requests..." +N1_RESULT=$(aws_via "$LOCAL_IP" ec2 describe-instance-types \ + --query 'InstanceTypes[0].InstanceType' --output text 2>/dev/null || echo "FAIL") +if [ "$N1_RESULT" != "FAIL" ] && [ -n "$N1_RESULT" ] && [ "$N1_RESULT" != "None" ]; then + echo " Node1: responding ($N1_RESULT)" + pass_test "Node1 survives node2 failure" +else + echo " ERROR: Node1 not responding after node2 failure" + fail_test "Node1 survives node2 failure" +fi + +echo " Verifying node3 still serves requests..." +N3_RESULT=$(aws_via "$NODE3_IP" ec2 describe-instance-types \ + --query 'InstanceTypes[0].InstanceType' --output text 2>/dev/null || echo "FAIL") +if [ "$N3_RESULT" != "FAIL" ] && [ -n "$N3_RESULT" ] && [ "$N3_RESULT" != "None" ]; then + echo " Node3: responding ($N3_RESULT)" + pass_test "Node3 survives node2 failure" +else + echo " ERROR: Node3 not responding after node2 failure" + fail_test "Node3 survives node2 failure" +fi + +# Check NATS degraded state (should have 1 route instead of 2) +NATS_DEGRADED=$(curl -s "http://${LOCAL_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null) +DEGRADED_PEERS=$(echo "$NATS_DEGRADED" | jq -r '[.routes[].remote_name] | unique | length' 2>/dev/null || echo "0") +echo " NATS peers during failure: $DEGRADED_PEERS (expected: 1)" +if [ "$DEGRADED_PEERS" -eq 1 ]; then + pass_test "NATS degraded mode" +else + echo " WARNING: Expected 1 NATS peer during node2 failure, got $DEGRADED_PEERS" + # Not fatal — NATS might take a moment to detect +fi + +# Verify describe-instances still works from surviving nodes +echo " Verifying describe-instances from surviving nodes..." +SURVIVING_COUNT=$(aws_via "$LOCAL_IP" ec2 describe-instances \ + --query 'length(Reservations[*].Instances[*][])' --output text 2>/dev/null || echo "0") +echo " Instances visible from node1: $SURVIVING_COUNT" +if [ "$SURVIVING_COUNT" -gt 0 ]; then + pass_test "Describe-instances during node failure" +else + fail_test "Describe-instances during node failure" fi -# Terminate all instances (including the crash-loop one in error state) echo "" + +# ========================================================================== +# Phase 9: Node Recovery +# ========================================================================== +echo "Phase 9: Node Recovery" +echo "========================================" +echo "Restarting services on node2 ($NODE2_IP)..." + +peer_ssh "$NODE2_IP" "cd ~/Development/mulga/hive && ./scripts/start-dev.sh" || { + echo " ERROR: Failed to restart services on node2" + fail_test "Node2 restart" +} + +# Wait for NATS to reform (2 routes again) +echo " Waiting for NATS cluster to reform..." +ATTEMPT=0 +REFORMED=false +while [ $ATTEMPT -lt 30 ]; do + NATS_RECOVER=$(curl -s "http://${LOCAL_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null) + RECOVER_PEERS=$(echo "$NATS_RECOVER" | jq -r '[.routes[].remote_name] | unique | length' 2>/dev/null || echo "0") + if [ "$RECOVER_PEERS" -ge 2 ]; then + echo " NATS cluster reformed ($RECOVER_PEERS peers)" + REFORMED=true + break + fi + echo " Waiting for NATS reform... ($((ATTEMPT + 1))/30, peers: $RECOVER_PEERS)" + sleep 2 + ATTEMPT=$((ATTEMPT + 1)) +done + +if [ "$REFORMED" = true ]; then + pass_test "NATS cluster reform" +else + echo " WARNING: NATS did not fully reform within timeout" + fail_test "NATS cluster reform" +fi + +# Verify node2 gateway is back +echo " Waiting for node2 gateway..." +ATTEMPT=0 +GW_BACK=false +while [ $ATTEMPT -lt 15 ]; do + if curl -k -s "https://${NODE2_IP}:${AWSGW_PORT}" > /dev/null 2>&1; then + echo " Node2 gateway is back" + GW_BACK=true + break + fi + sleep 2 + ATTEMPT=$((ATTEMPT + 1)) +done + +if [ "$GW_BACK" = true ]; then + pass_test "Node2 gateway recovery" +else + fail_test "Node2 gateway recovery" +fi + +# Verify hive get nodes shows 3 Ready again +echo " Checking hive get nodes after recovery..." +GET_NODES_RECOVER=$($HIVE_BIN get nodes --config "$HIVE_CONFIG" --timeout 10s 2>/dev/null || echo "") +echo "$GET_NODES_RECOVER" +READY_RECOVER=$(echo "$GET_NODES_RECOVER" | grep -c "Ready" || true) +if [ "$READY_RECOVER" -ge 3 ]; then + pass_test "All nodes Ready after recovery" +else + echo " WARNING: Only $READY_RECOVER Ready nodes after recovery (expected 3)" + fail_test "All nodes Ready after recovery" +fi + +# Verify node2 can serve requests +echo " Verifying node2 serves requests after recovery..." +N2_RESULT=$(aws_via "$NODE2_IP" ec2 describe-instance-types \ + --query 'InstanceTypes[0].InstanceType' --output text 2>/dev/null || echo "FAIL") +if [ "$N2_RESULT" != "FAIL" ] && [ -n "$N2_RESULT" ] && [ "$N2_RESULT" != "None" ]; then + echo " Node2 is serving requests again ($N2_RESULT)" + pass_test "Node2 serves requests after recovery" +else + fail_test "Node2 serves requests after recovery" +fi + +echo "" + +# ========================================================================== +# Phase 10: Cleanup +# ========================================================================== +echo "Phase 10: Cleanup" +echo "========================================" + echo "Terminating all instances..." -if ! terminate_and_wait "${INSTANCE_IDS[@]}"; then - echo "" - echo "ERROR: Some instances failed to terminate properly after restart" - dump_all_node_logs +if terminate_and_wait "${INSTANCE_IDS[@]}"; then + pass_test "Instance termination" +else + fail_test "Instance termination" +fi + +# ========================================================================== +# Summary +# ========================================================================== +echo "" +echo "========================================" +echo "Real Multi-Node E2E Test Summary" +echo "========================================" +echo " Passed: $TESTS_PASSED" +echo " Failed: $TESTS_FAILED" +if [ ${#FAILED_TESTS[@]} -gt 0 ]; then + echo " Failed tests:" + for t in "${FAILED_TESTS[@]}"; do + echo " - $t" + done +fi +echo "========================================" + +if [ $TESTS_FAILED -gt 0 ]; then + echo "SOME TESTS FAILED" exit 1 fi -echo " Instance relaunch + terminate after restart passed" - -echo "" -echo "========================================" -echo "Multi-Node E2E Tests Completed Successfully" -echo "========================================" +echo "All tests passed!" exit 0 diff --git a/tests/e2e/run-pseudo-multinode-e2e.sh b/tests/e2e/run-pseudo-multinode-e2e.sh new file mode 100755 index 000000000..8c8ac15e2 --- /dev/null +++ b/tests/e2e/run-pseudo-multinode-e2e.sh @@ -0,0 +1,2423 @@ +#!/bin/bash +set -e + +# Ensure Go is on PATH (SSH non-interactive shells don't source .bashrc) +export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" + +# Pseudo Multi-Node E2E test runner +# This script sets up a 3-node Hive cluster using simulated IPs on the loopback interface +# and runs distributed instance tests on a single VM. + +# Ensure we are in the project root +cd "$(dirname "$0")/../.." + +# Source helper functions +source ./tests/e2e/lib/multinode-helpers.sh + +# Cleanup function - ensure resources are cleaned up on exit +cleanup() { + local exit_code=$? + + echo "" + echo "Cleanup triggered (exit code: $exit_code)..." + + if [ $exit_code -ne 0 ]; then + dump_all_node_logs + fi + + # Kill any lingering formation background processes + [ -n "$LEADER_INIT_PID" ] && kill "$LEADER_INIT_PID" 2>/dev/null || true + [ -n "$JOIN2_PID" ] && kill "$JOIN2_PID" 2>/dev/null || true + [ -n "$JOIN3_PID" ] && kill "$JOIN3_PID" 2>/dev/null || true + + # Try coordinated shutdown first (only if NATS is likely still up) + if [ "$CLUSTER_SERVICES_STARTED" = "true" ]; then + echo "Attempting coordinated cluster shutdown..." + if timeout 60 ./bin/hive admin cluster shutdown --force --timeout 30s --config "$HOME/node1/config/hive.toml" 2>/dev/null; then + echo "Coordinated shutdown succeeded" + else + echo "Coordinated shutdown failed, falling back to per-node stop..." + stop_all_nodes || true + fi + else + stop_all_nodes || true + fi + + # Force-kill anything that survived and clean up stale locks + force_cleanup_all_nodes || true + + # Remove simulated IPs + remove_simulated_ips || true + + echo "Cleanup complete" +} +trap cleanup EXIT + +# PIDs for background formation processes (used in cleanup) +JOIN2_PID="" +JOIN3_PID="" + +# Track whether cluster services have been started (for cleanup trap) +CLUSTER_SERVICES_STARTED="false" + +# Use Hive profile for AWS CLI +export AWS_PROFILE=hive +# Trust Hive CA for all profiles (AWS CLI v2 bundles its own Python/certifi, ignores system CA store) +export AWS_CA_BUNDLE="$HOME/node1/config/ca.pem" + + +echo "========================================" +echo "Multi-Node E2E Test Suite" +echo "========================================" +echo "" + +# Phase 1: Environment Setup +echo "Phase 1: Environment Setup" +echo "========================================" + +# Check for KVM support +echo "Checking for KVM support..." +if [ -e /dev/kvm ]; then + echo " /dev/kvm exists" + if [ -w /dev/kvm ]; then + echo " /dev/kvm is writable" + else + echo " ERROR: /dev/kvm is NOT writable" + exit 1 + fi +else + echo " ERROR: /dev/kvm does NOT exist" + exit 1 +fi + +# Check for ip command (iproute2) +if ! command -v ip &> /dev/null; then + echo " ERROR: 'ip' command not found. Install iproute2." + exit 1 +fi + + +# Setup simulated network +echo "" +echo "Setting up simulated network..." +add_simulated_ips + +echo "" + +# Phase 2: Cluster Initialization +echo "Phase 2: Cluster Initialization" +echo "========================================" + +# Background init — starts formation server, generates certs first +echo "" +init_leader_node + +# Trust CA cert (exists before formation completes — cert generation is the first step) +echo "" +echo "Adding Hive CA certificate to system trust store..." +sudo cp ~/node1/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt +sudo update-ca-certificates + +# Background BOTH joins — they poll /formation/status until all 3 nodes have joined. +# Must be concurrent: each join blocks until formation is complete. +echo "" +echo "Joining follower nodes concurrently..." +join_follower_node 2 & +JOIN2_PID=$! +join_follower_node 3 & +JOIN3_PID=$! + +# Wait for formation to complete (all processes generate their configs) +echo "Waiting for cluster formation to complete..." +wait $JOIN2_PID || { echo "ERROR: Node 2 join failed"; exit 1; } +wait $JOIN3_PID || { echo "ERROR: Node 3 join failed"; exit 1; } +wait $LEADER_INIT_PID || { echo "ERROR: Leader init failed"; exit 1; } +echo "Cluster formation complete — all configs generated" + +# Now start services (configs exist for all nodes) +echo "" +echo "Starting node services..." +start_node_services 1 "$HOME/node1" +start_node_services 2 "$HOME/node2" +start_node_services 3 "$HOME/node3" +CLUSTER_SERVICES_STARTED="true" + +# Wait for all services to stabilize +echo "" +echo "Waiting for cluster to stabilize..." +sleep 5 + +# Phase 3: Cluster Health Verification +echo "" +echo "Phase 3: Cluster Health Verification" +echo "========================================" + +# Verify NATS cluster +echo "" +verify_nats_cluster 3 || { + echo "WARNING: NATS cluster verification failed, continuing anyway..." +} + +# Verify Predastore cluster +echo "" +verify_predastore_cluster 3 || { + echo "ERROR: Predastore cluster verification failed" + dump_all_node_logs + exit 1 +} + +# Wait for gateway on node1 (primary gateway) +echo "" +wait_for_gateway "${NODE1_IP}" 15 + +# Wait for daemon NATS subscriptions to be active +wait_for_daemon_ready "https://${NODE1_IP}:${AWSGW_PORT}" + +# Define AWS CLI args pointing to node1's gateway +AWS_EC2="aws --endpoint-url https://${NODE1_IP}:${AWSGW_PORT} ec2" +AWS_IAM="aws --endpoint-url https://${NODE1_IP}:${AWSGW_PORT} iam" + +# Discover the cluster's availability zone and region dynamically +HIVE_AZ=$($AWS_EC2 describe-availability-zones --query 'AvailabilityZones[0].ZoneName' --output text) +HIVE_REGION=$($AWS_EC2 describe-availability-zones --query 'AvailabilityZones[0].RegionName' --output text) +echo "Discovered AZ: $HIVE_AZ, Region: $HIVE_REGION" + +# Phase 3b: Cluster Stats CLI (Multi-Node) +echo "" +echo "Phase 3b: Cluster Stats CLI (Multi-Node)" +echo "========================================" + +# Test hive get nodes — should show all 3 nodes as Ready +echo "Testing hive get nodes..." +GET_NODES_OUTPUT=$(./bin/hive get nodes --config "$HOME/node1/config/hive.toml" --timeout 5s 2>/dev/null) +echo "$GET_NODES_OUTPUT" +READY_COUNT=$(echo "$GET_NODES_OUTPUT" | grep -c "Ready" || true) +if [ "$READY_COUNT" -lt 3 ]; then + echo "WARNING: hive get nodes shows $READY_COUNT Ready nodes (expected 3)" +fi +echo "hive get nodes passed ($READY_COUNT Ready nodes)" + +# Test hive top nodes — should show resource stats for all nodes +echo "Testing hive top nodes..." +TOP_NODES_OUTPUT=$(./bin/hive top nodes --config "$HOME/node1/config/hive.toml" --timeout 5s 2>/dev/null) +echo "$TOP_NODES_OUTPUT" +if ! echo "$TOP_NODES_OUTPUT" | grep -q "INSTANCE TYPE"; then + echo "WARNING: hive top nodes did not show instance type capacity table" +fi +echo "hive top nodes passed" + +# Test hive get vms — should show no VMs yet +echo "Testing hive get vms (empty)..." +GET_VMS_OUTPUT=$(./bin/hive get vms --config "$HOME/node1/config/hive.toml" --timeout 5s 2>/dev/null) +echo "$GET_VMS_OUTPUT" +echo "hive get vms (empty) passed" + +# Verify gateway responds +echo "" +echo "Testing gateway connectivity..." +$AWS_EC2 describe-regions | jq -e '.Regions | length > 0' || { + echo "ERROR: Gateway not responding correctly" + exit 1 +} +echo " Gateway is responding" + +# Phase 4: Image and Key Setup +echo "" +echo "Phase 4: Image and Key Setup" +echo "========================================" + +# Discover instance types +echo "Discovering available instance types..." +AVAILABLE_TYPES=$($AWS_EC2 describe-instance-types --query 'InstanceTypes[*].InstanceType' --output text) +echo " Available: $AVAILABLE_TYPES" + +# Pick nano instance type +INSTANCE_TYPE=$(echo $AVAILABLE_TYPES | tr ' ' '\n' | grep -m1 'nano') +if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" == "None" ]; then + echo "ERROR: No instance types found" + exit 1 +fi +echo " Selected: $INSTANCE_TYPE" + +# Get architecture +ARCH=$($AWS_EC2 describe-instance-types --instance-types "$INSTANCE_TYPE" \ + --query 'InstanceTypes[0].ProcessorInfo.SupportedArchitectures[0]' --output text) +echo " Architecture: $ARCH" + +# Create test key +echo "" +echo "Creating test key pair..." +KEY_MATERIAL=$($AWS_EC2 create-key-pair --key-name multinode-test-key --query 'KeyMaterial' --output text) +echo "$KEY_MATERIAL" > multinode-test-key.pem +chmod 600 multinode-test-key.pem +echo " Key created: multinode-test-key" + +# Import Ubuntu image (use node1's config and hive-dir) +echo "" +echo "Importing Ubuntu image..." +IMPORT_LOG=$(./bin/hive admin images import \ + --file ~/images/ubuntu-24.04.img \ + --arch "$ARCH" \ + --distro ubuntu \ + --version 24.04 \ + --config "$HOME/node1/config/hive.toml" \ + --hive-dir "$HOME/node1/" \ + --force) +echo "Import output: $IMPORT_LOG" +AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+') + +if [ -z "$AMI_ID" ]; then + echo "ERROR: Failed to capture AMI ID" + exit 1 +fi +echo " AMI ID: $AMI_ID" + +# Verify AMI +$AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.ImageId==\"$AMI_ID\")" > /dev/null +echo " AMI verified" + +# Phase 4b: Multi-Node Key Pair Operations +echo "" +echo "Phase 4b: Multi-Node Key Pair Operations" +echo "========================================" +echo "Testing key pair CRUD across the cluster..." + +# Import a key pair (goes to a random node via queue group) +echo " Generating local RSA key for import..." +ssh-keygen -t rsa -b 2048 -f multinode-test-key-2-local -N "" +$AWS_EC2 import-key-pair --key-name multinode-test-key-2 --public-key-material "fileb://multinode-test-key-2-local.pub" +echo " Imported key: multinode-test-key-2" + +# Describe key pairs — verify both keys are visible +echo " Verifying both keys via describe-key-pairs..." +KEY_LIST=$($AWS_EC2 describe-key-pairs --query 'KeyPairs[*].KeyName' --output text) +echo " Keys found: $KEY_LIST" +echo "$KEY_LIST" | grep -q "multinode-test-key" || { + echo " ERROR: multinode-test-key not found in describe-key-pairs" + exit 1 +} +echo "$KEY_LIST" | grep -q "multinode-test-key-2" || { + echo " ERROR: multinode-test-key-2 not found in describe-key-pairs" + exit 1 +} +echo " Both keys visible" + +# Delete the imported key +echo " Deleting multinode-test-key-2..." +$AWS_EC2 delete-key-pair --key-name multinode-test-key-2 + +# Verify deletion +REMAINING_KEYS=$($AWS_EC2 describe-key-pairs --query 'KeyPairs[*].KeyName' --output text) +if echo "$REMAINING_KEYS" | grep -q "multinode-test-key-2"; then + echo " ERROR: multinode-test-key-2 was not deleted" + exit 1 +fi +echo " Key deletion verified" +echo " Multi-node key pair operations passed" + +# Phase 5: Multi-Node Instance Tests +echo "" +echo "Phase 5: Multi-Node Instance Tests" +echo "========================================" + +# Test 1: Instance Distribution +echo "" +echo "Test 1: Instance Distribution" +echo "----------------------------------------" +echo "Launching 3 instances to test distribution across nodes..." + +INSTANCE_IDS=() +for i in 1 2 3; do + echo " Launching instance $i..." + RUN_OUTPUT=$($AWS_EC2 run-instances \ + --image-id "$AMI_ID" \ + --instance-type "$INSTANCE_TYPE" \ + --key-name multinode-test-key) + + INSTANCE_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId') + if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "null" ]; then + echo " ERROR: Failed to launch instance $i" + echo " Output: $RUN_OUTPUT" + exit 1 + fi + echo " Launched: $INSTANCE_ID" + INSTANCE_IDS+=("$INSTANCE_ID") + + # Small delay between launches to encourage distribution + sleep 2 +done + +# Wait for all instances to be running +echo "" +echo "Waiting for instances to reach running state..." +for instance_id in "${INSTANCE_IDS[@]}"; do + wait_for_instance_state "$instance_id" "running" 30 || { + echo "ERROR: Instance $instance_id failed to start" + exit 1 + } +done + +# Check distribution +echo "" +check_instance_distribution + +# Verify hive get vms shows all running instances +echo "" +echo "Verifying hive get vms (with running VMs)..." +GET_VMS_OUTPUT=$(./bin/hive get vms --config "$HOME/node1/config/hive.toml" --timeout 5s 2>/dev/null) +echo "$GET_VMS_OUTPUT" +for instance_id in "${INSTANCE_IDS[@]}"; do + if ! echo "$GET_VMS_OUTPUT" | grep -q "$instance_id"; then + echo "WARNING: hive get vms did not show instance $instance_id" + fi +done +echo "hive get vms shows launched instances" + +# Test 1a-ii: SSH Connectivity & Volume Verification +echo "" +echo "Test 1a-ii: SSH Connectivity & Volume Verification" +echo "----------------------------------------" +echo "Testing SSH into all 3 instances..." + +# Arrays to store SSH details for post-termination verification +SSH_PORTS=() +SSH_HOSTS=() + +for idx in "${!INSTANCE_IDS[@]}"; do + instance_id="${INSTANCE_IDS[$idx]}" + echo "" + echo " Instance $((idx + 1)): $instance_id" + + # Get SSH connection details from QEMU process + echo " Getting SSH port..." + SSH_PORT=$(get_ssh_port "$instance_id") + if [ -z "$SSH_PORT" ]; then + echo " ERROR: Failed to get SSH port for instance $instance_id" + exit 1 + fi + SSH_HOST=$(get_ssh_host "$instance_id") + echo " SSH endpoint: $SSH_HOST:$SSH_PORT" + + SSH_PORTS+=("$SSH_PORT") + SSH_HOSTS+=("$SSH_HOST") + + # Wait for SSH to become ready (VM boot + cloud-init) + wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 30 + + # Test basic SSH connectivity + test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" + + # Check root volume size via lsblk + echo " Verifying root volume size from inside the VM..." + ROOT_VOL_ID_SSH=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ + --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) + ROOT_VOL_SIZE_API=$($AWS_EC2 describe-volumes --volume-ids "$ROOT_VOL_ID_SSH" \ + --query 'Volumes[0].Size' --output text) + # Find the disk backing the root filesystem (avoids picking up floppy/cdrom devices) + # 1. findmnt gets the source device for / (e.g. /dev/vda1) + # 2. lsblk PKNAME resolves to parent disk name (e.g. vda) + # 3. lsblk -b -d gets that disk's byte size + ROOT_DISK_BYTES=$(ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=5 \ + -o BatchMode=yes \ + -p "$SSH_PORT" \ + -i "multinode-test-key.pem" \ + ec2-user@"$SSH_HOST" 'SRC=$(findmnt -n -o SOURCE /); PKN=$(lsblk -n -o PKNAME "$SRC" 2>/dev/null | head -1); DEV=${PKN:-$(basename "$SRC")}; lsblk -b -d -n -o SIZE "/dev/$DEV"' | tr -d '[:space:]') + if [ -z "$ROOT_DISK_BYTES" ] || [ "$ROOT_DISK_BYTES" = "0" ]; then + echo " ERROR: Failed to get root disk size from VM (got: '$ROOT_DISK_BYTES')" + echo " lsblk debug output:" + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ + -o ConnectTimeout=5 -o BatchMode=yes -p "$SSH_PORT" -i "multinode-test-key.pem" \ + ec2-user@"$SSH_HOST" 'lsblk -b -d; echo "---"; findmnt -n -o SOURCE /; cat /proc/partitions' || true + exit 1 + fi + ROOT_DISK_GIB=$((ROOT_DISK_BYTES / 1073741824)) + echo " Root disk size from VM: ${ROOT_DISK_GIB}GiB (API reports: ${ROOT_VOL_SIZE_API}GiB)" + if [ "$ROOT_DISK_GIB" -ne "$ROOT_VOL_SIZE_API" ]; then + echo " ERROR: Root volume size mismatch: VM reports ${ROOT_DISK_GIB}GiB, API reports ${ROOT_VOL_SIZE_API}GiB" + exit 1 + fi + echo " Root volume size verified" + + # Verify hostname contains instance ID + echo " Verifying hostname inside the VM..." + VM_HOSTNAME=$(ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 \ + -o BatchMode=yes \ + -p "$SSH_PORT" \ + -i "multinode-test-key.pem" \ + ec2-user@"$SSH_HOST" 'hostname' 2>/dev/null) + echo " VM hostname: $VM_HOSTNAME" + # Hostname uses truncated ID: hive-vm- + SHORT_ID=$(echo "$instance_id" | sed 's/^i-//' | cut -c1-8) + if echo "$VM_HOSTNAME" | grep -q "$SHORT_ID"; then + echo " Hostname contains instance ID prefix ($SHORT_ID)" + else + echo " WARNING: Hostname '$VM_HOSTNAME' does not contain instance ID prefix '$SHORT_ID' (non-fatal)" + fi +done + +echo "" +echo " SSH connectivity and volume verification passed for all instances" + +# Test 1b: Volume Lifecycle (Attach/Detach) +echo "" +echo "Test 1b: Volume Lifecycle (Attach/Detach)" +echo "----------------------------------------" +echo "Testing volume create -> resize -> attach -> detach -> delete..." + +# Create a test volume +echo " Creating 10GB volume in ${HIVE_AZ}..." +CREATE_OUTPUT=$($AWS_EC2 create-volume --size 10 --availability-zone "$HIVE_AZ") +TEST_VOLUME_ID=$(echo "$CREATE_OUTPUT" | jq -r '.VolumeId') + +if [ -z "$TEST_VOLUME_ID" ] || [ "$TEST_VOLUME_ID" == "null" ]; then + echo " ERROR: Failed to create test volume" + echo " Output: $CREATE_OUTPUT" + exit 1 +fi +echo " Created volume: $TEST_VOLUME_ID" + +# Resize to 20GB +NEW_SIZE=20 +echo " Modifying volume to ${NEW_SIZE}GB..." +$AWS_EC2 modify-volume --volume-id "$TEST_VOLUME_ID" --size "$NEW_SIZE" + +# Verify resize +echo " Verifying resize..." +COUNT=0 +while [ $COUNT -lt 15 ]; do + VOLUME_SIZE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ + --query 'Volumes[0].Size' --output text) + + if [ "$VOLUME_SIZE" -eq "$NEW_SIZE" ]; then + echo " Volume resized successfully to ${NEW_SIZE}GB" + break + fi + + sleep 2 + COUNT=$((COUNT + 1)) +done + +if [ "$VOLUME_SIZE" -ne "$NEW_SIZE" ]; then + echo " ERROR: Volume failed to resize to ${NEW_SIZE}GB (current: ${VOLUME_SIZE}GB)" + exit 1 +fi + +# Attach volume to the first running instance +echo " Attaching volume $TEST_VOLUME_ID to instance ${INSTANCE_IDS[0]}..." +$AWS_EC2 attach-volume --volume-id "$TEST_VOLUME_ID" --instance-id "${INSTANCE_IDS[0]}" --device /dev/sdf + +# Verify attachment +echo " Verifying volume attachment..." +COUNT=0 +while [ $COUNT -lt 15 ]; do + ATTACH_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ + --query 'Volumes[0].Attachments[0].State' --output text) + ATTACH_INSTANCE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ + --query 'Volumes[0].Attachments[0].InstanceId' --output text) + VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ + --query 'Volumes[0].State' --output text) + + if [ "$VOL_STATE" == "in-use" ] && [ "$ATTACH_STATE" == "attached" ] && [ "$ATTACH_INSTANCE" == "${INSTANCE_IDS[0]}" ]; then + echo " Volume attached successfully (State=$VOL_STATE, AttachState=$ATTACH_STATE, Instance=$ATTACH_INSTANCE)" + break + fi + + sleep 2 + COUNT=$((COUNT + 1)) +done + +if [ "$ATTACH_STATE" != "attached" ] || [ "$ATTACH_INSTANCE" != "${INSTANCE_IDS[0]}" ]; then + echo " ERROR: Volume attachment verification failed (AttachState=$ATTACH_STATE, Instance=$ATTACH_INSTANCE)" + exit 1 +fi + +# Detach volume (without --instance-id to test gateway resolution path) +echo " Detaching volume $TEST_VOLUME_ID..." +$AWS_EC2 detach-volume --volume-id "$TEST_VOLUME_ID" + +# Verify detachment +echo " Verifying volume detachment..." +COUNT=0 +while [ $COUNT -lt 15 ]; do + VOL_STATE=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ + --query 'Volumes[0].State' --output text) + + if [ "$VOL_STATE" == "available" ]; then + echo " Volume detached successfully (State=$VOL_STATE)" + break + fi + + sleep 2 + COUNT=$((COUNT + 1)) +done + +if [ "$VOL_STATE" != "available" ]; then + echo " ERROR: Volume detachment verification failed (State=$VOL_STATE)" + exit 1 +fi + +# Delete the test volume +echo " Deleting test volume $TEST_VOLUME_ID..." +$AWS_EC2 delete-volume --volume-id "$TEST_VOLUME_ID" + +# Verify deletion +echo " Verifying volume deletion..." +COUNT=0 +while [ $COUNT -lt 15 ]; do + set +e + VOLUME_CHECK=$($AWS_EC2 describe-volumes --volume-ids "$TEST_VOLUME_ID" \ + --query 'Volumes[0].VolumeId' --output text 2>&1) + DESCRIBE_EXIT=$? + set -e + + if [ $DESCRIBE_EXIT -ne 0 ] || [ "$VOLUME_CHECK" == "None" ] || [ -z "$VOLUME_CHECK" ]; then + echo " Volume deleted successfully" + break + fi + + sleep 2 + COUNT=$((COUNT + 1)) +done + +if [ $COUNT -ge 15 ]; then + echo " ERROR: Volume deletion verification timed out" + exit 1 +fi + +echo " Volume lifecycle test passed (create -> resize -> attach -> detach -> delete)" + +# Test 1c: Snapshot Lifecycle +echo "" +echo "Test 1c: Snapshot Lifecycle" +echo "----------------------------------------" +echo "Testing snapshot create -> describe -> copy -> delete..." + +# Use the root volume of the first instance — it's already attached and mounted +# in viperblockd, which is required for create-snapshot. +SNAP_VOL_ID=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[0]}" \ + --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) +echo " Using root volume $SNAP_VOL_ID (attached to ${INSTANCE_IDS[0]})" +SNAP_VOL_SIZE=$($AWS_EC2 describe-volumes --volume-ids "$SNAP_VOL_ID" \ + --query 'Volumes[0].Size' --output text) + +# Create a snapshot +echo " Creating snapshot from volume $SNAP_VOL_ID..." +SNAP_OUTPUT=$($AWS_EC2 create-snapshot --volume-id "$SNAP_VOL_ID" --description "multinode-e2e-snapshot") +SNAPSHOT_ID=$(echo "$SNAP_OUTPUT" | jq -r '.SnapshotId') + +if [ -z "$SNAPSHOT_ID" ] || [ "$SNAPSHOT_ID" == "null" ]; then + echo " ERROR: Failed to create snapshot" + echo " Output: $SNAP_OUTPUT" + exit 1 +fi +echo " Created snapshot: $SNAPSHOT_ID" + +# Verify create response fields +SNAP_STATE=$(echo "$SNAP_OUTPUT" | jq -r '.State') +SNAP_VOL_REF=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeId') +SNAP_SIZE=$(echo "$SNAP_OUTPUT" | jq -r '.VolumeSize') + +if [ "$SNAP_VOL_REF" != "$SNAP_VOL_ID" ]; then + echo " ERROR: Snapshot VolumeId mismatch: expected $SNAP_VOL_ID, got $SNAP_VOL_REF" + exit 1 +fi +if [ "$SNAP_SIZE" -ne "$SNAP_VOL_SIZE" ]; then + echo " ERROR: Snapshot VolumeSize mismatch: expected $SNAP_VOL_SIZE, got $SNAP_SIZE" + exit 1 +fi +echo " Create response verified (State=$SNAP_STATE, VolumeId=$SNAP_VOL_REF, Size=$SNAP_SIZE)" + +# Poll until completed +echo " Waiting for snapshot to complete..." +COUNT=0 +while [ $COUNT -lt 15 ]; do + SNAP_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ + --query 'Snapshots[0].State' --output text) + + if [ "$SNAP_STATE" == "completed" ]; then + echo " Snapshot completed" + break + fi + + sleep 2 + COUNT=$((COUNT + 1)) +done + +if [ "$SNAP_STATE" != "completed" ]; then + echo " ERROR: Snapshot failed to reach completed state (State=$SNAP_STATE)" + exit 1 +fi + +# Describe by ID and verify +echo " Verifying snapshot via describe-snapshots..." +DESCRIBE_SNAP=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID") +DESC_VOL_ID=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeId') +DESC_SIZE=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].VolumeSize') +DESC_DESC=$(echo "$DESCRIBE_SNAP" | jq -r '.Snapshots[0].Description') + +if [ "$DESC_VOL_ID" != "$SNAP_VOL_ID" ]; then + echo " ERROR: Describe VolumeId mismatch: expected $SNAP_VOL_ID, got $DESC_VOL_ID" + exit 1 +fi +if [ "$DESC_SIZE" -ne "$SNAP_VOL_SIZE" ]; then + echo " ERROR: Describe VolumeSize mismatch: expected $SNAP_VOL_SIZE, got $DESC_SIZE" + exit 1 +fi +if [ "$DESC_DESC" != "multinode-e2e-snapshot" ]; then + echo " ERROR: Describe Description mismatch: expected 'multinode-e2e-snapshot', got '$DESC_DESC'" + exit 1 +fi +echo " Describe verified (VolumeId=$DESC_VOL_ID, Size=$DESC_SIZE, Description=$DESC_DESC)" + +# Copy the snapshot +echo " Copying snapshot $SNAPSHOT_ID..." +COPY_OUTPUT=$($AWS_EC2 copy-snapshot --source-snapshot-id "$SNAPSHOT_ID" --source-region "$HIVE_REGION" --description "multinode-e2e-copy") +COPY_SNAPSHOT_ID=$(echo "$COPY_OUTPUT" | jq -r '.SnapshotId') + +if [ -z "$COPY_SNAPSHOT_ID" ] || [ "$COPY_SNAPSHOT_ID" == "null" ]; then + echo " ERROR: Failed to copy snapshot" + echo " Output: $COPY_OUTPUT" + exit 1 +fi +echo " Copied snapshot: $COPY_SNAPSHOT_ID" + +if [ "$COPY_SNAPSHOT_ID" == "$SNAPSHOT_ID" ]; then + echo " ERROR: Copy snapshot ID should differ from original" + exit 1 +fi + +# Verify both exist +TOTAL_SNAPS=$($AWS_EC2 describe-snapshots \ + --snapshot-ids "$SNAPSHOT_ID" "$COPY_SNAPSHOT_ID" \ + --query 'length(Snapshots)' --output text) + +if [ "$TOTAL_SNAPS" -ne 2 ]; then + echo " ERROR: Expected 2 snapshots, got $TOTAL_SNAPS" + exit 1 +fi +echo " Both snapshots visible via describe-snapshots" + +# Verify copy description +COPY_DESC=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \ + --query 'Snapshots[0].Description' --output text) +if [ "$COPY_DESC" != "multinode-e2e-copy" ]; then + echo " ERROR: Copy description mismatch: expected 'multinode-e2e-copy', got '$COPY_DESC'" + exit 1 +fi + +# Delete original +echo " Deleting original snapshot $SNAPSHOT_ID..." +$AWS_EC2 delete-snapshot --snapshot-id "$SNAPSHOT_ID" + +# Verify original gone, copy remains +echo " Verifying snapshot deletion..." +COUNT=0 +while [ $COUNT -lt 15 ]; do + set +e + SNAP_CHECK=$($AWS_EC2 describe-snapshots --snapshot-ids "$SNAPSHOT_ID" \ + --query 'Snapshots[0].SnapshotId' --output text 2>&1) + SNAP_EXIT=$? + set -e + + if [ $SNAP_EXIT -ne 0 ] || [ "$SNAP_CHECK" == "None" ] || [ -z "$SNAP_CHECK" ]; then + echo " Original snapshot deleted successfully" + break + fi + + sleep 2 + COUNT=$((COUNT + 1)) +done + +if [ $COUNT -ge 15 ]; then + echo " ERROR: Snapshot deletion verification timed out" + exit 1 +fi + +# Verify copy still exists +COPY_STATE=$($AWS_EC2 describe-snapshots --snapshot-ids "$COPY_SNAPSHOT_ID" \ + --query 'Snapshots[0].State' --output text) +if [ "$COPY_STATE" != "completed" ]; then + echo " ERROR: Copy snapshot should still exist (State=$COPY_STATE)" + exit 1 +fi +echo " Copy snapshot intact after original deletion" + +# Delete copy +echo " Deleting copy snapshot $COPY_SNAPSHOT_ID..." +$AWS_EC2 delete-snapshot --snapshot-id "$COPY_SNAPSHOT_ID" + +echo " Snapshot lifecycle test passed (create -> describe -> copy -> delete)" + +# Test 1c-ii: Verify Snapshot-Backed Instance Launch +echo "" +echo "Test 1c-ii: Verify Snapshot-Backed Instance Launch" +echo "----------------------------------------" +echo "All run-instances calls go through cloneAMIToVolume() -> OpenFromSnapshot()," +echo "so the Test 1 instances are already snapshot-backed. Verify their volume configs." + +AWS_S3="aws --endpoint-url https://${NODE1_IP}:${PREDASTORE_PORT} s3" + +# Verify the AMI snapshot exists in Predastore +echo " Checking AMI snapshot in Predastore..." +SNAP_PREFIX="snap-$AMI_ID" +SNAP_FILES=$($AWS_S3 ls "s3://predastore/$SNAP_PREFIX/" 2>&1 || echo "") +if echo "$SNAP_FILES" | grep -q "config.json"; then + echo " AMI snapshot config found at $SNAP_PREFIX/" +else + echo " ERROR: AMI snapshot config not found at $SNAP_PREFIX/" + exit 1 +fi + +# Verify the first instance's root volume has SnapshotID and SourceVolumeName +SNAP_ROOT_VOL=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[0]}" \ + --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) +echo " Verifying root volume $SNAP_ROOT_VOL is snapshot-backed via Predastore config..." +VOL_CONFIG=$($AWS_S3 cp "s3://predastore/$SNAP_ROOT_VOL/config.json" - 2>/dev/null || echo "{}") +VOL_SNAPSHOT_ID=$(echo "$VOL_CONFIG" | jq -r '.SnapshotID // empty') +VOL_SOURCE_NAME=$(echo "$VOL_CONFIG" | jq -r '.SourceVolumeName // empty') + +if [ -z "$VOL_SNAPSHOT_ID" ]; then + echo " ERROR: Volume config missing SnapshotID — launch was NOT snapshot-backed" + exit 1 +fi +if [ -z "$VOL_SOURCE_NAME" ]; then + echo " ERROR: Volume config missing SourceVolumeName — launch was NOT snapshot-backed" + exit 1 +fi +echo " Volume is snapshot-backed (SnapshotID=$VOL_SNAPSHOT_ID, SourceVolumeName=$VOL_SOURCE_NAME)" + +echo " Snapshot-backed instance launch verified" + +# Test 1d: Tag Management +echo "" +echo "Test 1d: Tag Management" +echo "----------------------------------------" +echo "Testing create-tags -> describe-tags -> delete-tags..." + +# Use the first instance for tag tests +TAG_INSTANCE="${INSTANCE_IDS[0]}" + +# Create tags on instance +echo " Creating tags on instance $TAG_INSTANCE..." +$AWS_EC2 create-tags --resources "$TAG_INSTANCE" --tags Key=Name,Value=multinode-test Key=Environment,Value=testing Key=DeleteMe,Value=please + +# Verify tags with describe-tags (resource-id filter) +echo " Verifying tags on instance..." +TAG_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$TAG_COUNT" -ne 3 ]; then + echo " ERROR: Expected 3 tags on instance, got $TAG_COUNT" + exit 1 +fi +echo " Instance has $TAG_COUNT tags" + +# Filter by key +echo " Testing key filter..." +ENV_TAGS=$($AWS_EC2 describe-tags --filters "Name=key,Values=Environment" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$ENV_TAGS" -lt 1 ]; then + echo " ERROR: Expected at least 1 'Environment' tag, got $ENV_TAGS" + exit 1 +fi +echo " Key filter returned $ENV_TAGS tags" + +# Filter by resource-type +echo " Testing resource-type filter..." +INSTANCE_TAGS=$($AWS_EC2 describe-tags --filters "Name=resource-type,Values=instance" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$INSTANCE_TAGS" -lt 3 ]; then + echo " ERROR: Expected at least 3 instance tags, got $INSTANCE_TAGS" + exit 1 +fi +echo " Resource-type filter returned $INSTANCE_TAGS instance tags" + +# Overwrite a tag value +echo " Overwriting Name tag..." +$AWS_EC2 create-tags --resources "$TAG_INSTANCE" --tags Key=Name,Value=multinode-updated +UPDATED_NAME=$($AWS_EC2 describe-tags \ + --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Name" \ + --query 'Tags[0].Value' --output text) +if [ "$UPDATED_NAME" != "multinode-updated" ]; then + echo " ERROR: Tag overwrite failed: expected 'multinode-updated', got '$UPDATED_NAME'" + exit 1 +fi +echo " Tag overwrite verified" + +# Delete tag by key (unconditional) +echo " Deleting DeleteMe tag unconditionally..." +$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=DeleteMe +REMAINING=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$REMAINING" -ne 2 ]; then + echo " ERROR: Expected 2 tags after unconditional delete, got $REMAINING" + exit 1 +fi +echo " Unconditional delete verified ($REMAINING tags remaining)" + +# Delete tag with wrong value (should NOT delete) +echo " Attempting delete with wrong value (should be no-op)..." +$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=Environment,Value=production +ENV_STILL=$($AWS_EC2 describe-tags \ + --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Environment" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$ENV_STILL" -ne 1 ]; then + echo " ERROR: Value-conditional delete incorrectly removed tag" + exit 1 +fi +echo " Value-conditional mismatch preserved tag" + +# Delete tag with correct value +echo " Deleting Environment tag with correct value..." +$AWS_EC2 delete-tags --resources "$TAG_INSTANCE" --tags Key=Environment,Value=testing +ENV_GONE=$($AWS_EC2 describe-tags \ + --filters "Name=resource-id,Values=$TAG_INSTANCE" "Name=key,Values=Environment" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$ENV_GONE" -ne 0 ]; then + echo " ERROR: Value-conditional delete failed to remove matching tag" + exit 1 +fi +echo " Value-conditional match deleted tag" + +# Verify only Name tag remains +FINAL_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_INSTANCE" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$FINAL_COUNT" -ne 1 ]; then + echo " ERROR: Expected 1 tag remaining, got $FINAL_COUNT" + exit 1 +fi +echo " Instance tag tests passed" + +# Test 1d-ii: Tags on Volumes (multi-node) +echo "" +echo "Test 1d-ii: Tags on Volumes (Multi-Node)" +echo "----------------------------------------" + +# Get the root volume of the first instance +TAG_VOL_ID=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[0]}" \ + --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) +echo " Tagging volume $TAG_VOL_ID..." + +# Create tags on volume +$AWS_EC2 create-tags --resources "$TAG_VOL_ID" --tags Key=Name,Value=multinode-root-vol Key=Environment,Value=testing + +# Verify tags on volume +VOL_TAG_COUNT=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_VOL_ID" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$VOL_TAG_COUNT" -ne 2 ]; then + echo " ERROR: Expected 2 tags on volume, got $VOL_TAG_COUNT" + exit 1 +fi +echo " Volume has $VOL_TAG_COUNT tags" + +# Filter by resource-type=volume +VOL_TYPE_TAGS=$($AWS_EC2 describe-tags --filters "Name=resource-type,Values=volume" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$VOL_TYPE_TAGS" -lt 2 ]; then + echo " ERROR: Expected at least 2 volume tags, got $VOL_TYPE_TAGS" + exit 1 +fi +echo " Volume resource-type filter returned $VOL_TYPE_TAGS tags" + +# Delete tags from volume +$AWS_EC2 delete-tags --resources "$TAG_VOL_ID" --tags Key=Name Key=Environment +VOL_TAG_AFTER=$($AWS_EC2 describe-tags --filters "Name=resource-id,Values=$TAG_VOL_ID" \ + --query 'length(Tags || `[]`)' --output text) +if [ "$VOL_TAG_AFTER" -ne 0 ]; then + echo " ERROR: Expected 0 tags after delete, got $VOL_TAG_AFTER" + exit 1 +fi +echo " Volume tag cleanup verified" + +echo " Tag management tests passed (instances + volumes)" + +# Test 2: DescribeInstances Aggregation +echo "" +echo "Test 2: DescribeInstances Aggregation" +echo "----------------------------------------" +echo "Verifying all instances are returned via fan-out query..." + +DESCRIBE_OUTPUT=$($AWS_EC2 describe-instances --query 'Reservations[*].Instances[*].InstanceId' --output text) +DESCRIBED_COUNT=$(echo "$DESCRIBE_OUTPUT" | wc -w) + +echo " Launched: ${#INSTANCE_IDS[@]} instances" +echo " Described: $DESCRIBED_COUNT instances" + +if [ "$DESCRIBED_COUNT" -lt "${#INSTANCE_IDS[@]}" ]; then + echo "ERROR: DescribeInstances did not return all instances" + echo " Expected: ${#INSTANCE_IDS[@]}, Got: $DESCRIBED_COUNT" + exit 1 +fi +echo " Aggregation test passed" + +# Test 3: Cross-Node Operations +echo "" +echo "Test 3: Cross-Node Operations" +echo "----------------------------------------" +echo "Testing stop/start/terminate via gateway regardless of instance location..." + +# Pick first instance for cross-node operations +TEST_INSTANCE="${INSTANCE_IDS[0]}" +echo " Test instance: $TEST_INSTANCE" + +# Stop instance +echo " Stopping instance..." +$AWS_EC2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null +wait_for_instance_state "$TEST_INSTANCE" "stopped" 30 + +# Start instance +echo " Starting instance..." +$AWS_EC2 start-instances --instance-ids "$TEST_INSTANCE" > /dev/null +wait_for_instance_state "$TEST_INSTANCE" "running" 30 + +echo " Cross-node operations test passed" + +# Test 4: NATS Cluster Health (Post-Operations) +echo "" +echo "Test 4: NATS Cluster Health (Post-Operations)" +echo "----------------------------------------" +echo "Verifying NATS cluster is still healthy after operations..." + +verify_nats_cluster 3 || { + echo "WARNING: NATS cluster verification failed after operations" +} + +# Test 5: VM Crash Recovery (kill -9 → detect → auto-restart) +echo "" +echo "Test 5: VM Crash Recovery" +echo "----------------------------------------" +echo "Testing QEMU crash detection and auto-restart..." + +# Use the second instance for crash testing (first was used for stop/start in Test 3) +CRASH_INSTANCE="${INSTANCE_IDS[1]}" +echo " Crash test instance: $CRASH_INSTANCE" + +# Verify instance is running before crash +CRASH_STATE=$($AWS_EC2 describe-instances --instance-ids "$CRASH_INSTANCE" \ + --query 'Reservations[0].Instances[0].State.Name' --output text) +if [ "$CRASH_STATE" != "running" ]; then + echo " ERROR: Instance not in running state before crash test (state: $CRASH_STATE)" + exit 1 +fi +echo " Instance is running" + +# Get the QEMU PID +QEMU_PID=$(get_qemu_pid "$CRASH_INSTANCE") +if [ -z "$QEMU_PID" ]; then + echo " ERROR: Could not find QEMU PID for $CRASH_INSTANCE" + exit 1 +fi +echo " QEMU PID: $QEMU_PID" + +# Kill QEMU with SIGKILL (simulates OOM kill) +echo " Killing QEMU process with SIGKILL (simulating OOM kill)..." +kill -9 "$QEMU_PID" + +# Brief pause for the daemon to detect the crash +sleep 3 + +# Verify the daemon detected the crash (state should be error or already recovering) +echo " Checking post-crash state..." +POST_CRASH_STATE=$($AWS_EC2 describe-instances --instance-ids "$CRASH_INSTANCE" \ + --query 'Reservations[0].Instances[0].State.Name' --output text) +echo " Post-crash state: $POST_CRASH_STATE" + +if [ "$POST_CRASH_STATE" == "running" ]; then + # Might have recovered very quickly, check if it's a new PID + NEW_PID=$(get_qemu_pid "$CRASH_INSTANCE" || echo "") + if [ "$NEW_PID" == "$QEMU_PID" ]; then + echo " ERROR: Instance still has same PID after kill -9, crash not detected" + exit 1 + fi + echo " Instance already recovered with new PID: $NEW_PID (was: $QEMU_PID)" +else + # Wait for auto-restart (backoff starts at 5s) + echo " Instance in $POST_CRASH_STATE state, waiting for auto-restart..." + wait_for_instance_recovery "$CRASH_INSTANCE" 30 || { + echo " ERROR: Instance failed to recover from crash" + # Dump daemon logs for debugging + for i in 1 2 3; do + if [ -f "$HOME/node$i/logs/hive.log" ]; then + echo "" + echo " --- node$i daemon log (last 30 lines) ---" + tail -30 "$HOME/node$i/logs/hive.log" + fi + done + exit 1 + } +fi + +# Verify the instance is running with a new QEMU process +RECOVERED_PID=$(get_qemu_pid "$CRASH_INSTANCE") +if [ -z "$RECOVERED_PID" ]; then + echo " ERROR: No QEMU process found after recovery" + exit 1 +fi + +if [ "$RECOVERED_PID" == "$QEMU_PID" ]; then + echo " ERROR: QEMU PID unchanged after crash recovery (expected new process)" + exit 1 +fi +echo " New QEMU PID: $RECOVERED_PID (was: $QEMU_PID)" + +# Verify describe-instances shows running state +FINAL_STATE=$($AWS_EC2 describe-instances --instance-ids "$CRASH_INSTANCE" \ + --query 'Reservations[0].Instances[0].State.Name' --output text) +if [ "$FINAL_STATE" != "running" ]; then + echo " ERROR: Instance not running after recovery (state: $FINAL_STATE)" + exit 1 +fi +echo " Instance is running after crash recovery" + +# Verify SSH works after recovery (VM rebooted with fresh OS from root volume) +echo " Verifying SSH after crash recovery..." +CRASH_SSH_PORT=$(get_ssh_port "$CRASH_INSTANCE" 30) +CRASH_SSH_HOST=$(get_ssh_host "$CRASH_INSTANCE") +if [ -n "$CRASH_SSH_PORT" ]; then + echo " SSH endpoint after recovery: $CRASH_SSH_HOST:$CRASH_SSH_PORT" + # Update SSH details for later termination verification + SSH_PORTS[1]="$CRASH_SSH_PORT" + SSH_HOSTS[1]="$CRASH_SSH_HOST" + wait_for_ssh "$CRASH_SSH_HOST" "$CRASH_SSH_PORT" "multinode-test-key.pem" 30 || { + echo " WARNING: SSH not ready after crash recovery (non-fatal, VM may still be booting)" + } +else + echo " WARNING: Could not determine SSH port after recovery (non-fatal)" +fi + +# Test 5b: Crash Loop Prevention (kill 4 times rapidly to exceed max restarts) +echo "" +echo "Test 5b: Crash Loop Prevention" +echo "----------------------------------------" +echo "Testing that crash loop is detected and restarts stop after max attempts..." + +# Use the third instance for crash loop testing +LOOP_INSTANCE="${INSTANCE_IDS[2]}" +echo " Crash loop test instance: $LOOP_INSTANCE" + +# Verify instance is running +LOOP_STATE=$($AWS_EC2 describe-instances --instance-ids "$LOOP_INSTANCE" \ + --query 'Reservations[0].Instances[0].State.Name' --output text) +if [ "$LOOP_STATE" != "running" ]; then + echo " ERROR: Instance not running before crash loop test (state: $LOOP_STATE)" + exit 1 +fi + +# Kill QEMU repeatedly to exhaust restart attempts (max 3 in 10 min window) +for crash_num in 1 2 3 4; do + echo " Crash $crash_num/4: killing QEMU..." + LOOP_PID=$(get_qemu_pid "$LOOP_INSTANCE" || echo "") + if [ -z "$LOOP_PID" ]; then + echo " No QEMU process found (instance may be in error state)" + break + fi + + kill -9 "$LOOP_PID" + + if [ $crash_num -lt 4 ]; then + # Wait for restart (backoff increases: 5s, 10s, 20s) + # Give generous time for each restart cycle + local_max=$((15 + crash_num * 10)) + echo " Waiting up to ${local_max}s for restart or error state..." + attempt=0 + while [ $attempt -lt $local_max ]; do + state=$($AWS_EC2 describe-instances --instance-ids "$LOOP_INSTANCE" \ + --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "unknown") + if [ "$state" == "running" ]; then + echo " Instance restarted (crash $crash_num)" + break + fi + if [ "$state" == "error" ] && [ $crash_num -ge 3 ]; then + echo " Instance in error state after crash $crash_num (restart limit may be reached)" + break + fi + sleep 2 + attempt=$((attempt + 2)) + done + fi +done + +# After 4 rapid crashes, the instance should be in error state (exceeded max 3 restarts) +sleep 5 +LOOP_FINAL_STATE=$($AWS_EC2 describe-instances --instance-ids "$LOOP_INSTANCE" \ + --query 'Reservations[0].Instances[0].State.Name' --output text) +echo " Final state after crash loop: $LOOP_FINAL_STATE" + +if [ "$LOOP_FINAL_STATE" == "error" ]; then + echo " Crash loop prevention working: instance stayed in error state" +else + echo " WARNING: Instance in state '$LOOP_FINAL_STATE' (expected 'error' after exceeding max restarts)" + echo " This may be expected if timing allowed restarts to spread across the window" +fi + +echo " Crash recovery tests passed" + +# Phase 5c: IAM Accounts & Cross-Account Isolation +echo "" +echo "Phase 5c: IAM Accounts & Cross-Account Isolation" +echo "========================================" + +AWS_EP="--endpoint-url https://${NODE1_IP}:${AWSGW_PORT}" + +# Step 1: Create accounts +echo "" +echo "Step 1: Create Accounts" +echo "----------------------------------------" + +echo " Creating Team Alpha account..." +ALPHA_OUTPUT=$(./bin/hive admin account create --name "Team Alpha" --config "$HOME/node1/config/hive.toml" 2>&1) +echo "$ALPHA_OUTPUT" +ALPHA_ACCOUNT=$(echo "$ALPHA_OUTPUT" | grep "Account ID:" | awk '{print $NF}') +ALPHA_KEY_ID=$(echo "$ALPHA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}') +ALPHA_SECRET=$(echo "$ALPHA_OUTPUT" | grep "Secret Access Key:" | awk '{print $NF}') + +if [ -z "$ALPHA_ACCOUNT" ] || [ -z "$ALPHA_KEY_ID" ]; then + echo " ERROR: Failed to parse Team Alpha account output" + exit 1 +fi +echo " Team Alpha: account=$ALPHA_ACCOUNT key=$ALPHA_KEY_ID" + +# Configure alpha profile (in case auto-config used a different endpoint) +aws configure set aws_access_key_id "$ALPHA_KEY_ID" --profile hive-team-alpha +aws configure set aws_secret_access_key "$ALPHA_SECRET" --profile hive-team-alpha +aws configure set region us-east-1 --profile hive-team-alpha + +echo " Creating Team Beta account..." +BETA_OUTPUT=$(./bin/hive admin account create --name "Team Beta" --config "$HOME/node1/config/hive.toml" 2>&1) +echo "$BETA_OUTPUT" +BETA_ACCOUNT=$(echo "$BETA_OUTPUT" | grep "Account ID:" | awk '{print $NF}') +BETA_KEY_ID=$(echo "$BETA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}') +BETA_SECRET=$(echo "$BETA_OUTPUT" | grep "Secret Access Key:" | awk '{print $NF}') + +if [ -z "$BETA_ACCOUNT" ] || [ -z "$BETA_KEY_ID" ]; then + echo " ERROR: Failed to parse Team Beta account output" + exit 1 +fi +echo " Team Beta: account=$BETA_ACCOUNT key=$BETA_KEY_ID" + +aws configure set aws_access_key_id "$BETA_KEY_ID" --profile hive-team-beta +aws configure set aws_secret_access_key "$BETA_SECRET" --profile hive-team-beta +aws configure set region us-east-1 --profile hive-team-beta + +# Verify sequential IDs +echo " Verifying sequential account IDs..." +if [ "$ALPHA_ACCOUNT" == "000000000001" ] && [ "$BETA_ACCOUNT" == "000000000002" ]; then + echo " Sequential IDs correct: $ALPHA_ACCOUNT, $BETA_ACCOUNT" +else + echo " WARNING: Account IDs may not be sequential: $ALPHA_ACCOUNT, $BETA_ACCOUNT" +fi + +# List accounts +echo " Listing accounts..." +ACCOUNT_LIST=$(./bin/hive admin account list --config "$HOME/node1/config/hive.toml" 2>&1) +echo "$ACCOUNT_LIST" +if echo "$ACCOUNT_LIST" | grep -q "Team Alpha" && echo "$ACCOUNT_LIST" | grep -q "Team Beta"; then + echo " Account list verified" +else + echo " ERROR: Account list missing expected accounts" + exit 1 +fi + +# Step 2: Account admin auth +echo "" +echo "Step 2: Account Admin Auth" +echo "----------------------------------------" + +echo " Testing alpha admin auth..." +aws $AWS_EP ec2 describe-instances --profile hive-team-alpha > /dev/null +echo " Alpha admin: ec2 OK" +aws $AWS_EP iam list-users --profile hive-team-alpha > /dev/null +echo " Alpha admin: iam OK" + +echo " Testing beta admin auth..." +aws $AWS_EP ec2 describe-instances --profile hive-team-beta > /dev/null +echo " Beta admin: ec2 OK" +aws $AWS_EP iam list-users --profile hive-team-beta > /dev/null +echo " Beta admin: iam OK" + +# Step 3: Account-scoped users +echo "" +echo "Step 3: Account-Scoped Users" +echo "----------------------------------------" + +# Same username in different accounts +echo " Creating alice in both accounts..." +aws $AWS_EP iam create-user --user-name alice --profile hive-team-alpha > /dev/null +ALPHA_ALICE_ARN=$(aws $AWS_EP iam get-user --user-name alice --profile hive-team-alpha | jq -r '.User.Arn') +echo " Alpha alice: $ALPHA_ALICE_ARN" + +aws $AWS_EP iam create-user --user-name alice --profile hive-team-beta > /dev/null +BETA_ALICE_ARN=$(aws $AWS_EP iam get-user --user-name alice --profile hive-team-beta | jq -r '.User.Arn') +echo " Beta alice: $BETA_ALICE_ARN" + +# Additional users +aws $AWS_EP iam create-user --user-name team-member --profile hive-team-alpha > /dev/null +aws $AWS_EP iam create-user --user-name dev-user --profile hive-team-beta > /dev/null + +# List users — scoped +echo " Verifying scoped user lists..." +ALPHA_USERS=$(aws $AWS_EP iam list-users --profile hive-team-alpha | jq -r '.Users[].UserName' | sort | tr '\n' ',') +echo " Alpha users: $ALPHA_USERS" +BETA_USERS=$(aws $AWS_EP iam list-users --profile hive-team-beta | jq -r '.Users[].UserName' | sort | tr '\n' ',') +echo " Beta users: $BETA_USERS" + +# Cross-account isolation +echo " Verifying cross-account isolation..." +set +e +CROSS_CHECK=$(aws $AWS_EP iam get-user --user-name dev-user --profile hive-team-alpha 2>&1) +set -e +if echo "$CROSS_CHECK" | grep -q "NoSuchEntity"; then + echo " Alpha cannot see Beta's dev-user — isolation OK" +else + echo " ERROR: Alpha can see Beta's user (cross-account leak)" + exit 1 +fi + +set +e +CROSS_CHECK2=$(aws $AWS_EP iam get-user --user-name team-member --profile hive-team-beta 2>&1) +set -e +if echo "$CROSS_CHECK2" | grep -q "NoSuchEntity"; then + echo " Beta cannot see Alpha's team-member — isolation OK" +else + echo " ERROR: Beta can see Alpha's user (cross-account leak)" + exit 1 +fi + +# Step 4: Account-scoped access keys +echo "" +echo "Step 4: Account-Scoped Access Keys" +echo "----------------------------------------" + +ALPHA_ALICE_KEY=$(aws $AWS_EP iam create-access-key --user-name alice --profile hive-team-alpha) +ALPHA_ALICE_KEY_ID=$(echo "$ALPHA_ALICE_KEY" | jq -r '.AccessKey.AccessKeyId') +ALPHA_ALICE_SECRET=$(echo "$ALPHA_ALICE_KEY" | jq -r '.AccessKey.SecretAccessKey') +echo " Alpha alice key: $ALPHA_ALICE_KEY_ID" + +aws configure set aws_access_key_id "$ALPHA_ALICE_KEY_ID" --profile hive-alpha-alice +aws configure set aws_secret_access_key "$ALPHA_ALICE_SECRET" --profile hive-alpha-alice +aws configure set region us-east-1 --profile hive-alpha-alice + +BETA_ALICE_KEY=$(aws $AWS_EP iam create-access-key --user-name alice --profile hive-team-beta) +BETA_ALICE_KEY_ID=$(echo "$BETA_ALICE_KEY" | jq -r '.AccessKey.AccessKeyId') +BETA_ALICE_SECRET=$(echo "$BETA_ALICE_KEY" | jq -r '.AccessKey.SecretAccessKey') +echo " Beta alice key: $BETA_ALICE_KEY_ID" + +aws configure set aws_access_key_id "$BETA_ALICE_KEY_ID" --profile hive-beta-alice +aws configure set aws_secret_access_key "$BETA_ALICE_SECRET" --profile hive-beta-alice +aws configure set region us-east-1 --profile hive-beta-alice + +# Step 5: Account-scoped policies & enforcement +echo "" +echo "Step 5: Account-Scoped Policies & Enforcement" +echo "----------------------------------------" + +# Alpha: narrow EC2 read-only +echo " Creating EC2ReadOnly in Alpha (narrow)..." +aws $AWS_EP iam create-policy \ + --policy-name EC2ReadOnly \ + --policy-document '{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["ec2:DescribeInstances", "ec2:DescribeVpcs"], + "Resource": "*" + }] + }' --profile hive-team-alpha > /dev/null + +# Beta: broad EC2 Describe* wildcard +echo " Creating EC2ReadOnly in Beta (broad Describe*)..." +aws $AWS_EP iam create-policy \ + --policy-name EC2ReadOnly \ + --policy-document '{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": "ec2:Describe*", + "Resource": "*" + }] + }' --profile hive-team-beta > /dev/null + +# Attach policies +echo " Attaching policies..." +aws $AWS_EP iam attach-user-policy --user-name alice \ + --policy-arn "arn:aws:iam::${ALPHA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-alpha +aws $AWS_EP iam attach-user-policy --user-name alice \ + --policy-arn "arn:aws:iam::${BETA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-beta + +# Verify scoped enforcement +echo " Verifying scoped enforcement..." + +# Alpha alice: narrow policy +aws $AWS_EP ec2 describe-instances --profile hive-alpha-alice > /dev/null +echo " Alpha alice: describe-instances — allowed" +aws $AWS_EP ec2 describe-vpcs --profile hive-alpha-alice > /dev/null +echo " Alpha alice: describe-vpcs — allowed" +expect_error "AccessDenied" aws $AWS_EP ec2 describe-key-pairs --profile hive-alpha-alice +echo " Alpha alice: describe-key-pairs — denied (narrow policy)" + +# Beta alice: broad policy +aws $AWS_EP ec2 describe-instances --profile hive-beta-alice > /dev/null +echo " Beta alice: describe-instances — allowed" +aws $AWS_EP ec2 describe-key-pairs --profile hive-beta-alice > /dev/null +echo " Beta alice: describe-key-pairs — allowed (Describe* wildcard)" + +# Both denied non-Describe +expect_error "AccessDenied" aws $AWS_EP ec2 create-key-pair --key-name x --profile hive-alpha-alice +echo " Alpha alice: create-key-pair — denied" +expect_error "AccessDenied" aws $AWS_EP ec2 create-key-pair --key-name x --profile hive-beta-alice +echo " Beta alice: create-key-pair — denied" + +echo " Scoped enforcement verified" + +# Step 6: Cross-account delete isolation +echo "" +echo "Step 6: Cross-Account Delete Isolation" +echo "----------------------------------------" + +# Delete Alpha's alice +echo " Deleting Alpha's alice..." +aws $AWS_EP iam detach-user-policy --user-name alice \ + --policy-arn "arn:aws:iam::${ALPHA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-alpha +aws $AWS_EP iam delete-access-key --user-name alice \ + --access-key-id "$ALPHA_ALICE_KEY_ID" --profile hive-team-alpha +aws $AWS_EP iam delete-user --user-name alice --profile hive-team-alpha + +# Verify Alpha alice gone +set +e +ALPHA_CHECK=$(aws $AWS_EP iam get-user --user-name alice --profile hive-team-alpha 2>&1) +set -e +if echo "$ALPHA_CHECK" | grep -q "NoSuchEntity"; then + echo " Alpha alice deleted" +else + echo " ERROR: Alpha alice still exists after delete" + exit 1 +fi + +# Verify Beta alice unaffected +aws $AWS_EP iam get-user --user-name alice --profile hive-team-beta > /dev/null +echo " Beta alice still exists (isolation OK)" + +aws $AWS_EP ec2 describe-instances --profile hive-beta-alice > /dev/null +echo " Beta alice auth still works (isolation OK)" + +# ========================================================================== +# Step 7: EC2 Resource Scoping +# ========================================================================== +# Tests that EC2 resources are properly isolated between accounts. +# Reuses Alpha/Beta accounts created in Step 1. +# Skips: Section 6 (CreateImage — mulga-612), instance tags (mulga-613) +echo "" +echo "Step 7: EC2 Resource Scoping" +echo "========================================" + +# --- 7a: Instance Scoping --- +echo "" +echo "Step 7a: Instance Scoping" +echo "----------------------------------------" + +# Create per-account key pairs (key pairs are account-scoped, root's multinode-test-key is invisible) +$AWS_EC2 create-key-pair --key-name alpha-instance-key --profile hive-team-alpha > /dev/null +$AWS_EC2 create-key-pair --key-name beta-instance-key --profile hive-team-beta > /dev/null +echo " Created per-account key pairs for instance launches" + +echo " Alpha launching instance..." +ALPHA_INST_RUN=$($AWS_EC2 run-instances \ + --image-id "$AMI_ID" \ + --instance-type "$INSTANCE_TYPE" \ + --key-name alpha-instance-key \ + --profile hive-team-alpha) +ALPHA_INST=$(echo "$ALPHA_INST_RUN" | jq -r '.Instances[0].InstanceId') +echo " Alpha instance: $ALPHA_INST" + +echo " Beta launching instance..." +BETA_INST_RUN=$($AWS_EC2 run-instances \ + --image-id "$AMI_ID" \ + --instance-type "$INSTANCE_TYPE" \ + --key-name beta-instance-key \ + --profile hive-team-beta) +BETA_INST=$(echo "$BETA_INST_RUN" | jq -r '.Instances[0].InstanceId') +echo " Beta instance: $BETA_INST" + +# Wait for running (inline — wait_for_instance_state uses root profile, can't see tenant instances) +echo " Waiting for instances to reach running state..." +COUNT=0 +while [ $COUNT -lt 30 ]; do + A_STATE=$($AWS_EC2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \ + --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending") + B_STATE=$($AWS_EC2 describe-instances --instance-ids "$BETA_INST" --profile hive-team-beta \ + --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "pending") + echo " Alpha=$A_STATE, Beta=$B_STATE" + if [ "$A_STATE" == "running" ] && [ "$B_STATE" == "running" ]; then break; fi + sleep 2 + COUNT=$((COUNT + 1)) +done +if [ "$A_STATE" != "running" ] || [ "$B_STATE" != "running" ]; then + echo " ERROR: Instances failed to reach running state" + exit 1 +fi +echo " Both instances running" + +# Describe isolation +ALPHA_DESC=$($AWS_EC2 describe-instances --profile hive-team-alpha \ + --query 'Reservations[].Instances[].InstanceId' --output text) +if echo "$ALPHA_DESC" | grep -q "$BETA_INST"; then + echo " ERROR: Alpha can see Beta's instance" + exit 1 +fi +echo " Alpha sees only own instances" + +BETA_DESC=$($AWS_EC2 describe-instances --profile hive-team-beta \ + --query 'Reservations[].Instances[].InstanceId' --output text) +if echo "$BETA_DESC" | grep -q "$ALPHA_INST"; then + echo " ERROR: Beta can see Alpha's instance" + exit 1 +fi +echo " Beta sees only own instances" + +# OwnerId verification +ALPHA_OWNER=$($AWS_EC2 describe-instances --profile hive-team-alpha \ + --query 'Reservations[0].OwnerId' --output text) +if [ "$ALPHA_OWNER" != "$ALPHA_ACCOUNT" ]; then + echo " ERROR: Alpha OwnerId mismatch: expected $ALPHA_ACCOUNT, got $ALPHA_OWNER" + exit 1 +fi +echo " Alpha OwnerId correct: $ALPHA_OWNER" + +# Cross-account operations +expect_error "InvalidInstanceID.NotFound" \ + $AWS_EC2 stop-instances --instance-ids "$BETA_INST" --profile hive-team-alpha +echo " Alpha cannot stop Beta's instance" + +expect_error "InvalidInstanceID.NotFound" \ + $AWS_EC2 terminate-instances --instance-ids "$ALPHA_INST" --profile hive-team-beta +echo " Beta cannot terminate Alpha's instance" + +expect_error "InvalidInstanceID.NotFound" \ + $AWS_EC2 get-console-output --instance-id "$ALPHA_INST" --profile hive-team-beta +echo " Beta cannot get console output of Alpha's instance" + +echo " Instance scoping passed" + +# --- 7b: Volume Scoping --- +echo "" +echo "Step 7b: Volume Scoping" +echo "----------------------------------------" + +ALPHA_VOL=$($AWS_EC2 create-volume --availability-zone "$HIVE_AZ" --size 10 \ + --volume-type gp3 --profile hive-team-alpha | jq -r '.VolumeId') +echo " Alpha volume: $ALPHA_VOL" + +BETA_VOL=$($AWS_EC2 create-volume --availability-zone "$HIVE_AZ" --size 10 \ + --volume-type gp3 --profile hive-team-beta | jq -r '.VolumeId') +echo " Beta volume: $BETA_VOL" + +# Describe isolation +ALPHA_VOLS=$($AWS_EC2 describe-volumes --profile hive-team-alpha \ + --query 'Volumes[].VolumeId' --output text) +if echo "$ALPHA_VOLS" | grep -q "$BETA_VOL"; then + echo " ERROR: Alpha can see Beta's volume" + exit 1 +fi +echo " Alpha sees only own volumes" + +# Cross-account operations +expect_error "InvalidVolume.NotFound" \ + $AWS_EC2 describe-volumes --volume-ids "$BETA_VOL" --profile hive-team-alpha +echo " Alpha cannot describe Beta's volume by ID" + +expect_error "InvalidVolume.NotFound" \ + $AWS_EC2 delete-volume --volume-id "$ALPHA_VOL" --profile hive-team-beta +echo " Beta cannot delete Alpha's volume" + +expect_error "InvalidVolume.NotFound" \ + $AWS_EC2 attach-volume --volume-id "$ALPHA_VOL" \ + --instance-id "$BETA_INST" --device /dev/sdf --profile hive-team-beta +echo " Beta cannot attach Alpha's volume" + +# Attach Alpha's volume, then test cross-account detach +$AWS_EC2 attach-volume --volume-id "$ALPHA_VOL" \ + --instance-id "$ALPHA_INST" --device /dev/sdf --profile hive-team-alpha > /dev/null +sleep 2 + +expect_error "InvalidVolume.NotFound" \ + $AWS_EC2 detach-volume --volume-id "$ALPHA_VOL" --profile hive-team-beta +echo " Beta cannot detach Alpha's volume" + +expect_error "InvalidVolume.NotFound" \ + $AWS_EC2 modify-volume --volume-id "$ALPHA_VOL" --size 20 --profile hive-team-beta +echo " Beta cannot modify Alpha's volume" + +# Detach for later cleanup +$AWS_EC2 detach-volume --volume-id "$ALPHA_VOL" --profile hive-team-alpha > /dev/null +sleep 2 + +echo " Volume scoping passed" + +# --- 7c: Key Pair Scoping --- +echo "" +echo "Step 7c: Key Pair Scoping" +echo "----------------------------------------" + +$AWS_EC2 create-key-pair --key-name alpha-key --profile hive-team-alpha > /dev/null +ALPHA_KEYPAIR_ID=$($AWS_EC2 describe-key-pairs --key-names alpha-key \ + --profile hive-team-alpha --query 'KeyPairs[0].KeyPairId' --output text) +echo " Alpha key: alpha-key ($ALPHA_KEYPAIR_ID)" + +$AWS_EC2 create-key-pair --key-name beta-key --profile hive-team-beta > /dev/null +echo " Beta key: beta-key" + +# Describe isolation +ALPHA_KEYS=$($AWS_EC2 describe-key-pairs --profile hive-team-alpha \ + --query 'KeyPairs[].KeyName' --output text) +if echo "$ALPHA_KEYS" | grep -q "beta-key"; then + echo " ERROR: Alpha can see Beta's key" + exit 1 +fi +echo " Alpha sees only own keys" + +# Same name, different accounts +$AWS_EC2 create-key-pair --key-name shared-name --profile hive-team-alpha > /dev/null +$AWS_EC2 create-key-pair --key-name shared-name --profile hive-team-beta > /dev/null +ALPHA_SHARED_ID=$($AWS_EC2 describe-key-pairs --key-names shared-name \ + --profile hive-team-alpha --query 'KeyPairs[0].KeyPairId' --output text) +BETA_SHARED_ID=$($AWS_EC2 describe-key-pairs --key-names shared-name \ + --profile hive-team-beta --query 'KeyPairs[0].KeyPairId' --output text) +if [ "$ALPHA_SHARED_ID" == "$BETA_SHARED_ID" ]; then + echo " ERROR: Same KeyPairId for shared-name in both accounts" + exit 1 +fi +echo " Namespace isolation: alpha=$ALPHA_SHARED_ID, beta=$BETA_SHARED_ID" + +# Cross-account delete (idempotent, but shouldn't affect other account) +$AWS_EC2 delete-key-pair --key-name alpha-key --profile hive-team-beta +ALPHA_KEY_CHECK=$($AWS_EC2 describe-key-pairs --key-names alpha-key \ + --profile hive-team-alpha --query 'KeyPairs[0].KeyPairId' --output text) +if [ "$ALPHA_KEY_CHECK" != "$ALPHA_KEYPAIR_ID" ]; then + echo " ERROR: Beta's delete affected Alpha's key" + exit 1 +fi +echo " Cross-account delete had no effect on Alpha's key" + +# Import key pair — account scoped +ssh-keygen -t ed25519 -f /tmp/test-import-key -N "" -q +$AWS_EC2 import-key-pair --key-name imported-key \ + --public-key-material fileb:///tmp/test-import-key.pub --profile hive-team-alpha > /dev/null +BETA_IMPORT_CHECK=$($AWS_EC2 describe-key-pairs --profile hive-team-beta \ + --query 'KeyPairs[].KeyName' --output text) +if echo "$BETA_IMPORT_CHECK" | grep -q "imported-key"; then + echo " ERROR: Beta can see Alpha's imported key" + exit 1 +fi +echo " Imported key invisible to Beta" +rm -f /tmp/test-import-key /tmp/test-import-key.pub + +echo " Key pair scoping passed" + +# --- 7d: Snapshot Scoping --- +echo "" +echo "Step 7d: Snapshot Scoping" +echo "----------------------------------------" + +ALPHA_SNAP=$($AWS_EC2 create-snapshot --volume-id "$ALPHA_VOL" \ + --description "Alpha snapshot" --profile hive-team-alpha | jq -r '.SnapshotId') +echo " Alpha snapshot: $ALPHA_SNAP" + +BETA_SNAP=$($AWS_EC2 create-snapshot --volume-id "$BETA_VOL" \ + --description "Beta snapshot" --profile hive-team-beta | jq -r '.SnapshotId') +echo " Beta snapshot: $BETA_SNAP" + +# Describe isolation +ALPHA_SNAPS=$($AWS_EC2 describe-snapshots --owner-ids self --profile hive-team-alpha \ + --query 'Snapshots[].SnapshotId' --output text) +if echo "$ALPHA_SNAPS" | grep -q "$BETA_SNAP"; then + echo " ERROR: Alpha can see Beta's snapshot" + exit 1 +fi +echo " Alpha sees only own snapshots" + +# OwnerId verification +ALPHA_SNAP_OWNER=$($AWS_EC2 describe-snapshots --owner-ids self --profile hive-team-alpha \ + --query 'Snapshots[0].OwnerId' --output text) +if [ "$ALPHA_SNAP_OWNER" != "$ALPHA_ACCOUNT" ]; then + echo " ERROR: Snapshot OwnerId mismatch: expected $ALPHA_ACCOUNT, got $ALPHA_SNAP_OWNER" + exit 1 +fi +echo " Alpha snapshot OwnerId correct" + +# Cross-account delete +expect_error "UnauthorizedOperation" \ + $AWS_EC2 delete-snapshot --snapshot-id "$ALPHA_SNAP" --profile hive-team-beta +echo " Beta cannot delete Alpha's snapshot" + +# Cross-account snapshot from other's volume +expect_error "InvalidVolume.NotFound" \ + $AWS_EC2 create-snapshot --volume-id "$ALPHA_VOL" \ + --description "stolen" --profile hive-team-beta +echo " Beta cannot snapshot Alpha's volume" + +echo " Snapshot scoping passed" + +# --- 7e: VPC/Subnet Scoping --- +echo "" +echo "Step 7e: VPC/Subnet Scoping" +echo "----------------------------------------" + +ALPHA_VPC=$($AWS_EC2 create-vpc --cidr-block 10.0.0.0/16 \ + --profile hive-team-alpha --query 'Vpc.VpcId' --output text) +echo " Alpha VPC: $ALPHA_VPC" + +BETA_VPC=$($AWS_EC2 create-vpc --cidr-block 10.0.0.0/16 \ + --profile hive-team-beta --query 'Vpc.VpcId' --output text) +echo " Beta VPC: $BETA_VPC (same CIDR — no conflict)" + +# Describe isolation +ALPHA_VPCS=$($AWS_EC2 describe-vpcs --profile hive-team-alpha \ + --query 'Vpcs[].VpcId' --output text) +if echo "$ALPHA_VPCS" | grep -q "$BETA_VPC"; then + echo " ERROR: Alpha can see Beta's VPC" + exit 1 +fi +echo " VPC describe isolation OK" + +expect_error "InvalidVpcID.NotFound" \ + $AWS_EC2 describe-vpcs --vpc-ids "$BETA_VPC" --profile hive-team-alpha +echo " Alpha cannot describe Beta's VPC by ID" + +expect_error "InvalidVpcID.NotFound" \ + $AWS_EC2 delete-vpc --vpc-id "$ALPHA_VPC" --profile hive-team-beta +echo " Beta cannot delete Alpha's VPC" + +# Create subnets +ALPHA_SUBNET=$($AWS_EC2 create-subnet --vpc-id "$ALPHA_VPC" --cidr-block 10.0.1.0/24 \ + --profile hive-team-alpha --query 'Subnet.SubnetId' --output text) +echo " Alpha subnet: $ALPHA_SUBNET" + +BETA_SUBNET=$($AWS_EC2 create-subnet --vpc-id "$BETA_VPC" --cidr-block 10.0.1.0/24 \ + --profile hive-team-beta --query 'Subnet.SubnetId' --output text) +echo " Beta subnet: $BETA_SUBNET" + +# Subnet describe isolation +ALPHA_SUBNETS=$($AWS_EC2 describe-subnets --profile hive-team-alpha \ + --query 'Subnets[].SubnetId' --output text) +if echo "$ALPHA_SUBNETS" | grep -q "$BETA_SUBNET"; then + echo " ERROR: Alpha can see Beta's subnet" + exit 1 +fi +echo " Subnet describe isolation OK" + +expect_error "InvalidVpcID.NotFound" \ + $AWS_EC2 create-subnet --vpc-id "$ALPHA_VPC" --cidr-block 10.0.2.0/24 \ + --profile hive-team-beta +echo " Beta cannot create subnet in Alpha's VPC" + +expect_error "InvalidSubnetID.NotFound" \ + $AWS_EC2 delete-subnet --subnet-id "$ALPHA_SUBNET" --profile hive-team-beta +echo " Beta cannot delete Alpha's subnet" + +echo " VPC/Subnet scoping passed" + +# --- 7f: IGW + EIGW Scoping --- +echo "" +echo "Step 7f: IGW + EIGW Scoping" +echo "----------------------------------------" + +ALPHA_IGW=$($AWS_EC2 create-internet-gateway --profile hive-team-alpha \ + --query 'InternetGateway.InternetGatewayId' --output text) +echo " Alpha IGW: $ALPHA_IGW" + +BETA_IGW=$($AWS_EC2 create-internet-gateway --profile hive-team-beta \ + --query 'InternetGateway.InternetGatewayId' --output text) +echo " Beta IGW: $BETA_IGW" + +# IGW describe isolation +ALPHA_IGWS=$($AWS_EC2 describe-internet-gateways --profile hive-team-alpha \ + --query 'InternetGateways[].InternetGatewayId' --output text) +if echo "$ALPHA_IGWS" | grep -q "$BETA_IGW"; then + echo " ERROR: Alpha can see Beta's IGW" + exit 1 +fi +echo " IGW describe isolation OK" + +expect_error "InvalidInternetGatewayID.NotFound" \ + $AWS_EC2 describe-internet-gateways --internet-gateway-ids "$BETA_IGW" \ + --profile hive-team-alpha +echo " Alpha cannot describe Beta's IGW by ID" + +expect_error "InvalidInternetGatewayID.NotFound" \ + $AWS_EC2 delete-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ + --profile hive-team-beta +echo " Beta cannot delete Alpha's IGW" + +expect_error "InvalidInternetGatewayID.NotFound" \ + $AWS_EC2 attach-internet-gateway --internet-gateway-id "$BETA_IGW" \ + --vpc-id "$ALPHA_VPC" --profile hive-team-alpha +echo " Alpha cannot attach Beta's IGW to own VPC" + +# Attach Alpha's IGW, test cross-account detach +$AWS_EC2 attach-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ + --vpc-id "$ALPHA_VPC" --profile hive-team-alpha > /dev/null +expect_error "InvalidInternetGatewayID.NotFound" \ + $AWS_EC2 detach-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ + --vpc-id "$ALPHA_VPC" --profile hive-team-beta +echo " Beta cannot detach Alpha's IGW" + +# EIGW +ALPHA_EIGW=$($AWS_EC2 create-egress-only-internet-gateway --vpc-id "$ALPHA_VPC" \ + --profile hive-team-alpha \ + --query 'EgressOnlyInternetGateway.EgressOnlyInternetGatewayId' --output text) +echo " Alpha EIGW: $ALPHA_EIGW" + +BETA_EIGW=$($AWS_EC2 create-egress-only-internet-gateway --vpc-id "$BETA_VPC" \ + --profile hive-team-beta \ + --query 'EgressOnlyInternetGateway.EgressOnlyInternetGatewayId' --output text) +echo " Beta EIGW: $BETA_EIGW" + +# EIGW describe isolation +ALPHA_EIGWS=$($AWS_EC2 describe-egress-only-internet-gateways --profile hive-team-alpha \ + --query 'EgressOnlyInternetGateways[].EgressOnlyInternetGatewayId' --output text) +if echo "$ALPHA_EIGWS" | grep -q "$BETA_EIGW"; then + echo " ERROR: Alpha can see Beta's EIGW" + exit 1 +fi +echo " EIGW describe isolation OK" + +# Cross-account EIGW delete (verify Alpha's still exists after Beta's attempt) +set +e +$AWS_EC2 delete-egress-only-internet-gateway \ + --egress-only-internet-gateway-id "$ALPHA_EIGW" --profile hive-team-beta 2>/dev/null +set -e +ALPHA_EIGW_CHECK=$($AWS_EC2 describe-egress-only-internet-gateways --profile hive-team-alpha \ + --query 'EgressOnlyInternetGateways[].EgressOnlyInternetGatewayId' --output text) +if ! echo "$ALPHA_EIGW_CHECK" | grep -q "$ALPHA_EIGW"; then + echo " ERROR: Alpha's EIGW was deleted by Beta" + exit 1 +fi +echo " Beta cannot delete Alpha's EIGW" + +echo " IGW + EIGW scoping passed" + +# --- 7g: Account Settings --- +echo "" +echo "Step 7g: Account Settings" +echo "----------------------------------------" + +$AWS_EC2 enable-ebs-encryption-by-default --profile hive-team-alpha > /dev/null +BETA_ENC=$($AWS_EC2 get-ebs-encryption-by-default --profile hive-team-beta \ + --query 'EbsEncryptionByDefault' --output text) +if [ "$BETA_ENC" != "False" ]; then + echo " ERROR: Alpha's encryption setting leaked to Beta (got $BETA_ENC)" + exit 1 +fi +echo " Alpha enable did not affect Beta" + +$AWS_EC2 enable-ebs-encryption-by-default --profile hive-team-beta > /dev/null +$AWS_EC2 disable-ebs-encryption-by-default --profile hive-team-alpha > /dev/null +ALPHA_ENC=$($AWS_EC2 get-ebs-encryption-by-default --profile hive-team-alpha \ + --query 'EbsEncryptionByDefault' --output text) +BETA_ENC=$($AWS_EC2 get-ebs-encryption-by-default --profile hive-team-beta \ + --query 'EbsEncryptionByDefault' --output text) +if [ "$ALPHA_ENC" != "False" ] || [ "$BETA_ENC" != "True" ]; then + echo " ERROR: Independent settings failed: alpha=$ALPHA_ENC beta=$BETA_ENC" + exit 1 +fi +echo " Independent toggle verified: alpha=$ALPHA_ENC, beta=$BETA_ENC" +$AWS_EC2 disable-ebs-encryption-by-default --profile hive-team-beta > /dev/null + +echo " Account settings scoping passed" + +# --- 7h: Global Resources --- +echo "" +echo "Step 7h: Global Resources" +echo "----------------------------------------" + +ALPHA_REGIONS=$($AWS_EC2 describe-regions --profile hive-team-alpha \ + --query 'Regions[].RegionName' --output text) +BETA_REGIONS=$($AWS_EC2 describe-regions --profile hive-team-beta \ + --query 'Regions[].RegionName' --output text) +if [ "$ALPHA_REGIONS" != "$BETA_REGIONS" ]; then + echo " ERROR: Regions differ between accounts" + exit 1 +fi +echo " Regions identical" + +ALPHA_AZS=$($AWS_EC2 describe-availability-zones --profile hive-team-alpha \ + --query 'AvailabilityZones[].ZoneName' --output text) +BETA_AZS=$($AWS_EC2 describe-availability-zones --profile hive-team-beta \ + --query 'AvailabilityZones[].ZoneName' --output text) +if [ "$ALPHA_AZS" != "$BETA_AZS" ]; then + echo " ERROR: AZs differ between accounts" + exit 1 +fi +echo " Availability zones identical" + +ALPHA_TYPES=$($AWS_EC2 describe-instance-types --profile hive-team-alpha \ + --query 'InstanceTypes[].InstanceType' --output text | tr '\t' '\n' | sort) +BETA_TYPES=$($AWS_EC2 describe-instance-types --profile hive-team-beta \ + --query 'InstanceTypes[].InstanceType' --output text | tr '\t' '\n' | sort) +if [ "$ALPHA_TYPES" != "$BETA_TYPES" ]; then + echo " ERROR: Instance types differ between accounts" + exit 1 +fi +echo " Instance types identical" + +echo " Global resources passed" + +# --- Step 8: Edge Cases --- +echo "" +echo "Step 8: Edge Cases" +echo "----------------------------------------" + +# Empty account (Gamma) +echo " Creating empty Gamma account..." +GAMMA_OUTPUT=$(./bin/hive admin account create --name "Team Gamma" --config "$HOME/node1/config/hive.toml" 2>&1) +GAMMA_KEY_ID=$(echo "$GAMMA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}') +GAMMA_SECRET=$(echo "$GAMMA_OUTPUT" | grep "Secret Access Key:" | awk '{print $NF}') +aws configure set aws_access_key_id "$GAMMA_KEY_ID" --profile hive-team-gamma +aws configure set aws_secret_access_key "$GAMMA_SECRET" --profile hive-team-gamma +aws configure set region us-east-1 --profile hive-team-gamma + +GAMMA_INSTANCES=$($AWS_EC2 describe-instances --profile hive-team-gamma \ + --query 'Reservations' --output text) +if [ -n "$GAMMA_INSTANCES" ] && [ "$GAMMA_INSTANCES" != "None" ]; then + echo " ERROR: Gamma has instances" + exit 1 +fi +echo " Gamma: no instances" + +# Skip volume check: root-account volumes (empty TenantID) are visible to all accounts by design +echo " Gamma: volumes skipped (root legacy volumes visible to all)" + +GAMMA_KEYS=$($AWS_EC2 describe-key-pairs --profile hive-team-gamma \ + --query 'KeyPairs' --output text) +if [ -n "$GAMMA_KEYS" ] && [ "$GAMMA_KEYS" != "None" ]; then + echo " ERROR: Gamma has key pairs" + exit 1 +fi +echo " Gamma: no key pairs" + +# Root isolation from tenants +echo " Verifying root isolation from tenants..." +ROOT_INSTANCE_CHECK=$($AWS_EC2 describe-instances \ + --query 'Reservations[].Instances[].InstanceId' --output text) +if echo "$ROOT_INSTANCE_CHECK" | grep -q "$ALPHA_INST"; then + echo " ERROR: Root can see Alpha's instance" + exit 1 +fi +echo " Root cannot see tenant instances" + +# Non-existent resource IDs — same error as cross-account +expect_error "InvalidVolume.NotFound" \ + $AWS_EC2 delete-volume --volume-id vol-00000000000000000 --profile hive-team-alpha +echo " Non-existent volume: same error as cross-account" + +expect_error "InvalidSnapshot.NotFound" \ + $AWS_EC2 delete-snapshot --snapshot-id snap-00000000000000000 --profile hive-team-alpha +echo " Non-existent snapshot: same error as cross-account" + +# Race condition: parallel key creation from both accounts +echo " Testing parallel key creation (race condition)..." +for i in $(seq 1 5); do + $AWS_EC2 create-key-pair --key-name "race-alpha-$i" --profile hive-team-alpha > /dev/null & + $AWS_EC2 create-key-pair --key-name "race-beta-$i" --profile hive-team-beta > /dev/null & +done +wait + +ALPHA_RACE_COUNT=$($AWS_EC2 describe-key-pairs --profile hive-team-alpha \ + --query 'KeyPairs[].KeyName' --output text | tr '\t' '\n' | grep -c "race-alpha" || true) +if [ "$ALPHA_RACE_COUNT" -ne 5 ]; then + echo " ERROR: Expected 5 race-alpha keys, got $ALPHA_RACE_COUNT" + exit 1 +fi +ALPHA_RACE_LEAK=$($AWS_EC2 describe-key-pairs --profile hive-team-alpha \ + --query 'KeyPairs[].KeyName' --output text | tr '\t' '\n' | grep -c "race-beta" || true) +if [ "$ALPHA_RACE_LEAK" -ne 0 ]; then + echo " ERROR: Alpha sees Beta's race keys (cross-contamination)" + exit 1 +fi +echo " Parallel key creation: no cross-contamination ($ALPHA_RACE_COUNT alpha keys, 0 beta leaks)" + +echo " Edge cases passed" + +# --- Step 9: EC2 + IAM Cleanup --- +echo "" +echo "Step 9: EC2 + IAM Cleanup" +echo "========================================" + +# Terminate instances +echo " Terminating account-scoped instances..." +$AWS_EC2 terminate-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha > /dev/null +$AWS_EC2 terminate-instances --instance-ids "$BETA_INST" --profile hive-team-beta > /dev/null + +COUNT=0 +while [ $COUNT -lt 30 ]; do + A_STATE=$($AWS_EC2 describe-instances --instance-ids "$ALPHA_INST" --profile hive-team-alpha \ + --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "terminated") + B_STATE=$($AWS_EC2 describe-instances --instance-ids "$BETA_INST" --profile hive-team-beta \ + --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "terminated") + if [ "$A_STATE" == "terminated" ] && [ "$B_STATE" == "terminated" ]; then + break + fi + sleep 2 + COUNT=$((COUNT + 1)) +done +echo " Instances terminated" + +# Delete snapshots +echo " Deleting snapshots..." +$AWS_EC2 delete-snapshot --snapshot-id "$ALPHA_SNAP" --profile hive-team-alpha 2>/dev/null || true +$AWS_EC2 delete-snapshot --snapshot-id "$BETA_SNAP" --profile hive-team-beta 2>/dev/null || true + +# Delete volumes +sleep 3 +echo " Deleting volumes..." +$AWS_EC2 delete-volume --volume-id "$ALPHA_VOL" --profile hive-team-alpha 2>/dev/null || true +$AWS_EC2 delete-volume --volume-id "$BETA_VOL" --profile hive-team-beta 2>/dev/null || true + +# Delete key pairs +echo " Deleting key pairs..." +for key in alpha-key alpha-instance-key shared-name imported-key; do + $AWS_EC2 delete-key-pair --key-name "$key" --profile hive-team-alpha 2>/dev/null || true +done +for key in beta-key beta-instance-key shared-name; do + $AWS_EC2 delete-key-pair --key-name "$key" --profile hive-team-beta 2>/dev/null || true +done +for i in $(seq 1 5); do + $AWS_EC2 delete-key-pair --key-name "race-alpha-$i" --profile hive-team-alpha 2>/dev/null || true + $AWS_EC2 delete-key-pair --key-name "race-beta-$i" --profile hive-team-beta 2>/dev/null || true +done + +# Delete EIGWs +echo " Deleting EIGWs..." +$AWS_EC2 delete-egress-only-internet-gateway \ + --egress-only-internet-gateway-id "$ALPHA_EIGW" --profile hive-team-alpha 2>/dev/null || true +$AWS_EC2 delete-egress-only-internet-gateway \ + --egress-only-internet-gateway-id "$BETA_EIGW" --profile hive-team-beta 2>/dev/null || true + +# Detach + delete IGWs +echo " Deleting IGWs..." +$AWS_EC2 detach-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ + --vpc-id "$ALPHA_VPC" --profile hive-team-alpha 2>/dev/null || true +$AWS_EC2 delete-internet-gateway --internet-gateway-id "$ALPHA_IGW" \ + --profile hive-team-alpha 2>/dev/null || true +$AWS_EC2 delete-internet-gateway --internet-gateway-id "$BETA_IGW" \ + --profile hive-team-beta 2>/dev/null || true + +# Delete subnets +echo " Deleting subnets..." +$AWS_EC2 delete-subnet --subnet-id "$ALPHA_SUBNET" --profile hive-team-alpha 2>/dev/null || true +$AWS_EC2 delete-subnet --subnet-id "$BETA_SUBNET" --profile hive-team-beta 2>/dev/null || true + +# Delete VPCs +echo " Deleting VPCs..." +$AWS_EC2 delete-vpc --vpc-id "$ALPHA_VPC" --profile hive-team-alpha 2>/dev/null || true +$AWS_EC2 delete-vpc --vpc-id "$BETA_VPC" --profile hive-team-beta 2>/dev/null || true + +echo " EC2 resources cleaned up" + +# IAM cleanup (originally Step 7) +echo " Cleaning up IAM resources..." + +# Beta cleanup +aws $AWS_EP iam detach-user-policy --user-name alice \ + --policy-arn "arn:aws:iam::${BETA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-beta +aws $AWS_EP iam delete-access-key --user-name alice \ + --access-key-id "$BETA_ALICE_KEY_ID" --profile hive-team-beta +aws $AWS_EP iam delete-user --user-name alice --profile hive-team-beta +aws $AWS_EP iam delete-user --user-name dev-user --profile hive-team-beta +aws $AWS_EP iam delete-policy \ + --policy-arn "arn:aws:iam::${BETA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-beta + +# Alpha cleanup (alice already deleted in Step 6) +aws $AWS_EP iam delete-user --user-name team-member --profile hive-team-alpha +aws $AWS_EP iam delete-policy \ + --policy-arn "arn:aws:iam::${ALPHA_ACCOUNT}:policy/EC2ReadOnly" --profile hive-team-alpha + +# Clean up AWS CLI profiles +for p in hive-team-alpha hive-team-beta hive-alpha-alice hive-beta-alice hive-team-gamma; do + aws configure set aws_access_key_id "" --profile $p 2>/dev/null || true + aws configure set aws_secret_access_key "" --profile $p 2>/dev/null || true +done + +echo " IAM + EC2 account cleanup complete" +echo "" +echo " IAM Accounts & Cross-Account Isolation tests passed" + +# Phase 5d: VPC Networking +echo "" +echo "Phase 5d: VPC Networking" +echo "========================================" +echo "Testing VPC instance launch, PrivateIpAddress, and same-subnet connectivity..." + +# Step 1: Create VPC and Subnet +echo "" +echo "Step 1: Create VPC + Subnet" +echo "----------------------------------------" + +VPC_OUTPUT=$($AWS_EC2 create-vpc --cidr-block 10.100.0.0/16) +VPC_ID=$(echo "$VPC_OUTPUT" | jq -r '.Vpc.VpcId') +if [ -z "$VPC_ID" ] || [ "$VPC_ID" == "null" ]; then + echo " ERROR: Failed to create VPC" + echo " Output: $VPC_OUTPUT" + exit 1 +fi +echo " Created VPC: $VPC_ID (10.100.0.0/16)" + +SUBNET_OUTPUT=$($AWS_EC2 create-subnet --vpc-id "$VPC_ID" --cidr-block 10.100.1.0/24) +SUBNET_ID=$(echo "$SUBNET_OUTPUT" | jq -r '.Subnet.SubnetId') +if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" == "null" ]; then + echo " ERROR: Failed to create subnet" + echo " Output: $SUBNET_OUTPUT" + exit 1 +fi +echo " Created Subnet: $SUBNET_ID (10.100.1.0/24)" + +# Brief pause for OVN topology to be programmed (logical switch + router port + DHCP) +sleep 2 + +# Step 2: Launch 3 VPC instances +echo "" +echo "Step 2: Launch 3 VPC instances" +echo "----------------------------------------" + +VPC_INSTANCE_IDS=() +for i in 1 2 3; do + echo " Launching VPC instance $i with subnet $SUBNET_ID..." + RUN_OUTPUT=$($AWS_EC2 run-instances \ + --image-id "$AMI_ID" \ + --instance-type "$INSTANCE_TYPE" \ + --key-name multinode-test-key \ + --subnet-id "$SUBNET_ID") + + VPC_INST_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId') + VPC_INST_IP=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].PrivateIpAddress // empty') + + if [ -z "$VPC_INST_ID" ] || [ "$VPC_INST_ID" == "null" ]; then + echo " ERROR: Failed to launch VPC instance $i" + echo " Output: $RUN_OUTPUT" + exit 1 + fi + echo " Launched: $VPC_INST_ID (PrivateIpAddress: ${VPC_INST_IP:-not yet assigned})" + VPC_INSTANCE_IDS+=("$VPC_INST_ID") + + sleep 2 +done + +# Wait for all VPC instances to be running +echo "" +echo "Waiting for VPC instances to reach running state..." +for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + wait_for_instance_state "$vpc_inst" "running" 30 || { + echo "ERROR: VPC instance $vpc_inst failed to start" + exit 1 + } +done + +# Step 3: Verify PrivateIpAddress in DescribeInstances +echo "" +echo "Step 3: Verify PrivateIpAddress in DescribeInstances" +echo "----------------------------------------" + +VPC_PRIVATE_IPS=() +for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + DESCRIBE_OUT=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst") + PRIVATE_IP=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].PrivateIpAddress // empty') + INST_SUBNET=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].SubnetId // empty') + INST_VPC=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].VpcId // empty') + ENI_COUNT=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].NetworkInterfaces | length') + + if [ -z "$PRIVATE_IP" ]; then + echo " ERROR: $vpc_inst has no PrivateIpAddress" + echo " Describe output: $(echo "$DESCRIBE_OUT" | jq -c '.Reservations[0].Instances[0] | {PrivateIpAddress, SubnetId, VpcId, NetworkInterfaces}')" + exit 1 + fi + + echo " $vpc_inst: IP=$PRIVATE_IP, Subnet=$INST_SUBNET, VPC=$INST_VPC, ENIs=$ENI_COUNT" + + if [ "$INST_SUBNET" != "$SUBNET_ID" ]; then + echo " ERROR: SubnetId mismatch (expected $SUBNET_ID, got $INST_SUBNET)" + exit 1 + fi + if [ "$INST_VPC" != "$VPC_ID" ]; then + echo " ERROR: VpcId mismatch (expected $VPC_ID, got $INST_VPC)" + exit 1 + fi + if [ "$ENI_COUNT" -lt 1 ]; then + echo " ERROR: No NetworkInterfaces found" + exit 1 + fi + + VPC_PRIVATE_IPS+=("$PRIVATE_IP") +done + +# Verify all IPs are unique and in the subnet range (10.100.1.x) +echo "" +echo " Verifying IP uniqueness and subnet range..." +UNIQUE_IPS=$(printf '%s\n' "${VPC_PRIVATE_IPS[@]}" | sort -u | wc -l) +if [ "$UNIQUE_IPS" -ne "${#VPC_PRIVATE_IPS[@]}" ]; then + echo " ERROR: Duplicate IPs detected: ${VPC_PRIVATE_IPS[*]}" + exit 1 +fi +for ip in "${VPC_PRIVATE_IPS[@]}"; do + if ! echo "$ip" | grep -qE '^10\.100\.1\.[0-9]+$'; then + echo " ERROR: IP $ip not in expected subnet 10.100.1.0/24" + exit 1 + fi +done +echo " All IPs unique and in correct subnet: ${VPC_PRIVATE_IPS[*]}" + +# Step 4: SSH via DEV_NETWORKING hostfwd and test ping connectivity +# DISABLED: VPC instances use OVN DHCP which takes ~6 min per instance to get SSH ready. +# With 3 instances this adds ~18 min to the E2E run. The SSH+ping test is best-effort +# anyway (OVN overlay not fully programmed in Docker single-host mode). IP allocation +# and subnet correctness are already verified in Step 3. +echo "" +echo "Step 4: SSH + Ping Connectivity (SKIPPED — OVN DHCP wait too slow in CI)" +echo "----------------------------------------" + +# Step 5: Stop/Start IP persistence +echo "" +echo "Step 5: Stop/Start IP Persistence" +echo "----------------------------------------" +echo "Verifying private IPs persist through stop/start cycle (AWS behavior)..." + +# Record IPs before stop +echo " IPs before stop: ${VPC_PRIVATE_IPS[*]}" + +# Stop all VPC instances +echo "" +echo " Stopping all VPC instances..." +for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + echo " Stopping $vpc_inst..." + $AWS_EC2 stop-instances --instance-ids "$vpc_inst" > /dev/null +done + +for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + wait_for_instance_state "$vpc_inst" "stopped" 30 || { + echo " ERROR: VPC instance $vpc_inst failed to stop" + exit 1 + } +done +echo " All VPC instances stopped" + +# Verify IPs are still present in DescribeInstances while stopped +echo "" +echo " Verifying IPs persist in stopped state..." +for idx in "${!VPC_INSTANCE_IDS[@]}"; do + vpc_inst="${VPC_INSTANCE_IDS[$idx]}" + expected_ip="${VPC_PRIVATE_IPS[$idx]}" + + STOPPED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \ + --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text) + + if [ "$STOPPED_IP" != "$expected_ip" ]; then + echo " ERROR: $vpc_inst IP changed while stopped (expected $expected_ip, got $STOPPED_IP)" + exit 1 + fi + echo " $vpc_inst: IP=$STOPPED_IP (unchanged)" +done + +# Start all VPC instances +echo "" +echo " Starting all VPC instances..." +for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + echo " Starting $vpc_inst..." + $AWS_EC2 start-instances --instance-ids "$vpc_inst" > /dev/null +done + +for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do + wait_for_instance_state "$vpc_inst" "running" 30 || { + echo " ERROR: VPC instance $vpc_inst failed to restart" + exit 1 + } +done +echo " All VPC instances restarted" + +# Verify IPs are identical after restart +echo "" +echo " Verifying IPs persist after restart..." +IP_MISMATCHES=0 +for idx in "${!VPC_INSTANCE_IDS[@]}"; do + vpc_inst="${VPC_INSTANCE_IDS[$idx]}" + expected_ip="${VPC_PRIVATE_IPS[$idx]}" + + RESTARTED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \ + --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text) + + if [ "$RESTARTED_IP" == "$expected_ip" ]; then + echo " $vpc_inst: IP=$RESTARTED_IP (matches pre-stop)" + else + echo " ERROR: $vpc_inst IP changed after restart (expected $expected_ip, got $RESTARTED_IP)" + IP_MISMATCHES=$((IP_MISMATCHES + 1)) + fi +done + +if [ "$IP_MISMATCHES" -gt 0 ]; then + echo " ERROR: $IP_MISMATCHES instances had IP changes — ENI not persisting through stop/start" + exit 1 +fi + +echo " Stop/start IP persistence verified — all IPs match" + +# Step 6: Clean up VPC instances +echo "" +echo "Step 6: Clean up VPC resources" +echo "----------------------------------------" + +terminate_and_wait "${VPC_INSTANCE_IDS[@]}" || true + +# Clean up subnet and VPC +echo " Deleting subnet $SUBNET_ID..." +$AWS_EC2 delete-subnet --subnet-id "$SUBNET_ID" 2>/dev/null || echo " (subnet delete failed, may have ENIs)" + +echo " Deleting VPC $VPC_ID..." +$AWS_EC2 delete-vpc --vpc-id "$VPC_ID" 2>/dev/null || echo " (vpc delete failed, may have subnets)" + +echo " VPC networking tests passed" + +# Phase 6: Cluster Shutdown + Restart +echo "" +echo "Phase 6: Cluster Shutdown + Restart" +echo "========================================" +echo "Testing hive admin cluster shutdown command..." + +# Test 6a: Dry-run shutdown +echo "" +echo "Test 6a: Dry-Run Shutdown" +echo "----------------------------------------" +echo "Running cluster shutdown in dry-run mode..." + +DRY_RUN_OUTPUT=$(./bin/hive admin cluster shutdown --dry-run --config "$HOME/node1/config/hive.toml" 2>&1) +echo "$DRY_RUN_OUTPUT" + +# Validate dry-run output contains expected phases +for phase in GATE DRAIN STORAGE PERSIST INFRA; do + if echo "$DRY_RUN_OUTPUT" | grep -qi "$phase"; then + echo " Phase $phase found in shutdown plan" + else + echo " WARNING: Phase $phase not found in dry-run output" + fi +done +echo " Dry-run shutdown test passed" + +# Test 6b: Real coordinated shutdown +echo "" +echo "Test 6b: Coordinated Cluster Shutdown" +echo "----------------------------------------" +echo "Running cluster shutdown..." + +./bin/hive admin cluster shutdown --force --timeout 30s --config "$HOME/node1/config/hive.toml" 2>&1 || { + echo " WARNING: Cluster shutdown command returned non-zero exit code" +} +CLUSTER_SERVICES_STARTED="false" + +# Verify all services are down +echo " Waiting for services to stop..." +sleep 2 +if ! verify_all_services_down; then + echo " Some services still running, force-cleaning..." + force_cleanup_all_nodes +fi + +# Test 6c: Restart and recovery +echo "" +echo "Test 6c: Cluster Restart + Recovery" +echo "----------------------------------------" +echo "Restarting all node services concurrently..." + +# Cluster restart requires concurrent startup: NATS needs route peers to form, +# Predastore needs Raft quorum (2/3), and the daemon needs JetStream. +# Sequential start would leave node1 waiting for quorum that never arrives. +start_node_services 1 "$HOME/node1" & +start_node_services 2 "$HOME/node2" & +start_node_services 3 "$HOME/node3" & +wait +CLUSTER_SERVICES_STARTED="true" + +echo "" +echo "Waiting for cluster to stabilize..." +sleep 5 + +# Verify NATS cluster reformed +echo "" +verify_nats_cluster 3 || { + echo "WARNING: NATS cluster verification failed after restart" +} + +# Wait for gateway +echo "" +wait_for_gateway "${NODE1_IP}" 15 + +# Wait for daemon readiness +wait_for_daemon_ready "https://${NODE1_IP}:${AWSGW_PORT}" + +# Smoke test: describe-instance-types +echo "" +echo "Running post-restart smoke test..." +SMOKE_OUTPUT=$($AWS_EC2 describe-instance-types --query 'InstanceTypes[*].InstanceType' --output text 2>/dev/null) +if [ -n "$SMOKE_OUTPUT" ] && [ "$SMOKE_OUTPUT" != "None" ]; then + echo " Smoke test passed: describe-instance-types returned: $SMOKE_OUTPUT" +else + echo " ERROR: Smoke test failed: describe-instance-types returned empty/None" + exit 1 +fi + +echo " Cluster shutdown + restart test passed" + +# Test 6d: Instance relaunch and terminate after restart +echo "" +echo "Test 6d: Instance Relaunch + Terminate" +echo "----------------------------------------" +echo "Waiting for instances to relaunch after cluster restart..." + +# Two instances were running before shutdown — the daemon will relaunch them. +# Must wait for them to finish launching (pending → running) before terminate +# will work, because the NATS per-instance subscription is only created after +# QEMU starts. +for instance_id in "${INSTANCE_IDS[0]}" "${INSTANCE_IDS[1]}"; do + echo " Waiting for $instance_id to finish relaunching..." + COUNT=0 + while [ $COUNT -lt 30 ]; do + STATE=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ + --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "unknown") + if [ "$STATE" = "running" ]; then + echo " $instance_id relaunched successfully: $STATE" + break + fi + sleep 2 + COUNT=$((COUNT + 1)) + done + if [ $COUNT -ge 30 ]; then + echo " WARNING: $instance_id still in $STATE after 60s" + fi +done + +# Verify crash-loop instance (INSTANCE_IDS[2]) stayed in error — daemon should +# not relaunch an instance that exceeded its crash restart limit. +CRASH_LOOP_STATE=$($AWS_EC2 describe-instances --instance-ids "${INSTANCE_IDS[2]}" \ + --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo "unknown") +echo " Crash-loop instance ${INSTANCE_IDS[2]} state: $CRASH_LOOP_STATE" +if [ "$CRASH_LOOP_STATE" != "error" ]; then + echo " WARNING: Expected crash-loop instance to remain in error state (got: $CRASH_LOOP_STATE)" +fi + +# Terminate all instances (including the crash-loop one in error state) +echo "" +echo "Terminating all instances..." +if ! terminate_and_wait "${INSTANCE_IDS[@]}"; then + echo "" + echo "ERROR: Some instances failed to terminate properly after restart" + dump_all_node_logs + exit 1 +fi + +echo " Instance relaunch + terminate after restart passed" + +echo "" +echo "========================================" +echo "Multi-Node E2E Tests Completed Successfully" +echo "========================================" +exit 0