feat: multi node e2e (#73)

This commit is contained in:
Julian Sommer
2026-03-13 09:11:26 +10:00
committed by GitHub
parent 4466de04d3
commit 3ffe67271f
14 changed files with 3688 additions and 2743 deletions
+163 -47
View File
@@ -1,4 +1,4 @@
name: Hive E2E Test
name: Hive E2E Tests
on:
push:
@@ -11,70 +11,186 @@ on:
- go.sum
- Makefile
- tests/e2e/**
- scripts/**
- .github/workflows/e2e.yml
workflow_dispatch:
env:
GIT_BRANCH: ${{ github.ref_name }}
jobs:
build_e2e_image:
name: Build E2E Image
runs-on: self-hosted
e2e_single:
name: Single-Node E2E
runs-on: [self-hosted, ci-single]
timeout-minutes: 25
env:
TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
steps:
- name: Checkout Code
uses: actions/checkout@v6
- name: Clean workspace
run: rm -rf "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v2
id: app-token
with:
path: hive
app-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- name: Clone Dependencies
run: ./hive/scripts/clone-deps.sh
- uses: actions/checkout@v6
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- name: Build E2E Image
- name: Provision VM
working-directory: mulga/scripts/tofu-cluster
run: |
docker build \
-t hive-e2e:latest \
-f hive/tests/e2e/Dockerfile.e2e \
.
export TF_VAR_git_branch="${GIT_BRANCH}"
tofu init -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
hive_e2e_test:
name: Hive Single-Node E2E Test
runs-on: self-hosted
needs: build_e2e_image
- name: Bootstrap Cluster
working-directory: mulga/scripts/tofu-cluster
run: ./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1
steps:
- name: Run E2E Tests
working-directory: mulga/scripts/tofu-cluster
run: |
docker run --privileged --rm \
-v /dev/kvm:/dev/kvm \
--name hive-e2e-test \
hive-e2e:latest
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
'cd ~/Development/mulga/hive && ./tests/e2e/run-e2e.sh'
hive_multinode_e2e_test:
name: Hive Multi-Node E2E Test
runs-on: self-hosted
needs: build_e2e_image
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
e2e_pseudo_multi:
name: Pseudo Multi-Node E2E
runs-on: [self-hosted, ci-pseudo]
timeout-minutes: 25
env:
TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log
steps:
- name: Run Multi-Node E2E Tests
- name: Clean workspace
run: rm -rf "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v2
id: app-token
with:
app-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v6
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- name: Provision VM
working-directory: mulga/scripts/tofu-cluster
run: |
docker run --privileged --rm \
-v /dev/kvm:/dev/kvm \
--cap-add=NET_ADMIN \
--name hive-multinode-e2e-test \
hive-e2e:latest \
./tests/e2e/run-multinode-e2e.sh
export TF_VAR_git_branch="${GIT_BRANCH}"
tofu init -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
cleanup:
name: Cleanup
runs-on: self-hosted
needs: [hive_e2e_test, hive_multinode_e2e_test]
if: always()
- name: Bootstrap Cluster
working-directory: mulga/scripts/tofu-cluster
run: ./bootstrap.sh env4 >> "$TOFU_LOG" 2>&1
- name: Run E2E Tests
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
'cd ~/Development/mulga/hive && ./tests/e2e/run-pseudo-multinode-e2e.sh'
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true
e2e_real_multi:
name: Real Multi-Node E2E
runs-on: [self-hosted, ci-multi]
timeout-minutes: 25
env:
TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
steps:
- name: Remove Stale Containers
run: |
docker rm -f hive-e2e-test 2>/dev/null || true
docker rm -f hive-multinode-e2e-test 2>/dev/null || true
- name: Clean workspace
run: rm -rf "$GITHUB_WORKSPACE"/*
- name: Prune Docker Images
run: docker image prune -f
- uses: actions/create-github-app-token@v2
id: app-token
with:
app-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v6
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- name: Provision VMs
working-directory: mulga/scripts/tofu-cluster
run: |
export TF_VAR_git_branch="${GIT_BRANCH}"
tofu init -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
- name: Bootstrap Cluster
working-directory: mulga/scripts/tofu-cluster
run: ./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1
- name: Run E2E Tests
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
# Build node IP list
NODE_IPS=""
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
for i in $(seq 0 $((NODE_COUNT - 1))); do
NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
done
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
"cd ~/Development/mulga/hive && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS}"
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
+1 -34
View File
@@ -193,39 +193,6 @@ security-check:
$(_Q)set -o pipefail && go tool staticcheck -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-SA1019,-SA9005" ./... $(_SECQ) tests/staticcheck-report.txt $(if $(QUIET),|| { cat tests/staticcheck-report.txt; exit 1; })
@echo " staticcheck ok"
# Docker E2E tests (mirrors GitHub Actions e2e.yml)
# Usage: make test-docker # both suites
# make test-docker-single # single-node only
# make test-docker-multi # multi-node only
PARENT_DIR := $(shell cd .. && pwd)
E2E_IMAGE := hive-e2e:latest
test-docker-build:
@echo -e "\n....Building E2E Docker image...."
@for dep in viperblock predastore; do \
if [ ! -d "$(PARENT_DIR)/$$dep" ]; then \
echo "Missing sibling repo $$dep — running clone-deps.sh"; \
./scripts/clone-deps.sh; \
break; \
fi; \
done
docker build -t $(E2E_IMAGE) -f tests/e2e/Dockerfile.e2e $(PARENT_DIR)
test-docker-single: test-docker-build
@echo -e "\n....Running Single-Node E2E Tests...."
docker run --privileged --rm -v /dev/kvm:/dev/kvm $(E2E_IMAGE)
test-docker-multi: test-docker-build
@echo -e "\n....Running Multi-Node E2E Tests...."
docker run --privileged --rm -v /dev/kvm:/dev/kvm --cap-add=NET_ADMIN $(E2E_IMAGE) ./tests/e2e/run-multinode-e2e.sh
test-docker: test-docker-build
@echo -e "\n....Running Single-Node E2E Tests...."
docker run --privileged --rm -v /dev/kvm:/dev/kvm $(E2E_IMAGE)
@echo -e "\n....Running Multi-Node E2E Tests...."
docker run --privileged --rm -v /dev/kvm:/dev/kvm --cap-add=NET_ADMIN $(E2E_IMAGE) ./tests/e2e/run-multinode-e2e.sh
.PHONY: build build-ui go_build go_run preflight test test-cover test-race diff-coverage bench run clean \
install-system install-go install-aws quickinstall \
format check-format modernize check-modernize vet security-check \
test-docker-build test-docker-single test-docker-multi test-docker
format check-format modernize check-modernize vet security-check
+1 -1
View File
@@ -1 +1 @@
24.12.0
24.14.0
+1 -1
View File
@@ -48,7 +48,7 @@
"@vitejs/plugin-react": "^5.1.4",
"@vitest/coverage-v8": "^4.0.18",
"babel-plugin-react-compiler": "^1.0.0",
"jsdom": "^28.1.0",
"happy-dom": "^20.8.3",
"typescript": "^5.9.3",
"ultracite": "^7.2.5",
"vite": "^7.3.1",
+124 -28
View File
@@ -104,13 +104,13 @@ importers:
version: 5.1.4(vite@7.3.1(@types/node@24.12.0)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0))
'@vitest/coverage-v8':
specifier: ^4.0.18
version: 4.0.18(vitest@4.0.18(@types/node@24.12.0)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0))
version: 4.0.18(vitest@4.0.18(@types/node@24.12.0)(happy-dom@20.8.3)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0))
babel-plugin-react-compiler:
specifier: ^1.0.0
version: 1.0.0
jsdom:
specifier: ^28.1.0
version: 28.1.0
happy-dom:
specifier: ^20.8.3
version: 20.8.3
typescript:
specifier: ^5.9.3
version: 5.9.3
@@ -122,7 +122,7 @@ importers:
version: 7.3.1(@types/node@24.12.0)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)
vitest:
specifier: ^4.0.18
version: 4.0.18(@types/node@24.12.0)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0)
version: 4.0.18(@types/node@24.12.0)(happy-dom@20.8.3)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0)
packages:
@@ -1338,6 +1338,12 @@ packages:
'@types/react@19.2.14':
resolution: {integrity: sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==}
'@types/whatwg-mimetype@3.0.2':
resolution: {integrity: sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==}
'@types/ws@8.18.1':
resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}
'@typescript/native-preview-darwin-arm64@7.0.0-dev.20260307.1':
resolution: {integrity: sha512-VpnrMP4iDLSTT9Hg/KrHwuIHLZr5dxYPMFErfv3ZDA0tv48u2H1lBhHVVMMopCuskuX3C35EOJbxLkxCJd6zDw==}
cpu: [arm64]
@@ -1608,6 +1614,10 @@ packages:
resolution: {integrity: sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==}
engines: {node: '>=0.12'}
entities@7.0.1:
resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==}
engines: {node: '>=0.12'}
es-module-lexer@1.7.0:
resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==}
@@ -1679,6 +1689,10 @@ packages:
graceful-fs@4.2.11:
resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
happy-dom@20.8.3:
resolution: {integrity: sha512-lMHQRRwIPyJ70HV0kkFT7jH/gXzSI7yDkQFe07E2flwmNDFoWUTRMKpW2sglsnpeA7b6S2TJPp98EbQxai8eaQ==}
engines: {node: '>=20.0.0'}
has-flag@4.0.0:
resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
engines: {node: '>=8'}
@@ -2262,6 +2276,10 @@ packages:
webpack-virtual-modules@0.6.2:
resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==}
whatwg-mimetype@3.0.0:
resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==}
engines: {node: '>=12'}
whatwg-mimetype@5.0.0:
resolution: {integrity: sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==}
engines: {node: '>=20'}
@@ -2275,6 +2293,18 @@ packages:
engines: {node: '>=8'}
hasBin: true
ws@8.19.0:
resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==}
engines: {node: '>=10.0.0'}
peerDependencies:
bufferutil: ^4.0.1
utf-8-validate: '>=5.0.2'
peerDependenciesMeta:
bufferutil:
optional: true
utf-8-validate:
optional: true
xml-name-validator@5.0.0:
resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==}
engines: {node: '>=18'}
@@ -2293,7 +2323,8 @@ packages:
snapshots:
'@acemir/cssom@0.9.31': {}
'@acemir/cssom@0.9.31':
optional: true
'@adobe/css-tools@4.4.4': {}
@@ -2304,6 +2335,7 @@ snapshots:
'@csstools/css-parser-algorithms': 4.0.0(@csstools/css-tokenizer@4.0.0)
'@csstools/css-tokenizer': 4.0.0
lru-cache: 11.2.6
optional: true
'@asamuzakjp/dom-selector@6.8.1':
dependencies:
@@ -2312,8 +2344,10 @@ snapshots:
css-tree: 3.2.1
is-potential-custom-element-name: 1.0.1
lru-cache: 11.2.6
optional: true
'@asamuzakjp/nwsapi@2.3.9': {}
'@asamuzakjp/nwsapi@2.3.9':
optional: true
'@aws-crypto/crc32@5.2.0':
dependencies:
@@ -3066,6 +3100,7 @@ snapshots:
'@bramus/specificity@2.4.2':
dependencies:
css-tree: 3.2.1
optional: true
'@clack/core@1.1.0':
dependencies:
@@ -3076,12 +3111,14 @@ snapshots:
'@clack/core': 1.1.0
sisteransi: 1.0.5
'@csstools/color-helpers@6.0.2': {}
'@csstools/color-helpers@6.0.2':
optional: true
'@csstools/css-calc@3.1.1(@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.0))(@csstools/css-tokenizer@4.0.0)':
dependencies:
'@csstools/css-parser-algorithms': 4.0.0(@csstools/css-tokenizer@4.0.0)
'@csstools/css-tokenizer': 4.0.0
optional: true
'@csstools/css-color-parser@4.0.2(@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.0))(@csstools/css-tokenizer@4.0.0)':
dependencies:
@@ -3089,14 +3126,18 @@ snapshots:
'@csstools/css-calc': 3.1.1(@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.0))(@csstools/css-tokenizer@4.0.0)
'@csstools/css-parser-algorithms': 4.0.0(@csstools/css-tokenizer@4.0.0)
'@csstools/css-tokenizer': 4.0.0
optional: true
'@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.0)':
dependencies:
'@csstools/css-tokenizer': 4.0.0
optional: true
'@csstools/css-syntax-patches-for-csstree@1.1.0': {}
'@csstools/css-syntax-patches-for-csstree@1.1.0':
optional: true
'@csstools/css-tokenizer@4.0.0': {}
'@csstools/css-tokenizer@4.0.0':
optional: true
'@esbuild/aix-ppc64@0.27.3':
optional: true
@@ -3176,7 +3217,8 @@ snapshots:
'@esbuild/win32-x64@0.27.3':
optional: true
'@exodus/bytes@1.15.0': {}
'@exodus/bytes@1.15.0':
optional: true
'@floating-ui/core@1.7.5':
dependencies:
@@ -3877,6 +3919,12 @@ snapshots:
dependencies:
csstype: 3.2.3
'@types/whatwg-mimetype@3.0.2': {}
'@types/ws@8.18.1':
dependencies:
'@types/node': 24.12.0
'@typescript/native-preview-darwin-arm64@7.0.0-dev.20260307.1':
optional: true
@@ -3924,7 +3972,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
'@vitest/coverage-v8@4.0.18(vitest@4.0.18(@types/node@24.12.0)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0))':
'@vitest/coverage-v8@4.0.18(vitest@4.0.18(@types/node@24.12.0)(happy-dom@20.8.3)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0))':
dependencies:
'@bcoe/v8-coverage': 1.0.2
'@vitest/utils': 4.0.18
@@ -3936,7 +3984,7 @@ snapshots:
obug: 2.1.1
std-env: 3.10.0
tinyrainbow: 3.0.3
vitest: 4.0.18(@types/node@24.12.0)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0)
vitest: 4.0.18(@types/node@24.12.0)(happy-dom@20.8.3)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0)
'@vitest/expect@4.0.18':
dependencies:
@@ -3979,7 +4027,8 @@ snapshots:
acorn@8.16.0: {}
agent-base@7.1.4: {}
agent-base@7.1.4:
optional: true
ansi-regex@5.0.1: {}
@@ -4032,6 +4081,7 @@ snapshots:
bidi-js@1.0.3:
dependencies:
require-from-string: 2.0.2
optional: true
binary-extensions@2.3.0: {}
@@ -4092,6 +4142,7 @@ snapshots:
dependencies:
mdn-data: 2.27.1
source-map-js: 1.2.1
optional: true
css.escape@1.5.1: {}
@@ -4101,6 +4152,7 @@ snapshots:
'@csstools/css-syntax-patches-for-csstree': 1.1.0
css-tree: 3.2.1
lru-cache: 11.2.6
optional: true
csstype@3.2.3: {}
@@ -4110,12 +4162,14 @@ snapshots:
whatwg-url: 16.0.1
transitivePeerDependencies:
- '@noble/hashes'
optional: true
debug@4.4.3:
dependencies:
ms: 2.1.3
decimal.js@10.6.0: {}
decimal.js@10.6.0:
optional: true
deepmerge@4.3.1: {}
@@ -4136,7 +4190,10 @@ snapshots:
graceful-fs: 4.2.11
tapable: 2.3.0
entities@6.0.1: {}
entities@6.0.1:
optional: true
entities@7.0.1: {}
es-module-lexer@1.7.0: {}
@@ -4217,6 +4274,18 @@ snapshots:
graceful-fs@4.2.11: {}
happy-dom@20.8.3:
dependencies:
'@types/node': 24.12.0
'@types/whatwg-mimetype': 3.0.2
'@types/ws': 8.18.1
entities: 7.0.1
whatwg-mimetype: 3.0.0
ws: 8.19.0
transitivePeerDependencies:
- bufferutil
- utf-8-validate
has-flag@4.0.0: {}
html-encoding-sniffer@6.0.0:
@@ -4224,6 +4293,7 @@ snapshots:
'@exodus/bytes': 1.15.0
transitivePeerDependencies:
- '@noble/hashes'
optional: true
html-escaper@2.0.2: {}
@@ -4233,6 +4303,7 @@ snapshots:
debug: 4.4.3
transitivePeerDependencies:
- supports-color
optional: true
https-proxy-agent@7.0.6:
dependencies:
@@ -4240,6 +4311,7 @@ snapshots:
debug: 4.4.3
transitivePeerDependencies:
- supports-color
optional: true
ieee754@1.2.1: {}
@@ -4259,7 +4331,8 @@ snapshots:
is-number@7.0.0: {}
is-potential-custom-element-name@1.0.1: {}
is-potential-custom-element-name@1.0.1:
optional: true
isbot@5.1.35: {}
@@ -4308,6 +4381,7 @@ snapshots:
transitivePeerDependencies:
- '@noble/hashes'
- supports-color
optional: true
jsesc@3.1.0: {}
@@ -4390,7 +4464,8 @@ snapshots:
dependencies:
semver: 7.7.4
mdn-data@2.27.1: {}
mdn-data@2.27.1:
optional: true
min-indent@1.0.1: {}
@@ -4419,6 +4494,7 @@ snapshots:
parse5@8.0.0:
dependencies:
entities: 6.0.1
optional: true
path-scurry@2.0.2:
dependencies:
@@ -4447,7 +4523,8 @@ snapshots:
ansi-styles: 5.2.0
react-is: 17.0.2
punycode@2.3.1: {}
punycode@2.3.1:
optional: true
react-dom@19.2.4(react@19.2.4):
dependencies:
@@ -4487,7 +4564,8 @@ snapshots:
indent-string: 4.0.0
strip-indent: 3.0.0
require-from-string@2.0.2: {}
require-from-string@2.0.2:
optional: true
reselect@5.1.1: {}
@@ -4529,6 +4607,7 @@ snapshots:
saxes@6.0.0:
dependencies:
xmlchars: 2.2.0
optional: true
scheduler@0.27.0: {}
@@ -4575,7 +4654,8 @@ snapshots:
dependencies:
has-flag: 4.0.0
symbol-tree@3.2.4: {}
symbol-tree@3.2.4:
optional: true
tabbable@6.4.0: {}
@@ -4600,11 +4680,13 @@ snapshots:
tinyrainbow@3.0.3: {}
tldts-core@7.0.25: {}
tldts-core@7.0.25:
optional: true
tldts@7.0.25:
dependencies:
tldts-core: 7.0.25
optional: true
to-regex-range@5.0.1:
dependencies:
@@ -4613,10 +4695,12 @@ snapshots:
tough-cookie@6.0.0:
dependencies:
tldts: 7.0.25
optional: true
tr46@6.0.0:
dependencies:
punycode: 2.3.1
optional: true
tslib@2.8.1: {}
@@ -4640,7 +4724,8 @@ snapshots:
undici-types@7.16.0: {}
undici@7.22.0: {}
undici@7.22.0:
optional: true
unplugin@2.3.11:
dependencies:
@@ -4676,7 +4761,7 @@ snapshots:
lightningcss: 1.31.1
tsx: 4.21.0
vitest@4.0.18(@types/node@24.12.0)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0):
vitest@4.0.18(@types/node@24.12.0)(happy-dom@20.8.3)(jiti@2.6.1)(jsdom@28.1.0)(lightningcss@1.31.1)(tsx@4.21.0):
dependencies:
'@vitest/expect': 4.0.18
'@vitest/mocker': 4.0.18(vite@7.3.1(@types/node@24.12.0)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0))
@@ -4700,6 +4785,7 @@ snapshots:
why-is-node-running: 2.3.0
optionalDependencies:
'@types/node': 24.12.0
happy-dom: 20.8.3
jsdom: 28.1.0
transitivePeerDependencies:
- jiti
@@ -4717,12 +4803,17 @@ snapshots:
w3c-xmlserializer@5.0.0:
dependencies:
xml-name-validator: 5.0.0
optional: true
webidl-conversions@8.0.1: {}
webidl-conversions@8.0.1:
optional: true
webpack-virtual-modules@0.6.2: {}
whatwg-mimetype@5.0.0: {}
whatwg-mimetype@3.0.0: {}
whatwg-mimetype@5.0.0:
optional: true
whatwg-url@16.0.1:
dependencies:
@@ -4731,15 +4822,20 @@ snapshots:
webidl-conversions: 8.0.1
transitivePeerDependencies:
- '@noble/hashes'
optional: true
why-is-node-running@2.3.0:
dependencies:
siginfo: 2.0.0
stackback: 0.0.2
xml-name-validator@5.0.0: {}
ws@8.19.0: {}
xmlchars@2.2.0: {}
xml-name-validator@5.0.0:
optional: true
xmlchars@2.2.0:
optional: true
yallist@3.1.1: {}
@@ -5,8 +5,10 @@ import { useCopyToClipboard } from "./use-copy-to-clipboard"
beforeEach(() => {
vi.useFakeTimers()
Object.assign(navigator, {
clipboard: { writeText: vi.fn().mockResolvedValue(undefined) },
Object.defineProperty(navigator, "clipboard", {
value: { writeText: vi.fn().mockResolvedValue(undefined) },
writable: true,
configurable: true,
})
})
+1 -1
View File
@@ -45,7 +45,7 @@ export default defineConfig({
},
test: {
globals: true,
environment: "jsdom",
environment: "happy-dom",
setupFiles: "./src/test/setup.ts",
coverage: {
all: true,
-103
View File
@@ -1,103 +0,0 @@
#!/bin/bash
# Run E2E tests locally using Docker, matching the CI process (e2e.yml).
# Requires: Docker, /dev/kvm
#
# Usage:
# ./scripts/run-e2e-local.sh # Run both suites
# ./scripts/run-e2e-local.sh single # Single-node only
# ./scripts/run-e2e-local.sh multi # Multi-node only
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
PARENT_DIR="$(cd "$PROJECT_ROOT/.." && pwd)"
IMAGE_NAME="hive-e2e:latest"
# Determine which suites to run
SUITE="${1:-both}"
case "$SUITE" in
single|multi|both) ;;
*)
echo "Usage: $0 [single|multi|both]"
exit 1
;;
esac
# Verify /dev/kvm exists
if [ ! -e /dev/kvm ]; then
echo "ERROR: /dev/kvm not found. KVM is required for E2E tests."
exit 1
fi
# Verify sibling repos exist, clone if missing
for dep in viperblock predastore; do
if [ ! -d "$PARENT_DIR/$dep" ]; then
echo "Sibling repo '$dep' not found at $PARENT_DIR/$dep"
echo "Running clone-deps.sh..."
"$PROJECT_ROOT/scripts/clone-deps.sh"
break
fi
done
# Build Docker image (same context as CI — parent directory)
echo "Building E2E Docker image..."
docker build \
-t "$IMAGE_NAME" \
-f "$PROJECT_ROOT/tests/e2e/Dockerfile.e2e" \
"$PARENT_DIR"
SINGLE_PASS=""
MULTI_PASS=""
# Run single-node E2E
if [ "$SUITE" = "single" ] || [ "$SUITE" = "both" ]; then
echo ""
echo "========================================"
echo "Running Single-Node E2E Tests"
echo "========================================"
if docker run --privileged --rm \
-v /dev/kvm:/dev/kvm \
--name hive-e2e-local \
"$IMAGE_NAME"; then
SINGLE_PASS="PASS"
echo "Single-node E2E: PASS"
else
SINGLE_PASS="FAIL"
echo "Single-node E2E: FAIL"
fi
fi
# Run multi-node E2E
if [ "$SUITE" = "multi" ] || [ "$SUITE" = "both" ]; then
echo ""
echo "========================================"
echo "Running Multi-Node E2E Tests"
echo "========================================"
if docker run --privileged --rm \
-v /dev/kvm:/dev/kvm \
--cap-add=NET_ADMIN \
--name hive-multinode-e2e-local \
"$IMAGE_NAME" \
./tests/e2e/run-multinode-e2e.sh; then
MULTI_PASS="PASS"
echo "Multi-node E2E: PASS"
else
MULTI_PASS="FAIL"
echo "Multi-node E2E: FAIL"
fi
fi
# Summary
echo ""
echo "========================================"
echo "E2E Results"
echo "========================================"
[ -n "$SINGLE_PASS" ] && echo " Single-node: $SINGLE_PASS"
[ -n "$MULTI_PASS" ] && echo " Multi-node: $MULTI_PASS"
# Exit with failure if any suite failed
if [ "$SINGLE_PASS" = "FAIL" ] || [ "$MULTI_PASS" = "FAIL" ]; then
exit 1
fi
-76
View File
@@ -1,76 +0,0 @@
FROM ubuntu:24.04
# Avoid prompts
ENV DEBIAN_FRONTEND=noninteractive
# Install make and iproute2 (for multi-node IP management)
RUN apt-get update && apt-get install -y make sudo iproute2 && rm -rf /var/lib/apt/lists/*
# Set up working directory for Makefile targets
WORKDIR /tmp/hive-setup
# Copy only the Makefile for dependency installation
COPY hive/Makefile .
# Install system dependencies in separate layers for better caching
RUN make install-system
RUN make install-go
RUN make install-aws
# Ensure Go is in the PATH
ENV PATH="/usr/local/go/bin:/root/go/bin:${PATH}"
# Set up working directory structure to match development layout
WORKDIR /root/Development/mulga
# Pre-download Ubuntu cloud image for e2e tests (cached in Docker layer)
# Placed before dependency copy so go.mod changes don't invalidate this layer
RUN mkdir -p /root/images && \
ARCH=$(dpkg --print-architecture) && \
if [ "$ARCH" = "amd64" ]; then \
curl -L -o /root/images/ubuntu-24.04.img \
https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img; \
elif [ "$ARCH" = "arm64" ]; then \
curl -L -o /root/images/ubuntu-24.04.img \
https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-arm64.img; \
fi
# 1. Pre-copy dependency manifests for cache layer
# go.work is gitignored (generated by init-workspace.sh), so create it inline
RUN printf 'go 1.26.1\n\nuse (\n\t./hive\n\t./predastore\n\t./viperblock\n)\n' > go.work
COPY hive/go.mod hive/go.sum ./hive/
COPY viperblock/go.mod viperblock/go.sum ./viperblock/
COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/
COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/
COPY predastore/go.mod predastore/go.sum ./predastore/
# 2. Download dependencies (using cache mount)
RUN --mount=type=cache,target=/root/go/pkg/mod \
(cd hive && go mod download) && \
(cd viperblock && go mod download) && \
(cd viperblock/nbd && go mod download) && \
(cd viperblock/nbd/libguestfs.org/nbdkit && go mod download) && \
(cd predastore && go mod download)
# 3. Copy the rest of the source code
COPY hive/ ./hive/
COPY viperblock/ ./viperblock/
COPY predastore/ ./predastore/
# Set working directory to hive
WORKDIR /root/Development/mulga/hive
# Ensure scripts are executable
RUN chmod +x ./scripts/*.sh ./tests/e2e/*.sh ./tests/e2e/lib/*.sh
# 4. Build everything (using cache mounts)
RUN --mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/root/go/pkg/mod \
./scripts/dev-setup.sh
# Set environment variables for runtime
ENV PATH="/root/Development/mulga/hive/bin:${PATH}"
ENV HOME="/root"
# Run the E2E test runner
CMD ["./tests/e2e/run-e2e.sh"]
+87 -2
View File
@@ -298,12 +298,12 @@ Tests that EC2 resources are properly isolated between tenant accounts (Alpha, B
---
## Multi-Node (`run-multinode-e2e.sh`)
## Pseudo Multi-Node (`run-pseudo-multinode-e2e.sh`)
### Phase 1: Environment Setup
- KVM support check
- Simulated network IPs (10.11.12.{1,2,3} on loopback)
- Create ramdisk mount
- Simulated network IPs (no ramdisk — start-dev.sh uses disk-backed WAL/VB in CI)
### Phase 2: Cluster Initialization
- `hive admin init` (leader node1)
@@ -550,6 +550,91 @@ Tests that EC2 resources are properly isolated between the Alpha/Beta accounts.
---
## Real Multi-Node (`run-multinode-e2e.sh`)
Runs on a real 3-node libvirt cluster provisioned by OpenTofu (`scripts/tofu-cluster/`). Each node is a separate VM with its own OVN, NATS, Predastore, and Hive daemon. Bootstrap (`bootstrap.sh`) handles all provisioning before the test script runs.
### Bootstrap (pre-test)
1. OpenTofu provisions 3 libvirt VMs (bottlebrush, ironbark, casuarina) with cloud-init
2. Wait for all nodes to be SSH-reachable
3. Copy SSH key to node1 for peer access
4. `git fetch` + checkout test branch on all nodes (all 3 repos)
5. `make build` on all nodes (parallel)
6. `setup-ovn.sh` — primary gets `--management`, secondaries connect to primary's OVN central
7. `hive admin init` on primary, `hive admin join` on secondaries (formation server handshake)
8. `start-dev.sh` on all nodes
9. Wait for `/health` (awsgw=ok) on all nodes
10. Install Hive CA certificate on all nodes
11. `import-key-pair` (hive-key) + `hive admin images import` (Ubuntu 24.04)
12. `create-vpc` (10.200.0.0/16) + `create-subnet` (10.200.1.0/24)
### Phase 1: Pre-flight Validation
- KVM support check (`/dev/kvm` writable)
- SSH connectivity to all peer nodes
### Phase 2: Cluster Health
- NATS cluster: verify 2 unique route peers (3-node quorum)
- Predastore reachable on all 3 nodes
- AWS gateway reachable on all 3 nodes
- Daemon readiness (`describe-instance-types` returns results)
- `hive get nodes` — verify all 3 nodes show Ready
- `hive get vms` — verify empty (no instances yet)
### Phase 3: Instance Lifecycle + Distribution
- Discover nano instance type and AMI (from bootstrap import)
- `run-instances` x3 with staggered launches
- Poll all instances to running state
- Check instance distribution across physical nodes (QEMU process check)
- Verify at least 2 different hosting nodes (non-deterministic, non-fatal if all on 1)
- `hive get vms` — verify all 3 instances visible
### Phase 4: SSH into Guest VMs
- For each instance across all nodes:
- Find hosting node via QEMU process scan
- Extract SSH hostfwd port from QEMU command line on remote node
- Wait for SSH readiness (up to 60s)
- Verify SSH connectivity (`id` returns ec2-user)
- Verify block device (`lsblk`)
### Phase 5: Volume Lifecycle
- `create-volume` (10GB)
- `attach-volume` (to first instance, /dev/sdf)
- Poll for in-use + attached state
- `detach-volume` (poll for available)
- `delete-volume` (verify gone)
### Phase 6: Cross-Node Gateway Access
- `describe-instances` via node1 gateway (baseline count)
- `describe-instances` via node2 and node3 gateways
- Verify all gateways return the same instance count (fan-out aggregation)
### Phase 7: Cross-Node Operations
- Find which node hosts instance 0
- Pick a different node's gateway for stop
- `stop-instances` via remote gateway (poll -> stopped)
- Pick a third node's gateway for start
- `start-instances` via another remote gateway (poll -> running)
### Phase 8: Node Failure
- `stop-dev.sh` on node2 (simulate node failure)
- Verify node1 still serves requests (`describe-instance-types`)
- Verify node3 still serves requests
- Check NATS degraded state (1 peer instead of 2)
- `describe-instances` still works from surviving nodes
### Phase 9: Node Recovery
- `start-dev.sh` on node2 (restart failed node)
- Wait for NATS cluster to reform (2 peers again)
- Verify node2 gateway is back
- `hive get nodes` — verify all 3 nodes Ready again
- Verify node2 serves requests after recovery
### Phase 10: Cleanup
- `terminate-instances` (all 3 instances, poll -> terminated)
---
## VPC (`run-vpc-e2e.sh`)
Standalone VPC networking test suite. Runs against a running Hive endpoint (configurable via `ENDPOINT` env var). OVN topology tests are skipped when OVN is unavailable.
+2 -113
View File
@@ -5,117 +5,6 @@
# and verifying NATS cluster health.
# Bootstrap OVN/OVS services inside a Docker container (no systemd).
# Starts ovsdb-server, ovs-vswitchd, OVN central (NB/SB DBs + northd),
# creates br-int, and starts ovn-controller.
# All 3 simulated nodes share one br-int and one ovn-controller.
bootstrap_ovn_docker() {
echo "Bootstrapping OVN/OVS for Docker (no systemd)..."
# Start ovsdb-server
mkdir -p /var/run/openvswitch
if [ ! -f /etc/openvswitch/conf.db ]; then
ovsdb-tool create /etc/openvswitch/conf.db /usr/share/openvswitch/vswitch.ovsschema
fi
ovsdb-server --remote=punix:/var/run/openvswitch/db.sock \
--remote=db:Open_vSwitch,Open_vSwitch,manager_options \
--pidfile --detach --log-file=/var/log/openvswitch/ovsdb-server.log \
/etc/openvswitch/conf.db
ovs-vsctl --no-wait init
# Start ovs-vswitchd
ovs-vswitchd --pidfile --detach --log-file=/var/log/openvswitch/ovs-vswitchd.log
# Create br-int with secure fail-mode
ovs-vsctl --may-exist add-br br-int
ovs-vsctl set Bridge br-int fail-mode=secure
ovs-vsctl set Bridge br-int other-config:disable-in-band=true
ip link set br-int up
# Start OVN central (NB + SB DBs + northd)
mkdir -p /var/run/ovn /var/lib/ovn /var/log/ovn
# NB database
if [ ! -f /var/lib/ovn/ovnnb_db.db ]; then
ovsdb-tool create /var/lib/ovn/ovnnb_db.db /usr/share/ovn/ovn-nb.ovsschema
fi
ovsdb-server --remote=punix:/var/run/ovn/ovnnb_db.sock \
--remote=ptcp:6641 \
--pidfile=/var/run/ovn/ovnnb_db.pid \
--detach --log-file=/var/log/ovn/ovnnb_db.log \
/var/lib/ovn/ovnnb_db.db
# SB database
if [ ! -f /var/lib/ovn/ovnsb_db.db ]; then
ovsdb-tool create /var/lib/ovn/ovnsb_db.db /usr/share/ovn/ovn-sb.ovsschema
fi
ovsdb-server --remote=punix:/var/run/ovn/ovnsb_db.sock \
--remote=ptcp:6642 \
--pidfile=/var/run/ovn/ovnsb_db.pid \
--detach --log-file=/var/log/ovn/ovnsb_db.log \
/var/lib/ovn/ovnsb_db.db
# Start ovn-northd
ovn-northd --pidfile=/var/run/ovn/ovn-northd.pid \
--detach --log-file=/var/log/ovn/ovn-northd.log \
--ovnnb-db=unix:/var/run/ovn/ovnnb_db.sock \
--ovnsb-db=unix:/var/run/ovn/ovnsb_db.sock
# Configure OVS external_ids for OVN
local chassis_id="chassis-docker"
ovs-vsctl set Open_vSwitch . \
external_ids:system-id="$chassis_id" \
external_ids:ovn-remote="tcp:127.0.0.1:6642" \
external_ids:ovn-encap-ip="127.0.0.1" \
external_ids:ovn-encap-type="geneve"
# Start ovn-controller with pidfile in OVS rundir
ovn-controller --pidfile=/var/run/openvswitch/ovn-controller.pid \
--detach --log-file=/var/log/ovn/ovn-controller.log
# Apply sysctl for overlay networking
sysctl -w net.ipv4.ip_forward=1 >/dev/null 2>&1 || true
sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null 2>&1 || true
sysctl -w net.ipv4.conf.default.rp_filter=0 >/dev/null 2>&1 || true
# Load geneve kernel module
modprobe geneve 2>/dev/null || true
# Wait for ovn-controller to become responsive.
# ovn-controller creates its ctl socket at /var/run/ovn/ovn-controller.PID.ctl
# but ovs-appctl -t ovn-controller looks in /var/run/openvswitch/, so we
# symlink it once the socket appears.
echo " Waiting for ovn-controller to start..."
local ovn_pid
ovn_pid=$(cat /var/run/openvswitch/ovn-controller.pid 2>/dev/null || true)
local attempt=0
while [ $attempt -lt 15 ]; do
# Create symlink once the ctl socket appears
if [ -n "$ovn_pid" ] && [ -S "/var/run/ovn/ovn-controller.${ovn_pid}.ctl" ] \
&& [ ! -e "/var/run/openvswitch/ovn-controller.${ovn_pid}.ctl" ]; then
ln -sf "/var/run/ovn/ovn-controller.${ovn_pid}.ctl" \
"/var/run/openvswitch/ovn-controller.${ovn_pid}.ctl"
fi
if ovs-appctl -t ovn-controller version >/dev/null 2>&1; then
break
fi
sleep 1
attempt=$((attempt + 1))
done
# Verify
if ovs-vsctl br-exists br-int && ovs-appctl -t ovn-controller version >/dev/null 2>&1; then
echo " OVN/OVS bootstrap complete (br-int up, ovn-controller running)"
else
echo " ERROR: OVN bootstrap failed"
ovs-vsctl br-exists br-int && echo " br-int: OK" || echo " br-int: MISSING"
ovs-appctl -t ovn-controller version >/dev/null 2>&1 && echo " ovn-controller: OK" || echo " ovn-controller: NOT RUNNING"
# Dump ovn-controller log for debugging
echo " --- ovn-controller.log ---"
cat /var/log/ovn/ovn-controller.log 2>/dev/null | tail -20 || true
return 1
fi
}
# Network configuration
SIMULATED_NETWORK="10.11.12"
@@ -139,7 +28,7 @@ add_simulated_ips() {
for i in 1 2 3; do
local ip="${SIMULATED_NETWORK}.$i"
if ! ip addr show lo | grep -q "$ip"; then
ip addr add "${ip}/24" dev lo
sudo ip addr add "${ip}/24" dev lo
echo " Added $ip to lo"
else
echo " $ip already exists on lo"
@@ -168,7 +57,7 @@ remove_simulated_ips() {
for i in 1 2 3; do
local ip="${SIMULATED_NETWORK}.$i"
if ip addr show lo | grep -q "$ip"; then
ip addr del "${ip}/24" dev lo 2>/dev/null || true
sudo ip addr del "${ip}/24" dev lo 2>/dev/null || true
echo " Removed $ip from lo"
fi
done
+82 -40
View File
@@ -1,7 +1,13 @@
#!/bin/bash
set -e
# Source helper functions for SSH testing
# Ensure Go is on PATH (SSH non-interactive shells don't source .bashrc)
export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH"
# Ensure we are in the project root
cd "$(dirname "$0")/../.."
# Source helper functions
source ./tests/e2e/lib/multinode-helpers.sh
# Ensure services are stopped on exit and print logs on failure
@@ -34,7 +40,10 @@ cleanup() {
tail -200 ~/hive/logs/awsgw.log 2>/dev/null
fi
fi
./scripts/stop-dev.sh
# Only stop services if we started them (not when bootstrapped)
if [ "$BOOTSTRAPPED" != "true" ]; then
./scripts/stop-dev.sh
fi
exit $EXIT_CODE
}
trap cleanup EXIT
@@ -42,6 +51,33 @@ trap cleanup EXIT
# Use Hive profile (hive admin init sets endpoint_url, ca_bundle, region in ~/.aws/config)
export AWS_PROFILE=hive
# Detect bootstrapped environment from hive.toml config
BOOTSTRAPPED="false"
GATEWAY_HOST="localhost"
PREDASTORE_HOST="localhost"
if [ -f ~/hive/config/hive.toml ]; then
# Resolve gateway host from AWS profile endpoint_url (set by hive admin init)
ENDPOINT_URL=$(aws configure get endpoint_url 2>/dev/null || true)
if [ -n "$ENDPOINT_URL" ]; then
DETECTED_GW_HOST=$(echo "$ENDPOINT_URL" | sed 's|https\?://||;s|:.*||')
if [ -n "$DETECTED_GW_HOST" ]; then
GATEWAY_HOST="$DETECTED_GW_HOST"
fi
fi
# Resolve predastore host from hive.toml
DETECTED_PS_HOST=$(awk -F'"' '/\[nodes\..*\.predastore\]/{found=1} found && /^host/{print $2; exit}' ~/hive/config/hive.toml)
if [ -n "$DETECTED_PS_HOST" ]; then
PREDASTORE_HOST="${DETECTED_PS_HOST%%:*}"
fi
# Check if gateway is actually responding
if curl -sk "https://${GATEWAY_HOST}:9999" > /dev/null 2>&1; then
BOOTSTRAPPED="true"
echo "Detected bootstrapped environment — skipping cluster setup"
echo " Gateway host: $GATEWAY_HOST"
echo " Predastore host: $PREDASTORE_HOST"
fi
fi
# Helper: set up an AWS CLI profile with credentials + endpoint/CA config from the hive profile
setup_test_profile() {
local profile="$1" key_id="$2" secret="$3"
@@ -52,9 +88,6 @@ setup_test_profile() {
aws configure set ca_bundle "$(aws configure get ca_bundle)" --profile "$profile"
}
# Ensure we are in the project root
cd "$(dirname "$0")/../.."
# Phase 1: Environment Setup
echo "Phase 1: Environment Setup"
@@ -73,29 +106,26 @@ else
exit 1
fi
# Bootstrap OVN/OVS (required — start-dev.sh will block without it)
echo "Bootstrapping OVN/OVS networking..."
bootstrap_ovn_docker
if [ "$BOOTSTRAPPED" != "true" ]; then
./bin/hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
./bin/hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
# Trust the Hive CA certificate for AWS CLI SSL verification
echo "Adding Hive CA certificate to system trust store..."
sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt
sudo update-ca-certificates
# Trust the Hive CA certificate for AWS CLI SSL verification
echo "Adding Hive CA certificate to system trust store..."
sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt
sudo update-ca-certificates
# Start all services
# Ensure logs directory exists for start-dev.sh
mkdir -p ~/hive/logs
./scripts/start-dev.sh
fi
# Start all services
# Ensure logs directory exists for start-dev.sh
mkdir -p ~/hive/logs
mkdir -p /mnt/ramdisk
./scripts/start-dev.sh
# Wait for health checks on https://localhost:9999 (AWS Gateway)
# Wait for health checks on AWS Gateway
echo "Waiting for AWS Gateway..."
MAX_RETRIES=15
COUNT=0
until curl -s https://localhost:9999 > /dev/null || [ $COUNT -eq $MAX_RETRIES ]; do
until curl -sk "https://${GATEWAY_HOST}:9999" > /dev/null || [ $COUNT -eq $MAX_RETRIES ]; do
echo "Waiting for gateway... ($COUNT/$MAX_RETRIES)"
sleep 2
COUNT=$((COUNT + 1))
@@ -107,7 +137,12 @@ if [ $COUNT -eq $MAX_RETRIES ]; then
fi
# Wait for daemon NATS subscriptions to be active
wait_for_daemon_ready "https://localhost:9999"
wait_for_daemon_ready "https://${GATEWAY_HOST}:9999"
# Discover the cluster's availability zone and region dynamically
HIVE_AZ=$(aws ec2 describe-availability-zones --query 'AvailabilityZones[0].ZoneName' --output text)
HIVE_REGION=$(aws ec2 describe-availability-zones --query 'AvailabilityZones[0].RegionName' --output text)
echo "Discovered AZ: $HIVE_AZ, Region: $HIVE_REGION"
# No need for AWS_EC2/AWS_IAM vars — endpoint_url and ca_bundle are in the profile config
@@ -260,17 +295,23 @@ fi
echo "Using image: $IMAGE_NAME"
# Import the pre-downloaded Ubuntu image using file-based import
echo "Importing pre-cached Ubuntu image..."
IMPORT_LOG=$(./bin/hive admin images import \
--file /root/images/ubuntu-24.04.img \
--arch "$ARCH" \
--distro ubuntu \
--version 24.04 \
--force 2>/dev/null)
AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+')
# When bootstrapped, the image is already imported — discover the existing AMI
if [ "$BOOTSTRAPPED" = "true" ]; then
echo "Discovering existing AMI (imported by bootstrap)..."
AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
else
echo "Importing pre-cached Ubuntu image..."
IMPORT_LOG=$(./bin/hive admin images import \
--file ~/images/ubuntu-24.04.img \
--arch "$ARCH" \
--distro ubuntu \
--version 24.04 \
--force 2>/dev/null)
AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+')
fi
if [ -z "$AMI_ID" ]; then
echo "Failed to capture AMI ID from import command"
if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
echo "Failed to capture AMI ID"
exit 1
fi
echo "Captured AMI ID: $AMI_ID"
@@ -477,8 +518,8 @@ echo "Phase 5b: Volume Lifecycle (Attach/Detach)"
echo "Testing volume create -> resize -> attach -> detach -> delete..."
# Create a test volume
echo "Creating 10GB volume in ap-southeast-2a..."
CREATE_OUTPUT=$(aws ec2 create-volume --size 10 --availability-zone ap-southeast-2a)
echo "Creating 10GB volume in ${HIVE_AZ}..."
CREATE_OUTPUT=$(aws ec2 create-volume --size 10 --availability-zone "$HIVE_AZ")
TEST_VOLUME_ID=$(echo "$CREATE_OUTPUT" | jq -r '.VolumeId')
if [ -z "$TEST_VOLUME_ID" ] || [ "$TEST_VOLUME_ID" == "null" ]; then
@@ -694,7 +735,7 @@ echo "Describe snapshot verified (VolumeId=$DESC_VOL_ID, Size=$DESC_SIZE, Descri
# Copy the snapshot
echo "Copying snapshot $SNAPSHOT_ID..."
COPY_OUTPUT=$(aws ec2 copy-snapshot --source-snapshot-id "$SNAPSHOT_ID" --source-region ap-southeast-2 --description "e2e-copy")
COPY_OUTPUT=$(aws ec2 copy-snapshot --source-snapshot-id "$SNAPSHOT_ID" --source-region "$HIVE_REGION" --description "e2e-copy")
COPY_SNAPSHOT_ID=$(echo "$COPY_OUTPUT" | jq -r '.SnapshotId')
if [ -z "$COPY_SNAPSHOT_ID" ] || [ "$COPY_SNAPSHOT_ID" == "null" ]; then
@@ -777,7 +818,7 @@ echo "Phase 5d: Verify Snapshot-Backed Instance Launch"
echo "All run-instances calls go through cloneAMIToVolume() -> OpenFromSnapshot(),"
echo "so the Phase 5 instance is already snapshot-backed. Verify its volume config."
AWS_S3="aws --endpoint-url https://localhost:8443 s3"
AWS_S3="aws --endpoint-url https://${PREDASTORE_HOST}:8443 s3"
# Verify the AMI snapshot exists in Predastore
echo "Checking AMI snapshot in Predastore..."
@@ -973,7 +1014,7 @@ echo "Reboot-stopped correctly rejected"
# Phase 7a: Attach volume to stopped instance (should fail)
echo "Phase 7a: Attach Volume to Stopped Instance (Error Path)"
echo "Creating a volume to test attach-to-stopped..."
STOPPED_VOL_OUTPUT=$(aws ec2 create-volume --size 10 --availability-zone ap-southeast-2a)
STOPPED_VOL_OUTPUT=$(aws ec2 create-volume --size 10 --availability-zone "$HIVE_AZ")
STOPPED_VOL_ID=$(echo "$STOPPED_VOL_OUTPUT" | jq -r '.VolumeId')
echo "Created volume: $STOPPED_VOL_ID"
@@ -990,8 +1031,9 @@ echo "Cleaned up test volume $STOPPED_VOL_ID"
echo "Phase 7b: ModifyInstanceAttribute"
echo "Instance is stopped — modifying instance type to verify changes take effect on restart"
# Derive an upsized type in the same family: nano → xlarge (4 vCPUs instead of 2)
MODIFY_TYPE="${INSTANCE_TYPE%.nano}.xlarge"
# Derive an upsized type in the same family: nano → small (same vCPUs, more RAM)
# Note: xlarge needs 16GB RAM which exceeds the 8GB CI VM — use small (2GB) instead
MODIFY_TYPE="${INSTANCE_TYPE%.nano}.small"
echo "Changing instance type from $INSTANCE_TYPE to $MODIFY_TYPE..."
# Get expected vCPU and memory for the new type
@@ -1222,7 +1264,7 @@ expect_error "InvalidSnapshot.NotFound" aws ec2 delete-snapshot \
# 8f: Call an unsupported Action (use raw curl to send an invalid Action)
echo "8f: Unsupported Action..."
set +e
UNSUPPORTED_OUTPUT=$(curl -s -k -X POST https://localhost:9999/ \
UNSUPPORTED_OUTPUT=$(curl -s -k -X POST "https://${GATEWAY_HOST}:9999/" \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "Action=DescribeFakeThings&Version=2016-11-15" 2>&1)
set -e
File diff suppressed because it is too large Load Diff
+2423
View File
File diff suppressed because it is too large Load Diff