feat: e2e test

2026-10-19 19:01:42 +00:00 · 2026-01-20 04:40:38 +00:00
parent 896b70e508
commit e8e9b72cd3
6 changed files with 386 additions and 13 deletions
@@ -6,9 +6,17 @@ Notes for development environment installation.

 For running on Ubuntu 22.04, 24.04 and 25.10.

+### Quick Install
+
+```bash
+make quickinstall
+```
+
+### Manual Install
+
 ```bash
 sudo add-apt-repository universe
-sudo apt install nbdkit nbdkit-plugin-dev pkg-config qemu-system qemu-utils qemu-kvm libvirt-daemon-system libvirt-clients libvirt-dev make gcc
+sudo apt install nbdkit nbdkit-plugin-dev pkg-config qemu-system qemu-utils qemu-kvm libvirt-daemon-system libvirt-clients libvirt-dev make gcc unzip xz-utils file
 ```

 Ensure the Go toolkit is installed, recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/).
@@ -22,10 +30,12 @@ go version
 Hive provides AWS API/SDK layer functionality, which requires the AWS CLI tool to be installed to interface with the system.

 ```bash
-sudo apt install awscli
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+unzip awscliv2.zip
+sudo ./aws/install
 ```

-Confirm awscli version > 2.0 is installed which is required by Hive. If your OS package manager installed an older version, e.g 1.X, install the latest version of the AWS CLI tool directly.
+Confirm awscli version > 2.0 is installed which is required by Hive.

 [https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)

@@ -33,10 +43,6 @@ Confirm awscli version > 2.0 is installed which is required by Hive. If your OS
 aws --version
 ```

-```
-aws-cli/2.32.4
-```
-
 ## Build

 Create the base directory for the Hive development environment.
@@ -70,6 +70,29 @@ run:
 clean:
 	rm ./bin/$(GO_PROJECT_NAME)

+quickinstall:
+	@echo "\n....Installing system dependencies...."
+	sudo apt-get update && sudo apt-get install -y \
+		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-utils qemu-kvm \
+		libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
+		iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file
+	@echo "\n....Installing AWS CLI v2...."
+	@if ! command -v aws >/dev/null 2>&1; then \
+		curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"; \
+		unzip -o awscliv2.zip; \
+		sudo ./aws/install; \
+		rm -rf awscliv2.zip aws/; \
+	else \
+		echo "AWS CLI already installed"; \
+	fi
+	@echo "\n....Installing Go 1.25.5...."
+	@if [ ! -d "/usr/local/go" ]; then \
+		curl -L https://go.dev/dl/go1.25.5.linux-amd64.tar.gz | sudo tar -C /usr/local -xz; \
+	else \
+		echo "Go already installed in /usr/local/go"; \
+	fi
+	@echo "\n✅ Quickinstall complete. Please ensure /usr/local/go/bin is in your PATH."
+
 security: $(GOVULNCHECK) $(GOSECCHECK) $(GOSTATICCHECK)
 	@echo "\n....Running security checks for $(GO_PROJECT_NAME)...."

@@ -86,4 +109,17 @@ security: $(GOVULNCHECK) $(GOSECCHECK) $(GOSTATICCHECK)
 	go vet ./... 2>&1 | tee tests/govet-report.txt || true
 	@echo "Go vet report saved to tests/govet-report.txt"

-.PHONY: go_build go_run build run test clean
+e2e-test:
+	@echo "\n....Ensuring E2E Base Image exists...."
+	@if ! sudo docker image inspect hive-base:latest >/dev/null 2>&1; then \
+		echo "Building hive-base:latest..."; \
+		sudo docker build -t hive-base -f tests/e2e/Dockerfile.base .; \
+	fi
+	@echo "\n....Removing old E2E image if it exists...."
+	-sudo docker rmi hive-e2e:latest 2>/dev/null || true
+	@echo "\n....Building E2E Docker image (building everything inside)...."
+	cd .. && sudo docker build -t hive-e2e -f hive/tests/e2e/Dockerfile.e2e .
+	@echo "\n....Running E2E Docker container...."
+	sudo docker run --privileged --rm -v /dev/kvm:/dev/kvm --name hive-e2e-test hive-e2e
+
+.PHONY: build go_build go_run test bench run clean quickinstall security e2e-test
@@ -36,7 +36,7 @@ fi


 if [ -d "/mnt/ramdisk" ]; then
-    
+
    # Check if /mnt/ramdisk is mounted, if not mount it as tmpfs
    if ! mountpoint -q /mnt/ramdisk; then
        echo "💾 Mounting /mnt/ramdisk as tmpfs"
@@ -111,12 +111,16 @@ check_service() {
    done

    echo "   ⚠️  $name may not be responding on port $port (continuing anyway)"
-    
+
 }

 # Pre-flight, compile latest
-echo "1️✈️ Pre-flight, compiling latest..."
-make build
+if [ "$HIVE_SKIP_BUILD" != "true" ]; then
+    echo "1️✈️ Pre-flight, compiling latest..."
+    make build
+else
+    echo "1️✈️ Skipping build (HIVE_SKIP_BUILD=true)"
+fi

 # 1️⃣ Start NATS server
 echo ""
@@ -268,4 +272,4 @@ echo ""

 # This will only be reached if air/daemon exits normally
 #echo ""
-#echo "🛑 Hive development environment stopped"
+#echo "🛑 Hive development environment stopped"
@@ -0,0 +1,19 @@
+FROM ubuntu:24.04
+
+# Avoid prompts
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install make first so we can use the Makefile
+RUN apt-get update && apt-get install -y make sudo && rm -rf /var/lib/apt/lists/*
+
+# Set up working directory
+WORKDIR /app/hive
+
+# Copy only the Makefile to run quickinstall
+COPY Makefile .
+
+# Run the quickinstall target from the Makefile
+RUN make quickinstall
+
+# Ensure Go is in the PATH
+ENV PATH="/usr/local/go/bin:${PATH}"
@@ -0,0 +1,45 @@
+# syntax=docker/dockerfile:1
+# This image assumes hive-base:latest exists locally or in a registry
+FROM hive-base:latest
+
+# Set up working directory structure to match INSTALL.md
+WORKDIR /root/Development/mulga
+
+# 1. Pre-copy dependency manifests
+# We copy these first to create a dedicated dependency layer
+COPY hive/go.mod hive/go.sum hive/go.work hive/go.work.sum ./hive/
+COPY viperblock/go.mod viperblock/go.sum ./viperblock/
+COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/
+COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/
+COPY predastore/go.mod predastore/go.sum ./predastore/
+
+# 2. Download dependencies (Using Cache Mount)
+RUN --mount=type=cache,target=/root/go/pkg/mod \
+    (cd hive && go mod download) && \
+    (cd viperblock && go mod download) && \
+    (cd viperblock/nbd && go mod download) && \
+    (cd viperblock/nbd/libguestfs.org/nbdkit && go mod download) && \
+    (cd predastore && go mod download)
+
+# 3. Copy the rest of the source code
+COPY hive/ ./hive/
+COPY viperblock/ ./viperblock/
+COPY predastore/ ./predastore/
+
+# Set working directory to hive
+WORKDIR /root/Development/mulga/hive
+
+# Ensure scripts are executable
+RUN chmod +x ./scripts/*.sh ./tests/e2e/run-e2e.sh
+
+# 4. Build everything
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/root/go/pkg/mod \
+    ./scripts/dev-setup.sh
+
+# Set environment variables for runtime
+ENV PATH="/root/Development/mulga/hive/bin:${PATH}"
+ENV HOME="/root"
+
+# Run the E2E test runner
+CMD ["./tests/e2e/run-e2e.sh"]
@@ -0,0 +1,263 @@
+#!/bin/bash
+set -ex
+
+# Use Hive profile
+export AWS_PROFILE=hive
+
+# Ensure we are in the project root
+cd "$(dirname "$0")/../.."
+
+# Phase 1: Environment Setup
+echo "Phase 1: Environment Setup"
+
+# Check for KVM support inside the container
+echo "Checking for KVM support..."
+if [ -e /dev/kvm ]; then
+    echo "✅ /dev/kvm exists"
+    if [ -w /dev/kvm ]; then
+        echo "✅ /dev/kvm is writable"
+    else
+        echo "❌ /dev/kvm is NOT writable. QEMU will fail."
+        exit 1
+    fi
+else
+    echo "❌ /dev/kvm does NOT exist. Ensure --privileged and -v /dev/kvm:/dev/kvm are used."
+    exit 1
+fi
+
+./bin/hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
+
+# Start all services
+# Ensure logs directory exists for start-dev.sh
+mkdir -p ~/hive/logs
+HIVE_SKIP_BUILD=true ./scripts/start-dev.sh
+
+# Wait for health checks on https://localhost:9999 (AWS Gateway)
+echo "Waiting for AWS Gateway..."
+MAX_RETRIES=30
+COUNT=0
+# Using -k because of self-signed certs
+until curl -k -s https://localhost:9999 > /dev/null || [ $COUNT -eq $MAX_RETRIES ]; do
+    echo "Waiting for gateway... ($COUNT/$MAX_RETRIES)"
+    sleep 2
+    COUNT=$((COUNT + 1))
+done
+
+if [ $COUNT -eq $MAX_RETRIES ]; then
+    echo "Gateway failed to start"
+    ./scripts/stop-dev.sh
+    exit 1
+fi
+
+# Define common AWS CLI args
+# Use --endpoint-url and --no-verify-ssl to be safe in the container environment
+# Suppress InsecureRequestWarning from urllib3
+export PYTHONWARNINGS="ignore:Unverified HTTPS request"
+AWS_EC2="aws --endpoint-url https://localhost:9999 --no-verify-ssl ec2"
+
+# Phase 2: Discovery & Metadata
+echo "Phase 2: Discovery & Metadata"
+# Verify describe-regions (just ensure it returns at least one region)
+$AWS_EC2 describe-regions | jq -e '.Regions | length > 0'
+
+# Discover available instance types from Hive
+# Hive generates these based on the host CPU (e.g., m7i.micro, m8g.small, etc.)
+echo "Discovering available instance types..."
+AVAILABLE_TYPES=$($AWS_EC2 describe-instance-types --query 'InstanceTypes[*].InstanceType' --output text)
+echo "Available instance types: $AVAILABLE_TYPES"
+
+# Pick the first one as our primary test type
+INSTANCE_TYPE=$(echo $AVAILABLE_TYPES | awk '{print $1}')
+if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" == "None" ]; then
+    echo "No instance types found!"
+    exit 1
+fi
+echo "Selected instance type for test: $INSTANCE_TYPE"
+
+# Get architecture for the selected instance type
+ARCH=$($AWS_EC2 describe-instance-types --instance-types "$INSTANCE_TYPE" --query 'InstanceTypes[0].ProcessorInfo.SupportedArchitectures[0]' --output text)
+echo "Detected architecture for $INSTANCE_TYPE: $ARCH"
+
+# Verify describe-instance-types (ensure the chosen type is available)
+$AWS_EC2 describe-instance-types | jq -e ".InstanceTypes[] | select(.InstanceType==\"$INSTANCE_TYPE\")"
+
+# Phase 3: SSH Key Management
+echo "Phase 3: SSH Key Management"
+# Create test-key-1 (create-key-pair) and verify private key material is returned
+KEY_MATERIAL=$($AWS_EC2 create-key-pair --key-name test-key-1 --query 'KeyMaterial' --output text)
+if [ -z "$KEY_MATERIAL" ] || [ "$KEY_MATERIAL" == "None" ]; then
+    echo "Failed to create key pair test-key-1"
+    exit 1
+fi
+echo "$KEY_MATERIAL" > test-key-1.pem
+chmod 600 test-key-1.pem
+
+# Generate a local RSA key and import it as test-key-2 (import-key-pair)
+ssh-keygen -t rsa -b 2048 -f test-key-2-local -N ""
+$AWS_EC2 import-key-pair --key-name test-key-2 --public-key-material "fileb://test-key-2-local.pub"
+
+# Verify both keys are present (describe-key-pairs)
+$AWS_EC2 describe-key-pairs --query 'KeyPairs[*].KeyName' --output text | grep test-key-1
+$AWS_EC2 describe-key-pairs --query 'KeyPairs[*].KeyName' --output text | grep test-key-2
+
+# Delete test-key-2 (delete-key-pair) and verify only one remains
+$AWS_EC2 delete-key-pair --key-name test-key-2
+REMAINING_KEYS=$($AWS_EC2 describe-key-pairs --query 'KeyPairs[*].KeyName' --output text)
+echo "Remaining keys: $REMAINING_KEYS"
+echo "$REMAINING_KEYS" | grep test-key-1
+if echo "$REMAINING_KEYS" | grep -q test-key-2; then
+    echo "test-key-2 was not deleted"
+    exit 1
+fi
+
+# Phase 4: Image Management
+echo "Phase 4: Image Management"
+# Detect correct image name based on architecture
+if [ "$ARCH" = "x86_64" ]; then
+    IMAGE_NAME="ubuntu-24.04-x86_64"
+else
+    IMAGE_NAME="ubuntu-24.04-arm64"
+fi
+echo "Using image: $IMAGE_NAME"
+
+# Import a reliable Ubuntu image and capture the AMI ID from the output
+echo "Importing image $IMAGE_NAME..."
+echo "May appear stalled here, just takes a while to import..."
+IMPORT_LOG=$(./bin/hive admin images import --name "$IMAGE_NAME" --force)
+AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+')
+
+if [ -z "$AMI_ID" ]; then
+    echo "Failed to capture AMI ID from import command"
+    ./scripts/stop-dev.sh
+    exit 1
+fi
+echo "Captured AMI ID: $AMI_ID"
+
+# Verify the AMI exists using its ID (describe-images)
+echo "Verifying AMI availability..."
+$AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.ImageId==\"$AMI_ID\")"
+
+# Phase 5: Instance Lifecycle
+# echo "Phase 5: Instance Lifecycle"
+# # Launch a VM (run-instances)
+# INSTANCE_ID=$($AWS_EC2 run-instances --image-id "$AMI_ID" --instance-type "$INSTANCE_TYPE" --key-name test-key-1 --query 'Instances[0].InstanceId' --output text)
+# if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "None" ]; then
+#     echo "Failed to launch instance"
+#     ./scripts/stop-dev.sh
+#     exit 1
+# fi
+# echo "Launched Instance ID: $INSTANCE_ID"
+
+# # Poll until state is running (describe-instances)
+# echo "Polling for instance running state..."
+# COUNT=0
+# STATE="unknown"
+# while [ $COUNT -lt 60 ]; do
+#     # Capture full output to check if instance even exists in the response
+#     DESCRIBE_OUTPUT=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID") || {
+#         echo "⚠️  Gateway request failed, retrying... ($COUNT/60)"
+#         sleep 5
+#         COUNT=$((COUNT + 1))
+#         continue
+#     }
+
+#     if [ -z "$DESCRIBE_OUTPUT" ]; then
+#         echo "⚠️  Gateway returned empty response, retrying..."
+#         sleep 5
+#         COUNT=$((COUNT + 1))
+#         continue
+#     fi
+
+#     # Extract state using jq
+#     STATE=$(echo "$DESCRIBE_OUTPUT" | jq -r '.Reservations[0].Instances[0].State.Name // "not-found"')
+
+#     echo "Instance state: $STATE"
+#     if [ "$STATE" == "running" ]; then
+#         break
+#     fi
+
+#     if [ "$STATE" == "terminated" ]; then
+#         echo "❌ Instance terminated unexpectedly!"
+#         ./scripts/stop-dev.sh
+#         exit 1
+#     fi
+
+#     sleep 5
+#     COUNT=$((COUNT + 1))
+# done
+
+# if [ "$STATE" != "running" ]; then
+#     echo "Instance failed to reach running state"
+#     ./scripts/stop-dev.sh
+#     exit 1
+# fi
+
+# # Verify root volume attached to the instance (describe-volumes)
+# VOLUME_ID=$($AWS_EC2 describe-volumes --query 'Volumes[0].VolumeId' --output text)
+# if [ -z "$VOLUME_ID" ] || [ "$VOLUME_ID" == "None" ]; then
+#     echo "Failed to find volume for instance $INSTANCE_ID"
+#     ./scripts/stop-dev.sh
+#     exit 1
+# fi
+# echo "Volume ID: $VOLUME_ID"
+
+# # Stop instance (stop-instances) and verify transition to stopped (describe-instances)
+# echo "Stopping instance..."
+# $AWS_EC2 stop-instances --instance-ids "$INSTANCE_ID"
+# COUNT=0
+# while [ $COUNT -lt 30 ]; do
+#     STATE=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text)
+#     echo "Instance state: $STATE"
+#     if [ "$STATE" == "stopped" ]; then
+#         break
+#     fi
+#     sleep 5
+#     COUNT=$((COUNT + 1))
+# done
+
+# if [ "$STATE" != "stopped" ]; then
+#     echo "Instance failed to reach stopped state"
+#     ./scripts/stop-dev.sh
+#     exit 1
+# fi
+
+# # Start instance (start-instances) and verify transition back to running (describe-instances)
+# echo "Starting instance..."
+# $AWS_EC2 start-instances --instance-ids "$INSTANCE_ID"
+# COUNT=0
+# while [ $COUNT -lt 30 ]; do
+#     STATE=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text)
+#     echo "Instance state: $STATE"
+#     if [ "$STATE" == "running" ]; then
+#         break
+#     fi
+#     sleep 5
+#     COUNT=$((COUNT + 1))
+# done
+
+# if [ "$STATE" != "running" ]; then
+#     echo "Instance failed to reach running state after restart"
+#     ./scripts/stop-dev.sh
+#     exit 1
+# fi
+
+# # Terminate instance (terminate-instances) and verify termination (describe-instances)
+# echo "Terminating instance..."
+# $AWS_EC2 terminate-instances --instance-ids "$INSTANCE_ID"
+# COUNT=0
+# while [ $COUNT -lt 30 ]; do
+#     STATE=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text)
+#     echo "Instance state: $STATE"
+#     if [ "$STATE" == "terminated" ] || [ "$STATE" == "None" ]; then
+#         break
+#     fi
+#     sleep 5
+#     COUNT=$((COUNT + 1))
+# done
+
+# Phase 6: Cleanup (Inside Container)
+echo "Phase 6: Cleanup"
+./scripts/stop-dev.sh
+
+echo "E2E Test Completed Successfully"
+exit 0