diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..12e7c6a50 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,43 @@ +name: E2E Test Hive + +on: + push: + branches: [ main, e2e ] + workflow_dispatch: + +jobs: + hive_e2e_test: + name: Hive E2E Test + runs-on: self-hosted + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + with: + path: hive + + - name: Clone Dependencies + run: ./hive/scripts/clone-deps.sh + + - name: Cleanup Stale Containers + run: docker rm -f hive-e2e-test 2>/dev/null || true + + - name: Build E2E Image + run: | + docker build \ + -t hive-e2e:latest \ + -f hive/tests/e2e/Dockerfile.e2e \ + . + + - name: Run E2E Tests + run: | + docker run --privileged --rm \ + -v /dev/kvm:/dev/kvm \ + --name hive-e2e-test \ + hive-e2e:latest + + - name: Cleanup + if: always() + run: | + docker rm -f hive-e2e-test 2>/dev/null || true + docker image prune -f diff --git a/Makefile b/Makefile index fffda0e3c..ea275d3c4 100644 --- a/Makefile +++ b/Makefile @@ -40,30 +40,34 @@ run: clean: rm ./bin/$(GO_PROJECT_NAME) -quickinstall: +install-system: @echo "\n....Installing system dependencies...." - sudo apt-get update && sudo apt-get install -y \ + apt-get update && apt-get install -y \ nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-utils qemu-kvm \ libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \ iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file +install-go: + @echo "\n....Installing Go 1.25.5...." + @if [ ! -d "/usr/local/go" ]; then \ + curl -L https://go.dev/dl/go1.25.5.linux-amd64.tar.gz | tar -C /usr/local -xz; \ + else \ + echo "Go already installed in /usr/local/go"; \ + fi + @echo "Go version: $$(go version)" + +install-aws: @echo "\n....Installing AWS CLI v2...." @if ! command -v aws >/dev/null 2>&1; then \ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"; \ unzip -o awscliv2.zip; \ - sudo ./aws/install; \ + ./aws/install; \ rm -rf awscliv2.zip aws/; \ else \ echo "AWS CLI already installed"; \ fi - @echo "\n....Installing Go 1.25.5...." - @if [ ! -d "/usr/local/go" ]; then \ - curl -L https://go.dev/dl/go1.25.5.linux-amd64.tar.gz | sudo tar -C /usr/local -xz; \ - else \ - echo "Go already installed in /usr/local/go"; \ - fi - +quickinstall: install-system install-go install-aws @echo "\nāœ… Quickinstall complete. Please ensure /usr/local/go/bin is in your PATH." security: @@ -81,20 +85,4 @@ security: go vet ./... 2>&1 | tee tests/govet-report.txt || true @echo "Go vet report saved to tests/govet-report.txt" -e2e-test: - @echo "\n....Ensuring E2E Base Image exists...." - @if ! sudo docker image inspect hive-base:latest >/dev/null 2>&1; then \ - echo "Building hive-base:latest..."; \ - sudo docker build -t hive-base -f tests/e2e/Dockerfile.base .; \ - fi - - @echo "\n....Removing old E2E image if it exists...." - -sudo docker rmi hive-e2e:latest 2>/dev/null || true - - @echo "\n....Building E2E Docker image (building everything inside)...." - cd .. && sudo docker build -t hive-e2e -f hive/tests/e2e/Dockerfile.e2e . - - @echo "\n....Running E2E Docker container...." - sudo docker run --privileged --rm -v /dev/kvm:/dev/kvm --name hive-e2e-test hive-e2e - -.PHONY: build go_build go_run test bench run clean quickinstall security e2e-test +.PHONY: build go_build go_run test bench run clean install-system install-go install-aws quickinstall security diff --git a/compute/geniso.sh b/compute/geniso.sh old mode 100644 new mode 100755 diff --git a/compute/nbdkit.sh b/compute/nbdkit.sh old mode 100644 new mode 100755 diff --git a/compute/vm-nbd-daemon.sh b/compute/vm-nbd-daemon.sh old mode 100644 new mode 100755 diff --git a/go.mod b/go.mod index ee5126269..755932545 100644 --- a/go.mod +++ b/go.mod @@ -94,8 +94,6 @@ require ( golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.12.0 // indirect golang.org/x/tools v0.39.0 // indirect - golang.org/x/tools/go/expect v0.1.1-deprecated // indirect - golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect golang.org/x/vuln v1.1.4 // indirect google.golang.org/genai v1.37.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect diff --git a/go.sum b/go.sum index e7b383e78..6f2c3b9a0 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,6 @@ github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/mulgadc/predastore v0.1.5 h1:7SySkQQ5MQ1NTcKcB14pCUIJipq0YVgCJZeKM6ytrLM= github.com/mulgadc/predastore v0.1.5/go.mod h1:BhMymRV3zMTu2S+5pJnNDDm5+AGakjhSc/PKX0q8Xfg= -github.com/mulgadc/viperblock v0.1.3 h1:xooK1uTUI0RLK23PHAuVInirC3TrBo6/6ezkt+1vMJo= -github.com/mulgadc/viperblock v0.1.3/go.mod h1:eFolZVfX4iBFBnHAfOoYILfiIjNoI/NhmKht3OIPRR8= github.com/mulgadc/viperblock v0.1.4 h1:YnRHEC6XNbhnSk6S+ET3X7DTMRezdO3o1O3nO/R9Syg= github.com/mulgadc/viperblock v0.1.4/go.mod h1:JNooDHb6ZuPvNPWaf0gChlft/D8xs5jHJFln+P3B9lQ= github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI= diff --git a/scripts/start-multi-nats.sh b/scripts/start-multi-nats.sh old mode 100644 new mode 100755 diff --git a/tests/e2e/Dockerfile.base b/tests/e2e/Dockerfile.base deleted file mode 100644 index 38fc02f07..000000000 --- a/tests/e2e/Dockerfile.base +++ /dev/null @@ -1,19 +0,0 @@ -FROM ubuntu:24.04 - -# Avoid prompts -ENV DEBIAN_FRONTEND=noninteractive - -# Install make first so we can use the Makefile -RUN apt-get update && apt-get install -y make sudo && rm -rf /var/lib/apt/lists/* - -# Set up working directory -WORKDIR /app/hive - -# Copy only the Makefile to run quickinstall -COPY Makefile . - -# Run the quickinstall target from the Makefile -RUN make quickinstall - -# Ensure Go is in the PATH -ENV PATH="/usr/local/go/bin:${PATH}" diff --git a/tests/e2e/Dockerfile.e2e b/tests/e2e/Dockerfile.e2e index 8caee7037..94768ed3a 100644 --- a/tests/e2e/Dockerfile.e2e +++ b/tests/e2e/Dockerfile.e2e @@ -1,19 +1,36 @@ -# syntax=docker/dockerfile:1 -# This image assumes hive-base:latest exists locally or in a registry -FROM hive-base:latest +FROM ubuntu:24.04 -# Set up working directory structure to match INSTALL.md +# Avoid prompts +ENV DEBIAN_FRONTEND=noninteractive + +# Install make first so we can use the Makefile +RUN apt-get update && apt-get install -y make sudo && rm -rf /var/lib/apt/lists/* + +# Set up working directory for Makefile targets +WORKDIR /tmp/hive-setup + +# Copy only the Makefile for dependency installation +COPY hive/Makefile . + +# Install system dependencies in separate layers for better caching +RUN make install-system +RUN make install-go +RUN make install-aws + +# Ensure Go is in the PATH +ENV PATH="/usr/local/go/bin:/root/go/bin:${PATH}" + +# Set up working directory structure to match development layout WORKDIR /root/Development/mulga -# 1. Pre-copy dependency manifests -# We copy these first to create a dedicated dependency layer -COPY hive/go.mod hive/go.sum hive/go.work hive/go.work.sum ./hive/ +# 1. Pre-copy dependency manifests for cache layer +COPY hive/go.mod hive/go.sum ./hive/ COPY viperblock/go.mod viperblock/go.sum ./viperblock/ COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/ COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/ COPY predastore/go.mod predastore/go.sum ./predastore/ -# 2. Download dependencies (Using Cache Mount) +# 2. Download dependencies (using cache mount) RUN --mount=type=cache,target=/root/go/pkg/mod \ (cd hive && go mod download) && \ (cd viperblock && go mod download) && \ @@ -32,7 +49,7 @@ WORKDIR /root/Development/mulga/hive # Ensure scripts are executable RUN chmod +x ./scripts/*.sh ./tests/e2e/run-e2e.sh -# 4. Build everything +# 4. Build everything (using cache mounts) RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/root/go/pkg/mod \ ./scripts/dev-setup.sh diff --git a/tests/e2e/run-e2e.sh b/tests/e2e/run-e2e.sh old mode 100644 new mode 100755 index 0219a6650..e902b029b --- a/tests/e2e/run-e2e.sh +++ b/tests/e2e/run-e2e.sh @@ -1,5 +1,18 @@ #!/bin/bash -set -ex +set -e + +# Ensure services are stopped on exit and print logs on failure +cleanup() { + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "=== Hive Service Log ===" + if [ -f ~/hive/logs/hive.log ]; then + cat ~/hive/logs/hive.log 2>/dev/null + fi + fi + ./scripts/stop-dev.sh +} +trap cleanup EXIT # Use Hive profile export AWS_PROFILE=hive @@ -30,6 +43,7 @@ fi # Start all services # Ensure logs directory exists for start-dev.sh mkdir -p ~/hive/logs +mkdir -p /mnt/ramdisk HIVE_SKIP_BUILD=true ./scripts/start-dev.sh # Wait for health checks on https://localhost:9999 (AWS Gateway) @@ -45,15 +59,11 @@ done if [ $COUNT -eq $MAX_RETRIES ]; then echo "Gateway failed to start" - ./scripts/stop-dev.sh exit 1 fi # Define common AWS CLI args -# Use --endpoint-url and --no-verify-ssl to be safe in the container environment -# Suppress InsecureRequestWarning from urllib3 -export PYTHONWARNINGS="ignore:Unverified HTTPS request" -AWS_EC2="aws --endpoint-url https://localhost:9999 --no-verify-ssl ec2" +AWS_EC2="aws --endpoint-url https://localhost:9999 ec2" # Phase 2: Discovery & Metadata echo "Phase 2: Discovery & Metadata" @@ -66,8 +76,8 @@ echo "Discovering available instance types..." AVAILABLE_TYPES=$($AWS_EC2 describe-instance-types --query 'InstanceTypes[*].InstanceType' --output text) echo "Available instance types: $AVAILABLE_TYPES" -# Pick the first one as our primary test type -INSTANCE_TYPE=$(echo $AVAILABLE_TYPES | awk '{print $1}') +# Pick the nano instance type for minimal resource usage in tests +INSTANCE_TYPE=$(echo $AVAILABLE_TYPES | tr ' ' '\n' | grep -m1 'nano') if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" == "None" ]; then echo "No instance types found!" exit 1 @@ -123,12 +133,11 @@ echo "Using image: $IMAGE_NAME" # Import a reliable Ubuntu image and capture the AMI ID from the output echo "Importing image $IMAGE_NAME..." echo "May appear stalled here, just takes a while to import..." -IMPORT_LOG=$(./bin/hive admin images import --name "$IMAGE_NAME" --force) +IMPORT_LOG=$(./bin/hive admin images import --name "$IMAGE_NAME" --force 2>/dev/null) AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+') if [ -z "$AMI_ID" ]; then echo "Failed to capture AMI ID from import command" - ./scripts/stop-dev.sh exit 1 fi echo "Captured AMI ID: $AMI_ID" @@ -138,17 +147,16 @@ echo "Verifying AMI availability..." $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.ImageId==\"$AMI_ID\")" # Phase 5: Instance Lifecycle -# echo "Phase 5: Instance Lifecycle" -# # Launch a VM (run-instances) +echo "Phase 5: Instance Lifecycle" +# Launch a VM (run-instances) # INSTANCE_ID=$($AWS_EC2 run-instances --image-id "$AMI_ID" --instance-type "$INSTANCE_TYPE" --key-name test-key-1 --query 'Instances[0].InstanceId' --output text) # if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "None" ]; then # echo "Failed to launch instance" -# ./scripts/stop-dev.sh # exit 1 # fi # echo "Launched Instance ID: $INSTANCE_ID" -# # Poll until state is running (describe-instances) +# Poll until state is running (describe-instances) # echo "Polling for instance running state..." # COUNT=0 # STATE="unknown" @@ -178,7 +186,6 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima # if [ "$STATE" == "terminated" ]; then # echo "āŒ Instance terminated unexpectedly!" -# ./scripts/stop-dev.sh # exit 1 # fi @@ -188,7 +195,6 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima # if [ "$STATE" != "running" ]; then # echo "Instance failed to reach running state" -# ./scripts/stop-dev.sh # exit 1 # fi @@ -196,7 +202,6 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima # VOLUME_ID=$($AWS_EC2 describe-volumes --query 'Volumes[0].VolumeId' --output text) # if [ -z "$VOLUME_ID" ] || [ "$VOLUME_ID" == "None" ]; then # echo "Failed to find volume for instance $INSTANCE_ID" -# ./scripts/stop-dev.sh # exit 1 # fi # echo "Volume ID: $VOLUME_ID" @@ -217,7 +222,6 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima # if [ "$STATE" != "stopped" ]; then # echo "Instance failed to reach stopped state" -# ./scripts/stop-dev.sh # exit 1 # fi @@ -237,7 +241,6 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima # if [ "$STATE" != "running" ]; then # echo "Instance failed to reach running state after restart" -# ./scripts/stop-dev.sh # exit 1 # fi @@ -255,9 +258,5 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima # COUNT=$((COUNT + 1)) # done -# Phase 6: Cleanup (Inside Container) -echo "Phase 6: Cleanup" -./scripts/stop-dev.sh - echo "E2E Test Completed Successfully" exit 0