mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 09:01:31 +00:00
feat: E2E tests in GitHub Actions
This commit is contained in:
Julian Sommer
@@ -0,0 +1,43 @@
|
||||
name: E2E Test Hive
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, e2e ]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
hive_e2e_test:
|
||||
name: Hive E2E Test
|
||||
runs-on: self-hosted
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
path: hive
|
||||
|
||||
- name: Clone Dependencies
|
||||
run: ./hive/scripts/clone-deps.sh
|
||||
|
||||
- name: Cleanup Stale Containers
|
||||
run: docker rm -f hive-e2e-test 2>/dev/null || true
|
||||
|
||||
- name: Build E2E Image
|
||||
run: |
|
||||
docker build \
|
||||
-t hive-e2e:latest \
|
||||
-f hive/tests/e2e/Dockerfile.e2e \
|
||||
.
|
||||
|
||||
- name: Run E2E Tests
|
||||
run: |
|
||||
docker run --privileged --rm \
|
||||
-v /dev/kvm:/dev/kvm \
|
||||
--name hive-e2e-test \
|
||||
hive-e2e:latest
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f hive-e2e-test 2>/dev/null || true
|
||||
docker image prune -f
|
||||
@@ -40,30 +40,34 @@ run:
|
||||
clean:
|
||||
rm ./bin/$(GO_PROJECT_NAME)
|
||||
|
||||
quickinstall:
|
||||
install-system:
|
||||
@echo "\n....Installing system dependencies...."
|
||||
sudo apt-get update && sudo apt-get install -y \
|
||||
apt-get update && apt-get install -y \
|
||||
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-utils qemu-kvm \
|
||||
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
|
||||
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file
|
||||
|
||||
install-go:
|
||||
@echo "\n....Installing Go 1.25.5...."
|
||||
@if [ ! -d "/usr/local/go" ]; then \
|
||||
curl -L https://go.dev/dl/go1.25.5.linux-amd64.tar.gz | tar -C /usr/local -xz; \
|
||||
else \
|
||||
echo "Go already installed in /usr/local/go"; \
|
||||
fi
|
||||
@echo "Go version: $$(go version)"
|
||||
|
||||
install-aws:
|
||||
@echo "\n....Installing AWS CLI v2...."
|
||||
@if ! command -v aws >/dev/null 2>&1; then \
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"; \
|
||||
unzip -o awscliv2.zip; \
|
||||
sudo ./aws/install; \
|
||||
./aws/install; \
|
||||
rm -rf awscliv2.zip aws/; \
|
||||
else \
|
||||
echo "AWS CLI already installed"; \
|
||||
fi
|
||||
|
||||
@echo "\n....Installing Go 1.25.5...."
|
||||
@if [ ! -d "/usr/local/go" ]; then \
|
||||
curl -L https://go.dev/dl/go1.25.5.linux-amd64.tar.gz | sudo tar -C /usr/local -xz; \
|
||||
else \
|
||||
echo "Go already installed in /usr/local/go"; \
|
||||
fi
|
||||
|
||||
quickinstall: install-system install-go install-aws
|
||||
@echo "\n✅ Quickinstall complete. Please ensure /usr/local/go/bin is in your PATH."
|
||||
|
||||
security:
|
||||
@@ -81,20 +85,4 @@ security:
|
||||
go vet ./... 2>&1 | tee tests/govet-report.txt || true
|
||||
@echo "Go vet report saved to tests/govet-report.txt"
|
||||
|
||||
e2e-test:
|
||||
@echo "\n....Ensuring E2E Base Image exists...."
|
||||
@if ! sudo docker image inspect hive-base:latest >/dev/null 2>&1; then \
|
||||
echo "Building hive-base:latest..."; \
|
||||
sudo docker build -t hive-base -f tests/e2e/Dockerfile.base .; \
|
||||
fi
|
||||
|
||||
@echo "\n....Removing old E2E image if it exists...."
|
||||
-sudo docker rmi hive-e2e:latest 2>/dev/null || true
|
||||
|
||||
@echo "\n....Building E2E Docker image (building everything inside)...."
|
||||
cd .. && sudo docker build -t hive-e2e -f hive/tests/e2e/Dockerfile.e2e .
|
||||
|
||||
@echo "\n....Running E2E Docker container...."
|
||||
sudo docker run --privileged --rm -v /dev/kvm:/dev/kvm --name hive-e2e-test hive-e2e
|
||||
|
||||
.PHONY: build go_build go_run test bench run clean quickinstall security e2e-test
|
||||
.PHONY: build go_build go_run test bench run clean install-system install-go install-aws quickinstall security
|
||||
|
||||
Regular → Executable
Regular → Executable
Regular → Executable
@@ -94,8 +94,6 @@ require (
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
golang.org/x/tools v0.39.0 // indirect
|
||||
golang.org/x/tools/go/expect v0.1.1-deprecated // indirect
|
||||
golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect
|
||||
golang.org/x/vuln v1.1.4 // indirect
|
||||
google.golang.org/genai v1.37.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
|
||||
|
||||
@@ -126,8 +126,6 @@ github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD
|
||||
github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/mulgadc/predastore v0.1.5 h1:7SySkQQ5MQ1NTcKcB14pCUIJipq0YVgCJZeKM6ytrLM=
|
||||
github.com/mulgadc/predastore v0.1.5/go.mod h1:BhMymRV3zMTu2S+5pJnNDDm5+AGakjhSc/PKX0q8Xfg=
|
||||
github.com/mulgadc/viperblock v0.1.3 h1:xooK1uTUI0RLK23PHAuVInirC3TrBo6/6ezkt+1vMJo=
|
||||
github.com/mulgadc/viperblock v0.1.3/go.mod h1:eFolZVfX4iBFBnHAfOoYILfiIjNoI/NhmKht3OIPRR8=
|
||||
github.com/mulgadc/viperblock v0.1.4 h1:YnRHEC6XNbhnSk6S+ET3X7DTMRezdO3o1O3nO/R9Syg=
|
||||
github.com/mulgadc/viperblock v0.1.4/go.mod h1:JNooDHb6ZuPvNPWaf0gChlft/D8xs5jHJFln+P3B9lQ=
|
||||
github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI=
|
||||
|
||||
Regular → Executable
@@ -1,19 +0,0 @@
|
||||
FROM ubuntu:24.04
|
||||
|
||||
# Avoid prompts
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Install make first so we can use the Makefile
|
||||
RUN apt-get update && apt-get install -y make sudo && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set up working directory
|
||||
WORKDIR /app/hive
|
||||
|
||||
# Copy only the Makefile to run quickinstall
|
||||
COPY Makefile .
|
||||
|
||||
# Run the quickinstall target from the Makefile
|
||||
RUN make quickinstall
|
||||
|
||||
# Ensure Go is in the PATH
|
||||
ENV PATH="/usr/local/go/bin:${PATH}"
|
||||
@@ -1,19 +1,36 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# This image assumes hive-base:latest exists locally or in a registry
|
||||
FROM hive-base:latest
|
||||
FROM ubuntu:24.04
|
||||
|
||||
# Set up working directory structure to match INSTALL.md
|
||||
# Avoid prompts
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Install make first so we can use the Makefile
|
||||
RUN apt-get update && apt-get install -y make sudo && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set up working directory for Makefile targets
|
||||
WORKDIR /tmp/hive-setup
|
||||
|
||||
# Copy only the Makefile for dependency installation
|
||||
COPY hive/Makefile .
|
||||
|
||||
# Install system dependencies in separate layers for better caching
|
||||
RUN make install-system
|
||||
RUN make install-go
|
||||
RUN make install-aws
|
||||
|
||||
# Ensure Go is in the PATH
|
||||
ENV PATH="/usr/local/go/bin:/root/go/bin:${PATH}"
|
||||
|
||||
# Set up working directory structure to match development layout
|
||||
WORKDIR /root/Development/mulga
|
||||
|
||||
# 1. Pre-copy dependency manifests
|
||||
# We copy these first to create a dedicated dependency layer
|
||||
COPY hive/go.mod hive/go.sum hive/go.work hive/go.work.sum ./hive/
|
||||
# 1. Pre-copy dependency manifests for cache layer
|
||||
COPY hive/go.mod hive/go.sum ./hive/
|
||||
COPY viperblock/go.mod viperblock/go.sum ./viperblock/
|
||||
COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/
|
||||
COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/
|
||||
COPY predastore/go.mod predastore/go.sum ./predastore/
|
||||
|
||||
# 2. Download dependencies (Using Cache Mount)
|
||||
# 2. Download dependencies (using cache mount)
|
||||
RUN --mount=type=cache,target=/root/go/pkg/mod \
|
||||
(cd hive && go mod download) && \
|
||||
(cd viperblock && go mod download) && \
|
||||
@@ -32,7 +49,7 @@ WORKDIR /root/Development/mulga/hive
|
||||
# Ensure scripts are executable
|
||||
RUN chmod +x ./scripts/*.sh ./tests/e2e/run-e2e.sh
|
||||
|
||||
# 4. Build everything
|
||||
# 4. Build everything (using cache mounts)
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
--mount=type=cache,target=/root/go/pkg/mod \
|
||||
./scripts/dev-setup.sh
|
||||
|
||||
Regular → Executable
+22
-23
@@ -1,5 +1,18 @@
|
||||
#!/bin/bash
|
||||
set -ex
|
||||
set -e
|
||||
|
||||
# Ensure services are stopped on exit and print logs on failure
|
||||
cleanup() {
|
||||
EXIT_CODE=$?
|
||||
if [ $EXIT_CODE -ne 0 ]; then
|
||||
echo "=== Hive Service Log ==="
|
||||
if [ -f ~/hive/logs/hive.log ]; then
|
||||
cat ~/hive/logs/hive.log 2>/dev/null
|
||||
fi
|
||||
fi
|
||||
./scripts/stop-dev.sh
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Use Hive profile
|
||||
export AWS_PROFILE=hive
|
||||
@@ -30,6 +43,7 @@ fi
|
||||
# Start all services
|
||||
# Ensure logs directory exists for start-dev.sh
|
||||
mkdir -p ~/hive/logs
|
||||
mkdir -p /mnt/ramdisk
|
||||
HIVE_SKIP_BUILD=true ./scripts/start-dev.sh
|
||||
|
||||
# Wait for health checks on https://localhost:9999 (AWS Gateway)
|
||||
@@ -45,15 +59,11 @@ done
|
||||
|
||||
if [ $COUNT -eq $MAX_RETRIES ]; then
|
||||
echo "Gateway failed to start"
|
||||
./scripts/stop-dev.sh
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Define common AWS CLI args
|
||||
# Use --endpoint-url and --no-verify-ssl to be safe in the container environment
|
||||
# Suppress InsecureRequestWarning from urllib3
|
||||
export PYTHONWARNINGS="ignore:Unverified HTTPS request"
|
||||
AWS_EC2="aws --endpoint-url https://localhost:9999 --no-verify-ssl ec2"
|
||||
AWS_EC2="aws --endpoint-url https://localhost:9999 ec2"
|
||||
|
||||
# Phase 2: Discovery & Metadata
|
||||
echo "Phase 2: Discovery & Metadata"
|
||||
@@ -66,8 +76,8 @@ echo "Discovering available instance types..."
|
||||
AVAILABLE_TYPES=$($AWS_EC2 describe-instance-types --query 'InstanceTypes[*].InstanceType' --output text)
|
||||
echo "Available instance types: $AVAILABLE_TYPES"
|
||||
|
||||
# Pick the first one as our primary test type
|
||||
INSTANCE_TYPE=$(echo $AVAILABLE_TYPES | awk '{print $1}')
|
||||
# Pick the nano instance type for minimal resource usage in tests
|
||||
INSTANCE_TYPE=$(echo $AVAILABLE_TYPES | tr ' ' '\n' | grep -m1 'nano')
|
||||
if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" == "None" ]; then
|
||||
echo "No instance types found!"
|
||||
exit 1
|
||||
@@ -123,12 +133,11 @@ echo "Using image: $IMAGE_NAME"
|
||||
# Import a reliable Ubuntu image and capture the AMI ID from the output
|
||||
echo "Importing image $IMAGE_NAME..."
|
||||
echo "May appear stalled here, just takes a while to import..."
|
||||
IMPORT_LOG=$(./bin/hive admin images import --name "$IMAGE_NAME" --force)
|
||||
IMPORT_LOG=$(./bin/hive admin images import --name "$IMAGE_NAME" --force 2>/dev/null)
|
||||
AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+')
|
||||
|
||||
if [ -z "$AMI_ID" ]; then
|
||||
echo "Failed to capture AMI ID from import command"
|
||||
./scripts/stop-dev.sh
|
||||
exit 1
|
||||
fi
|
||||
echo "Captured AMI ID: $AMI_ID"
|
||||
@@ -138,17 +147,16 @@ echo "Verifying AMI availability..."
|
||||
$AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.ImageId==\"$AMI_ID\")"
|
||||
|
||||
# Phase 5: Instance Lifecycle
|
||||
# echo "Phase 5: Instance Lifecycle"
|
||||
# # Launch a VM (run-instances)
|
||||
echo "Phase 5: Instance Lifecycle"
|
||||
# Launch a VM (run-instances)
|
||||
# INSTANCE_ID=$($AWS_EC2 run-instances --image-id "$AMI_ID" --instance-type "$INSTANCE_TYPE" --key-name test-key-1 --query 'Instances[0].InstanceId' --output text)
|
||||
# if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "None" ]; then
|
||||
# echo "Failed to launch instance"
|
||||
# ./scripts/stop-dev.sh
|
||||
# exit 1
|
||||
# fi
|
||||
# echo "Launched Instance ID: $INSTANCE_ID"
|
||||
|
||||
# # Poll until state is running (describe-instances)
|
||||
# Poll until state is running (describe-instances)
|
||||
# echo "Polling for instance running state..."
|
||||
# COUNT=0
|
||||
# STATE="unknown"
|
||||
@@ -178,7 +186,6 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima
|
||||
|
||||
# if [ "$STATE" == "terminated" ]; then
|
||||
# echo "❌ Instance terminated unexpectedly!"
|
||||
# ./scripts/stop-dev.sh
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
@@ -188,7 +195,6 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima
|
||||
|
||||
# if [ "$STATE" != "running" ]; then
|
||||
# echo "Instance failed to reach running state"
|
||||
# ./scripts/stop-dev.sh
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
@@ -196,7 +202,6 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima
|
||||
# VOLUME_ID=$($AWS_EC2 describe-volumes --query 'Volumes[0].VolumeId' --output text)
|
||||
# if [ -z "$VOLUME_ID" ] || [ "$VOLUME_ID" == "None" ]; then
|
||||
# echo "Failed to find volume for instance $INSTANCE_ID"
|
||||
# ./scripts/stop-dev.sh
|
||||
# exit 1
|
||||
# fi
|
||||
# echo "Volume ID: $VOLUME_ID"
|
||||
@@ -217,7 +222,6 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima
|
||||
|
||||
# if [ "$STATE" != "stopped" ]; then
|
||||
# echo "Instance failed to reach stopped state"
|
||||
# ./scripts/stop-dev.sh
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
@@ -237,7 +241,6 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima
|
||||
|
||||
# if [ "$STATE" != "running" ]; then
|
||||
# echo "Instance failed to reach running state after restart"
|
||||
# ./scripts/stop-dev.sh
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
@@ -255,9 +258,5 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima
|
||||
# COUNT=$((COUNT + 1))
|
||||
# done
|
||||
|
||||
# Phase 6: Cleanup (Inside Container)
|
||||
echo "Phase 6: Cleanup"
|
||||
./scripts/stop-dev.sh
|
||||
|
||||
echo "E2E Test Completed Successfully"
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user