mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 10:52:04 +00:00
merge latest
This commit is contained in:
Ben Duncan
@@ -8,7 +8,7 @@ require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/kdomanski/iso9660 v0.4.0
|
||||
github.com/mulgadc/predastore v0.1.5
|
||||
github.com/mulgadc/viperblock v0.1.4
|
||||
github.com/mulgadc/viperblock v0.1.5
|
||||
github.com/nats-io/nats-server/v2 v2.11.6
|
||||
github.com/nats-io/nats.go v1.43.0
|
||||
github.com/pelletier/go-toml/v2 v2.2.4
|
||||
@@ -54,6 +54,7 @@ require (
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/highwayhash v1.0.3 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.7.4 // indirect
|
||||
github.com/nats-io/nkeys v0.4.11 // indirect
|
||||
|
||||
@@ -122,12 +122,16 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q=
|
||||
github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/mulgadc/predastore v0.1.5 h1:7SySkQQ5MQ1NTcKcB14pCUIJipq0YVgCJZeKM6ytrLM=
|
||||
github.com/mulgadc/predastore v0.1.5/go.mod h1:BhMymRV3zMTu2S+5pJnNDDm5+AGakjhSc/PKX0q8Xfg=
|
||||
github.com/mulgadc/viperblock v0.1.4 h1:YnRHEC6XNbhnSk6S+ET3X7DTMRezdO3o1O3nO/R9Syg=
|
||||
github.com/mulgadc/viperblock v0.1.4/go.mod h1:JNooDHb6ZuPvNPWaf0gChlft/D8xs5jHJFln+P3B9lQ=
|
||||
github.com/mulgadc/viperblock v0.1.5 h1:vS7ImGApQvxKuU1+fJNmwOycduwr9fx/xyofxWYKPcI=
|
||||
github.com/mulgadc/viperblock v0.1.5/go.mod h1:JNooDHb6ZuPvNPWaf0gChlft/D8xs5jHJFln+P3B9lQ=
|
||||
github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI=
|
||||
github.com/nats-io/jwt/v2 v2.7.4/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
|
||||
github.com/nats-io/nats-server/v2 v2.11.6 h1:4VXRjbTUFKEB+7UoaKL3F5Y83xC7MxPoIONOnGgpkHw=
|
||||
|
||||
@@ -1465,12 +1465,15 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er
|
||||
// Wait for all shutdowns to finish
|
||||
wg.Wait()
|
||||
|
||||
// Unsubscribe from NATS subjects that match instances
|
||||
for _, instance := range instances {
|
||||
slog.Info("Unsubscribing from NATS subject", "instance", instance.ID)
|
||||
d.natsSubscriptions[fmt.Sprintf("ec2.cmd.%s", instance.ID)].Unsubscribe()
|
||||
// TODO: Remove redundant subscription if not used
|
||||
//d.natsSubscriptions[fmt.Sprintf("ec2.describe.%s", instance.ID)].Unsubscribe()
|
||||
// Only unsubscribe from NATS subjects when terminating (deleteVolume=true)
|
||||
// For stop operations, keep the subscription so we can receive start commands
|
||||
if deleteVolume {
|
||||
for _, instance := range instances {
|
||||
slog.Info("Unsubscribing from NATS subject", "instance", instance.ID)
|
||||
d.natsSubscriptions[fmt.Sprintf("ec2.cmd.%s", instance.ID)].Unsubscribe()
|
||||
// TODO: Remove redundant subscription if not used
|
||||
//d.natsSubscriptions[fmt.Sprintf("ec2.describe.%s", instance.ID)].Unsubscribe()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ func (gw *GatewayConfig) EC2_Request(ctx *fiber.Ctx) error {
|
||||
}
|
||||
|
||||
// Convert to XML
|
||||
payload := utils.GenerateXMLPayload("RunInstanceResponse", output)
|
||||
payload := utils.GenerateXMLPayload("RunInstancesResponse", output)
|
||||
xmlOutput, err = utils.MarshalToXML(payload)
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -23,6 +23,18 @@ ENV PATH="/usr/local/go/bin:/root/go/bin:${PATH}"
|
||||
# Set up working directory structure to match development layout
|
||||
WORKDIR /root/Development/mulga
|
||||
|
||||
# Pre-download Ubuntu cloud image for e2e tests (cached in Docker layer)
|
||||
# Placed before dependency copy so go.mod changes don't invalidate this layer
|
||||
RUN mkdir -p /root/images && \
|
||||
ARCH=$(dpkg --print-architecture) && \
|
||||
if [ "$ARCH" = "amd64" ]; then \
|
||||
curl -L -o /root/images/ubuntu-24.04.img \
|
||||
https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img; \
|
||||
elif [ "$ARCH" = "arm64" ]; then \
|
||||
curl -L -o /root/images/ubuntu-24.04.img \
|
||||
https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-arm64.img; \
|
||||
fi
|
||||
|
||||
# 1. Pre-copy dependency manifests for cache layer
|
||||
COPY hive/go.mod hive/go.sum ./hive/
|
||||
COPY viperblock/go.mod viperblock/go.sum ./viperblock/
|
||||
|
||||
+137
-101
@@ -5,9 +5,30 @@ set -e
|
||||
cleanup() {
|
||||
EXIT_CODE=$?
|
||||
if [ $EXIT_CODE -ne 0 ]; then
|
||||
echo "=== Hive Service Log ==="
|
||||
echo ""
|
||||
echo "=== NATS Service Log ==="
|
||||
if [ -f ~/hive/logs/nats.log ]; then
|
||||
tail -100 ~/hive/logs/nats.log 2>/dev/null
|
||||
fi
|
||||
echo ""
|
||||
echo "=== Predastore Service Log ==="
|
||||
if [ -f ~/hive/logs/predastore.log ]; then
|
||||
tail -100 ~/hive/logs/predastore.log 2>/dev/null
|
||||
fi
|
||||
echo ""
|
||||
echo "=== Viperblock Service Log ==="
|
||||
if [ -f ~/hive/logs/viperblock.log ]; then
|
||||
tail -100 ~/hive/logs/viperblock.log 2>/dev/null
|
||||
fi
|
||||
echo ""
|
||||
echo "=== Hive Daemon Log ==="
|
||||
if [ -f ~/hive/logs/hive.log ]; then
|
||||
cat ~/hive/logs/hive.log 2>/dev/null
|
||||
tail -200 ~/hive/logs/hive.log 2>/dev/null
|
||||
fi
|
||||
echo ""
|
||||
echo "=== AWS Gateway Log ==="
|
||||
if [ -f ~/hive/logs/awsgw.log ]; then
|
||||
tail -200 ~/hive/logs/awsgw.log 2>/dev/null
|
||||
fi
|
||||
fi
|
||||
./scripts/stop-dev.sh
|
||||
@@ -130,11 +151,14 @@ else
|
||||
fi
|
||||
echo "Using image: $IMAGE_NAME"
|
||||
|
||||
# Import a reliable Ubuntu image and capture the AMI ID from the output
|
||||
# TODO: Cache the import image, see Docker layer or another method.
|
||||
echo "Importing image $IMAGE_NAME..."
|
||||
echo "May appear stalled here, just takes a while to import..."
|
||||
IMPORT_LOG=$(./bin/hive admin images import --name "$IMAGE_NAME" --force 2>/dev/null)
|
||||
# Import the pre-downloaded Ubuntu image using file-based import
|
||||
echo "Importing pre-cached Ubuntu image..."
|
||||
IMPORT_LOG=$(./bin/hive admin images import \
|
||||
--file /root/images/ubuntu-24.04.img \
|
||||
--arch "$ARCH" \
|
||||
--distro ubuntu \
|
||||
--version 24.04 \
|
||||
--force 2>/dev/null)
|
||||
AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+')
|
||||
|
||||
if [ -z "$AMI_ID" ]; then
|
||||
@@ -150,114 +174,126 @@ $AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.Ima
|
||||
# Phase 5: Instance Lifecycle
|
||||
echo "Phase 5: Instance Lifecycle"
|
||||
# Launch a VM (run-instances)
|
||||
# INSTANCE_ID=$($AWS_EC2 run-instances --image-id "$AMI_ID" --instance-type "$INSTANCE_TYPE" --key-name test-key-1 --query 'Instances[0].InstanceId' --output text)
|
||||
# if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "None" ]; then
|
||||
# echo "Failed to launch instance"
|
||||
# exit 1
|
||||
# fi
|
||||
# echo "Launched Instance ID: $INSTANCE_ID"
|
||||
echo "Running: aws ec2 run-instances --image-id $AMI_ID --instance-type $INSTANCE_TYPE --key-name test-key-1"
|
||||
# Capture full output for debugging
|
||||
set +e # Temporarily disable exit on error to capture output
|
||||
RUN_OUTPUT=$($AWS_EC2 run-instances --image-id "$AMI_ID" --instance-type "$INSTANCE_TYPE" --key-name test-key-1 2>&1)
|
||||
RUN_EXIT_CODE=$?
|
||||
set -e # Re-enable exit on error
|
||||
echo "Run instances exit code: $RUN_EXIT_CODE"
|
||||
echo "Run instances output: $RUN_OUTPUT"
|
||||
if [ $RUN_EXIT_CODE -ne 0 ]; then
|
||||
echo "❌ Failed to launch instance - AWS CLI returned error"
|
||||
exit 1
|
||||
fi
|
||||
INSTANCE_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId')
|
||||
if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "None" ] || [ "$INSTANCE_ID" == "null" ]; then
|
||||
echo "Failed to launch instance - no InstanceId in response"
|
||||
exit 1
|
||||
fi
|
||||
echo "Launched Instance ID: $INSTANCE_ID"
|
||||
|
||||
# Poll until state is running (describe-instances)
|
||||
# echo "Polling for instance running state..."
|
||||
# COUNT=0
|
||||
# STATE="unknown"
|
||||
# while [ $COUNT -lt 60 ]; do
|
||||
# # Capture full output to check if instance even exists in the response
|
||||
# DESCRIBE_OUTPUT=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID") || {
|
||||
# echo "⚠️ Gateway request failed, retrying... ($COUNT/60)"
|
||||
# sleep 5
|
||||
# COUNT=$((COUNT + 1))
|
||||
# continue
|
||||
# }
|
||||
echo "Polling for instance running state..."
|
||||
COUNT=0
|
||||
STATE="unknown"
|
||||
while [ $COUNT -lt 60 ]; do
|
||||
# Capture full output to check if instance even exists in the response
|
||||
DESCRIBE_OUTPUT=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID") || {
|
||||
echo "⚠️ Gateway request failed, retrying... ($COUNT/60)"
|
||||
sleep 5
|
||||
COUNT=$((COUNT + 1))
|
||||
continue
|
||||
}
|
||||
|
||||
# if [ -z "$DESCRIBE_OUTPUT" ]; then
|
||||
# echo "⚠️ Gateway returned empty response, retrying..."
|
||||
# sleep 5
|
||||
# COUNT=$((COUNT + 1))
|
||||
# continue
|
||||
# fi
|
||||
if [ -z "$DESCRIBE_OUTPUT" ]; then
|
||||
echo "⚠️ Gateway returned empty response, retrying..."
|
||||
sleep 5
|
||||
COUNT=$((COUNT + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# # Extract state using jq
|
||||
# STATE=$(echo "$DESCRIBE_OUTPUT" | jq -r '.Reservations[0].Instances[0].State.Name // "not-found"')
|
||||
# Extract state using jq
|
||||
STATE=$(echo "$DESCRIBE_OUTPUT" | jq -r '.Reservations[0].Instances[0].State.Name // "not-found"')
|
||||
|
||||
# echo "Instance state: $STATE"
|
||||
# if [ "$STATE" == "running" ]; then
|
||||
# break
|
||||
# fi
|
||||
echo "Instance state: $STATE"
|
||||
if [ "$STATE" == "running" ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
# if [ "$STATE" == "terminated" ]; then
|
||||
# echo "❌ Instance terminated unexpectedly!"
|
||||
# exit 1
|
||||
# fi
|
||||
if [ "$STATE" == "terminated" ]; then
|
||||
echo "❌ Instance terminated unexpectedly!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# sleep 5
|
||||
# COUNT=$((COUNT + 1))
|
||||
# done
|
||||
sleep 5
|
||||
COUNT=$((COUNT + 1))
|
||||
done
|
||||
|
||||
# if [ "$STATE" != "running" ]; then
|
||||
# echo "Instance failed to reach running state"
|
||||
# exit 1
|
||||
# fi
|
||||
if [ "$STATE" != "running" ]; then
|
||||
echo "Instance failed to reach running state"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# # Verify root volume attached to the instance (describe-volumes)
|
||||
# VOLUME_ID=$($AWS_EC2 describe-volumes --query 'Volumes[0].VolumeId' --output text)
|
||||
# if [ -z "$VOLUME_ID" ] || [ "$VOLUME_ID" == "None" ]; then
|
||||
# echo "Failed to find volume for instance $INSTANCE_ID"
|
||||
# exit 1
|
||||
# fi
|
||||
# echo "Volume ID: $VOLUME_ID"
|
||||
# Verify root volume attached to the instance (describe-volumes)
|
||||
VOLUME_ID=$($AWS_EC2 describe-volumes --query 'Volumes[0].VolumeId' --output text)
|
||||
if [ -z "$VOLUME_ID" ] || [ "$VOLUME_ID" == "None" ]; then
|
||||
echo "Failed to find volume for instance $INSTANCE_ID"
|
||||
exit 1
|
||||
fi
|
||||
echo "Volume ID: $VOLUME_ID"
|
||||
|
||||
# # Stop instance (stop-instances) and verify transition to stopped (describe-instances)
|
||||
# echo "Stopping instance..."
|
||||
# $AWS_EC2 stop-instances --instance-ids "$INSTANCE_ID"
|
||||
# COUNT=0
|
||||
# while [ $COUNT -lt 30 ]; do
|
||||
# STATE=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text)
|
||||
# echo "Instance state: $STATE"
|
||||
# if [ "$STATE" == "stopped" ]; then
|
||||
# break
|
||||
# fi
|
||||
# sleep 5
|
||||
# COUNT=$((COUNT + 1))
|
||||
# done
|
||||
# Stop instance (stop-instances) and verify transition to stopped (describe-instances)
|
||||
echo "Stopping instance..."
|
||||
$AWS_EC2 stop-instances --instance-ids "$INSTANCE_ID"
|
||||
COUNT=0
|
||||
while [ $COUNT -lt 30 ]; do
|
||||
STATE=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text)
|
||||
echo "Instance state: $STATE"
|
||||
if [ "$STATE" == "stopped" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
COUNT=$((COUNT + 1))
|
||||
done
|
||||
|
||||
# if [ "$STATE" != "stopped" ]; then
|
||||
# echo "Instance failed to reach stopped state"
|
||||
# exit 1
|
||||
# fi
|
||||
if [ "$STATE" != "stopped" ]; then
|
||||
echo "Instance failed to reach stopped state"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# # Start instance (start-instances) and verify transition back to running (describe-instances)
|
||||
# echo "Starting instance..."
|
||||
# $AWS_EC2 start-instances --instance-ids "$INSTANCE_ID"
|
||||
# COUNT=0
|
||||
# while [ $COUNT -lt 30 ]; do
|
||||
# STATE=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text)
|
||||
# echo "Instance state: $STATE"
|
||||
# if [ "$STATE" == "running" ]; then
|
||||
# break
|
||||
# fi
|
||||
# sleep 5
|
||||
# COUNT=$((COUNT + 1))
|
||||
# done
|
||||
# Start instance (start-instances) and verify transition back to running (describe-instances)
|
||||
echo "Starting instance..."
|
||||
$AWS_EC2 start-instances --instance-ids "$INSTANCE_ID"
|
||||
COUNT=0
|
||||
while [ $COUNT -lt 30 ]; do
|
||||
STATE=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text)
|
||||
echo "Instance state: $STATE"
|
||||
if [ "$STATE" == "running" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
COUNT=$((COUNT + 1))
|
||||
done
|
||||
|
||||
# if [ "$STATE" != "running" ]; then
|
||||
# echo "Instance failed to reach running state after restart"
|
||||
# exit 1
|
||||
# fi
|
||||
if [ "$STATE" != "running" ]; then
|
||||
echo "Instance failed to reach running state after restart"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# # Terminate instance (terminate-instances) and verify termination (describe-instances)
|
||||
# echo "Terminating instance..."
|
||||
# $AWS_EC2 terminate-instances --instance-ids "$INSTANCE_ID"
|
||||
# COUNT=0
|
||||
# while [ $COUNT -lt 30 ]; do
|
||||
# STATE=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text)
|
||||
# echo "Instance state: $STATE"
|
||||
# if [ "$STATE" == "terminated" ] || [ "$STATE" == "None" ]; then
|
||||
# break
|
||||
# fi
|
||||
# sleep 5
|
||||
# COUNT=$((COUNT + 1))
|
||||
# done
|
||||
# Terminate instance (terminate-instances) and verify termination (describe-instances)
|
||||
echo "Terminating instance..."
|
||||
$AWS_EC2 terminate-instances --instance-ids "$INSTANCE_ID"
|
||||
COUNT=0
|
||||
while [ $COUNT -lt 30 ]; do
|
||||
STATE=$($AWS_EC2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].State.Name' --output text)
|
||||
echo "Instance state: $STATE"
|
||||
if [ "$STATE" == "terminated" ] || [ "$STATE" == "None" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
COUNT=$((COUNT + 1))
|
||||
done
|
||||
|
||||
echo "E2E Test Completed Successfully"
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user