gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-20 03:11:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: nightly e2e (#155)

Browse Source
This commit is contained in:
Julian Sommer
2026-04-24 12:25:03 +10:00
committed by GitHub
parent 068aef5589
commit c7d4040dd3
16 changed files with 613 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/e2e-ddil.yml
+1 -1
View File
@@ -1,4 +1,4 @@
name: Spinifex DDIL E2E
name: E2E DDIL
# Phase 1 trigger: workflow_dispatch only. Nightly cron + on-main-merge
# land in mulga-siv-5.8 once ≥4 scenarios are asserting. See
.github/workflows/e2e-nightly.yml
+188
View File
@@ -0,0 +1,188 @@
name: E2E Nightly
on:
schedule:
- cron: "0 14 * * *" # 00:00 AEST = 14:00 UTC
workflow_dispatch:
inputs:
cells:
description: "Comma-separated cell numbers to run (blank = all)"
required: false
permissions:
contents: read
env:
GIT_BRANCH: ${{ github.ref_name }}
UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
jobs:
matrix:
name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{
matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
strategy:
fail-fast: false
max-parallel: 3
matrix:
include:
- { cell: 2, install: binary, topo: single, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
- { cell: 9, install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi, extra: "" }
- { cell: 17, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-single, extra: "tofu-examples" }
runs-on: [ self-hosted, "${{ matrix.runner }}" ]
timeout-minutes: 35
env:
TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
steps:
# blank or scheduled run = full matrix.
# Input must be comma-separated with no whitespace (e.g. '1,2,17', not '1, 2').
- name: Gate on Dispatch Input
id: gate
env:
CELL: ${{ matrix.cell }}
CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
run: |
if [ -z "$CELLS" ] || [[ ",${CELLS}," == *",${CELL},"* ]]; then
echo "selected=true" >> "$GITHUB_OUTPUT"
echo "cell ${CELL} selected"
else
echo "selected=false" >> "$GITHUB_OUTPUT"
echo "cell ${CELL} not in '${CELLS}' — subsequent steps will skip"
fi
- name: Clean Workspace
if: steps.gate.outputs.selected == 'true'
run: rm -rf "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
if: steps.gate.outputs.selected == 'true'
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- name: Resolve Mulga Branch
if: steps.gate.outputs.selected == 'true'
id: mulga-branch
env:
GH_TOKEN: ${{ steps.app-token.outputs.token }}
BRANCH: ${{ github.ref_name }}
run: |
if gh api "repos/mulgadc/mulga/branches/${BRANCH}" >/dev/null 2>&1; then
echo "ref=${BRANCH}" >> "$GITHUB_OUTPUT"
echo "mulga: using branch ${BRANCH}"
else
echo "ref=main" >> "$GITHUB_OUTPUT"
echo "mulga: branch ${BRANCH} not found, falling back to main"
fi
- uses: actions/checkout@v6
if: steps.gate.outputs.selected == 'true'
with:
repository: mulgadc/mulga
ref: ${{ steps.mulga-branch.outputs.ref }}
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- name: Build Install Artifacts
if: steps.gate.outputs.selected == 'true'
working-directory: mulga/scripts/tofu-cluster
run: |
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar"
- name: Provision VMs
if: steps.gate.outputs.selected == 'true'
working-directory: mulga/scripts/tofu-cluster
env:
MATRIX_HOST: ${{ matrix.host }}
run: |
export TF_VAR_git_branch="${GIT_BRANCH}"
TOFU_VAR_ARGS=""
if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
fi
tofu init -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
- name: Bootstrap via Binary Install
if: steps.gate.outputs.selected == 'true'
working-directory: mulga/scripts/tofu-cluster
env:
MATRIX_NET: ${{ matrix.net }}
run: |
if [ "$MATRIX_NET" = "dhcp" ]; then
unset SPX_EXTERNAL_POOL
export SPX_EXTERNAL_SOURCE=dhcp
else
export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
fi
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Run Single Node E2E
if: steps.gate.outputs.selected == 'true' && matrix.topo == 'single' && matrix.extra != 'tofu-examples'
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
"cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh"
- name: Run Multi Node E2E
if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
NODE_IPS=""
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
for i in $(seq 0 $((NODE_COUNT - 1))); do
NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
done
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
"cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}"
- name: Run Tofu Workbooks
if: matrix.extra == 'tofu-examples'
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
"cd ~/e2e-tests && ./run-tofu-examples-e2e.sh"
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
env:
MATRIX_HOST: ${{ matrix.host }}
run: |
TOFU_VAR_ARGS=""
if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
fi
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
.github/workflows/e2e.yml
+1 -1
View File
@@ -1,4 +1,4 @@
name: Spinifex E2E Tests
name: E2E
on:
push:
.github/workflows/spinifex.yml
+4 -4
View File
@@ -1,4 +1,4 @@
name: Spinifex Build and Test
name: Build and Test
on:
push:
@@ -17,7 +17,7 @@ on:
jobs:
build_and_test:
name: Spinifex Build and Test
name: Build and Test
runs-on: ubuntu-24.04
steps:
@@ -58,7 +58,7 @@ jobs:
run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
race_detection:
name: Spinifex Race Detection
name: Race Detection
runs-on: ubuntu-24.04
steps:
@@ -92,7 +92,7 @@ jobs:
run: make -C spinifex test-race
lint_and_security:
name: Spinifex Lint and Security
name: Lint and Security
runs-on: ubuntu-24.04
steps:
docs/terraform-workbooks/bastion-private-subnet/README.md
+4 -2
View File
@@ -56,7 +56,7 @@ Deploy a VPC with public and private subnets where the private subnet has no int
**Prerequisites:**
- Spinifex installed and running (see [Installing Spinifex](/docs/install))
- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
- OpenTofu or Terraform installed
## Instructions
@@ -78,6 +78,8 @@ Or create a `main.tf` file and paste the full configuration below.
### Step 2. Deploy
The workbook defaults to `t3.small` (2 vCPU, 2 GiB) for both bastion and private instance. On clusters without that type registered, override with `TF_VAR_instance_type` — query what's available with `aws ec2 describe-instance-types`.
```bash
export AWS_PROFILE=spinifex
tofu init
@@ -147,7 +149,7 @@ The private route table should have no `0.0.0.0/0` route. Also verify the privat
### AMI Not Found
Ensure you have imported a Debian 12 image:
Ensure you have imported an Ubuntu 24.04 image:
```bash
aws ec2 describe-images --owners 000000000000 --profile spinifex
docs/terraform-workbooks/bastion-private-subnet/main.tf
+12 -7
View File
@@ -52,9 +52,14 @@ variable "region" {
default = "ap-southeast-2"
}
variable "instance_type" {
type = string
default = "t3.small"
}
variable "spinifex_endpoint" {
type = string
default = "https://localhost:9999"
default = "https://127.0.0.1:9999"
description = "Spinifex AWS gateway endpoint"
}
@@ -85,13 +90,13 @@ data "aws_availability_zones" "available" {
state = "available"
}
data "aws_ami" "debian12" {
data "aws_ami" "ubuntu" {
most_recent = true
owners = ["000000000000"]
filter {
name = "name"
values = ["*debian-12*"]
values = ["*ubuntu-24.04*"]
}
filter {
@@ -275,8 +280,8 @@ resource "aws_security_group" "private" {
# ---------------------------------------------------------------------------
resource "aws_instance" "bastion" {
ami = data.aws_ami.debian12.id
instance_type = "t3.small"
ami = data.aws_ami.ubuntu.id
instance_type = var.instance_type
subnet_id = aws_subnet.public.id
vpc_security_group_ids = [aws_security_group.bastion.id]
@@ -307,8 +312,8 @@ resource "aws_instance" "bastion" {
# ---------------------------------------------------------------------------
resource "aws_instance" "private" {
ami = data.aws_ami.debian12.id
instance_type = "t3.small"
ami = data.aws_ami.ubuntu.id
instance_type = var.instance_type
subnet_id = aws_subnet.private.id
vpc_security_group_ids = [aws_security_group.private.id]
docs/terraform-workbooks/nginx-alb/README.md
+4 -4
View File
@@ -58,7 +58,7 @@ Deploy two Nginx web servers behind an internet-facing Application Load Balancer
| Elastic IP | `nginx-alb-nat-eip` | Public address for the NAT Gateway |
| NAT Gateway | `nginx-alb-nat` | Outbound internet for the private subnets (cloud-init apt bootstrap) |
| Security Group | `nginx-alb-sg` | Allows SSH (22) and HTTP (80) inbound |
| EC2 Instances | `nginx-alb-1`, `nginx-alb-2` | Debian 12 with Nginx via cloud-init (private subnets) |
| EC2 Instances | `nginx-alb-1`, `nginx-alb-2` | Ubuntu 24.04 with Nginx via cloud-init (private subnets) |
| ALB | `nginx-alb` | Internet-facing Application Load Balancer on port 80 |
| Target Group | `nginx-alb-tg` | HTTP health-checked group for both instances |
| Listener | HTTP :80 | Forwards traffic to the target group |
@@ -66,7 +66,7 @@ Deploy two Nginx web servers behind an internet-facing Application Load Balancer
**Prerequisites:**
- Spinifex installed and running (see [Installing Spinifex](/docs/install))
- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
- OpenTofu or Terraform installed
## Instructions
@@ -176,7 +176,7 @@ tofu destroy
### AMI Not Found
Ensure you have imported a Debian 12 image. Check available AMIs:
Ensure you have imported an Ubuntu 24.04 image. Check available AMIs:
```bash
aws ec2 describe-images --owners 000000000000 --profile spinifex
@@ -185,7 +185,7 @@ aws ec2 describe-images --owners 000000000000 --profile spinifex
If missing import:
```bash
spx admin images import --name debian-12-x86_64
spx admin images import --name ubuntu-24.04-x86_64
```
### Provider Connection Refused
docs/terraform-workbooks/nginx-alb/main.tf
+5 -5
View File
@@ -64,7 +64,7 @@ variable "instance_type" {
variable "spinifex_endpoint" {
type = string
default = "https://localhost:9999"
default = "https://127.0.0.1:9999"
description = "Spinifex AWS gateway endpoint"
}
@@ -96,13 +96,13 @@ data "aws_availability_zones" "available" {
state = "available"
}
data "aws_ami" "debian12" {
data "aws_ami" "ubuntu" {
most_recent = true
owners = ["000000000000"] # Spinifex system images
filter {
name = "name"
values = ["*debian-12*"]
values = ["*ubuntu-24.04*"]
}
}
@@ -247,7 +247,7 @@ resource "aws_security_group" "web" {
# ---------------------------------------------------------------------------
resource "aws_instance" "nginx_1" {
ami = data.aws_ami.debian12.id
ami = data.aws_ami.ubuntu.id
instance_type = var.instance_type
subnet_id = aws_subnet.public_a.id
@@ -287,7 +287,7 @@ resource "aws_instance" "nginx_1" {
}
resource "aws_instance" "nginx_2" {
ami = data.aws_ami.debian12.id
ami = data.aws_ami.ubuntu.id
instance_type = var.instance_type
subnet_id = aws_subnet.public_b.id
docs/terraform-workbooks/nginx-webserver/README.md
+4 -2
View File
@@ -43,7 +43,7 @@ Deploy a complete Nginx web server on Spinifex using Terraform/OpenTofu. This wo
**Prerequisites:**
- Spinifex installed and running (see [Installing Spinifex](/docs/install))
- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
- OpenTofu or Terraform installed
## Instructions
@@ -65,6 +65,8 @@ Or create a `main.tf` file and paste the full configuration below.
### Step 2. Deploy
The workbook defaults to `t3.small` (2 vCPU, 2 GiB). On clusters without that type registered, override with `TF_VAR_instance_type` — query what's available with `aws ec2 describe-instance-types`.
```bash
export AWS_PROFILE=spinifex
tofu init
@@ -94,7 +96,7 @@ tofu destroy
### AMI Not Found
Ensure you have imported a Debian 12 image. Check available AMIs:
Ensure you have imported an Ubuntu 24.04 image. Check available AMIs:
```bash
aws ec2 describe-images --owners 000000000000 --profile spinifex
docs/terraform-workbooks/nginx-webserver/main.tf
+11 -6
View File
@@ -41,9 +41,14 @@ variable "region" {
default = "ap-southeast-2"
}
variable "instance_type" {
type = string
default = "t3.small"
}
variable "spinifex_endpoint" {
type = string
default = "https://localhost:9999"
default = "https://127.0.0.1:9999"
description = "Spinifex AWS gateway endpoint"
}
@@ -74,13 +79,13 @@ data "aws_availability_zones" "available" {
state = "available"
}
data "aws_ami" "debian12" {
data "aws_ami" "ubuntu" {
most_recent = true
owners = ["000000000000"] # Spinifex system images
filter {
name = "name"
values = ["*debian-12*"]
values = ["*ubuntu-24.04*"]
}
filter {
@@ -215,12 +220,12 @@ resource "aws_security_group" "web" {
}
# ---------------------------------------------------------------------------
# EC2 Instance — Debian 12 with Nginx installed via cloud-init
# EC2 Instance — Ubuntu 24.04 with Nginx installed via cloud-init
# ---------------------------------------------------------------------------
resource "aws_instance" "nginx" {
ami = data.aws_ami.debian12.id
instance_type = "t3.small"
ami = data.aws_ami.ubuntu.id
instance_type = var.instance_type
subnet_id = aws_subnet.public.id
vpc_security_group_ids = [aws_security_group.web.id]
docs/terraform-workbooks/s3-webapp/README.md
+4 -2
View File
@@ -53,7 +53,7 @@ Deploy an EC2 instance running a Flask file-sharing web application backed by S3
- Spinifex installed and running (see [Installing Spinifex](/docs/install))
- Predastore running (S3 API on port 8443)
- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
- OpenTofu or Terraform installed
- The EC2 instance must be able to reach Predastore — use the host's br-wan IP, not localhost
@@ -84,6 +84,8 @@ Before deploying, create a `terraform.tfvars` with your Predastore credentials.
### Step 4. Deploy
The workbook defaults to `t3.small` (2 vCPU, 2 GiB). On clusters without that type registered, override with `TF_VAR_instance_type` — query what's available with `aws ec2 describe-instance-types`.
```bash
export AWS_PROFILE=spinifex
tofu init
@@ -151,7 +153,7 @@ cat /opt/webapp/.env
### AMI Not Found
Ensure you have imported a Debian 12 image:
Ensure you have imported an Ubuntu 24.04 image:
```bash
aws ec2 describe-images --owners 000000000000 --profile spinifex
docs/terraform-workbooks/s3-webapp/main.tf
+11 -6
View File
@@ -56,15 +56,20 @@ variable "region" {
default = "ap-southeast-2"
}
variable "instance_type" {
type = string
default = "t3.small"
}
variable "spinifex_endpoint" {
type = string
default = "https://localhost:9999"
default = "https://127.0.0.1:9999"
description = "Spinifex AWS gateway endpoint (EC2/IAM)"
}
variable "predastore_endpoint" {
type = string
default = "https://localhost:8443"
default = "https://127.0.0.1:8443"
description = "Predastore S3 endpoint (for Terraform to create buckets)"
}
@@ -121,13 +126,13 @@ data "aws_availability_zones" "available" {
state = "available"
}
data "aws_ami" "debian12" {
data "aws_ami" "ubuntu" {
most_recent = true
owners = ["000000000000"]
filter {
name = "name"
values = ["*debian-12*"]
values = ["*ubuntu-24.04*"]
}
filter {
@@ -262,8 +267,8 @@ resource "aws_security_group" "webapp" {
# ---------------------------------------------------------------------------
resource "aws_instance" "webapp" {
ami = data.aws_ami.debian12.id
instance_type = "t3.small"
ami = data.aws_ami.ubuntu.id
instance_type = var.instance_type
subnet_id = aws_subnet.public.id
vpc_security_group_ids = [aws_security_group.webapp.id]
spinifex/handlers/ec2/image/service_impl.go
+18 -8
View File
@@ -74,14 +74,16 @@ func NewImageServiceImplWithStore(store objectstore.ObjectStore, bucketName stri
// describeImagesValidFilters defines the set of filter names accepted by DescribeImages.
var describeImagesValidFilters = map[string]bool{
"name": true,
"state": true,
"architecture": true,
"image-id": true,
"is-public": true,
"owner-id": true,
"description": true,
"image-type": true,
"name": true,
"state": true,
"architecture": true,
"image-id": true,
"is-public": true,
"owner-id": true,
"description": true,
"image-type": true,
"virtualization-type": true,
"root-device-type": true,
}
// DescribeImages lists available AMI images by reading config.json files from S3
@@ -340,6 +342,14 @@ func imageMatchesFilters(image *ec2.Image, filters map[string][]string, tags map
if image.ImageType != nil {
field = *image.ImageType
}
case "virtualization-type":
if image.VirtualizationType != nil {
field = *image.VirtualizationType
}
case "root-device-type":
if image.RootDeviceType != nil {
field = *image.RootDeviceType
}
default:
return false
}
spinifex/handlers/ec2/image/service_impl_test.go
+42
View File
@@ -1031,6 +1031,48 @@ func TestDescribeImages_FilterByState(t *testing.T) {
assert.Empty(t, out.Images)
}
func TestDescribeImages_FilterByVirtualizationType(t *testing.T) {
svc, store := setupTestImageService(t)
createTestAMIConfigFull(t, store, viperblock.AMIMetadata{
ImageID: "ami-hvm", Name: "hvm-img", Architecture: "x86_64",
Virtualization: "hvm", RootDeviceType: "ebs", VolumeSizeGiB: 8,
})
createTestAMIConfigFull(t, store, viperblock.AMIMetadata{
ImageID: "ami-pv", Name: "pv-img", Architecture: "x86_64",
Virtualization: "paravirtual", RootDeviceType: "ebs", VolumeSizeGiB: 8,
})
out, err := svc.DescribeImages(&ec2.DescribeImagesInput{
Filters: []*ec2.Filter{
{Name: aws.String("virtualization-type"), Values: []*string{aws.String("hvm")}},
},
}, testAccountID)
require.NoError(t, err)
assert.Len(t, out.Images, 1)
assert.Equal(t, "ami-hvm", *out.Images[0].ImageId)
}
func TestDescribeImages_FilterByRootDeviceType(t *testing.T) {
svc, store := setupTestImageService(t)
createTestAMIConfigFull(t, store, viperblock.AMIMetadata{
ImageID: "ami-ebs", Name: "ebs-img", Architecture: "x86_64",
Virtualization: "hvm", RootDeviceType: "ebs", VolumeSizeGiB: 8,
})
createTestAMIConfigFull(t, store, viperblock.AMIMetadata{
ImageID: "ami-is", Name: "is-img", Architecture: "x86_64",
Virtualization: "hvm", RootDeviceType: "instance-store", VolumeSizeGiB: 8,
})
out, err := svc.DescribeImages(&ec2.DescribeImagesInput{
Filters: []*ec2.Filter{
{Name: aws.String("root-device-type"), Values: []*string{aws.String("ebs")}},
},
}, testAccountID)
require.NoError(t, err)
assert.Len(t, out.Images, 1)
assert.Equal(t, "ami-ebs", *out.Images[0].ImageId)
}
// --- RegisterImage tests ---
// putTestSnapshotConfig writes a SnapshotConfig at {snapshotID}/metadata.json,
tests/e2e/run-multinode-e2e.sh
+70
View File
@@ -91,6 +91,75 @@ aws_via_retry() {
# Default AWS CLI shorthand (via local node)
AWS_EC2="aws --endpoint-url https://${LOCAL_IP}:${AWSGW_PORT} ec2"
# Dump SSH-related diagnostics when a guest SSH attempt fails.
# Args: <instance_id> <host_ip> <ssh_host> [ssh_port]
# ssh_host is the address we tried (EIP or node IP), ssh_port is hostfwd port if applicable.
dump_guest_ssh_diagnostics() {
local instance_id="$1" host_ip="$2" ssh_host="$3" ssh_port="${4:-}"
echo ""
echo " === Diagnostics for ${instance_id} (host=${host_ip}, target=${ssh_host}:${ssh_port:-22}) ==="
echo " --- describe-instances ---"
$AWS_EC2 describe-instances --instance-ids "$instance_id" \
--query 'Reservations[0].Instances[0].[InstanceId,State.Name,PrivateIpAddress,PublicIpAddress,SecurityGroups[].GroupId,NetworkInterfaces[0].MacAddress,VpcId,SubnetId]' \
--output json 2>&1 || true
local sg_ids priv_ip mac vpc_id
sg_ids=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \
--query 'Reservations[0].Instances[0].SecurityGroups[].GroupId' --output text 2>/dev/null || true)
priv_ip=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \
--query 'Reservations[0].Instances[0].PrivateIpAddress' --output text 2>/dev/null || true)
mac=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \
--query 'Reservations[0].Instances[0].NetworkInterfaces[0].MacAddress' --output text 2>/dev/null || true)
vpc_id=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \
--query 'Reservations[0].Instances[0].VpcId' --output text 2>/dev/null || true)
if [ -n "$sg_ids" ] && [ "$sg_ids" != "None" ]; then
echo " --- describe-security-groups (${sg_ids}) ---"
# shellcheck disable=SC2086
$AWS_EC2 describe-security-groups --group-ids $sg_ids \
--query 'SecurityGroups[].[GroupId,GroupName,IpPermissions]' --output json 2>&1 || true
fi
echo " --- local ARP for ${ssh_host} ---"
ip neigh show "$ssh_host" 2>&1 || true
echo " --- local ping ${ssh_host} (2 pkts) ---"
ping -c 2 -W 2 "$ssh_host" 2>&1 || true
if [ -n "$host_ip" ] && [ "$host_ip" != "unknown" ]; then
echo " --- on hosting node ${host_ip}: QEMU process ---"
peer_ssh "$host_ip" "pgrep -af 'qemu.*${instance_id}' || echo '(no qemu process)'" 2>&1 || true
if [ -n "$priv_ip" ] && [ "$priv_ip" != "None" ]; then
echo " --- on hosting node ${host_ip}: ping VM private IP ${priv_ip} ---"
peer_ssh "$host_ip" "ping -c 2 -W 2 ${priv_ip}" 2>&1 || true
fi
echo " --- on hosting node ${host_ip}: OVN chassis + port bindings ---"
peer_ssh "$host_ip" "sudo ovs-vsctl get open_vswitch . external_ids:system-id 2>/dev/null || true" 2>&1 || true
fi
if [ -n "$vpc_id" ] && [ "$vpc_id" != "None" ]; then
echo " --- OVN NAT rules for vpc-${vpc_id} (from primary) ---"
peer_ssh "$LOCAL_IP" "sudo ovn-nbctl lr-nat-list vpc-${vpc_id} 2>&1 || true" 2>&1 || true
fi
if [ -n "$mac" ] && [ "$mac" != "None" ]; then
echo " --- OVN port binding for MAC ${mac} (from primary) ---"
peer_ssh "$LOCAL_IP" "sudo ovn-sbctl --columns=logical_port,chassis,mac find port_binding mac~='${mac}' 2>&1 || true" 2>&1 || true
fi
echo " --- cloud-init / sshd from hosting node (via QEMU console log if available) ---"
peer_ssh "$host_ip" "ls -la /var/log/libvirt/qemu/ 2>/dev/null | head -20; \
for f in /tmp/spinifex/vms/${instance_id}/console.log /var/log/libvirt/qemu/${instance_id}.log; do \
[ -f \"\$f\" ] && echo \"--- \$f (tail 50) ---\" && sudo tail -50 \"\$f\"; \
done" 2>&1 || true
echo " === end diagnostics for ${instance_id} ==="
echo ""
}
# Dump logs from ALL nodes on failure
dump_all_node_logs() {
echo ""
@@ -559,6 +628,7 @@ for idx in "${!INSTANCE_IDS[@]}"; do
if [ "$SSH_READY" = false ]; then
echo " ERROR: SSH not ready after 60 attempts"
dump_guest_ssh_diagnostics "$instance_id" "$host_ip" "$SSH_HOST" "$SSH_PORT"
fail_test "Guest SSH ($instance_id)"
continue
fi
tests/e2e/run-tofu-examples-e2e.sh
Executable
+234
View File
@@ -0,0 +1,234 @@
#!/bin/bash
# run-tofu-examples-e2e.sh — smoke-test the public terraform workbooks.
#
# Invoked by the nightly matrix (cell #17) after bootstrap-install.sh has
# brought up a single-node cluster. For each workbook, the driver does a
# clean tofu init/apply, runs one smoke assertion, then tofu destroy.
#
# Expects the workbooks to be available at $WORKBOOK_DIR (defaults to the
# tarball-included ./workbooks/ directory beside this script).
set -u
cd "$(dirname "$0")"
SCRIPT_DIR="$(pwd)"
export AWS_PROFILE=spinifex
WORKBOOK_DIR="${WORKBOOK_DIR:-${SCRIPT_DIR}/workbooks}"
OPENTOFU_VERSION="${OPENTOFU_VERSION:-1.11.5}"
# awsgw and predastore bind to the WAN IP (bootstrap-install passes --bind
# ${PRIMARY_WAN}), so workbooks need the WAN IP for both the tofu provider
# endpoints and any CLI assertions that talk to S3 (:8443).
WAN_IP=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{print $7; exit}')
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR)
log() { echo "[$(date +%H:%M:%S)] $*"; }
install_tofu() {
command -v tofu >/dev/null 2>&1 && return 0
local arch
case "$(uname -m)" in
x86_64) arch=amd64 ;;
aarch64) arch=arm64 ;;
*) log "unsupported arch: $(uname -m)"; return 1 ;;
esac
local url="https://github.com/opentofu/opentofu/releases/download/v${OPENTOFU_VERSION}/tofu_${OPENTOFU_VERSION}_linux_${arch}.tar.gz"
log "Installing OpenTofu ${OPENTOFU_VERSION} (${arch})"
local tmp
tmp=$(mktemp -d)
curl -fsSL "$url" | tar -xz -C "$tmp" tofu
sudo install -m 0755 "$tmp/tofu" /usr/local/bin/tofu
rm -rf "$tmp"
}
wait_for_ssh() {
local key="$1" host="$2"
for _ in $(seq 1 30); do
if ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${host}" true 2>/dev/null; then
return 0
fi
sleep 5
done
return 1
}
wait_for_http_200() {
local url="$1" budget="${2:-150}"
local attempts=$((budget / 5))
for _ in $(seq 1 "$attempts"); do
if [ "$(curl -sk -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null)" = "200" ]; then
return 0
fi
sleep 5
done
return 1
}
wait_for_alb_healthy() {
local tg_arn="$1" budget="${2:-300}"
local attempts=$((budget / 10))
for _ in $(seq 1 "$attempts"); do
local healthy
healthy=$(aws elbv2 describe-target-health --target-group-arn "$tg_arn" \
--query 'length(TargetHealthDescriptions[?TargetHealth.State==`healthy`])' \
--output text 2>/dev/null || echo "0")
if [ "$healthy" -gt 0 ] 2>/dev/null; then
return 0
fi
sleep 10
done
return 1
}
assert_bastion_private_subnet() {
local bastion private key
bastion=$(tofu output -raw bastion_public_ip)
private=$(tofu output -raw private_instance_ip)
key="$(pwd)/bastion-demo.pem"
chmod 600 "$key"
wait_for_ssh "$key" "$bastion" || return 1
# sshd accepts before cloud-init finishes writing ~/.ssh/bastion-demo.pem,
# so wait for the hop key before attempting the inner ssh.
ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" \
'for _ in $(seq 1 36); do [ -s ~/.ssh/bastion-demo.pem ] && exit 0; sleep 5; done; exit 1' || return 1
ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" \
"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ~/.ssh/bastion-demo.pem ec2-user@${private} id" \
| grep -q '^uid='
}
assert_nginx_webserver() {
wait_for_http_200 "http://$(tofu output -raw public_ip)/"
}
assert_nginx_alb() {
# ALB DNS (*.elb.spinifex.local) isn't resolvable from the host — README
# documents fetching the public IP via elbv2 describe-load-balancers.
local name ip tg_arn
name=$(tofu output -raw alb_name)
ip=$(aws elbv2 describe-load-balancers --names "$name" \
--query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
--output text 2>/dev/null | awk '{print $1}')
if [ -z "$ip" ] || [ "$ip" = "None" ]; then
log " nginx-alb: no public IP for ${name}"
return 1
fi
tg_arn=$(aws elbv2 describe-target-groups --names nginx-alb-tg \
--query 'TargetGroups[0].TargetGroupArn' --output text 2>/dev/null)
if [ -z "$tg_arn" ] || [ "$tg_arn" = "None" ]; then
log " nginx-alb: could not resolve target group ARN"
return 1
fi
wait_for_alb_healthy "$tg_arn" 300 || return 1
wait_for_http_200 "http://${ip}/" 300
}
assert_s3_webapp() {
# aws s3 goes to predastore on :8443, not awsgw on :9999 — the workbook's
# provider sets s3 = predastore_endpoint but the CLI doesn't inherit that.
local bucket endpoint sentinel
bucket=$(tofu output -raw bucket_name)
endpoint="https://${WAN_IP}:8443"
sentinel="spinifex-nightly-$(date +%s).txt"
echo "nightly-smoke" | aws s3 cp --endpoint-url "$endpoint" - "s3://${bucket}/${sentinel}" >/dev/null
aws s3 ls --endpoint-url "$endpoint" "s3://${bucket}/" | grep -q "$sentinel"
}
# Pick an instance type available on this cluster. Workbooks default to
# t3.small (Intel); on AMD-only hosts t3 isn't registered, so query and
# fall back to the smallest type with ≥2 vCPU / ≥1 GiB RAM.
detect_instance_type() {
local endpoint="https://${WAN_IP}:9999"
local picked
picked=$(aws --endpoint-url "$endpoint" ec2 describe-instance-types \
--query "sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus==\`2\` && MemoryInfo.SizeInMiB>=\`1024\`], &MemoryInfo.SizeInMiB)[0].InstanceType" \
--output text 2>/dev/null || true)
if [ -z "$picked" ] || [ "$picked" = "None" ]; then
picked=$(aws --endpoint-url "$endpoint" ec2 describe-instance-types \
--query 'InstanceTypes[0].InstanceType' --output text 2>/dev/null || true)
fi
echo "$picked"
}
INSTANCE_TYPE=""
run_workbook() {
local example="$1"
local path="${WORKBOOK_DIR}/${example}"
if [ ! -d "$path" ]; then
log "SKIP ${example}: ${path} not found"
return 1
fi
log "=== ${example} ==="
cd "$path"
rm -rf .terraform terraform.tfstate* .terraform.lock.hcl
local apply_args=(
-input=false -no-color
"-var=spinifex_endpoint=https://${WAN_IP}:9999"
"-var=instance_type=${INSTANCE_TYPE}"
)
# s3-webapp has extra required vars; creds come from AWS_PROFILE=spinifex.
if [ "$example" = "s3-webapp" ]; then
local access_key secret_key
access_key=$(aws configure get aws_access_key_id --profile spinifex)
secret_key=$(aws configure get aws_secret_access_key --profile spinifex)
apply_args+=(
"-var=predastore_endpoint=https://${WAN_IP}:8443"
"-var=predastore_host=${WAN_IP}:8443"
"-var=s3_access_key=${access_key}"
"-var=s3_secret_key=${secret_key}"
)
fi
if ! tofu init -input=false -no-color; then
log " FAIL ${example}: tofu init"
return 1
fi
if ! tofu apply -auto-approve "${apply_args[@]}"; then
log " FAIL ${example}: tofu apply"
tofu destroy -auto-approve "${apply_args[@]}" >/dev/null 2>&1 || true
return 1
fi
local rc=0
if assert_"${example//-/_}"; then
log " PASS ${example}"
else
log " FAIL ${example}: assertion"
rc=1
fi
tofu destroy -auto-approve "${apply_args[@]}" >/dev/null 2>&1 || \
log " WARN ${example}: tofu destroy failed"
cd "$SCRIPT_DIR"
return "$rc"
}
install_tofu || { log "tofu install failed"; exit 1; }
INSTANCE_TYPE=$(detect_instance_type)
if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" = "None" ]; then
log "no instance type available from describe-instance-types"
exit 1
fi
log "Using instance_type=${INSTANCE_TYPE}"
# nginx-alb is disabled pending: (1) vpcd port-group live OVN client
# (CreatePortGroup/AddACL not yet implemented → SG rules are no-ops),
# (2) viperblock zero-write block-alignment bug that crashes lb-agent boot,
# (3) external-IP allocator reusing an ENI EIP for the ALB. Re-enable once
# those land.
for workbook in bastion-private-subnet nginx-webserver s3-webapp; do
if ! run_workbook "$workbook"; then
log "FAIL ${workbook} — aborting remaining workbooks"
exit 1
fi
done
log "All workbooks passed"