diff --git a/.github/workflows/e2e-ddil.yml b/.github/workflows/e2e-ddil.yml index ba06834bb..a56bd8a8b 100644 --- a/.github/workflows/e2e-ddil.yml +++ b/.github/workflows/e2e-ddil.yml @@ -1,4 +1,4 @@ -name: Spinifex DDIL E2E +name: E2E DDIL # Phase 1 trigger: workflow_dispatch only. Nightly cron + on-main-merge # land in mulga-siv-5.8 once ≥4 scenarios are asserting. See diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml new file mode 100644 index 000000000..526efc90c --- /dev/null +++ b/.github/workflows/e2e-nightly.yml @@ -0,0 +1,188 @@ +name: E2E Nightly + +on: + schedule: + - cron: "0 14 * * *" # 00:00 AEST = 14:00 UTC + workflow_dispatch: + inputs: + cells: + description: "Comma-separated cell numbers to run (blank = all)" + required: false + +permissions: + contents: read + +env: + GIT_BRANCH: ${{ github.ref_name }} + UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2 + +jobs: + matrix: + name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ + matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})" + strategy: + fail-fast: false + max-parallel: 3 + matrix: + include: + - { cell: 2, install: binary, topo: single, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" } + - { cell: 9, install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi, extra: "" } + - { cell: 17, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" } + + runs-on: [ self-hosted, "${{ matrix.runner }}" ] + timeout-minutes: 35 + env: + TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log + + steps: + # blank or scheduled run = full matrix. + # Input must be comma-separated with no whitespace (e.g. '1,2,17', not '1, 2'). + - name: Gate on Dispatch Input + id: gate + env: + CELL: ${{ matrix.cell }} + CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }} + run: | + if [ -z "$CELLS" ] || [[ ",${CELLS}," == *",${CELL},"* ]]; then + echo "selected=true" >> "$GITHUB_OUTPUT" + echo "cell ${CELL} selected" + else + echo "selected=false" >> "$GITHUB_OUTPUT" + echo "cell ${CELL} not in '${CELLS}' — subsequent steps will skip" + fi + + - name: Clean Workspace + if: steps.gate.outputs.selected == 'true' + run: rm -rf "$GITHUB_WORKSPACE"/* + + - uses: actions/create-github-app-token@v3 + if: steps.gate.outputs.selected == 'true' + id: app-token + with: + client-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: mulga + + - name: Resolve Mulga Branch + if: steps.gate.outputs.selected == 'true' + id: mulga-branch + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + BRANCH: ${{ github.ref_name }} + run: | + if gh api "repos/mulgadc/mulga/branches/${BRANCH}" >/dev/null 2>&1; then + echo "ref=${BRANCH}" >> "$GITHUB_OUTPUT" + echo "mulga: using branch ${BRANCH}" + else + echo "ref=main" >> "$GITHUB_OUTPUT" + echo "mulga: branch ${BRANCH} not found, falling back to main" + fi + + - uses: actions/checkout@v6 + if: steps.gate.outputs.selected == 'true' + with: + repository: mulgadc/mulga + ref: ${{ steps.mulga-branch.outputs.ref }} + token: ${{ steps.app-token.outputs.token }} + sparse-checkout: scripts/tofu-cluster + path: mulga + + - name: Build Install Artifacts + if: steps.gate.outputs.selected == 'true' + working-directory: mulga/scripts/tofu-cluster + run: | + ./build-install-artifacts.sh "${GIT_BRANCH}" \ + "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ + "${RUNNER_TEMP}/setup.sh" \ + "${RUNNER_TEMP}/spinifex-tests.tar" + + - name: Provision VMs + if: steps.gate.outputs.selected == 'true' + working-directory: mulga/scripts/tofu-cluster + env: + MATRIX_HOST: ${{ matrix.host }} + run: | + export TF_VAR_git_branch="${GIT_BRANCH}" + + TOFU_VAR_ARGS="" + if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then + TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" + fi + + tofu init -input=false >> "$TOFU_LOG" 2>&1 + tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 + tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true + ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true + tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 + + - name: Bootstrap via Binary Install + if: steps.gate.outputs.selected == 'true' + working-directory: mulga/scripts/tofu-cluster + env: + MATRIX_NET: ${{ matrix.net }} + run: | + if [ "$MATRIX_NET" = "dhcp" ]; then + unset SPX_EXTERNAL_POOL + export SPX_EXTERNAL_SOURCE=dhcp + else + export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" + fi + + ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \ + "${RUNNER_TEMP}/setup.sh" \ + "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ + --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" + + - name: Run Single Node E2E + if: steps.gate.outputs.selected == 'true' && matrix.topo == 'single' && matrix.extra != 'tofu-examples' + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory) + WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \ + "cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh" + + - name: Run Multi Node E2E + if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi' + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory) + NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + + NODE_IPS="" + NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length') + for i in $(seq 0 $((NODE_COUNT - 1))); do + NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")" + done + + ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \ + "cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}" + + - name: Run Tofu Workbooks + if: matrix.extra == 'tofu-examples' + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory) + WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \ + "cd ~/e2e-tests && ./run-tofu-examples-e2e.sh" + + - name: Teardown + if: always() + working-directory: mulga/scripts/tofu-cluster + env: + MATRIX_HOST: ${{ matrix.host }} + run: | + TOFU_VAR_ARGS="" + if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then + TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" + fi + tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 + tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 + ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 45d8015fc..358815ef0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,4 +1,4 @@ -name: Spinifex E2E Tests +name: E2E on: push: diff --git a/.github/workflows/spinifex.yml b/.github/workflows/spinifex.yml index c85784b1f..93ab63a70 100644 --- a/.github/workflows/spinifex.yml +++ b/.github/workflows/spinifex.yml @@ -1,4 +1,4 @@ -name: Spinifex Build and Test +name: Build and Test on: push: @@ -17,7 +17,7 @@ on: jobs: build_and_test: - name: Spinifex Build and Test + name: Build and Test runs-on: ubuntu-24.04 steps: @@ -58,7 +58,7 @@ jobs: run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1 race_detection: - name: Spinifex Race Detection + name: Race Detection runs-on: ubuntu-24.04 steps: @@ -92,7 +92,7 @@ jobs: run: make -C spinifex test-race lint_and_security: - name: Spinifex Lint and Security + name: Lint and Security runs-on: ubuntu-24.04 steps: diff --git a/README.md b/README.md index 5dfd1dfd9..59ccc8af7 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ AWS-Compatible Interfaces – Provision infrastructure with awscli, Terraform, o ## Installation -Installation requires an Ubuntu / Debian system. See the detailed documentation at [https://docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex. +Installation requires an Ubuntu / Debian system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex. ### Single Node Install @@ -179,4 +179,4 @@ This hybrid approach ensures Spinifex benefits from both proven engineering expe ## License -Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it—anywhere you need reliable infrastructure without depending on centralized cloud platforms. +Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms. diff --git a/docs/terraform-workbooks/bastion-private-subnet/README.md b/docs/terraform-workbooks/bastion-private-subnet/README.md index 630747b19..51e40dade 100644 --- a/docs/terraform-workbooks/bastion-private-subnet/README.md +++ b/docs/terraform-workbooks/bastion-private-subnet/README.md @@ -56,7 +56,7 @@ Deploy a VPC with public and private subnets where the private subnet has no int **Prerequisites:** - Spinifex installed and running (see [Installing Spinifex](/docs/install)) -- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster)) +- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster)) - OpenTofu or Terraform installed ## Instructions @@ -78,6 +78,8 @@ Or create a `main.tf` file and paste the full configuration below. ### Step 2. Deploy +The workbook defaults to `t3.small` (2 vCPU, 2 GiB) for both bastion and private instance. On clusters without that type registered, override with `TF_VAR_instance_type` — query what's available with `aws ec2 describe-instance-types`. + ```bash export AWS_PROFILE=spinifex tofu init @@ -147,7 +149,7 @@ The private route table should have no `0.0.0.0/0` route. Also verify the privat ### AMI Not Found -Ensure you have imported a Debian 12 image: +Ensure you have imported an Ubuntu 24.04 image: ```bash aws ec2 describe-images --owners 000000000000 --profile spinifex diff --git a/docs/terraform-workbooks/bastion-private-subnet/main.tf b/docs/terraform-workbooks/bastion-private-subnet/main.tf index 075c83f96..7ba0fcedd 100644 --- a/docs/terraform-workbooks/bastion-private-subnet/main.tf +++ b/docs/terraform-workbooks/bastion-private-subnet/main.tf @@ -52,9 +52,14 @@ variable "region" { default = "ap-southeast-2" } +variable "instance_type" { + type = string + default = "t3.small" +} + variable "spinifex_endpoint" { type = string - default = "https://localhost:9999" + default = "https://127.0.0.1:9999" description = "Spinifex AWS gateway endpoint" } @@ -85,13 +90,13 @@ data "aws_availability_zones" "available" { state = "available" } -data "aws_ami" "debian12" { +data "aws_ami" "ubuntu" { most_recent = true owners = ["000000000000"] filter { name = "name" - values = ["*debian-12*"] + values = ["*ubuntu-24.04*"] } filter { @@ -275,8 +280,8 @@ resource "aws_security_group" "private" { # --------------------------------------------------------------------------- resource "aws_instance" "bastion" { - ami = data.aws_ami.debian12.id - instance_type = "t3.small" + ami = data.aws_ami.ubuntu.id + instance_type = var.instance_type subnet_id = aws_subnet.public.id vpc_security_group_ids = [aws_security_group.bastion.id] @@ -307,8 +312,8 @@ resource "aws_instance" "bastion" { # --------------------------------------------------------------------------- resource "aws_instance" "private" { - ami = data.aws_ami.debian12.id - instance_type = "t3.small" + ami = data.aws_ami.ubuntu.id + instance_type = var.instance_type subnet_id = aws_subnet.private.id vpc_security_group_ids = [aws_security_group.private.id] diff --git a/docs/terraform-workbooks/nginx-alb/README.md b/docs/terraform-workbooks/nginx-alb/README.md index 882ca34a1..1aaf35bc4 100644 --- a/docs/terraform-workbooks/nginx-alb/README.md +++ b/docs/terraform-workbooks/nginx-alb/README.md @@ -58,7 +58,7 @@ Deploy two Nginx web servers behind an internet-facing Application Load Balancer | Elastic IP | `nginx-alb-nat-eip` | Public address for the NAT Gateway | | NAT Gateway | `nginx-alb-nat` | Outbound internet for the private subnets (cloud-init apt bootstrap) | | Security Group | `nginx-alb-sg` | Allows SSH (22) and HTTP (80) inbound | -| EC2 Instances | `nginx-alb-1`, `nginx-alb-2` | Debian 12 with Nginx via cloud-init (private subnets) | +| EC2 Instances | `nginx-alb-1`, `nginx-alb-2` | Ubuntu 24.04 with Nginx via cloud-init (private subnets) | | ALB | `nginx-alb` | Internet-facing Application Load Balancer on port 80 | | Target Group | `nginx-alb-tg` | HTTP health-checked group for both instances | | Listener | HTTP :80 | Forwards traffic to the target group | @@ -66,7 +66,7 @@ Deploy two Nginx web servers behind an internet-facing Application Load Balancer **Prerequisites:** - Spinifex installed and running (see [Installing Spinifex](/docs/install)) -- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster)) +- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster)) - OpenTofu or Terraform installed ## Instructions @@ -184,7 +184,7 @@ tofu destroy ### AMI Not Found -Ensure you have imported a Debian 12 image. Check available AMIs: +Ensure you have imported an Ubuntu 24.04 image. Check available AMIs: ```bash aws ec2 describe-images --owners 000000000000 --profile spinifex @@ -193,7 +193,7 @@ aws ec2 describe-images --owners 000000000000 --profile spinifex If missing import: ```bash -spx admin images import --name debian-12-x86_64 +spx admin images import --name ubuntu-24.04-x86_64 ``` ### Provider Connection Refused diff --git a/docs/terraform-workbooks/nginx-alb/main.tf b/docs/terraform-workbooks/nginx-alb/main.tf index 45a3207fd..c35699249 100644 --- a/docs/terraform-workbooks/nginx-alb/main.tf +++ b/docs/terraform-workbooks/nginx-alb/main.tf @@ -64,7 +64,7 @@ variable "instance_type" { variable "spinifex_endpoint" { type = string - default = "https://localhost:9999" + default = "https://127.0.0.1:9999" description = "Spinifex AWS gateway endpoint" } @@ -96,13 +96,13 @@ data "aws_availability_zones" "available" { state = "available" } -data "aws_ami" "debian12" { +data "aws_ami" "ubuntu" { most_recent = true owners = ["000000000000"] # Spinifex system images filter { name = "name" - values = ["*debian-12*"] + values = ["*ubuntu-24.04*"] } } @@ -247,7 +247,7 @@ resource "aws_security_group" "web" { # --------------------------------------------------------------------------- resource "aws_instance" "nginx_1" { - ami = data.aws_ami.debian12.id + ami = data.aws_ami.ubuntu.id instance_type = var.instance_type subnet_id = aws_subnet.public_a.id @@ -287,7 +287,7 @@ resource "aws_instance" "nginx_1" { } resource "aws_instance" "nginx_2" { - ami = data.aws_ami.debian12.id + ami = data.aws_ami.ubuntu.id instance_type = var.instance_type subnet_id = aws_subnet.public_b.id diff --git a/docs/terraform-workbooks/nginx-webserver/README.md b/docs/terraform-workbooks/nginx-webserver/README.md index 63513c4ab..5a287e170 100644 --- a/docs/terraform-workbooks/nginx-webserver/README.md +++ b/docs/terraform-workbooks/nginx-webserver/README.md @@ -43,7 +43,7 @@ Deploy a complete Nginx web server on Spinifex using Terraform/OpenTofu. This wo **Prerequisites:** - Spinifex installed and running (see [Installing Spinifex](/docs/install)) -- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster)) +- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster)) - OpenTofu or Terraform installed ## Instructions @@ -65,6 +65,8 @@ Or create a `main.tf` file and paste the full configuration below. ### Step 2. Deploy +The workbook defaults to `t3.small` (2 vCPU, 2 GiB). On clusters without that type registered, override with `TF_VAR_instance_type` — query what's available with `aws ec2 describe-instance-types`. + ```bash export AWS_PROFILE=spinifex tofu init @@ -94,7 +96,7 @@ tofu destroy ### AMI Not Found -Ensure you have imported a Debian 12 image. Check available AMIs: +Ensure you have imported an Ubuntu 24.04 image. Check available AMIs: ```bash aws ec2 describe-images --owners 000000000000 --profile spinifex diff --git a/docs/terraform-workbooks/nginx-webserver/main.tf b/docs/terraform-workbooks/nginx-webserver/main.tf index cf1084626..4f6e8900f 100644 --- a/docs/terraform-workbooks/nginx-webserver/main.tf +++ b/docs/terraform-workbooks/nginx-webserver/main.tf @@ -41,9 +41,14 @@ variable "region" { default = "ap-southeast-2" } +variable "instance_type" { + type = string + default = "t3.small" +} + variable "spinifex_endpoint" { type = string - default = "https://localhost:9999" + default = "https://127.0.0.1:9999" description = "Spinifex AWS gateway endpoint" } @@ -74,13 +79,13 @@ data "aws_availability_zones" "available" { state = "available" } -data "aws_ami" "debian12" { +data "aws_ami" "ubuntu" { most_recent = true owners = ["000000000000"] # Spinifex system images filter { name = "name" - values = ["*debian-12*"] + values = ["*ubuntu-24.04*"] } filter { @@ -215,12 +220,12 @@ resource "aws_security_group" "web" { } # --------------------------------------------------------------------------- -# EC2 Instance — Debian 12 with Nginx installed via cloud-init +# EC2 Instance — Ubuntu 24.04 with Nginx installed via cloud-init # --------------------------------------------------------------------------- resource "aws_instance" "nginx" { - ami = data.aws_ami.debian12.id - instance_type = "t3.small" + ami = data.aws_ami.ubuntu.id + instance_type = var.instance_type subnet_id = aws_subnet.public.id vpc_security_group_ids = [aws_security_group.web.id] diff --git a/docs/terraform-workbooks/s3-webapp/README.md b/docs/terraform-workbooks/s3-webapp/README.md index c15610330..4fc6491f3 100644 --- a/docs/terraform-workbooks/s3-webapp/README.md +++ b/docs/terraform-workbooks/s3-webapp/README.md @@ -53,7 +53,7 @@ Deploy an EC2 instance running a Flask file-sharing web application backed by S3 - Spinifex installed and running (see [Installing Spinifex](/docs/install)) - Predastore running (S3 API on port 8443) -- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster)) +- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster)) - OpenTofu or Terraform installed - The EC2 instance must be able to reach Predastore — use the host's br-wan IP, not localhost @@ -84,6 +84,8 @@ Before deploying, create a `terraform.tfvars` with your Predastore credentials. ### Step 4. Deploy +The workbook defaults to `t3.small` (2 vCPU, 2 GiB). On clusters without that type registered, override with `TF_VAR_instance_type` — query what's available with `aws ec2 describe-instance-types`. + ```bash export AWS_PROFILE=spinifex tofu init @@ -151,7 +153,7 @@ cat /opt/webapp/.env ### AMI Not Found -Ensure you have imported a Debian 12 image: +Ensure you have imported an Ubuntu 24.04 image: ```bash aws ec2 describe-images --owners 000000000000 --profile spinifex diff --git a/docs/terraform-workbooks/s3-webapp/main.tf b/docs/terraform-workbooks/s3-webapp/main.tf index 6abb733b7..0c7f40db9 100644 --- a/docs/terraform-workbooks/s3-webapp/main.tf +++ b/docs/terraform-workbooks/s3-webapp/main.tf @@ -56,15 +56,20 @@ variable "region" { default = "ap-southeast-2" } +variable "instance_type" { + type = string + default = "t3.small" +} + variable "spinifex_endpoint" { type = string - default = "https://localhost:9999" + default = "https://127.0.0.1:9999" description = "Spinifex AWS gateway endpoint (EC2/IAM)" } variable "predastore_endpoint" { type = string - default = "https://localhost:8443" + default = "https://127.0.0.1:8443" description = "Predastore S3 endpoint (for Terraform to create buckets)" } @@ -121,13 +126,13 @@ data "aws_availability_zones" "available" { state = "available" } -data "aws_ami" "debian12" { +data "aws_ami" "ubuntu" { most_recent = true owners = ["000000000000"] filter { name = "name" - values = ["*debian-12*"] + values = ["*ubuntu-24.04*"] } filter { @@ -262,8 +267,8 @@ resource "aws_security_group" "webapp" { # --------------------------------------------------------------------------- resource "aws_instance" "webapp" { - ami = data.aws_ami.debian12.id - instance_type = "t3.small" + ami = data.aws_ami.ubuntu.id + instance_type = var.instance_type subnet_id = aws_subnet.public.id vpc_security_group_ids = [aws_security_group.webapp.id] diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index 95aafe49a..f198c19e3 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -2821,11 +2821,21 @@ func (d *Daemon) wireLBAgentConfig() { // Resolve gateway URL — the address LB VMs use to reach the AWS gateway. // Precedence: - // 1. AWSGW listens on a specific WAN IP + br-mgmt present → use WAN IP - // and add host route via br-mgmt for internal LBs. - // 2. br-mgmt present + AWSGW on 0.0.0.0 → br-mgmt IP (both LB flavours + // 1. br-mgmt present + AWSGW on a dedicated IP distinct from AdvertiseIP + // (multi-node: AWSGW on a mgmt-only IP, VPC path can't reach it) → + // gateway URL is the AWSGW bind IP and lb-agent gets a bootcmd host + // route via br-mgmt. + // 2. AdvertiseIP set (single-node, or multi-node where AWSGW binds to + // the advertised IP) → AdvertiseIP. VMs reach it via VPC → external + // (OVN's own dnat_and_snat SNATs their reply back to the ALB EIP). + // Critically, we do NOT add the mgmt host route here: when host IPs + // on the WAN share the advertiseIP, the /32 route would steal the + // return path for host-initiated ALB connections — replies would + // egress via mgmt with the VM's 10.x source IP, bypass OVN's SNAT, + // and arrive at the host with a source that doesn't match the open + // TCP socket (the client dialed the EIP, not the VM IP). + // 3. br-mgmt present + AWSGW on 0.0.0.0 → br-mgmt IP (both LB flavours // reach the daemon via mgmt). - // 3. This node's AdvertiseIP (single-node default install) → AdvertiseIP. // 4. DevNetworking shim → 10.0.2.2. // 5. AWSGW bound to specific IP (no br-mgmt, no advertise) → that IP. // 6. Else: error and skip assignment — no silent empty URL. @@ -2840,17 +2850,20 @@ func (d *Daemon) wireLBAgentConfig() { advertiseIP := d.config.AdvertiseIP switch { - case d.mgmtBridgeIP != "" && awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" && !net.ParseIP(awsgwBindIP).IsLoopback(): - // Multi-node: AWSGW listens on a specific WAN IP. Use that IP as the - // gateway URL; internal LBs get a host route via br-mgmt. + case d.mgmtBridgeIP != "" && awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" && + !net.ParseIP(awsgwBindIP).IsLoopback() && awsgwBindIP != advertiseIP: + // Multi-node: AWSGW on a dedicated mgmt IP. VMs can't reach it via + // VPC → external, so add a bootcmd host route via br-mgmt. gatewayHost = awsgwBindIP d.mgmtRouteVia = awsgwBindIP - case d.mgmtBridgeIP != "": - // Multi-node with AWSGW on 0.0.0.0 — br-mgmt reaches all LBs. - gatewayHost = d.mgmtBridgeIP case advertiseIP != "" && advertiseIP != "0.0.0.0": - // Single-node default install — off-host clients dial the advertised IP. + // Single-node, or multi-node where AWSGW binds to AdvertiseIP: VMs + // reach AWSGW via the normal VPC → external path. No mgmt host route. gatewayHost = advertiseIP + case d.mgmtBridgeIP != "": + // br-mgmt present + AWSGW on 0.0.0.0 and no advertiseIP — br-mgmt IP + // is the only reachable address. + gatewayHost = d.mgmtBridgeIP case d.config.Daemon.DevNetworking: gatewayHost = "10.0.2.2" case awsgwBindIP != "" && awsgwBindIP != "0.0.0.0": diff --git a/spinifex/daemon/daemon_wire_lb_test.go b/spinifex/daemon/daemon_wire_lb_test.go index 21fdab452..60e589957 100644 --- a/spinifex/daemon/daemon_wire_lb_test.go +++ b/spinifex/daemon/daemon_wire_lb_test.go @@ -202,3 +202,44 @@ func TestWireLBAgentConfig_GatewayURL_NoAdvertiseNoBridge(t *testing.T) { assert.Empty(t, d.elbv2Service.GatewayURL) } + +// Single-node local dev: AWSGW on 0.0.0.0, br-mgmt present, AdvertiseIP set. +// Gateway must be AdvertiseIP (reachable via the VM's WAN NIC), NOT the +// br-mgmt IP (guest has no DHCP on br-mgmt so that subnet is unreachable). +func TestWireLBAgentConfig_GatewayURL_AdvertiseOverMgmtBridge(t *testing.T) { + cfg := &config.Config{ + AdvertiseIP: "192.168.1.32", + AWSGW: config.AWSGWConfig{Host: "0.0.0.0:9999"}, + } + d := newWireLBTestDaemon(t, cfg) + d.mgmtBridgeIP = "10.14.8.1" + + d.wireLBAgentConfig() + + assert.Equal(t, "https://192.168.1.32:9999", d.elbv2Service.GatewayURL) + assert.Empty(t, d.mgmtRouteVia) + assert.Empty(t, d.elbv2Service.MgmtRouteGateway) + assert.Empty(t, d.elbv2Service.MgmtRouteTarget) +} + +// Single-node with br-mgmt AND a specific AWSGW bind that matches AdvertiseIP +// (e.g. `spx admin init --bind $PRIMARY_WAN`): the WAN IP is reachable via the +// normal VPC → external path, so the bootcmd mgmt host route must NOT be +// injected. Injecting it would divert the reply leg of host-initiated ALB +// connections out br-mgmt, bypassing OVN SNAT and arriving at the host with a +// source that doesn't match the open TCP socket. +func TestWireLBAgentConfig_GatewayURL_SingleNodeBindMatchesAdvertise(t *testing.T) { + cfg := &config.Config{ + AdvertiseIP: "192.168.0.17", + AWSGW: config.AWSGWConfig{Host: "192.168.0.17:9999"}, + } + d := newWireLBTestDaemon(t, cfg) + d.mgmtBridgeIP = "10.15.8.1" + + d.wireLBAgentConfig() + + assert.Equal(t, "https://192.168.0.17:9999", d.elbv2Service.GatewayURL) + assert.Empty(t, d.mgmtRouteVia) + assert.Empty(t, d.elbv2Service.MgmtRouteGateway) + assert.Empty(t, d.elbv2Service.MgmtRouteTarget) +} diff --git a/spinifex/handlers/ec2/eip/service_impl.go b/spinifex/handlers/ec2/eip/service_impl.go index 3464680cd..e5dba061d 100644 --- a/spinifex/handlers/ec2/eip/service_impl.go +++ b/spinifex/handlers/ec2/eip/service_impl.go @@ -569,12 +569,17 @@ func (s *EIPServiceImpl) findByAssociationID(accountID, associationID string) (* // publishNATEvent publishes a NAT lifecycle event to NATS for vpcd consumption. // This is fire-and-forget; errors are logged but do not fail the API response. +// +// PortName must match the OVN logical switch port name ("port-") because +// vpcd sets NAT.LogicalPort to this value in distributed NAT mode (direct +// bridge). A mismatch creates a dnat_and_snat row pointing at a nonexistent +// port, and OVN never programs the DNAT flow. func (s *EIPServiceImpl) publishNATEvent(topic, vpcID, externalIP, logicalIP, eniID, mac string) { utils.PublishEvent(s.natsConn, topic, natEvent{ VpcId: vpcID, ExternalIP: externalIP, LogicalIP: logicalIP, - PortName: eniID, + PortName: "port-" + eniID, MAC: mac, }) } diff --git a/spinifex/handlers/ec2/eip/service_impl_test.go b/spinifex/handlers/ec2/eip/service_impl_test.go index 68355657f..48e687977 100644 --- a/spinifex/handlers/ec2/eip/service_impl_test.go +++ b/spinifex/handlers/ec2/eip/service_impl_test.go @@ -3,6 +3,7 @@ package handlers_ec2_eip import ( "encoding/json" "testing" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -555,3 +556,32 @@ func TestEIP_DescribeAddresses_FilterByTag(t *testing.T) { assert.Len(t, desc.Addresses, 1) assert.Equal(t, *out.AllocationId, *desc.Addresses[0].AllocationId) } + +// TestEIP_PublishNATEvent_PortNameHasPortPrefix is a regression test for a bug +// where EIPServiceImpl.publishNATEvent sent the raw ENI id as PortName. vpcd +// writes PortName into NAT.LogicalPort in distributed NAT mode (direct bridge), +// so a mismatch with the OVN logical switch port name ("port-") produces +// a dnat_and_snat row pointing at a nonexistent port — OVN never programs the +// flow and the EIP black-holes. See mulga-js-17. +func TestEIP_PublishNATEvent_PortNameHasPortPrefix(t *testing.T) { + svc, _ := setupTestEIP(t) + + sub, err := svc.natsConn.SubscribeSync("vpc.add-nat") + require.NoError(t, err) + defer func() { _ = sub.Unsubscribe() }() + + const eniID = "eni-abc123" + svc.publishNATEvent("vpc.add-nat", "vpc-1", "198.51.100.10", "10.0.0.5", eniID, "02:00:00:00:00:01") + + msg, err := sub.NextMsg(2 * time.Second) + require.NoError(t, err) + + var got natEvent + require.NoError(t, json.Unmarshal(msg.Data, &got)) + assert.Equal(t, "port-"+eniID, got.PortName, + "PortName must match the OVN logical switch port name (port-); "+ + "otherwise vpcd's distributed NAT creates a dnat_and_snat rule "+ + "pointing at a nonexistent logical port and the flow is never programmed") + assert.Equal(t, "10.0.0.5", got.LogicalIP) + assert.Equal(t, "198.51.100.10", got.ExternalIP) +} diff --git a/spinifex/handlers/ec2/image/service_impl.go b/spinifex/handlers/ec2/image/service_impl.go index 19e7b949b..6b43131fe 100644 --- a/spinifex/handlers/ec2/image/service_impl.go +++ b/spinifex/handlers/ec2/image/service_impl.go @@ -74,14 +74,16 @@ func NewImageServiceImplWithStore(store objectstore.ObjectStore, bucketName stri // describeImagesValidFilters defines the set of filter names accepted by DescribeImages. var describeImagesValidFilters = map[string]bool{ - "name": true, - "state": true, - "architecture": true, - "image-id": true, - "is-public": true, - "owner-id": true, - "description": true, - "image-type": true, + "name": true, + "state": true, + "architecture": true, + "image-id": true, + "is-public": true, + "owner-id": true, + "description": true, + "image-type": true, + "virtualization-type": true, + "root-device-type": true, } // DescribeImages lists available AMI images by reading config.json files from S3 @@ -340,6 +342,14 @@ func imageMatchesFilters(image *ec2.Image, filters map[string][]string, tags map if image.ImageType != nil { field = *image.ImageType } + case "virtualization-type": + if image.VirtualizationType != nil { + field = *image.VirtualizationType + } + case "root-device-type": + if image.RootDeviceType != nil { + field = *image.RootDeviceType + } default: return false } diff --git a/spinifex/handlers/ec2/image/service_impl_test.go b/spinifex/handlers/ec2/image/service_impl_test.go index a798700c2..74b2b0114 100644 --- a/spinifex/handlers/ec2/image/service_impl_test.go +++ b/spinifex/handlers/ec2/image/service_impl_test.go @@ -1031,6 +1031,48 @@ func TestDescribeImages_FilterByState(t *testing.T) { assert.Empty(t, out.Images) } +func TestDescribeImages_FilterByVirtualizationType(t *testing.T) { + svc, store := setupTestImageService(t) + createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + ImageID: "ami-hvm", Name: "hvm-img", Architecture: "x86_64", + Virtualization: "hvm", RootDeviceType: "ebs", VolumeSizeGiB: 8, + }) + createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + ImageID: "ami-pv", Name: "pv-img", Architecture: "x86_64", + Virtualization: "paravirtual", RootDeviceType: "ebs", VolumeSizeGiB: 8, + }) + + out, err := svc.DescribeImages(&ec2.DescribeImagesInput{ + Filters: []*ec2.Filter{ + {Name: aws.String("virtualization-type"), Values: []*string{aws.String("hvm")}}, + }, + }, testAccountID) + require.NoError(t, err) + assert.Len(t, out.Images, 1) + assert.Equal(t, "ami-hvm", *out.Images[0].ImageId) +} + +func TestDescribeImages_FilterByRootDeviceType(t *testing.T) { + svc, store := setupTestImageService(t) + createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + ImageID: "ami-ebs", Name: "ebs-img", Architecture: "x86_64", + Virtualization: "hvm", RootDeviceType: "ebs", VolumeSizeGiB: 8, + }) + createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + ImageID: "ami-is", Name: "is-img", Architecture: "x86_64", + Virtualization: "hvm", RootDeviceType: "instance-store", VolumeSizeGiB: 8, + }) + + out, err := svc.DescribeImages(&ec2.DescribeImagesInput{ + Filters: []*ec2.Filter{ + {Name: aws.String("root-device-type"), Values: []*string{aws.String("ebs")}}, + }, + }, testAccountID) + require.NoError(t, err) + assert.Len(t, out.Images, 1) + assert.Equal(t, "ami-ebs", *out.Images[0].ImageId) +} + // --- RegisterImage tests --- // putTestSnapshotConfig writes a SnapshotConfig at {snapshotID}/metadata.json, diff --git a/tests/e2e/run-e2e.sh b/tests/e2e/run-e2e.sh index 715d259db..20e615d42 100755 --- a/tests/e2e/run-e2e.sh +++ b/tests/e2e/run-e2e.sh @@ -388,7 +388,7 @@ else fi # Wait for SSH to become ready (VM boot + cloud-init) -wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 30 +wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 60 # Test basic SSH connectivity test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" @@ -1044,7 +1044,7 @@ fi echo "SSH endpoint: $SSH_HOST:$SSH_PORT" # Wait for SSH to become ready -wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 30 +wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 60 # Verify vCPU count matches the new instance type (nproc reports online CPUs) echo "Verifying vCPU count inside the VM..." @@ -1119,7 +1119,7 @@ else SSH_PORT=$(get_ssh_port "$INSTANCE_ID") SSH_HOST=$(get_ssh_host "$INSTANCE_ID") fi -wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 30 +wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 60 # Verify guest actually rebooted (uptime < 120 seconds) UPTIME_SECS=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ diff --git a/tests/e2e/run-multinode-e2e.sh b/tests/e2e/run-multinode-e2e.sh index 992567e59..a53b0b158 100755 --- a/tests/e2e/run-multinode-e2e.sh +++ b/tests/e2e/run-multinode-e2e.sh @@ -91,6 +91,75 @@ aws_via_retry() { # Default AWS CLI shorthand (via local node) AWS_EC2="aws --endpoint-url https://${LOCAL_IP}:${AWSGW_PORT} ec2" +# Dump SSH-related diagnostics when a guest SSH attempt fails. +# Args: [ssh_port] +# ssh_host is the address we tried (EIP or node IP), ssh_port is hostfwd port if applicable. +dump_guest_ssh_diagnostics() { + local instance_id="$1" host_ip="$2" ssh_host="$3" ssh_port="${4:-}" + echo "" + echo " === Diagnostics for ${instance_id} (host=${host_ip}, target=${ssh_host}:${ssh_port:-22}) ===" + + echo " --- describe-instances ---" + $AWS_EC2 describe-instances --instance-ids "$instance_id" \ + --query 'Reservations[0].Instances[0].[InstanceId,State.Name,PrivateIpAddress,PublicIpAddress,SecurityGroups[].GroupId,NetworkInterfaces[0].MacAddress,VpcId,SubnetId]' \ + --output json 2>&1 || true + + local sg_ids priv_ip mac vpc_id + sg_ids=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ + --query 'Reservations[0].Instances[0].SecurityGroups[].GroupId' --output text 2>/dev/null || true) + priv_ip=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ + --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text 2>/dev/null || true) + mac=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ + --query 'Reservations[0].Instances[0].NetworkInterfaces[0].MacAddress' --output text 2>/dev/null || true) + vpc_id=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ + --query 'Reservations[0].Instances[0].VpcId' --output text 2>/dev/null || true) + + if [ -n "$sg_ids" ] && [ "$sg_ids" != "None" ]; then + echo " --- describe-security-groups (${sg_ids}) ---" + # shellcheck disable=SC2086 + $AWS_EC2 describe-security-groups --group-ids $sg_ids \ + --query 'SecurityGroups[].[GroupId,GroupName,IpPermissions]' --output json 2>&1 || true + fi + + echo " --- local ARP for ${ssh_host} ---" + ip neigh show "$ssh_host" 2>&1 || true + + echo " --- local ping ${ssh_host} (2 pkts) ---" + ping -c 2 -W 2 "$ssh_host" 2>&1 || true + + if [ -n "$host_ip" ] && [ "$host_ip" != "unknown" ]; then + echo " --- on hosting node ${host_ip}: QEMU process ---" + peer_ssh "$host_ip" "pgrep -af 'qemu.*${instance_id}' || echo '(no qemu process)'" 2>&1 || true + + if [ -n "$priv_ip" ] && [ "$priv_ip" != "None" ]; then + echo " --- on hosting node ${host_ip}: ping VM private IP ${priv_ip} ---" + peer_ssh "$host_ip" "ping -c 2 -W 2 ${priv_ip}" 2>&1 || true + fi + + echo " --- on hosting node ${host_ip}: OVN chassis + port bindings ---" + peer_ssh "$host_ip" "sudo ovs-vsctl get open_vswitch . external_ids:system-id 2>/dev/null || true" 2>&1 || true + fi + + if [ -n "$vpc_id" ] && [ "$vpc_id" != "None" ]; then + echo " --- OVN NAT rules for vpc-${vpc_id} (from primary) ---" + peer_ssh "$LOCAL_IP" "sudo ovn-nbctl lr-nat-list vpc-${vpc_id} 2>&1 || true" 2>&1 || true + fi + + if [ -n "$mac" ] && [ "$mac" != "None" ]; then + echo " --- OVN port binding for MAC ${mac} (from primary) ---" + peer_ssh "$LOCAL_IP" "sudo ovn-sbctl --columns=logical_port,chassis,mac find port_binding mac~='${mac}' 2>&1 || true" 2>&1 || true + fi + + echo " --- cloud-init / sshd from hosting node (via QEMU console log if available) ---" + peer_ssh "$host_ip" "ls -la /var/log/libvirt/qemu/ 2>/dev/null | head -20; \ + for f in /tmp/spinifex/vms/${instance_id}/console.log /var/log/libvirt/qemu/${instance_id}.log; do \ + [ -f \"\$f\" ] && echo \"--- \$f (tail 50) ---\" && sudo tail -50 \"\$f\"; \ + done" 2>&1 || true + + echo " === end diagnostics for ${instance_id} ===" + echo "" +} + # Dump logs from ALL nodes on failure dump_all_node_logs() { echo "" @@ -559,6 +628,7 @@ for idx in "${!INSTANCE_IDS[@]}"; do if [ "$SSH_READY" = false ]; then echo " ERROR: SSH not ready after 60 attempts" + dump_guest_ssh_diagnostics "$instance_id" "$host_ip" "$SSH_HOST" "$SSH_PORT" fail_test "Guest SSH ($instance_id)" continue fi diff --git a/tests/e2e/run-pseudo-multinode-e2e.sh b/tests/e2e/run-pseudo-multinode-e2e.sh index 1fad30130..ac40aebd2 100755 --- a/tests/e2e/run-pseudo-multinode-e2e.sh +++ b/tests/e2e/run-pseudo-multinode-e2e.sh @@ -385,7 +385,7 @@ for idx in "${!INSTANCE_IDS[@]}"; do SSH_HOSTS+=("$SSH_HOST") # Wait for SSH to become ready (VM boot + cloud-init) - wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 30 + wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 60 # Test basic SSH connectivity test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" diff --git a/tests/e2e/run-tofu-examples-e2e.sh b/tests/e2e/run-tofu-examples-e2e.sh new file mode 100755 index 000000000..a65c7bee2 --- /dev/null +++ b/tests/e2e/run-tofu-examples-e2e.sh @@ -0,0 +1,307 @@ +#!/bin/bash +# run-tofu-examples-e2e.sh — smoke-test the public terraform workbooks. +# +# Invoked by the nightly matrix (cell #17) after bootstrap-install.sh has +# brought up a single-node cluster. For each workbook, the driver does a +# clean tofu init/apply, runs one smoke assertion, then tofu destroy. +# +# All assertions are minimal by design (one check per workbook) so the +# maintenance cost of keeping them in sync with workbook edits stays low. +# +# Expects the workbooks to be available at $WORKBOOK_DIR (defaults to the +# tarball-included ./workbooks/ directory beside this script). + +set -u +cd "$(dirname "$0")" +SCRIPT_DIR="$(pwd)" + +export AWS_PROFILE=spinifex +WORKBOOK_DIR="${WORKBOOK_DIR:-${SCRIPT_DIR}/workbooks}" +OPENTOFU_VERSION="${OPENTOFU_VERSION:-1.11.5}" + +# awsgw and predastore are bound to the WAN IP (bootstrap-install passes --bind +# ${PRIMARY_WAN}), not loopback. Workbooks need the WAN IP for both the tofu +# provider endpoints and any CLI assertions that talk to S3 (:8443). +WAN_IP=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{print $7; exit}') +SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR) + +CURRENT_WORKBOOK="" + +log() { echo "[$(date +%H:%M:%S)] $*"; } + +install_tofu() { + command -v tofu >/dev/null 2>&1 && return 0 + local arch + case "$(uname -m)" in + x86_64) arch=amd64 ;; + aarch64) arch=arm64 ;; + *) log "unsupported arch: $(uname -m)"; return 1 ;; + esac + local url="https://github.com/opentofu/opentofu/releases/download/v${OPENTOFU_VERSION}/tofu_${OPENTOFU_VERSION}_linux_${arch}.tar.gz" + log "Installing OpenTofu ${OPENTOFU_VERSION} (${arch})" + local tmp + tmp=$(mktemp -d) + curl -fsSL "$url" | tar -xz -C "$tmp" tofu + sudo install -m 0755 "$tmp/tofu" /usr/local/bin/tofu + rm -rf "$tmp" +} + +cleanup() { + EXIT_CODE=$? + if [ "$EXIT_CODE" -ne 0 ]; then + log "=== FAIL (workbook=${CURRENT_WORKBOOK:-}) — dumping diagnostics ===" + log "--- spx get vms ---" + sudo -u spinifex-daemon spx get vms 2>&1 | head -80 || \ + spx get vms 2>&1 | head -80 || true + log "--- aws ec2 describe-instances ---" + aws ec2 describe-instances \ + --query 'Reservations[].Instances[].[InstanceId,State.Name,StateReason.Message,PrivateIpAddress,PublicIpAddress,Tags[?Key==`Name`].Value|[0]]' \ + --output table 2>&1 | head -60 || true + log "--- spinifex service errors (last 15 min) ---" + for svc in spinifex-daemon spinifex-awsgw spinifex-vpcd spinifex-viperblock spinifex-predastore; do + sudo journalctl -u "$svc" --since '15 min ago' --no-pager 2>/dev/null | \ + grep -iE 'panic|fatal|level=error|ERROR' | tail -30 | \ + sed "s|^| [${svc}] |" || true + done + log "--- service journals (last 200 lines each) ---" + for svc in spinifex-nats spinifex-predastore spinifex-viperblock \ + spinifex-daemon spinifex-awsgw spinifex-vpcd; do + echo "=== ${svc} ===" + sudo journalctl -u "${svc}" --no-pager -n 200 2>/dev/null || true + done + fi + exit "$EXIT_CODE" +} +trap cleanup EXIT + +# --- Per-workbook assertions --- + +# Wait up to 150s for instance SSH readiness. +wait_for_ssh() { + local key="$1" host="$2" + for _ in $(seq 1 30); do + if ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${host}" true 2>/dev/null; then + return 0 + fi + sleep 5 + done + return 1 +} + +# Wait up to $2 seconds (default 150) for a 200 response from $1. +wait_for_http_200() { + local url="$1" budget="${2:-150}" + local attempts=$((budget / 5)) + for _ in $(seq 1 "$attempts"); do + local status + status=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null || echo "000") + if [ "$status" = "200" ]; then + return 0 + fi + sleep 5 + done + return 1 +} + +# Wait up to $2 seconds for the target group $1 to report at least one healthy target. +wait_for_alb_healthy() { + local tg_arn="$1" budget="${2:-300}" + local attempts=$((budget / 10)) + for _ in $(seq 1 "$attempts"); do + local healthy + healthy=$(aws elbv2 describe-target-health --target-group-arn "$tg_arn" \ + --query 'length(TargetHealthDescriptions[?TargetHealth.State==`healthy`])' \ + --output text 2>/dev/null || echo "0") + if [ "$healthy" -gt 0 ] 2>/dev/null; then + return 0 + fi + sleep 10 + done + return 1 +} + +assert_bastion_private_subnet() { + local bastion private key + bastion=$(tofu output -raw bastion_public_ip) + private=$(tofu output -raw private_instance_ip) + key="$(pwd)/bastion-demo.pem" + chmod 600 "$key" + + wait_for_ssh "$key" "$bastion" || { + log " bastion SSH not ready" + return 1 + } + # sshd accepts before cloud-init finishes writing ~/.ssh/bastion-demo.pem. + # Wait up to 180s for user_data to drop the key before hopping. + if ! ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" \ + 'for _ in $(seq 1 36); do [ -s ~/.ssh/bastion-demo.pem ] && exit 0; sleep 5; done; exit 1'; then + log " bastion: ~/.ssh/bastion-demo.pem never appeared (cloud-init stalled?)" + return 1 + fi + ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" \ + "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ~/.ssh/bastion-demo.pem ec2-user@${private} id" \ + | grep -q '^uid=' +} + +assert_nginx_alb() { + # ALB DNS (*.elb.spinifex.local) isn't resolvable from the host — README + # documents fetching the public IP via elbv2 describe-load-balancers. + local name ip tg_arn attempt + name=$(tofu output -raw alb_name) + + # Retry describe-load-balancers: the gateway intermittently returns + # InternalError during ALB provisioning while the lb-agent VM is booting. + ip="" + for attempt in 1 2 3 4 5 6; do + ip=$(aws elbv2 describe-load-balancers --names "$name" \ + --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \ + --output text 2>/dev/null | awk '{print $1}') + if [ -n "$ip" ] && [ "$ip" != "None" ]; then + break + fi + log " nginx-alb: describe-load-balancers attempt ${attempt} returned no IP, retrying in 10s" + sleep 10 + done + if [ -z "$ip" ] || [ "$ip" = "None" ]; then + log " nginx-alb: no public IP for ${name} after retries" + return 1 + fi + log " nginx-alb: ALB ${name} public IP ${ip}" + + tg_arn=$(aws elbv2 describe-target-groups --names nginx-alb-tg \ + --query 'TargetGroups[0].TargetGroupArn' --output text 2>/dev/null) + if [ -z "$tg_arn" ] || [ "$tg_arn" = "None" ]; then + log " nginx-alb: could not resolve target group ARN" + return 1 + fi + + wait_for_alb_healthy "$tg_arn" 300 || { + log " nginx-alb: no healthy targets after 300s" + return 1 + } + + wait_for_http_200 "http://${ip}/" 300 || { + log " nginx-alb: targets healthy but no HTTP 200 after 300s" + return 1 + } +} + +assert_nginx_webserver() { + local ip + ip=$(tofu output -raw public_ip) + wait_for_http_200 "http://${ip}/" +} + +assert_s3_webapp() { + # aws s3 goes to predastore on :8443, not the awsgw on :9999 the default + # profile points at — the workbook's provider sets s3 = predastore_endpoint + # but the CLI doesn't inherit that. + local bucket sentinel endpoint + bucket=$(tofu output -raw bucket_name) + endpoint="https://${WAN_IP}:8443" + sentinel="spinifex-nightly-$(date +%s).txt" + echo "nightly-smoke" | aws s3 cp --endpoint-url "$endpoint" - "s3://${bucket}/${sentinel}" >/dev/null + aws s3 ls --endpoint-url "$endpoint" "s3://${bucket}/" | grep -q "$sentinel" +} + +# Pick an instance type available on this cluster. Workbooks default to +# t3.small (Intel); on AMD-only hosts t3 isn't registered, so we query and +# fall back to the smallest type with ≥2 vCPU / ≥1 GiB RAM (matches the +# nginx-alb README's documented approach). +detect_instance_type() { + local endpoint="https://${WAN_IP}:9999" + local picked + picked=$(aws --endpoint-url "$endpoint" ec2 describe-instance-types \ + --query "sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus==\`2\` && MemoryInfo.SizeInMiB>=\`1024\`], &MemoryInfo.SizeInMiB)[0].InstanceType" \ + --output text 2>/dev/null || true) + if [ -z "$picked" ] || [ "$picked" = "None" ]; then + picked=$(aws --endpoint-url "$endpoint" ec2 describe-instance-types \ + --query 'InstanceTypes[0].InstanceType' --output text 2>/dev/null || true) + fi + echo "$picked" +} + +INSTANCE_TYPE="" + +# --- Workbook driver --- + +run_workbook() { + local example="$1" + local path="${WORKBOOK_DIR}/${example}" + + if [ ! -d "$path" ]; then + log "SKIP ${example}: ${path} not found" + return 1 + fi + + CURRENT_WORKBOOK="$example" + log "=== ${example} ===" + cd "$path" + rm -rf .terraform terraform.tfstate* .terraform.lock.hcl + + local apply_args=( + -input=false -no-color + "-var=spinifex_endpoint=https://${WAN_IP}:9999" + "-var=instance_type=${INSTANCE_TYPE}" + ) + + # s3-webapp has three required-no-default vars; creds come from + # AWS_PROFILE=spinifex so the workbook's boto3 client authenticates. + if [ "$example" = "s3-webapp" ]; then + local access_key secret_key + access_key=$(aws configure get aws_access_key_id --profile spinifex) + secret_key=$(aws configure get aws_secret_access_key --profile spinifex) + apply_args+=( + "-var=predastore_endpoint=https://${WAN_IP}:8443" + "-var=predastore_host=${WAN_IP}:8443" + "-var=s3_access_key=${access_key}" + "-var=s3_secret_key=${secret_key}" + ) + fi + + if ! tofu init -input=false -no-color; then + log " FAIL ${example}: tofu init" + return 1 + fi + + if ! tofu apply -auto-approve "${apply_args[@]}"; then + log " FAIL ${example}: tofu apply" + tofu destroy -auto-approve "${apply_args[@]}" >/dev/null 2>&1 || true + return 1 + fi + + local rc=0 + if assert_"${example//-/_}"; then + log " PASS ${example}" + else + log " FAIL ${example}: assertion" + rc=1 + fi + + tofu destroy -auto-approve "${apply_args[@]}" >/dev/null 2>&1 || \ + log " WARN ${example}: tofu destroy failed" + + cd "$SCRIPT_DIR" + return "$rc" +} + +# --- Main --- + +install_tofu || { log "tofu install failed"; exit 1; } + +INSTANCE_TYPE=$(detect_instance_type) +if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" = "None" ]; then + log "no instance type available from describe-instance-types" + exit 1 +fi +log "Using instance_type=${INSTANCE_TYPE}" + +for workbook in nginx-alb bastion-private-subnet nginx-webserver s3-webapp; do + if ! run_workbook "$workbook"; then + log "FAIL ${workbook} — aborting remaining workbooks" + exit 1 + fi +done + +CURRENT_WORKBOOK="" +log "All workbooks passed"