release: update go.mod for v1.4.0

feat: iso release flow (#229 )
feat(elbv2): QEMU microvm direct-boot for LB system instances (#228 )
2026-10-24 05:04:03 +00:00 · 2026-05-19 09:37:54 +10:00 · 2026-05-19 09:07:26 +10:00 · 2026-05-18 15:48:04 +10:00 · 2026-05-18 12:32:07 +10:00 · 2026-05-18 10:18:04 +10:00
185 changed files with 12501 additions and 4401 deletions
@@ -12,9 +12,9 @@ env:

 jobs:
  e2e_baremetal:
-    name: Bare-Metal Single-Node E2E
+    name: Bare-Metal Multi-Node E2E
    runs-on: [ self-hosted, ci-baremetal ]
-    timeout-minutes: 60
+    timeout-minutes: 120
    concurrency:
      group: e2e-baremetal-cluster
      cancel-in-progress: false
@@ -22,7 +22,7 @@ jobs:
      BM_ISO_CACHE: /var/cache/spinifex-iso-builder
    steps:
      - name: Clean Workspace
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/checkout@v6

@@ -44,9 +44,9 @@ jobs:

      - name: Install SSH Key
        run: |
-          install -m 0700 -d ~/.ssh
-          echo "${{ secrets.BM_SSH_PRIVATE_KEY }}" > ~/.ssh/bm-ci
-          chmod 0600 ~/.ssh/bm-ci
+          echo "${{ secrets.BM_SSH_PRIVATE_KEY }}" > "${RUNNER_TEMP}/bm-ci"
+          chmod 0600 "${RUNNER_TEMP}/bm-ci"
+          ssh-keygen -y -f "${RUNNER_TEMP}/bm-ci" > "${RUNNER_TEMP}/bm-ci.pub"

      - name: Build Artifacts
        run: |
@@ -66,11 +66,48 @@ jobs:
            VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
            VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always 2>/dev/null || echo dev)"

+      - name: Write ePDU credentials
+        run: |
+          install -m 0600 /dev/null "${RUNNER_TEMP}/epdu.env"
+          cat >> "${RUNNER_TEMP}/epdu.env" <<EOF
+          EPDU_HOST=192.168.1.63
+          EPDU_SNMP_USER=ci-bmpower
+          EPDU_SNMP_SEC_LEVEL=authPriv
+          EPDU_SNMP_AUTH_PROTO=MD5
+          EPDU_SNMP_AUTH_PASS=${{ secrets.BM_EPDU_AUTH_PASS }}
+          EPDU_SNMP_PRIV_PROTO=DES
+          EPDU_SNMP_PRIV_PASS=${{ secrets.BM_EPDU_PRIV_PASS }}
+          EOF
+
+      - name: Cache VM images
+        run: |
+          UBUNTU_IMG="${BM_ISO_CACHE}/ubuntu-24.04-cloudimg-amd64.img"
+          if [ ! -f "$UBUNTU_IMG" ]; then
+            echo "Downloading Ubuntu 24.04 cloud image..."
+            sudo curl -fsSL -o "$UBUNTU_IMG" \
+              "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+            echo "  Downloaded $(du -h "$UBUNTU_IMG" | cut -f1)"
+          else
+            echo "Ubuntu 24.04 cloud image cached ($(du -h "$UBUNTU_IMG" | cut -f1))"
+          fi
+
+          LB_IMG="${BM_ISO_CACHE}/lb-alpine-3.21.6-x86_64.raw"
+          if [ ! -f "$LB_IMG" ]; then
+            echo "Downloading lb-alpine image..."
+            sudo curl -fsSL -o "$LB_IMG" \
+              "https://iso.mulgadc.com/system-ami/lb-alpine-3.21.6-x86_64.raw"
+            echo "  Downloaded $(du -h "$LB_IMG" | cut -f1)"
+          else
+            echo "lb-alpine image cached ($(du -h "$LB_IMG" | cut -f1))"
+          fi
+
      - name: Run Bare-Metal E2E
        env:
          BM_CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
-          BM_SSH_KEY: /home/tf-user/.ssh/bm-ci
-          BM_SKIP_EPDU: "1"
+          BM_SSH_KEY: ${{ runner.temp }}/bm-ci
+          BM_EPDU_ENV: ${{ runner.temp }}/epdu.env
+          BM_UBUNTU_IMAGE: ${{ env.BM_ISO_CACHE }}/ubuntu-24.04-cloudimg-amd64.img
+          BM_LB_ALPINE_IMAGE: ${{ env.BM_ISO_CACHE }}/lb-alpine-3.21.6-x86_64.raw
          SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
        run: |
          sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
@@ -30,7 +30,7 @@ jobs:
      TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
    steps:
      - name: Clean Workspace
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -12,11 +12,10 @@ on:
 permissions:
  contents: read

-# Scheduled runs target dev (active development trunk). GitHub loads scheduled
-# workflow files from the repo's default branch, so this file must be kept in
-# sync on main for the schedule to use it.
+# Scheduled runs are loaded from the default branch (main), so the schedule
+# trigger must stay in sync on main for cron to fire.
 env:
-  GIT_BRANCH: dev
+  GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2

 jobs:
@@ -61,7 +60,7 @@ jobs:

      - name: Clean Workspace
        if: steps.gate.outputs.selected == 'true'
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        if: steps.gate.outputs.selected == 'true'
@@ -34,7 +34,7 @@ jobs:
      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
    steps:
      - name: Clean Workspace
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -103,7 +103,7 @@ jobs:
      TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log
    steps:
      - name: Clean Workspace
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -167,7 +167,7 @@ jobs:
      TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
    steps:
      - name: Clean Workspace
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -31,6 +31,14 @@ jobs:
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.26.3'
+
+      - name: Build microVM artifacts (lb-agent + kernel + initramfs)
+        run: cd spinifex && make build-microvm-image
+
      - name: Build amd64 Tarball
        env:
          VERSION: ${{ github.ref_name }}
@@ -192,3 +200,150 @@ jobs:
          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
+
+  iso:
+    needs: release
+    runs-on: [ self-hosted, ci-single ]
+    timeout-minutes: 90
+    steps:
+      - name: Clean Workspace
+        run: |
+          ROOTFS="${GITHUB_WORKSPACE}/mulga/scripts/iso-builder/.cache/rootfs"
+          # Mirror make clean's umount sequence unconditionally — lazy unmount
+          # detaches immediately even if the mount is busy, so rm -rf is safe.
+          sudo umount -l -R "${ROOTFS}" 2>/dev/null || true
+          sudo umount -l "${ROOTFS}/dev/pts" 2>/dev/null || true
+          sudo umount -l "${ROOTFS}/dev" 2>/dev/null || true
+          sudo umount -l "${ROOTFS}/sys" 2>/dev/null || true
+          sudo umount -l "${ROOTFS}/proc" 2>/dev/null || true
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - name: Checkout Spinifex
+        uses: actions/checkout@v6
+        with:
+          path: spinifex
+          ref: ${{ github.ref }}
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/predastore
+          path: predastore
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/viperblock
+          path: viperblock
+
+      - name: Checkout Mulga iso-builder
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/iso-builder
+          path: mulga
+
+      - name: Clean ISO builder cache
+        run: sudo make -C mulga/scripts/iso-builder clean
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Configure Go workspace
+        run: go work init ./spinifex ./predastore ./viperblock
+
+      - name: Install ISO builder dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y \
+            debootstrap squashfs-tools xorriso grub-pc-bin grub-efi-amd64-bin \
+            grub-common mtools busybox-static ovmf zstd \
+            nbdkit nbdkit-plugin-dev pkg-config gcc
+
+      - name: Build spinifex binaries
+        run: cd spinifex && make build
+
+      - name: Build viperblock
+        run: cd viperblock && make build
+
+      - name: Build release ISO
+        id: build-iso
+        env:
+          VERSION: ${{ github.ref_name }}
+        run: |
+          sudo make -C mulga/scripts/iso-builder release \
+            VERSION="${VERSION}" \
+            SPINIFEX_SRC="${GITHUB_WORKSPACE}/spinifex" \
+            VIPERBLOCK_SRC="${GITHUB_WORKSPACE}/viperblock" \
+            SPINIFEX_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spx" \
+            INSTALLER_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spinifex-installer"
+
+      - name: Wait for verify
+        id: wait-verify
+        if: steps.build-iso.outcome == 'success'
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          echo "Waiting for verify job to complete..."
+          while true; do
+            CONCLUSION=$(gh api \
+              "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \
+              --jq '.jobs[] | select(.name == "verify") | .conclusion // empty')
+            if [ -n "$CONCLUSION" ]; then
+              echo "Verify conclusion: $CONCLUSION"
+              [ "$CONCLUSION" = "success" ] && exit 0
+              exit 1
+            fi
+            echo "Verify still running, retrying in 30s..."
+            sleep 30
+          done
+
+      - name: Ensure AWS CLI is installed
+        if: steps.build-iso.outcome == 'success' && steps.wait-verify.outcome == 'success'
+        run: |
+          if ! command -v aws &>/dev/null; then
+            curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
+            unzip -q /tmp/awscliv2.zip -d /tmp/aws-install
+            sudo /tmp/aws-install/aws/install
+            rm -rf /tmp/awscliv2.zip /tmp/aws-install
+          fi
+
+      - name: Upload ISO to Cloudflare R2
+        id: upload
+        if: steps.build-iso.outcome == 'success' && steps.wait-verify.outcome == 'success'
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
+          VERSION: ${{ github.ref_name }}
+        run: |
+          aws --endpoint-url=https://e80e329f7dbcf75b80e019c1dce640f2.r2.cloudflarestorage.com \
+            s3 cp "mulga/scripts/iso-builder/dist/spinifex-${VERSION}.iso" \
+                  "s3://spinifex-iso/spinifex-${VERSION}.iso"
+          aws --endpoint-url=https://e80e329f7dbcf75b80e019c1dce640f2.r2.cloudflarestorage.com \
+            s3 cp "mulga/scripts/iso-builder/dist/spinifex-${VERSION}.iso.sha256" \
+                  "s3://spinifex-iso/spinifex-${VERSION}.iso.sha256"
+
+      - name: Mark ISO failed
+        if: always() && (steps.build-iso.outcome == 'failure' || steps.upload.outcome == 'failure')
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ github.ref_name }}
+          REPO: ${{ github.repository }}
+        run: |
+          EXISTING_BODY=$(gh release view "$TAG" --repo "$REPO" --json body -q .body)
+          NOTES_FILE=$(mktemp)
+          printf 'ISO build/upload failed — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
+          gh release edit "$TAG" --repo "$REPO" --notes-file "$NOTES_FILE"
+          rm -f "$NOTES_FILE"
@@ -36,12 +36,14 @@ jobs:
        uses: actions/checkout@v6
        with:
          repository: mulgadc/viperblock
+          ref: dev
          path: viperblock

      - name: Checkout Predastore
        uses: actions/checkout@v6
        with:
          repository: mulgadc/predastore
+          ref: dev
          path: predastore

      - name: Setup Go
@@ -76,12 +78,14 @@ jobs:
        uses: actions/checkout@v6
        with:
          repository: mulgadc/viperblock
+          ref: dev
          path: viperblock

      - name: Checkout Predastore
        uses: actions/checkout@v6
        with:
          repository: mulgadc/predastore
+          ref: dev
          path: predastore

      - name: Setup Go
@@ -110,12 +114,14 @@ jobs:
        uses: actions/checkout@v6
        with:
          repository: mulgadc/viperblock
+          ref: dev
          path: viperblock

      - name: Checkout Predastore
        uses: actions/checkout@v6
        with:
          repository: mulgadc/predastore
+          ref: dev
          path: predastore

      - name: Setup Go
@@ -17,6 +17,8 @@ go.work.sum
 # Build outputs
 bin/
 dist/
+build/microvm/vmlinuz
+build/microvm/initramfs.cpio.gz
 !spinifex/services/spinifexui/frontend/dist/
 tmp/
 *.test
@@ -63,15 +63,15 @@ LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSI

 go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
-	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	GOFIPS140=v1.0.0 go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go

 build-installer:
 	@echo -e "\n....Building spinifex-installer"
-	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
+	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go

 build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
-	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
+	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go

 build-system-image: ## Build a system image from manifest (use IMAGE=lb)
 ifndef IMAGE
@@ -79,8 +79,25 @@ ifndef IMAGE
 endif
 	./scripts/build-system-image.sh scripts/images/$(IMAGE).conf

-build-lb-image: ## Build LB Alpine image
-	$(MAKE) build-system-image IMAGE=lb
+MICROVM_OUT_DIR := build/microvm
+MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
+MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
+
+# Grouped target — script writes both files in one run.
+$(MICROVM_ARTIFACTS) &: $(MICROVM_INPUTS)
+	./scripts/build-microvm-image.sh
+
+# Only triggers when bin/lb-agent is missing; preserves the artifact's mtime so
+# build-microvm-image stays correctly stale-aware.
+bin/lb-agent:
+	$(MAKE) build-lb-agent
+
+build-microvm-image: $(MICROVM_ARTIFACTS) ## Build microVM kernel + initramfs (incremental — skips when up to date)
+
+install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
+	sudo install -d /usr/share/spinifex/microvm
+	sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
+	sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz

 go_run:
 	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
@@ -122,9 +139,18 @@ run:
 	$(MAKE) go_build
 	$(MAKE) go_run

-# Fast iteration: build + install binary + restart all services
+# Fast iteration: build + install binary + restart all services.
+# Microvm artifacts are reinstalled when they already exist on disk — the rule's
+# input timestamps drive a rebuild only if anything actually changed. On a fresh
+# checkout (no build/microvm/vmlinuz yet) the install step is skipped; run
+# `make install-microvm` explicitly the first time.
 deploy: build
 	sudo install -m 755 bin/spx /usr/local/bin/spx
+	@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
+		$(MAKE) install-microvm; \
+	else \
+		echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
+	fi
 	sudo systemctl daemon-reload
 	sudo systemctl restart spinifex.target

@@ -206,6 +232,14 @@ distro-amd64:
 		-f build/Dockerfile.distro \
 		--output type=local,dest=dist/amd64/ \
 		../
+	@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ]; then \
+		echo "[distro-amd64] staging microvm artifacts into tarball"; \
+		mkdir -p dist/amd64/microvm; \
+		cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz dist/amd64/microvm/; \
+	else \
+		echo "[distro-amd64] WARNING: microvm artifacts missing — tarball will not include them"; \
+		echo "[distro-amd64]          run 'make build-microvm-image' before 'make distro-amd64'"; \
+	fi
 	tar -czf dist/spinifex-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
 	sha256sum dist/spinifex-$(VERSION)-linux-amd64.tar.gz > dist/spinifex-$(VERSION)-linux-amd64.tar.gz.sha256

@@ -238,9 +272,12 @@ ansible-dev-install:
 ansible-dev-reset:
 	cd scripts/ansible && ansible-playbook playbooks/dev-reset.yml

-.PHONY: build build-ui build-installer build-lb-agent build-system-image build-lb-image go_build go_run preflight test test-cover test-race diff-coverage bench run \
+ansible-dev-deploy:
+	cd scripts/ansible && ansible-playbook playbooks/dev-deploy.yml
+
+.PHONY: build build-ui build-installer build-lb-agent build-system-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck \
 	distro distro-amd64 distro-arm64 distro-clean \
-	ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset
+	ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset ansible-dev-deploy
@@ -30,13 +30,13 @@ COPY viperblock/ ./viperblock/

 # Build spinifex binary (spx)
 RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \
-    cd spinifex && go build -ldflags "-s -w \
+    cd spinifex && GOFIPS140=v1.0.0 go build -ldflags "-s -w \
    -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=${VERSION} \
    -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=${COMMIT}" \
    -o /out/spx cmd/spinifex/main.go

 # Build nbdkit viperblock plugin (.so)
-RUN cd viperblock && go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-shared nbd/viperblock.go
+RUN cd viperblock && GOFIPS140=v1.0.0 go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-shared nbd/viperblock.go

 # Copy setup scripts and systemd units
 RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
@@ -0,0 +1,56 @@
+#!/bin/sh
+mount -t proc  proc  /proc
+mount -t sysfs sys   /sys
+mount -t devtmpfs dev /dev
+
+# virtio_mmio is built-in (CONFIG_VIRTIO_MMIO=y) so virtio-net devices are
+# already enumerated by the time PID 1 runs — no modprobe needed.
+# qemu_fw_cfg and virtio_net are =m in stock Alpine linux-virt, so load them.
+modprobe qemu_fw_cfg 2>/dev/null || modprobe fw_cfg_sysfs 2>/dev/null || true
+modprobe virtio_net 2>/dev/null || true
+
+for i in 1 2 3 4 5; do
+    [ -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ] && break
+    sleep 0.02
+done
+
+if [ ! -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ]; then
+    echo "[init] FATAL: fw_cfg netcfg not available" >&2
+    exit 1
+fi
+
+. /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
+
+for n in 0 1 2 3 4 5; do
+    eval mac=\$NIC${n}_MAC
+    [ -z "$mac" ] && continue
+
+    iface=""
+    for attempt in 1 2 3 4 5; do
+        iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
+        [ -n "$iface" ] && break
+        sleep 0.02
+    done
+
+    if [ -z "$iface" ]; then
+        echo "[init] WARNING: no interface found for MAC $mac (NIC$n), skipping" >&2
+        continue
+    fi
+
+    eval cidr=\$NIC${n}_CIDR
+    eval gw=\$NIC${n}_GW
+    eval default=\$NIC${n}_DEFAULT
+    eval rdst=\$NIC${n}_ROUTE_DST
+    eval rvia=\$NIC${n}_ROUTE_VIA
+
+    ip link set "$iface" up
+    [ -n "$cidr" ] && ip addr add "$cidr" dev "$iface"
+    [ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface"
+    [ -n "$rdst" ] && [ -n "$rvia" ] && ip route add "$rdst" via "$rvia" dev "$iface"
+done
+
+cp /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/lb-agent-env/raw /etc/conf.d/lb-agent
+mkdir -p /etc/ssl/certs
+cp /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw /etc/ssl/certs/ca-certificates.crt
+
+exec /sbin/init
@@ -0,0 +1,4 @@
+::sysinit:/etc/rc.local
+::respawn:/bin/sh -c 'set -a; . /etc/conf.d/lb-agent; set +a; exec /usr/local/bin/lb-agent'
+::ctrlaltdel:/sbin/reboot
+::shutdown:/bin/umount -a -r
@@ -143,7 +143,12 @@ func buildConfig() (*install.Config, error) {
 		role = "init"
 	}
 	cfg.ClusterRole = role
-	if role == "join" {
+	skipFormation := os.Getenv("SPINIFEX_SKIP_FORMATION") == "1"
+	if skipFormation {
+		cfg.SkipFormation = true
+	}
+
+	if role == "join" && !skipFormation {
 		joinAddr := os.Getenv("SPINIFEX_JOIN_ADDR")
 		if joinAddr == "" {
 			return nil, fmt.Errorf("SPINIFEX_JOIN_ADDR required when SPINIFEX_ROLE=join")
@@ -52,6 +52,10 @@ type Config struct {
 	// and network failures are swallowed so the install is not gated on
 	// controller reachability.
 	InstallCallback string
+	// SkipFormation skips spx admin init/join. Used when a provisioning
+	// controller (e.g. bm-bootstrap.sh) owns cluster formation and will call
+	// spx admin init/join itself with the correct multi-node parameters.
+	SkipFormation bool
 }

 // Write drops the firstboot script and systemd unit into root, which should be
@@ -232,6 +236,9 @@ fi
 }

 func buildClusterCmd(cfg Config) string {
+	if cfg.SkipFormation {
+		return `echo "[firstboot] cluster formation skipped — provisioning controller will handle it"`
+	}
 	emailFlag := ""
 	if cfg.Email != "" {
 		// shellEscapeSingle keeps the email safe if it ever contains a
@@ -55,6 +55,10 @@ type Config struct {
 	// JoinAddr is the primary node address (host:port) when ClusterRole is "join".
 	JoinAddr string

+	// SkipFormation skips spx admin init/join in firstboot; the provisioning
+	// controller (e.g. bm-bootstrap.sh) owns cluster formation instead.
+	SkipFormation bool
+
 	// CA certificate (PEM), optional.
 	HasCACert bool
 	CACert    string
@@ -23,6 +23,7 @@ import (
 	"io"
 	"log/slog"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -77,6 +78,7 @@ func Run(cfg *Config) error {
 	}

 	slog.Info("installation complete")
+	fireInstallCallback()
 	promptRemoveUSB()
 	return reboot()
 }
@@ -172,6 +174,7 @@ func copyRootfs() error {
 		"--exclude=/etc/openvswitch/",
 		"--exclude=/var/lib/openvswitch/",
 		"--exclude=/var/lib/dhcpcd/",
+		"--exclude=/etc/ssh/ssh_host_*",
 		"--exclude=/lost+found",
 		"--exclude=/boot/efi",
 		"/", mountRoot+"/",
@@ -555,6 +558,11 @@ func installBootloader(disk string) error {
 		}
 		return biosErr
 	}
+	// Copy splash image and unicode font from the ISO (mounted at /cdrom) so the
+	// installed GRUB shows the same branded background as the installer GRUB.
+	// The font must be at /boot/grub/fonts/unicode.pf2 so update-grub finds it
+	// there and emits the same loadfont path as the ISO's grub.cfg — GRUB 2.12
+	// (trixie) needs the font in the boot partition, not just /usr/share/grub/.
 	copySplashImage(mountRoot)
 	copyGrubFont(mountRoot)

@@ -644,6 +652,26 @@ func promptRemoveUSB() {
 	}
 }

+// fireInstallCallback notifies the boot controller that installation is done
+// so it clears the PXE install flag before the node reboots. Without this,
+// a PXE-first boot order causes the node to reinstall on every reboot until
+// firstboot fires the callback — which it can never do if it never runs.
+// No-op when SPINIFEX_INSTALL_CALLBACK is not set (ISO/USB installs).
+func fireInstallCallback() {
+	url := strings.TrimSpace(os.Getenv("SPINIFEX_INSTALL_CALLBACK"))
+	if url == "" {
+		return
+	}
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Get(url) //nolint:noctx // installer has no context; best-effort fire-and-forget
+	if err != nil {
+		slog.Warn("fireInstallCallback: request failed", "url", url, "err", err)
+		return
+	}
+	resp.Body.Close()
+	slog.Info("fireInstallCallback: notified boot controller", "url", url, "status", resp.StatusCode)
+}
+
 func reboot() error {
 	// sync filesystems before reboot so nothing is lost.
 	_ = run("sync")
@@ -668,6 +696,7 @@ func (c *Config) toFirstbootConfig() firstboot.Config {
 		JoinAddr:        c.JoinAddr,
 		Email:           c.Email,
 		InstallCallback: strings.TrimSpace(os.Getenv("SPINIFEX_INSTALL_CALLBACK")),
+		SkipFormation:   c.SkipFormation,
 	}
 }

@@ -713,9 +742,46 @@ func partitionPaths(disk string) (efi, root string) {
 	return disk + "2", disk + "3"
 }

-// copySplashImage copies the GRUB splash (embedded in the squashfs at build time by
-// inject-bins.sh) into the installed system so the post-install GRUB shows the same
-// branded background as the installer GRUB. Non-fatal — missing source is logged and skipped.
+// copyGrubFont copies the unicode.pf2 GRUB font into the installed system's
+// /boot/grub/fonts/ directory. This ensures update-grub finds the font at
+// /boot/grub/fonts/unicode.pf2 — the same path the ISO's grub.cfg uses —
+// so the generated grub.cfg enables gfxterm and the background image.
+// Without this, grub-mkconfig falls back to /usr/share/grub/unicode.pf2,
+// a path that GRUB 2.12 (trixie) may fail to resolve at boot time.
+// Non-fatal — a missing source is logged and skipped.
+func copyGrubFont(root string) {
+	candidates := []string{
+		"/cdrom/boot/grub/fonts/unicode.pf2", // ISO tree (preferred)
+		"/usr/share/grub/unicode.pf2",        // live system grub-common fallback
+	}
+	for _, src := range candidates {
+		in, err := os.Open(src)
+		if err != nil {
+			continue
+		}
+		defer in.Close()
+		dstDir := filepath.Join(root, "boot/grub/fonts")
+		if err := os.MkdirAll(dstDir, 0o755); err != nil {
+			slog.Warn("copyGrubFont: cannot create fonts dir", "err", err)
+			return
+		}
+		out, err := os.OpenFile(filepath.Join(dstDir, "unicode.pf2"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
+		if err != nil {
+			slog.Warn("copyGrubFont: cannot open destination", "err", err)
+			return
+		}
+		defer out.Close()
+		if _, err := io.Copy(out, in); err != nil {
+			slog.Warn("copyGrubFont: copy failed", "err", err)
+		}
+		return
+	}
+	slog.Warn("copyGrubFont: no unicode.pf2 found, splash may not display")
+}
+
+// copySplashImage copies the GRUB splash from the live ISO (/cdrom/boot/grub/splash.png)
+// into the installed system so the post-install GRUB shows the same branded background
+// as the installer. Non-fatal — a missing or unreadable source is logged and skipped.
 func copySplashImage(root string) {
 	const src = "/usr/share/spinifex/grub-splash.png"
 	in, err := os.Open(src)
@@ -741,34 +807,6 @@ func copySplashImage(root string) {
 	}
 }

-// copyGrubFont copies the unicode font into the installed system's
-// /boot/grub/fonts/ so the loadfont path in 05_spinifex resolves at boot.
-// grub-install does not copy fonts; we mirror what build-iso.sh does.
-func copyGrubFont(root string) {
-	const src = "/usr/share/grub/unicode.pf2"
-	in, err := os.Open(src)
-	if err != nil {
-		slog.Warn("copyGrubFont: font not found, graphical GRUB may not work", "path", src)
-		return
-	}
-	defer in.Close()
-
-	dstDir := filepath.Join(root, "boot/grub/fonts")
-	if err := os.MkdirAll(dstDir, 0o755); err != nil {
-		slog.Warn("copyGrubFont: cannot create fonts dir", "err", err)
-		return
-	}
-	out, err := os.OpenFile(filepath.Join(dstDir, "unicode.pf2"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
-	if err != nil {
-		slog.Warn("copyGrubFont: cannot open destination", "err", err)
-		return
-	}
-	defer out.Close()
-	if _, err := io.Copy(out, in); err != nil {
-		slog.Warn("copyGrubFont: copy failed", "err", err)
-	}
-}
-
 // maskSystemdUnit creates a symlink to /dev/null for the given unit, which is
 // the standard way to permanently disable a unit so systemd will never start it.
 func maskSystemdUnit(root, unit string) error {
@@ -24,6 +24,7 @@ import (
 	"github.com/mulgadc/spinifex/cmd/installer/autoinstall"
 	"github.com/mulgadc/spinifex/cmd/installer/install"
 	"github.com/mulgadc/spinifex/cmd/installer/ui"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )

 func main() {
@@ -19,6 +19,8 @@ import (
 	"syscall"

 	"github.com/mulgadc/spinifex/spinifex/lbagent"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )

 func main() {
@@ -993,8 +993,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	var predastoreNodeID int
 	if predastoreNodesStr != "" {
 		ips := strings.Split(predastoreNodesStr, ",")
-		if len(ips) < 3 {
-			fmt.Fprintf(os.Stderr, "❌ Error: --predastore-nodes requires at least 3 IPs, got %d\n", len(ips))
+		if len(ips) < 2 {
+			fmt.Fprintf(os.Stderr, "❌ Error: --predastore-nodes requires at least 2 IPs, got %d\n", len(ips))
 			os.Exit(1)
 		}

@@ -1113,6 +1113,20 @@ func runAdminInit(cmd *cobra.Command, args []string) {

 	finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP, advertiseIP)

+	// Write node.conf so spx admin banner works on source installs (not just ISO).
+	nodeHostname, _ := os.Hostname()
+	if nodeHostname == "" {
+		nodeHostname = node
+	}
+	nodeConfPath := filepath.Join(configDir, "node.conf")
+	if err := writeNodeConf(nodeConfPath, map[string]string{
+		"MANAGEMENT_IP":    advertiseIP,
+		"MANAGEMENT_IFACE": "br-wan",
+		"NODE_HOSTNAME":    nodeHostname,
+	}); err != nil {
+		fmt.Fprintf(os.Stderr, "⚠️  Warning: could not write %s: %v\n", nodeConfPath, err)
+	}
+
 	// Print success message
 	fmt.Println("\n🎉 Spinifex initialization complete!")
 	fmt.Println()
@@ -1269,7 +1283,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,

 	// Generate multi-node predastore config
 	var predastoreNodeID int
-	hasPredastoreConfig := len(predastoreNodes) >= 3
+	hasPredastoreConfig := len(predastoreNodes) >= 2
 	if hasPredastoreConfig {
 		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
 		if err != nil {
@@ -1682,7 +1696,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {

 	// Generate multi-node predastore config
 	var predastoreNodeID int
-	hasPredastoreConfig := len(predastoreNodes) >= 3
+	hasPredastoreConfig := len(predastoreNodes) >= 2

 	if hasPredastoreConfig {
 		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP)
@@ -2361,6 +2375,12 @@ func runAdminBanner(cmd *cobra.Command, _ []string) {
 	}

 	// Resolve current IP from the management interface at runtime.
+	// If node.conf is absent or has no MANAGEMENT_IFACE (source installs before
+	// the first spx admin init writes node.conf), fall back to br-wan which
+	// setup-ovn.sh always creates as the management bridge.
+	if iface == "" {
+		iface = "br-wan"
+	}
 	currentIP := resolveIfaceIP(iface)
 	if currentIP == "" {
 		currentIP = recordedIP // fall back to value recorded at install time
@@ -120,18 +120,16 @@ func runAdminGpuStatus(cmd *cobra.Command, _ []string) {
 		fmt.Printf("Passthrough:     enabled\n")
 		fmt.Printf("GPU pool:        %d/%d allocated\n", resp.AllocGPUs, resp.TotalGPUs)

-		var g5Types []string
+		var gpuTypes []string
 		for _, cap := range resp.InstanceTypes {
-			if strings.HasPrefix(cap.Name, "g5.") {
-				g5Types = append(g5Types, cap.Name)
-			}
+			gpuTypes = append(gpuTypes, cap.Name)
 		}
-		if len(g5Types) > 0 {
-			fmt.Printf("Instance types:  %s\n", strings.Join(g5Types, " "))
+		if len(gpuTypes) > 0 {
+			fmt.Printf("Instance types:  %s\n", strings.Join(gpuTypes, " "))
 		}
 	} else if resp.GPUCapable {
 		fmt.Printf("Passthrough:     disabled\n")
-		fmt.Printf("Instance types:  (none — run 'spx admin gpu enable' to activate g5.*)\n")
+		fmt.Printf("Instance types:  (none — run 'spx admin gpu enable' to activate)\n")
 	} else {
 		fmt.Printf("Passthrough:     disabled\n")
 		fmt.Printf("Instance types:  (prerequisites not met)\n")
@@ -209,6 +207,9 @@ func gpuToggle(_ *cobra.Command, enable bool) {
 		}
 		if r.GPUPassthrough == enable {
 			fmt.Println(" done.")
+			// Refresh the MOTD banner immediately so the login prompt
+			// reflects the new GPU passthrough state without requiring a reboot.
+			_ = exec.Command("spx", "admin", "banner", "--boot-check").Run()
 			runAdminGpuStatus(adminGpuStatusCmd, nil)
 			return
 		}
@@ -306,6 +307,30 @@ func runAdminGpuSetup(_ *cobra.Command, _ []string) {
 		fmt.Println("    Already blacklisted")
 	}

+	fmt.Println("==> Configuring amdgpu blacklist")
+	const amdgpuConf = "/etc/modprobe.d/blacklist-amdgpu.conf"
+	hasAMD := false
+	for _, d := range devices {
+		if d.Vendor == gpu.VendorAMD {
+			hasAMD = true
+			break
+		}
+	}
+	if hasAMD {
+		if _, err := os.Stat(amdgpuConf); os.IsNotExist(err) {
+			if err := os.WriteFile(amdgpuConf, []byte("blacklist amdgpu\n"), 0o644); err != nil {
+				fmt.Fprintf(os.Stderr, "Failed to write amdgpu blacklist: %v\n", err)
+				os.Exit(1)
+			}
+			fmt.Println("    amdgpu blacklisted")
+			rebootNeeded = true
+		} else {
+			fmt.Println("    Already blacklisted")
+		}
+	} else {
+		fmt.Println("    No AMD GPUs — skipping")
+	}
+
 	fmt.Println("==> Configuring vfio-pci early binding")
 	const vfioPCIConf = "/etc/modprobe.d/vfio-pci.conf"
 	existing, _ := os.ReadFile(vfioPCIConf)
@@ -350,7 +375,7 @@ func runAdminGpuSetup(_ *cobra.Command, _ []string) {
 		}
 		fmt.Println("\nSetup complete — reboot required.")
 		fmt.Println("  sudo reboot")
-		fmt.Println("  Then re-run: sudo spx admin gpu setup")
+		fmt.Println("  Then run: sudo spx admin gpu enable")
 		return
 	}

@@ -363,15 +388,29 @@ func runAdminGpuSetup(_ *cobra.Command, _ []string) {

 	fmt.Println("==> Verifying GPU driver binding")
 	for _, d := range devices {
-		driverLink := "/sys/bus/pci/devices/" + d.PCIAddress + "/driver"
-		target, err := os.Readlink(driverLink)
-		driver := filepath.Base(target)
-		if err != nil || driver != "vfio-pci" {
-			fmt.Fprintf(os.Stderr, "%s is bound to %q, not vfio-pci\n", d.PCIAddress, driver)
-			fmt.Fprintf(os.Stderr, "  Check: ls -la %s\n", driverLink)
+		current, err := gpu.CurrentDriver(d.PCIAddress)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Cannot read driver for %s: %v\n", d.PCIAddress, err)
 			os.Exit(1)
 		}
-		fmt.Printf("    %s - vfio-pci\n", d.PCIAddress)
+		if current == "vfio-pci" {
+			fmt.Printf("    %s → vfio-pci\n", d.PCIAddress)
+			continue
+		}
+		if current != "" {
+			// Bound to native driver — blacklist/initramfs changes may not have taken effect.
+			fmt.Fprintf(os.Stderr, "%s is bound to %q, not vfio-pci — reboot may be required\n", d.PCIAddress, current)
+			fmt.Fprintf(os.Stderr, "  Check: ls -la /sys/bus/pci/devices/%s/driver\n", d.PCIAddress)
+			os.Exit(1)
+		}
+		// Device is unbound (ids= did not auto-claim it) — bind explicitly via driver_override.
+		fmt.Printf("    %s unbound — binding to vfio-pci\n", d.PCIAddress)
+		if _, err := gpu.BindVFIO(d.PCIAddress); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to bind %s to vfio-pci: %v\n", d.PCIAddress, err)
+			fmt.Fprintf(os.Stderr, "  Check: ls -la /sys/bus/pci/devices/%s/driver\n", d.PCIAddress)
+			os.Exit(1)
+		}
+		fmt.Printf("    %s → vfio-pci\n", d.PCIAddress)
 	}

 	fmt.Println("==> Enabling GPU passthrough")
@@ -9,8 +9,13 @@ port = 8443
 debug = false

 [rs]
+{{- if eq (len .Nodes) 2}}
+data = 1
+parity = 1
+{{- else}}
 data = 2
 parity = 1
+{{- end}}
 {{- range .Nodes}}

 [[db]]
@@ -16,7 +16,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package main

-import "github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+import (
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)

 func main() {
 	cmd.Execute()
@@ -45,7 +45,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t

 | Command | Flags | Prerequisites | Basic Logic | Output | Status |
 |---------|-------|---------------|-------------|--------|--------|
-| `spx top nodes` | `--timeout` (default: 3s) | Cluster must be running (NATS) | Publishes to `spinifex.node.status` fan-out topic → collects CPU/memory usage per node → aggregates instance type capacity across all nodes → prints two tables: per-node resource usage and cluster-wide instance type availability | Table 1: NAME, CPU (used/total), MEM (used/total), VMs. Table 2: INSTANCE TYPE, AVAILABLE, VCPU, MEMORY | **DONE** |
+| `spx top nodes` | `--timeout` (default: 3s) | Cluster must be running (NATS) | Publishes to `spinifex.node.status` fan-out topic → collects CPU/memory usage per node → aggregates instance type capacity across all nodes → prints two tables: per-node resource usage and cluster-wide instance type availability | Table 1: NAME, CPU (used/total), MEM (used/total), GPU (used/total), VMs — GPU column shows `allocated/total` when passthrough is enabled, `0/count*` when the node is GPU-capable but passthrough is disabled, or `-` when no GPU is present. Table 2: INSTANCE TYPE, AVAILABLE, VCPU, MEMORY | **DONE** |

 ### Cluster Initialization

@@ -123,6 +123,15 @@ served. A mirror compromise that swaps both image and sums file is not
 detected; closing that gap requires GPG signature verification of the sums
 file, deferred to a later phase.

+### GPU Management
+
+| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------|---------------|-------------|------------|--------|
+| `spx admin gpu status` | `--node` (default: local node) | Cluster must be running (NATS) | Queries `spinifex.node.status` fan-out → finds the target node response → prints Node, GPU hardware (model list or "none detected"), IOMMU state, vfio-pci state, passthrough enabled/disabled, and GPU pool allocation (`allocated/total`). Also lists GPU-capable instance types when passthrough is active. | 1. Local node with passthrough enabled (shows pool and instance types)<br>2. Local node capable but passthrough disabled (shows "run enable")<br>3. Node without GPU hardware (shows "none detected")<br>4. `--node` targets a remote node<br>5. Node not found or not responding (error) | **DONE** |
+| `spx admin gpu enable` | — | Cluster must be running (NATS); node must be GPU-capable (IOMMU + vfio-pci configured) | Checks current passthrough state via NATS → errors if already enabled or prerequisites not met (directs to `setup`) → writes `gpu_passthrough = true` to `spinifex.toml` via `admin.SetGPUPassthrough` → sends SIGHUP to `spinifex-daemon` via `systemctl kill -s HUP` → polls node status for up to 30 s until daemon confirms new state → prints final `gpu status` output. Must be run directly on the target host. | 1. Enable on capable node<br>2. Already enabled (no-op)<br>3. Prerequisites not met (error, suggests `setup`)<br>4. Daemon does not confirm within 30 s (timeout error) | **DONE** |
+| `spx admin gpu disable` | — | Cluster must be running (NATS); no GPU instances may be running | Checks current passthrough state via NATS → errors if already disabled or if `AllocGPUs > 0` (must terminate GPU instances first) → writes `gpu_passthrough = false` → sends SIGHUP to `spinifex-daemon` → polls for up to 30 s → prints final `gpu status` output. Must be run directly on the target host. | 1. Disable with no GPU instances running<br>2. Already disabled (no-op)<br>3. GPU instances running (error with count)<br>4. Daemon does not confirm within 30 s (timeout error) | **DONE** |
+| `spx admin gpu setup` | — | Must be run as root; NVIDIA or AMD GPU must be present | Idempotent host configuration for GPU passthrough. Steps (skipped if already applied): detect GPUs via `gpu.Discover` → collect PCI IDs for all IOMMU-group siblings → check/enable IOMMU in GRUB (`intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt`) → write vfio udev rule (`/etc/udev/rules.d/99-spinifex-vfio.rules`) → blacklist nouveau (`/etc/modprobe.d/blacklist-nouveau.conf`) → blacklist amdgpu if AMD GPU present (`/etc/modprobe.d/blacklist-amdgpu.conf`) → write vfio-pci early binding config (`/etc/modprobe.d/vfio-pci.conf`) → add vfio modules to initramfs. If any change requires a reboot: runs `update-initramfs -u` and exits with reboot instructions. After reboot: verifies `vfio_pci` module is loaded → verifies each GPU is bound to `vfio-pci` (binds explicitly via `driver_override` if unbound) → calls `gpu enable` to activate passthrough. | 1. First run on unconfigured host (GRUB updated, reboot required)<br>2. Re-run after reboot (verifies bindings, enables passthrough)<br>3. Already fully configured (all steps skipped, enables passthrough)<br>4. AMD GPU present (blacklists amdgpu in addition to nouveau)<br>5. Not run as root (error)<br>6. No GPU hardware detected (error) | **DONE** |
+
 ## AWS Commands

 ### EC2 - Instance Management
@@ -133,13 +142,14 @@ file, deferred to a later phase.
 | `describe-instances` | `--instance-ids`, `--filters` (instance-state-name, instance-id, instance-type, vpc-id, subnet-id, tag:\*, tag-key, tag-value) | `--max-results`, `--next-token`, `--dry-run` | None | Gateway fans out NATS `ec2.DescribeInstances` to all nodes (no queue group) → each daemon returns local instances → gateway aggregates and returns combined list. Filters applied per-node before aggregation (reduces payload). Also applies to stopped/terminated instances via `describeInstancesFromKV()`. | 1. Describe all instances (no filter)<br>2. Describe by instance ID<br>3. Describe with filters (e.g. instance-state-name)<br>4. Instance not found returns empty set<br>5. Multi-node aggregation returns instances from all nodes<br>6. Filter by tag<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `start-instances` | `--instance-ids` | `--dry-run`, `--force` | `run-instances` (instance must exist in stopped state) | Gateway sends NATS `ec2.cmd.{instance-id}` → daemon restarts stopped QEMU process with same config → state transitions stopped→pending→running | 1. Start a stopped instance<br>2. Start already-running instance (error: IncorrectInstanceState)<br>3. Start with invalid instance ID<br>4. Verify volumes re-mount on start | **DONE** |
 | `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | `run-instances` (instance must be running) | Gateway sends NATS to target node → daemon issues QMP `system_powerdown` for graceful shutdown → monitors heartbeat until QEMU exits → state transitions running→stopping→stopped | 1. Graceful stop of running instance<br>2. Force stop (kills QEMU process)<br>3. Stop already-stopped instance (error)<br>4. Verify ~30s heartbeat detection | **DONE** |
-| `terminate-instances` | `--instance-ids`, `DeleteOnTermination` (per-volume flag, default true) | `--dry-run` | `run-instances` (instance must exist) | Gateway sends NATS to target node → daemon kills QEMU process → cleans up NBD mounts → deletes volumes with `DeleteOnTermination=true` via `volumeService.DeleteVolume()` (S3 cleanup of vol/, vol-efi/, vol-cloudinit/) → internal volumes (EFI, cloud-init) always cleaned up via `ebs.delete` NATS → volumes with `DeleteOnTermination=false` left in available state → state→terminated | 1. Terminate running instance<br>2. Terminate stopped instance<br>3. Terminate with DeleteOnTermination=true deletes volumes<br>4. Terminate with DeleteOnTermination=false preserves volumes<br>5. Terminate already-terminated (idempotent)<br>6. Internal volumes (EFI, cloud-init) always cleaned up<br>7. Invalid instance ID | **DONE** |
+| `terminate-instances` | `--instance-ids`, `DeleteOnTermination` (per-volume flag, default true) | `--dry-run` | `run-instances` (instance must exist) | Gateway sends NATS to target node → daemon checks `DisableApiTermination` first and rejects with `OperationNotPermitted` if set (running or stopped, no side effects) → kills QEMU process → cleans up NBD mounts → deletes volumes with `DeleteOnTermination=true` via `volumeService.DeleteVolume()` (S3 cleanup of vol/, vol-efi/, vol-cloudinit/) → internal volumes (EFI, cloud-init) always cleaned up via `ebs.delete` NATS → volumes with `DeleteOnTermination=false` left in available state → state→terminated | 1. Terminate running instance<br>2. Terminate stopped instance<br>3. Terminate with DeleteOnTermination=true deletes volumes<br>4. Terminate with DeleteOnTermination=false preserves volumes<br>5. Terminate already-terminated (idempotent)<br>6. Internal volumes (EFI, cloud-init) always cleaned up<br>7. Invalid instance ID<br>8. Running protected instance rejected (OperationNotPermitted, state stays running)<br>9. Stopped protected instance rejected (resources intact) | **DONE** |
 | `reboot-instances` | `--instance-ids` | `--dry-run` | `run-instances` (instance must be running) | Gateway validates instance IDs → sends EC2InstanceCommand with `RebootInstance=true` via NATS `ec2.cmd.{instanceId}` → daemon validates instance is in StateRunning (returns IncorrectInstanceState if stopped) → issues QMP `system_reset` → instance reboots without stopping, stays in running state | 1. Reboot running instance<br>2. Reboot multiple instances<br>3. Reboot stopped instance (error: IncorrectInstanceState)<br>4. Instance not found (error: InvalidInstanceID.NotFound)<br>5. Verify instance stays in running state after reboot | **DONE** |
 | `describe-instance-types` | `--filters` (capacity filter only) | `--instance-types`, `--max-results`, `--next-token`, `--dry-run`, all other filters | None | Gateway fans out NATS `ec2.DescribeInstanceTypes` to all nodes → each daemon reports supported types (t3.micro/small/medium/large) with vCPU/memory specs → gateway deduplicates and returns | 1. List all instance types<br>2. Filter by specific type<br>3. Filter with `capacity=true` shows available slots<br>4. Verify vCPU/memory specs match hardware | **DONE** |
-| `modify-instance-attribute` | `--instance-id`, `--instance-type`, `--user-data` | `--disable-api-termination`, `--ebs-optimized`, `--source-dest-check`, `--instance-initiated-shutdown-behavior`, `--block-device-mappings`, `--groups`, `--ena-support`, `--sriov-net-support` | Instance must be stopped (in NATS KV) | Gateway validates input (exactly one attribute per call, instance ID format) → NATS `ec2.ModifyInstanceAttribute` with `spinifex-workers` queue group → daemon loads stopped instance from JetStream KV → applies attribute change → writes back to KV → returns `{}` on success. **InstanceType**: updates vm.InstanceType, Config, and Instance fields; clears StateReason (enables recovery from instance-type-missing bug). **UserData**: stores decoded content in vm.UserData and re-encodes to base64 for RunInstancesInput (cloud-init on next start). No instance type pre-validation (matches AWS — invalid types accepted, fail at StartInstances time). | 1. Change instance type while stopped<br>2. Change user data while stopped<br>3. Modify running instance (error: NotFound — running instances not in KV)<br>4. Instance not found (error: InvalidInstanceID.NotFound)<br>5. Instance not stopped (error: IncorrectInstanceState)<br>6. Invalid instance type accepted (fails on start with InsufficientInstanceCapacity)<br>7. StateReason cleared on type change (recovery from capacity-unavailable)<br>8. Missing/malformed instance ID (error: InvalidInstanceID.Malformed)<br>9. No attribute set (error: InvalidParameterValue)<br>10. Multiple attributes in one call (error: InvalidParameterValue) | **DONE** |
+| `modify-instance-attribute` | `--instance-id`, `--instance-type`, `--user-data`, `--disable-api-termination` | `--ebs-optimized`, `--source-dest-check`, `--instance-initiated-shutdown-behavior`, `--block-device-mappings`, `--groups`, `--ena-support`, `--sriov-net-support` | Instance must exist; `instance-type` and `user-data` require stopped state, `disable-api-termination` works on running or stopped | Gateway validates input (exactly one attribute per call, instance ID format) → NATS `ec2.ModifyInstanceAttribute` with `spinifex-workers` queue group → daemon dispatches by state: stopped instances loaded from JetStream KV, running instances updated via `vm.Manager.UpdateAndPersist` (atomic flag check + persist under one lock to prevent Modify/Terminate race) → returns `{}` on success. **InstanceType**: updates vm.InstanceType, Config, and Instance fields; clears StateReason (enables recovery from instance-type-missing bug). **UserData**: stores decoded content in vm.UserData and re-encodes to base64 for RunInstancesInput (cloud-init on next start). **DisableApiTermination**: stamps `RunInstancesInput.DisableApiTermination`; flag survives stop/start and daemon restarts. No instance type pre-validation (matches AWS — invalid types accepted, fail at StartInstances time). | 1. Change instance type while stopped<br>2. Change user data while stopped<br>3. Toggle disable-api-termination on running instance<br>4. Toggle disable-api-termination on stopped instance<br>5. Modify instance-type on running instance (error: NotFound — running instances not in KV)<br>6. Instance not found (error: InvalidInstanceID.NotFound)<br>7. Instance not stopped for type/user-data change (error: IncorrectInstanceState)<br>8. Invalid instance type accepted (fails on start with InsufficientInstanceCapacity)<br>9. StateReason cleared on type change (recovery from capacity-unavailable)<br>10. Missing/malformed instance ID (error: InvalidInstanceID.Malformed)<br>11. No attribute set (error: InvalidParameterValue)<br>12. Multiple attributes in one call (error: InvalidParameterValue) | **DONE** |
 | `get-console-output` | `--instance-id` | `--latest` (always returns latest), `--dry-run` | Instance must be running on a node | Gateway sends NATS `ec2.{instanceId}.GetConsoleOutput` (per-instance topic, routed to owning node) → daemon reads console log file from disk → returns last 64KB base64-encoded with timestamp. Always available regardless of serial console access setting (matches AWS behavior). | 1. Get output from running instance<br>2. Empty log file returns empty output<br>3. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
-| `describe-instance-attribute` | `--instance-id`, `--attribute` (instanceType, userData, instanceInitiatedShutdownBehavior, disableApiTermination, disableApiStop, ebsOptimized, enaSupport, sourceDestCheck, rootDeviceName, kernel, ramdisk) | `--dry-run` | Instance must exist (running or stopped) | Gateway validates input → NATS `ec2.DescribeInstanceAttribute` with `spinifex-workers` queue group → daemon checks running instances first (`d.Instances.VMS`), then stopped instances in JetStream KV → returns single attribute per call (matches AWS behavior). Stored attributes (`instanceType`, `userData`) return real values; unstored attributes return AWS defaults (`instanceInitiatedShutdownBehavior`=stop, `disableApiTermination`=false, etc.) | 1. Get instanceType from running instance<br>2. Get userData from stopped instance<br>3. Get default disableApiTermination<br>4. Invalid attribute name (error)<br>5. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
+| `describe-instance-attribute` | `--instance-id`, `--attribute` (instanceType, userData, instanceInitiatedShutdownBehavior, disableApiTermination, disableApiStop, ebsOptimized, enaSupport, sourceDestCheck, rootDeviceName, kernel, ramdisk) | `--dry-run` | Instance must exist (running or stopped) | Gateway validates input → NATS `ec2.DescribeInstanceAttribute` with `spinifex-workers` queue group → daemon checks running instances first (`d.Instances.VMS`), then stopped instances in JetStream KV → returns single attribute per call (matches AWS behavior). Stored attributes (`instanceType`, `userData`, `disableApiTermination`) return real values from `RunInstancesInput`; remaining attributes return AWS defaults (`instanceInitiatedShutdownBehavior`=stop, etc.) | 1. Get instanceType from running instance<br>2. Get userData from stopped instance<br>3. Get disableApiTermination (reflects RunInstances flag + Modify updates)<br>4. Invalid attribute name (error)<br>5. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
 | `describe-instance-credit-specifications` | `--instance-ids` | `--filters`, `--max-results`, `--dry-run` | None | Gateway-only stub — returns `CpuCredits: "standard"` for each requested instance ID. No daemon round-trip. T-series credit mode is not persisted. | 1. Get credit spec for T-series instance<br>2. Multiple instance IDs | **DONE** |
+| `describe-instance-status` | `--instance-ids`, `--include-all-instances`, `--filters` (availability-zone, instance-state-code, instance-state-name, tag:\*) | `--max-results`, `--next-token`, `--dry-run` (silently ignored), `event.*` / `instance-status.*` / `system-status.*` filters (rejected with InvalidParameterValue) | None | Gateway fans out NATS `ec2.DescribeInstanceStatus` to all nodes (no queue group). Each daemon validates input (i- prefix, filter names), iterates local VMs and renders InstanceStatus: `Running`→ok/passed, everything-else-included→not-applicable. When `IncludeAllInstances=true`, gateway also queries `ec2.DescribeStoppedInstances` in parallel and transforms reservation entries into InstanceStatus with `not-applicable`. Aggregated results dedup'd by InstanceId (running > stopped). Terminated and error-state instances never returned. No real health probing — static values reflect Mulga's lifecycle state only. | 1. Status of running instance (ok/passed)<br>2. Filter by instance ID<br>3. Default excludes stopped instances<br>4. IncludeAllInstances=true includes stopped (not-applicable)<br>5. Cross-account instance excluded<br>6. Unknown filter returns InvalidParameterValue<br>7. Malformed instance ID rejected<br>8. Tag filter narrows correctly | **DONE** |
 | `monitor-instances` | — | `--instance-ids` | Instance must exist | Enable basic monitoring (CPU, disk, network) → store metrics in NATS KV → return monitoring state | 1. Enable monitoring<br>2. Verify monitoring state in describe-instances | **NOT STARTED** |
 | `unmonitor-instances` | — | `--instance-ids` | Instance must exist | Disable detailed monitoring → revert to basic monitoring → return monitoring state | 1. Disable monitoring<br>2. Verify monitoring state reverts in describe-instances | **NOT STARTED** |

@@ -336,6 +346,7 @@ Requires OVN/OVS (`apt install openvswitch-switch ovn-host`). VPC/Subnet/ENI/SG
 | `authorize-security-group-egress` | `--group-id`, `--ip-permissions` | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.AuthorizeSecurityGroupEgress` → daemon retrieves SG → appends rules to EgressRules → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Allow HTTPS outbound<br>2. Restrict to specific CIDR<br>3. Missing GroupId (MissingParameter)<br>4. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
 | `revoke-security-group-ingress` | `--group-id`, `--ip-permissions` (matching rules to remove) | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.RevokeSecurityGroupIngress` → daemon retrieves SG → removes matching rules from IngressRules (set difference) → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Revoke existing rule<br>2. Missing GroupId (MissingParameter)<br>3. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
 | `revoke-security-group-egress` | `--group-id`, `--ip-permissions` (matching rules to remove) | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.RevokeSecurityGroupEgress` → daemon retrieves SG → removes matching rules from EgressRules (set difference) → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Revoke existing rule<br>2. Missing GroupId (MissingParameter)<br>3. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
+| `describe-security-group-rules` | `--filters` (group-id, security-group-rule-id, tag:\*, tag-key), `--security-group-rule-ids` | `--max-results`, `--next-token`, `--dry-run` | None | Gateway validates `sgr-` format on SecurityGroupRuleIds (nil input = all rules in account) → NATS `ec2.DescribeSecurityGroupRules` → daemon iterates SG records in `spinifex-vpc-security-groups` KV under account → applies filters → emits one `SecurityGroupRule` per ingress (IsEgress=false) and egress (IsEgress=true) rule, each with stable `sgr-` ID, GroupOwnerId=accountID, ReferencedGroupInfo for SG-source rules → returns `InvalidSecurityGroupRuleId.NotFound` if any requested ID missing → v1→v2 KV migration backfills `sgr-` IDs on existing rules at daemon startup; MaxResults/NextToken accepted but ignored | 1. List all rules in account<br>2. Filter by group-id<br>3. Filter by security-group-rule-id<br>4. Non-existent rule ID (InvalidSecurityGroupRuleId.NotFound)<br>5. Malformed sgr- ID at gateway (InvalidParameterValue)<br>6. Tag filters match zero (Tags not yet populated on rules)<br>7. Unknown filter (InvalidParameterValue)<br>8. Migration backfills sgr- IDs on pre-v2 records | **DONE** |
 | `create-internet-gateway` | `--tag-specifications` | `--dry-run` | None | NATS `ec2.CreateInternetGateway` → daemon generates igw-ID → creates IGWRecord in `spinifex-igw` KV bucket with state=available (not attached) → returns InternetGateway with empty Attachments | 1. Create IGW<br>2. Verify in describe-internet-gateways<br>3. Tags applied at creation | **DONE** |
 | `attach-internet-gateway` | `--internet-gateway-id`, `--vpc-id` | `--dry-run` | IGW and VPC must exist | NATS `ec2.AttachInternetGateway` → daemon validates IGW exists and is not already attached (ResourceAlreadyAssociated) → sets VpcId on record → state=attached → publishes `vpc.igw-attach` event to vpcd (creates OVN external switch, gateway port, SNAT rules) | 1. Attach IGW to VPC<br>2. Attach already-attached IGW (ResourceAlreadyAssociated)<br>3. Non-existent IGW (error)<br>4. Missing params (MissingParameter) | **DONE** |
 | `detach-internet-gateway` | `--internet-gateway-id`, `--vpc-id` | `--dry-run` | IGW must be attached to specified VPC | NATS `ec2.DetachInternetGateway` → daemon validates IGW exists and is attached to specified VPC (GatewayNotAttached if wrong VPC) → clears VpcId → state=available → publishes `vpc.igw-detach` event to vpcd (removes OVN resources) | 1. Detach attached IGW<br>2. Detach from wrong VPC (GatewayNotAttached)<br>3. Detach unattached IGW (error) | **DONE** |
@@ -0,0 +1,223 @@
+---
+title: "GPU Passthrough"
+description: "Configure VFIO GPU passthrough on a Spinifex node to expose NVIDIA or AMD GPUs to EC2 instances."
+category: "Compute"
+tags:
+  - gpu
+  - passthrough
+  - vfio
+  - ec2
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS EC2 Accelerated Computing Instance Types"
+    url: "https://docs.aws.amazon.com/ec2/latest/instancetypes/ac.html"
+---
+
+# GPU Passthrough
+
+> Configure VFIO GPU passthrough on a Spinifex node to expose NVIDIA or AMD GPUs to EC2 instances.
+
+> **Note:** GPU passthrough is supported on x86_64 hosts only.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Host Setup](#host-setup)
+- [Enable Passthrough](#enable-passthrough)
+- [Status and Monitoring](#status-and-monitoring)
+- [GPU Instance Types](#gpu-instance-types)
+- [Launching GPU Instances](#launching-gpu-instances)
+- [Disable Passthrough](#disable-passthrough)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Spinifex supports VFIO-based GPU passthrough, binding NVIDIA and AMD GPUs directly to guest VMs via the `vfio-pci` kernel driver. Once configured, the node exposes GPU-enabled EC2 instance types and the GPU is allocated exclusively to individual instances.
+
+At startup, Spinifex always probes for GPU hardware and surfaces the result in the node banner and in `spx top nodes` — regardless of whether passthrough is enabled. This lets operators verify hardware before activating the feature.
+
+The passthrough state is reflected in the login banner:
+
+![GPU not yet enabled](../../../.github/assets/images/gpu-banner1.png)
+
+---
+
+## Prerequisites
+
+- IOMMU must be supported and enabled in the host BIOS/UEFI (`Intel VT-d` or `AMD-Vi`).
+- An NVIDIA or AMD GPU must be physically installed.
+- The Spinifex node must be running on bare metal (not inside a VM).
+
+---
+
+## Host Setup
+
+Run the one-time host configuration as root. This command is idempotent — safe to re-run after a reboot.
+
+```bash
+sudo spx admin gpu setup
+```
+
+What it does:
+
+1. Detects installed GPUs and their PCI IDs.
+2. Enables IOMMU in GRUB (`intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt`).
+3. Writes the vfio udev rule (`/etc/udev/rules.d/99-spinifex-vfio.rules`).
+4. Blacklists `nouveau` (NVIDIA) and `amdgpu` (AMD) so the host kernel does not claim the device.
+5. Configures `vfio-pci` early binding in `/etc/modprobe.d/vfio-pci.conf`.
+6. Adds vfio modules to initramfs.
+
+If any change requires a reboot, the command prints instructions and exits:
+
+```
+Setup complete — reboot required.
+  sudo reboot
+  Then run: sudo spx admin gpu enable
+```
+
+---
+
+## Enable Passthrough
+
+After setup and reboot, enable GPU passthrough:
+
+```bash
+sudo spx admin gpu enable
+```
+
+The command writes the configuration, signals the daemon (`SIGHUP`), and waits up to 30 seconds for the daemon to confirm the new state. On success it prints the current GPU status.
+
+
+![GPU enabled](../../../.github/assets/images/gpu-enabled.png)
+
+---
+
+## Status and Monitoring
+
+### Per-node status
+
+```bash
+spx admin gpu status
+# or for a specific node in the cluster:
+spx admin gpu status --node <node-name>
+```
+
+Output includes hardware detected, IOMMU state, vfio-pci binding, passthrough enabled/disabled, GPU pool allocation (`allocated/total`), and the GPU-capable EC2 instance types available on that node.
+
+### Cluster view
+
+`spx top nodes` includes a `GPU (used/total)` column:
+
+| Value | Meaning |
+|-------|---------|
+| `1/2` | Passthrough active — 1 of 2 GPUs allocated |
+| `0/1*` | Node has a GPU but passthrough is not enabled |
+| `-` | No GPU hardware detected |
+
+```bash
+spx top nodes
+```
+
+---
+
+## GPU Instance Types
+
+GPU instance types are derived from detected hardware.
+
+To list available GPU instance types on the current node:
+
+```bash
+aws ec2 describe-instance-types \
+  --filters Name=instance-type,Values=g*
+```
+
+---
+
+## Launching GPU Instances
+
+Import the GPU-enabled AMI before launching:
+
+```bash
+spx admin images import --name ubuntu-gpu-nvidia   # NVIDIA hosts
+spx admin images import --name ubuntu-gpu-amd      # AMD hosts
+```
+
+Launch a GPU instance:
+
+```bash
+aws ec2 run-instances \
+  --image-id $GPU_AMI \
+  --instance-type $INSTANCE_TYPE \
+  --key-name spinifex-key
+```
+
+To verify the GPU is visible from inside the instance, SSH in and run:
+
+```bash
+# NVIDIA
+nvidia-smi
+
+# AMD
+rocm-smi
+```
+
+---
+
+## Disable Passthrough
+
+Passthrough cannot be disabled while GPU instances are running. Terminate all GPU instances first, then:
+
+```bash
+sudo spx admin gpu disable
+```
+
+The command signals the daemon and waits for confirmation. The GPU is released back to the host kernel on the next reboot (the vfio-pci binding persists until `setup` is re-run without the blacklists).
+
+---
+
+## Troubleshooting
+
+### IOMMU not active after reboot
+
+Verify IOMMU is enabled in BIOS/UEFI (`Intel VT-d` or `AMD-Vi`). Then check GRUB picked up the parameters:
+
+```bash
+cat /proc/cmdline | grep iommu
+ls /sys/kernel/iommu_groups/
+```
+
+If `/sys/kernel/iommu_groups/` is empty, IOMMU is not active. Re-run `sudo spx admin gpu setup` after enabling it in firmware.
+
+### GPU bound to native driver instead of vfio-pci
+
+```bash
+lspci -k | grep -A3 -i "vga\|3d\|display"
+```
+
+If the driver is `amdgpu` or `nvidia` rather than `vfio-pci`, the blacklist or early binding did not take effect. Re-run setup and reboot:
+
+```bash
+sudo spx admin gpu setup
+sudo reboot
+```
+
+### `spx admin gpu enable` fails with "prerequisites not met"
+
+Run `setup` first to configure the host, then retry `enable`:
+
+```bash
+sudo spx admin gpu setup
+sudo spx admin gpu enable
+```
+
+### Daemon does not confirm within 30 seconds
+
+Check the daemon log for errors:
+
+```bash
+journalctl -u spinifex-daemon -n 50
+```
@@ -573,6 +573,54 @@ external_interface = "br-wan"
 DHCP server when VMs launch and releases them on terminate. Requires `dhclient`
 on the host (`apt install isc-dhcp-client`).

+## Host-Local Subnet (No Upstream Router)
+
+```
+Network: 192.168.10.0/24 — host-local, reachable from the host only
+Host WAN: 198.51.100.10/24 on br-wan (existing address — unchanged)
+Gateway: 192.168.10.1 — second address added to br-wan
+```
+
+Add the VM pool gateway as a second address on `br-wan` alongside the existing WAN
+IP. The host acts as the gateway for the pool — no upstream router or DHCP server
+needed for this range.
+
+```yaml
+# /etc/netplan/…
+bridges:
+  br-wan:
+    addresses:
+      - 192.168.10.1/24      # VM pool gateway — host-local
+      - 198.51.100.10/24     # existing WAN IP — unchanged
+    routes:
+      - to: default
+        via: 198.51.100.1
+```
+
+```toml
+[network]
+external_mode = "pool"
+
+[[network.external_pools]]
+name        = "wan"
+source      = "static"         # required — no upstream DHCP for this range
+range_start = "192.168.10.2"
+range_end   = "192.168.10.100"
+gateway     = "192.168.10.1"   # second address on br-wan
+prefix_len  = 24
+dns_servers = ["8.8.8.8"]
+```
+
+**Setup:** Apply with `sudo netplan apply`. VMs are reachable from the host at
+`192.168.10.x`. For internet access through the host's WAN interface:
+
+```bash
+sysctl -w net.ipv4.ip_forward=1
+iptables -t nat -A POSTROUTING -s 192.168.10.0/24 -o br-wan -j MASQUERADE
+```
+
+Persist via `/etc/sysctl.d/99-ip-forward.conf` and `netfilter-persistent save`.
+
 ## Datacenter / Colo (ISP Block)

 ```
@@ -60,7 +60,6 @@ spx admin images list
 NAME                    | DISTRO | VERSION | ARCH   | BOOT
 debian-13-arm64         | debian | 13      | arm64  | bios
 debian-13-x86_64        | debian | 13      | x86_64 | bios
-lb-alpine-3.21.6-x86_64 | alpine | 3.21.6  | x86_64 | bios
 ubuntu-26.04-arm64      | ubuntu | 26.04   | arm64  | bios
 ubuntu-26.04-x86_64     | ubuntu | 26.04   | x86_64 | bios
 ```
@@ -1,47 +0,0 @@
-# Configuring GPU Passthrough for Spinifex Nodes
-
-
-Spinifex is designed for bare-metal hardware, edge nodes and data-centre use. Follow this guide to configure your Spinifex node to enable GPU Passthrough to its VM instances if hardware is available.
-
->**Note:** this tutorial is for x86 architecture.
-
-
-
-### Getting Started - GPU Setup
-By default, Spinifex will probe for the presence of a GPU on its machine. After booting, the presence and availability of GPU hardware is printed in the info banner:
-![Installer complete](../../.github/assets/images/gpu-banner1.png)
-
-To enable GPU passthrough, you may first need to run:
-```bash
-sudo spx admin gpu setup
-```
-Note that this command will blacklist the GPU's drivers from the host machine such that they can be used by VM instances Therefore the GPU will **NOT** be available to the host machine until after the VM instance  It will also require rebooting the host machine before the changes take effect.
-
-### Enable Passthrough
-After reboot, the banner message should reflect that GPU passthrough is **ready** but not **enabled**.
-
-![Installer complete](../../.github/assets/images/gpu-banner2.png)
-
-This will require you to run:
-```bash
-sudo spx admin gpu enable
-```
-After running this command, you should see:
-
-![Installer complete](../../.github/assets/images/gpu-enabled.png)
-
-The host machine is now configured to provide GPU passthrough to EC2 instances on the host machine.
-
-Lastly, import the Spinifex GPU enabled AMI: [link]
-
-As a final check, spin up an EC2 instance, SSH into it, and check that the host machine's GPU is visible from the EC2 instance.
-
-### Multiple GPUs
-Spinifex is able to detect multiple GPUs. After running the setup commands, the total number of allocatable GPUs is printed.
-
-### Mapping GPUs to EC2 instances
-By defualt, any consumer grade GPUs are mapped to the g5 EC2 instance class, which is the most basic "Accelerated Computing" Ec2 instance provided by AWS. If using a consumer GPU, running:
-```bash
-aws ec2 describe-instance-types
-```
-will return the various sizes of g5 class. All other AWS "Accelerated Computing" EC2 instance classes map directly to their associated GPU, so the Spinifex host machine will need to have the specified GPU if a specific EC2 instance class is desired.
@@ -86,22 +86,16 @@ Or create a `main.tf` file and paste the full configuration below.

 <!-- INCLUDE: main.tf lang:hcl -->

-### Step 2. Install Load Balancer AMI
+### Step 2. Install Debian AMI

-Next install the load balancer AMI images, which is used as the AMI disk image for launching the load-balancer and a requirement.
-
-```bash
-spx admin images import --name lb-alpine-3.21.6-x86_64
-```
-
-### Step 2.1 Install Debian AMI
-
-Next, install the Debian 13 AMI which is used in the example to host the `nginx` webservers as an EC2 instance.
+Install the Debian 13 AMI which is used in the example to host the `nginx` webservers as an EC2 instance.

 ```bash
 spx admin images import --name debian-13-x86_64
 ```

+> **Note:** The load balancer itself runs as a direct-boot QEMU microvm using the kernel + initramfs bundled with the Spinifex distribution (`/usr/share/spinifex/microvm/`), so no separate LB AMI import is required.
+
 ### Step 3. Deploy

 ```bash
@@ -12,8 +12,8 @@ require (
 	github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mulgadc/predastore v1.3.0
-	github.com/mulgadc/viperblock v1.3.0
+	github.com/mulgadc/predastore v1.4.0
+	github.com/mulgadc/viperblock v1.4.0
 	github.com/nats-io/nats-server/v2 v2.14.0
 	github.com/nats-io/nats.go v1.52.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
@@ -104,7 +104,7 @@ require (
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.20.1 // indirect
-	github.com/quic-go/quic-go v0.59.0 // indirect
+	github.com/quic-go/quic-go v0.59.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.12.0 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
@@ -121,7 +121,7 @@ require (
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/mod v0.35.0 // indirect
-	golang.org/x/net v0.53.0 // indirect
+	golang.org/x/net v0.54.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.44.0 // indirect
 	golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect
@@ -286,10 +286,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/predastore v1.3.0 h1:Zf0rIuyQ1mi9Gpd4ihLqik1gqn8L/VKLf6P6Tk9Qs0Q=
-github.com/mulgadc/predastore v1.3.0/go.mod h1:nH5lBc8iXIw4fXXXNAnrJiRO47yZ+ZBCjaqjiCRoGOQ=
-github.com/mulgadc/viperblock v1.3.0 h1:vrDCNqNpLmolbTbgEHMSz8CBjoc8xkmQdtwKhZfrqew=
-github.com/mulgadc/viperblock v1.3.0/go.mod h1:WCP9h+u2DvN8/+ccai5oMOa48hOJPUzSbh7Lia0Q8nY=
+github.com/mulgadc/predastore v1.4.0 h1:txPZCaVT1cDpBotx8zHPXHZD3Mz62+n207zXISkK4uU=
+github.com/mulgadc/predastore v1.4.0/go.mod h1:fXIGz7yCKcHh68ybcw7aigdOdzIfyt19h/0diPF+JCU=
+github.com/mulgadc/viperblock v1.4.0 h1:3Yr6lRDnFgxkdMOWu+Tq1b97qWHQDNwkPWmCp38CIy8=
+github.com/mulgadc/viperblock v1.4.0/go.mod h1:hHdfo02iJgCdNktiPAq+x+VQE0ZNYOLJXw0abJ81LeE=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -351,8 +351,8 @@ github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5b
 github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s=
 github.com/pterm/pterm v0.12.83 h1:ie+YmGmA727VuhxBlyGr74Ks+7McV6kT99IB8EU80aA=
 github.com/pterm/pterm v0.12.83/go.mod h1:xlgc6bFWyJIMtmLJvGim+L7jhSReilOlOnodeIYe4Tk=
-github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
-github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
+github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic=
+github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -442,8 +442,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
-golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
+golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
+golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -0,0 +1,26 @@
+// Package fipsboot enforces FIPS 140-3 mode at process startup.
+//
+// Blank-import this package from every binary's main:
+//
+//	import _ "github.com/mulgadc/spinifex/internal/fipsboot"
+//
+// Builds with GOFIPS140=v1.0.0 link the Go Cryptographic Module v1.0.0
+// (NIST CMVP cert #5247). At runtime FIPS mode can still be turned off
+// via GODEBUG=fips140=off; init() panics in that case to close the hole.
+package fipsboot
+
+import (
+	"crypto/fips140"
+	"log/slog"
+)
+
+func init() {
+	if !fips140.Enabled() {
+		panic("fipsboot: FIPS 140-3 mode is not enabled at runtime — refusing to start. " +
+			"Build with GOFIPS140=v1.0.0 and do not set GODEBUG=fips140=off.")
+	}
+	slog.Info("FIPS 140-3 mode enabled",
+		"module", "Go Cryptographic Module v1.0.0",
+		"cmvp_cert", "5247",
+	)
+}
@@ -31,6 +31,7 @@ inventory and roles relative to CWD.
 | `playbooks/dev-teardown.yml` | Wipe state only (no reinstall) | `reset-dev-env.sh` §teardown |
 | `playbooks/dev-install.yml` | Build + install + init + smoketest on clean box | `dev-install.sh` |
 | `playbooks/dev-reset.yml` | Capture settings → teardown → build → install → init → smoketest | `reset-dev-env.sh` (full) |
+| `playbooks/dev-deploy.yml` | Rebuild + swap binaries/microvm artifacts + restart (no setup.sh, no smoketest) | `make deploy` |

 Upcoming (not yet implemented):

@@ -44,6 +45,7 @@ ansible-playbook playbooks/dev-preflight.yml
 ansible-playbook playbooks/dev-teardown.yml
 ansible-playbook playbooks/dev-install.yml
 ansible-playbook playbooks/dev-reset.yml
+ansible-playbook playbooks/dev-deploy.yml
 ```

 Or via `make` (from `spinifex/`):
@@ -53,8 +55,16 @@ make ansible-dev-preflight
 make ansible-dev-teardown
 make ansible-dev-install
 make ansible-dev-reset
+make ansible-dev-deploy
 ```

+### When to use which
+
+- First-time / clean box → `ansible-dev-install`
+- Iterate on Go code, microvm initramfs, lb-agent → `ansible-dev-deploy` (fast)
+- Changed systemd units, helper scripts, logrotate, setup.sh → `ansible-dev-reset` (slow, full rebuild)
+- Need a clean slate without reinstall → `ansible-dev-teardown`
+
 ## Variable overrides

 Defaults in `vars/defaults.yml`. Override per run with `-e`:
@@ -74,7 +84,6 @@ Useful overrides:
 | `spinifex_region` | `ap-southeast-2` | Region passed to `spx admin init` |
 | `spinifex_az` | `{{ region }}a` | AZ passed to `spx admin init` |
 | `spinifex_external_mode` | `pool` | `pool` \| `nat` external networking |
-| `spinifex_build_lb_image` | `true` | Build + import LB system image after init |

 `dev-reset.yml` automatically captures `region`, `az`, `external_mode`,
 pool range, gateway, prefix-len, nat gateway_ip from the existing
@@ -0,0 +1,14 @@
+---
+# In-place artifact swap: rebuilds + replaces binaries/microvm artifacts on
+# an already-installed node, then restarts spinifex.target. Use when iterating
+# on code without altering systemd units, users, dirs, or running the full
+# smoketest. For first-time or unit/script changes use dev-install / dev-reset.
+- name: Dev deploy — build + swap binaries + restart (no setup.sh, no smoketest)
+  hosts: localhost
+  gather_facts: true
+  vars_files:
+    - ../vars/defaults.yml
+
+  roles:
+    - build
+    - deploy
@@ -11,6 +11,18 @@
    chdir: "{{ mulga_root }}/viperblock"
  changed_when: true

+- name: build — detect microVM build script (present only on direct-boot branches)
+  ansible.builtin.stat:
+    path: "{{ spinifex_root }}/scripts/build-microvm-image.sh"
+  register: build_microvm_script
+
+- name: build — microVM kernel + initramfs (direct-boot LBs)
+  ansible.builtin.command:
+    cmd: make build-microvm-image
+    chdir: "{{ spinifex_root }}"
+  changed_when: true
+  when: build_microvm_script.stat.exists
+
 - name: build — verify build artifacts present
  ansible.builtin.stat:
    path: "{{ item }}"
@@ -40,9 +52,15 @@
    mkdir -p "$STAGING/scripts"
    cp "{{ spinifex_root }}/build/scripts/"* "$STAGING/scripts/"
    cp "{{ spinifex_root }}/build/logrotate/spinifex" "$STAGING/logrotate-spinifex"
+    # microVM artifacts ship only when the branch built them.
+    if [ -f "{{ spinifex_root }}/build/microvm/vmlinuz" ] && [ -f "{{ spinifex_root }}/build/microvm/initramfs.cpio.gz" ]; then
+        mkdir -p "$STAGING/microvm"
+        cp "{{ spinifex_root }}/build/microvm/vmlinuz" "$STAGING/microvm/"
+        cp "{{ spinifex_root }}/build/microvm/initramfs.cpio.gz" "$STAGING/microvm/"
+    fi
    tar czf "{{ build_tarball_path }}" -C "$STAGING" .
  changed_when: true

 - name: build — summary
  ansible.builtin.debug:
-    msg: "Built spx + nbdkit plugin, tarball at {{ build_tarball_path }}"
+    msg: "Built spx + nbdkit plugin{{ ' + microvm artifacts' if build_microvm_script.stat.exists else '' }}, tarball at {{ build_tarball_path }}"
@@ -0,0 +1,148 @@
+---
+# In-place artifact swap for an already-installed spinifex node. Mirrors
+# `make deploy`: replace binaries + microvm artifacts, restart services.
+# Does NOT run setup.sh (no user/dir/systemd-unit changes) or smoketest.
+
+- name: deploy — fail fast if spinifex isn't already installed
+  ansible.builtin.stat:
+    path: "{{ spinifex_config_file }}"
+  register: deploy_installed_stat
+  failed_when: not deploy_installed_stat.stat.exists
+
+- name: deploy — assert config present
+  ansible.builtin.assert:
+    that: deploy_installed_stat.stat.exists
+    fail_msg: |
+      {{ spinifex_config_file }} missing — this playbook only updates an
+      existing install. Run ansible-dev-install or ansible-dev-reset first.
+
+- name: deploy — verify build artifacts present
+  ansible.builtin.stat:
+    path: "{{ item }}"
+  register: deploy_artifacts
+  loop:
+    - "{{ spinifex_root }}/bin/spx"
+    - "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so"
+
+- name: deploy — assert artifacts exist after build role
+  ansible.builtin.assert:
+    that: item.stat.exists
+    fail_msg: "Build artifact missing: {{ item.item }} — build role should have produced it"
+  loop: "{{ deploy_artifacts.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+
+- name: deploy — detect microvm artifacts (direct-boot branches only)
+  ansible.builtin.stat:
+    path: "{{ spinifex_root }}/build/microvm/{{ item }}"
+  register: deploy_microvm_stat
+  loop:
+    - vmlinuz
+    - initramfs.cpio.gz
+
+- name: deploy — set microvm presence fact
+  ansible.builtin.set_fact:
+    deploy_microvm_present: "{{ deploy_microvm_stat.results | map(attribute='stat.exists') | min }}"
+
+- name: deploy — detect nbdkit plugin directory
+  ansible.builtin.shell: |
+    set -e
+    nbdkit --dump-config | awk -F= '/^plugindir=/{print $2; exit}'
+  register: deploy_plugindir_cmd
+  changed_when: false
+
+- name: deploy — set plugin path fact
+  ansible.builtin.set_fact:
+    deploy_plugindir: "{{ deploy_plugindir_cmd.stdout | trim }}"
+
+- name: deploy — assert plugindir resolved
+  ansible.builtin.assert:
+    that: deploy_plugindir | length > 0
+    fail_msg: "Could not detect nbdkit plugin directory via `nbdkit --dump-config`"
+
+- name: deploy — stop spinifex.target before binary swap
+  become: true
+  ansible.builtin.systemd:
+    name: spinifex.target
+    state: stopped
+  failed_when: false
+
+- name: deploy — install spx binary
+  become: true
+  ansible.builtin.copy:
+    src: "{{ spinifex_root }}/bin/spx"
+    dest: /usr/local/bin/spx
+    mode: "0755"
+    owner: root
+    group: root
+
+- name: deploy — install viperblock nbdkit plugin
+  become: true
+  ansible.builtin.copy:
+    src: "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so"
+    dest: "{{ deploy_plugindir }}/nbdkit-viperblock-plugin.so"
+    mode: "0755"
+    owner: root
+    group: root
+
+- name: deploy — ensure microvm dir
+  become: true
+  ansible.builtin.file:
+    path: /usr/share/spinifex/microvm
+    state: directory
+    mode: "0755"
+    owner: root
+    group: root
+  when: deploy_microvm_present
+
+- name: deploy — install microvm kernel
+  become: true
+  ansible.builtin.copy:
+    src: "{{ spinifex_root }}/build/microvm/vmlinuz"
+    dest: /usr/share/spinifex/microvm/vmlinuz
+    mode: "0644"
+    owner: root
+    group: root
+  when: deploy_microvm_present
+
+- name: deploy — install microvm initramfs
+  become: true
+  ansible.builtin.copy:
+    src: "{{ spinifex_root }}/build/microvm/initramfs.cpio.gz"
+    dest: /usr/share/spinifex/microvm/initramfs.cpio.gz
+    mode: "0644"
+    owner: root
+    group: root
+  when: deploy_microvm_present
+
+- name: deploy — daemon-reload (covers any unit changes shipped via setup.sh later)
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: deploy — start spinifex.target
+  become: true
+  ansible.builtin.systemd:
+    name: spinifex.target
+    state: started
+
+- name: deploy — wait for awsgw to respond
+  become: true
+  become_user: "{{ spinifex_invoking_user }}"
+  environment:
+    AWS_PROFILE: spinifex
+  ansible.builtin.command: aws ec2 describe-regions
+  register: deploy_awsgw
+  retries: 15
+  delay: 2
+  until: deploy_awsgw.rc == 0
+  changed_when: false
+
+- name: deploy — summary
+  ansible.builtin.debug:
+    msg: |
+      Deployed:
+        spx           → /usr/local/bin/spx
+        nbdkit plugin → {{ deploy_plugindir }}/nbdkit-viperblock-plugin.so
+        microvm       → {{ '/usr/share/spinifex/microvm/{vmlinuz,initramfs.cpio.gz}' if deploy_microvm_present else 'skipped (not built on this branch)' }}
+      spinifex.target restarted; awsgw responding.
@@ -74,19 +74,6 @@
  delay: 2
  changed_when: false

- name: init — build LB agent
-  ansible.builtin.command:
-    cmd: make build-lb-agent
-    chdir: "{{ spinifex_root }}"
-  changed_when: true
-  when: spinifex_build_lb_image | bool
-
- name: init — import LB system image
-  ansible.builtin.command:
-    cmd: "{{ spinifex_root }}/scripts/build-system-image.sh {{ spinifex_lb_image_conf }} --import --quiet"
-  changed_when: true
-  when: spinifex_build_lb_image | bool
-
 - name: init — summary
  ansible.builtin.debug:
    msg: |
@@ -95,4 +82,3 @@
        external_mode={{ spinifex_external_mode }}
        spinifex.target: started
        CA: {{ spinifex_ca_trust_path }}
-        LB image: {{ 'imported' if spinifex_build_lb_image else 'skipped' }}
@@ -79,10 +79,6 @@ mulga_root: "{{ lookup('pipe', 'readlink -f ' ~ playbook_dir ~ '/../../../..') }
 # Build output.
 build_tarball_path: /tmp/spinifex-local.tar.gz

-# LB system image (built + imported at end of init).
-spinifex_build_lb_image: true
-spinifex_lb_image_conf: "{{ spinifex_root }}/scripts/images/lb.conf"
-
 # Smoketest image — parameterised so tofu examples can share the same base.
 spinifex_smoketest_distro: debian
 spinifex_smoketest_version: "13"
@@ -0,0 +1,261 @@
+#!/bin/bash
+# build-microvm-image.sh — build a minimal Alpine-based microVM kernel + initramfs
+# for use with QEMU microvm machine type and Spinifex lb-agent.
+#
+# Outputs:
+#   $REPO_ROOT/build/microvm/vmlinuz
+#   $REPO_ROOT/build/microvm/initramfs.cpio.gz
+#
+# Rootfs strategy (first match wins):
+#   1. apk available on host → install directly into a temp chroot (Alpine CI/CD)
+#   2. docker available      → run alpine container, export filesystem
+#   3. podman available      → same via podman
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+BUILD_DIR="$REPO_ROOT/build/microvm"
+INIT_SH="$BUILD_DIR/init.sh"
+INITTAB="$BUILD_DIR/inittab"
+
+ALPINE_VERSION="${ALPINE_VERSION:-3.20}"
+ALPINE_MIRROR="${ALPINE_MIRROR:-https://dl-cdn.alpinelinux.org/alpine}"
+ALPINE_PACKAGES="busybox linux-virt haproxy iproute2 ca-certificates"
+
+echo "[build-microvm-image] repo root: $REPO_ROOT"
+echo "[build-microvm-image] build dir: $BUILD_DIR"
+mkdir -p "$BUILD_DIR"
+
+# --- Verify non-negotiable tools ---
+for tool in cpio gzip find; do
+    if ! command -v "$tool" >/dev/null 2>&1; then
+        echo "ERROR: required tool not found: $tool" >&2
+        exit 1
+    fi
+done
+
+# --- Create chroot ---
+CHROOT_DIR=$(mktemp -d)
+cleanup() {
+    echo "[build-microvm-image] cleaning up chroot: $CHROOT_DIR"
+    rm -rf "$CHROOT_DIR"
+}
+trap cleanup EXIT
+
+# --- Populate rootfs ---
+# Strategy 1: native apk (Alpine hosts, CI)
+if command -v apk >/dev/null 2>&1; then
+    echo "[build-microvm-image] rootfs: native apk"
+    mkdir -p \
+        "$CHROOT_DIR/etc/apk" \
+        "$CHROOT_DIR/lib/apk/db" \
+        "$CHROOT_DIR/var/cache/apk"
+    cat > "$CHROOT_DIR/etc/apk/repositories" <<EOF
+${ALPINE_MIRROR}/v${ALPINE_VERSION}/main
+${ALPINE_MIRROR}/v${ALPINE_VERSION}/community
+EOF
+    apk add \
+        --root "$CHROOT_DIR" \
+        --arch x86_64 \
+        --no-cache \
+        --allow-untrusted \
+        --initdb \
+        $ALPINE_PACKAGES
+
+# Strategy 2/3: container runtime (dev machines)
+else
+    CONTAINER_TOOL=""
+    if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
+        CONTAINER_TOOL=docker
+    elif command -v podman >/dev/null 2>&1; then
+        CONTAINER_TOOL=podman
+    else
+        echo "ERROR: no rootfs build tool available." >&2
+        echo "       Install one of: apk (Alpine), docker, podman" >&2
+        exit 1
+    fi
+
+    echo "[build-microvm-image] rootfs: $CONTAINER_TOOL export (alpine:${ALPINE_VERSION})"
+    cid=$($CONTAINER_TOOL run -d \
+        "alpine:${ALPINE_VERSION}" \
+        sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
+
+    exit_code=$($CONTAINER_TOOL wait "$cid")
+    if [ "$exit_code" != "0" ]; then
+        echo "ERROR: package installation failed in container (exit $exit_code)" >&2
+        $CONTAINER_TOOL rm "$cid" >/dev/null 2>&1 || true
+        exit 1
+    fi
+
+    $CONTAINER_TOOL export "$cid" | tar -x -C "$CHROOT_DIR"
+    $CONTAINER_TOOL rm "$cid" >/dev/null 2>&1
+fi
+
+# --- Fix /dev device nodes ---
+# tar extraction without root silently creates 0-byte regular files for device
+# nodes (mknod requires CAP_MKNOD). The kernel opens /dev/console to wire
+# init's stdio to the serial console; a regular file there causes all init
+# output to be silently discarded. Recreate the two nodes needed before
+# devtmpfs mounts (which provides the rest at runtime).
+rm -rf "$CHROOT_DIR/dev"
+mkdir "$CHROOT_DIR/dev"
+sudo mknod -m 600 "$CHROOT_DIR/dev/console" c 5 1
+sudo mknod -m 666 "$CHROOT_DIR/dev/null"    c 1 3
+
+# --- Place init script ---
+echo "[build-microvm-image] installing init.sh as /init..."
+if [ ! -f "$INIT_SH" ]; then
+    echo "ERROR: $INIT_SH not found — cannot build initramfs" >&2
+    exit 1
+fi
+install -m 0755 "$INIT_SH" "$CHROOT_DIR/init"
+
+# --- Place inittab ---
+if [ -f "$INITTAB" ]; then
+    echo "[build-microvm-image] installing inittab..."
+    install -m 0644 "$INITTAB" "$CHROOT_DIR/etc/inittab"
+fi
+
+# --- Copy lb-agent binary ---
+LB_AGENT_BIN="$REPO_ROOT/bin/lb-agent"
+if [ -f "$LB_AGENT_BIN" ]; then
+    echo "[build-microvm-image] copying lb-agent binary..."
+    mkdir -p "$CHROOT_DIR/usr/local/bin"
+    install -m 0755 "$LB_AGENT_BIN" "$CHROOT_DIR/usr/local/bin/lb-agent"
+else
+    echo "[build-microvm-image] WARNING: $LB_AGENT_BIN not found — lb-agent will be absent from initramfs" >&2
+fi
+
+# --- Create required directories for init ---
+mkdir -p \
+    "$CHROOT_DIR/proc" \
+    "$CHROOT_DIR/sys" \
+    "$CHROOT_DIR/dev" \
+    "$CHROOT_DIR/etc/ssl/certs" \
+    "$CHROOT_DIR/etc/conf.d"
+
+# --- Locate kernel ---
+KERNEL_IMG=$(find "$CHROOT_DIR/boot" -name "vmlinuz*" | sort | tail -1)
+if [ -z "$KERNEL_IMG" ]; then
+    echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
+    exit 1
+fi
+echo "[build-microvm-image] kernel image: $KERNEL_IMG"
+
+KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
+KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
+
+# --- Assert fw_cfg module ---
+if ! find "$CHROOT_DIR/lib/modules" \
+        -name "qemu_fw_cfg.ko*" \
+        -o -name "fw_cfg_sysfs.ko*" \
+    | grep -q .; then
+    if ! grep -q "CONFIG_FW_CFG_SYSFS=y" "$KERNEL_CONFIG" 2>/dev/null; then
+        echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
+        exit 1
+    fi
+    echo "[build-microvm-image] fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
+else
+    echo "[build-microvm-image] fw_cfg: module found in lib/modules"
+fi
+
+# --- Strip kernel modules (Phase 2 keep-list approach) ---
+# Only modules needed by a virtio-mmio microvm guest are kept:
+#   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
+#   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
+#   *failover*.ko* — net_failover + failover (virtio_net deps for live migration)
+# All other loadable modules are removed. Drivers compiled into the kernel
+# (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
+echo "[build-microvm-image] stripping kernel modules to virtio+fw_cfg only..."
+for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
+    kver="$(basename "$kver_dir")"
+    kernel_dir="$kver_dir/kernel"
+    [ -d "$kernel_dir" ] || continue
+
+    # Collect keeper modules and their relative paths from the kernel/ tree.
+    tmp_keep=$(mktemp -d)
+    find "$kernel_dir" -name "virtio*.ko.gz" -o -name "virtio*.ko" \
+                       -o -name "*fw_cfg*.ko.gz" -o -name "*fw_cfg*.ko" \
+                       -o -name "*failover*.ko.gz" -o -name "*failover*.ko" \
+        2>/dev/null | while read -r mod; do
+        rel="${mod#"$kernel_dir/"}"
+        dest_dir="$tmp_keep/$(dirname "$rel")"
+        mkdir -p "$dest_dir"
+        cp "$mod" "$dest_dir/"
+    done
+
+    # Replace kernel module tree with the keeper set.
+    rm -rf "$kernel_dir"
+    mkdir -p "$kernel_dir"
+    # Restore each kept file into the same relative path.
+    (cd "$tmp_keep" && find . -name "*.ko*" | while read -r f; do
+        dest_dir="$kernel_dir/$(dirname "${f#./}")"
+        mkdir -p "$dest_dir"
+        cp "$f" "$dest_dir/"
+    done)
+    rm -rf "$tmp_keep"
+
+    # Regenerate module dependency map from the surviving modules.
+    depmod -b "$CHROOT_DIR" "$kver" 2>/dev/null || true
+done
+
+# --- Strip package-manager artifacts (not needed at runtime) ---
+echo "[build-microvm-image] stripping package manager artifacts..."
+rm -rf \
+    "$CHROOT_DIR/lib/apk" \
+    "$CHROOT_DIR/var/cache/apk" \
+    "$CHROOT_DIR/etc/apk" \
+    "$CHROOT_DIR/usr/share/man" \
+    "$CHROOT_DIR/usr/share/doc" \
+    "$CHROOT_DIR/usr/share/apk" \
+    "$CHROOT_DIR/usr/include" \
+    "$CHROOT_DIR/usr/lib/pkgconfig"
+
+# --- Strip debug symbols from binaries ---
+echo "[build-microvm-image] stripping debug symbols from binaries..."
+find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
+     "$CHROOT_DIR/sbin" "$CHROOT_DIR/bin" \
+     "$CHROOT_DIR/usr/local/bin" \
+    -type f 2>/dev/null | while read -r bin; do
+    strip --strip-all "$bin" 2>/dev/null || true
+done
+
+# --- Copy vmlinuz out before stripping /boot from chroot ---
+VMLINUZ_OUT="$BUILD_DIR/vmlinuz"
+echo "[build-microvm-image] copying kernel: $VMLINUZ_OUT"
+cp "$KERNEL_IMG" "$VMLINUZ_OUT"
+
+# Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
+# and kernel config all live here (~25 MiB). None are needed at runtime: the
+# guest already has the kernel loaded by QEMU's -kernel, and Alpine's initramfs
+# is unused because we replace it with our own /init. Leaving /boot in the cpio
+# nearly doubles uncompressed initramfs size and forces a 512 MiB guest just to
+# survive decompression.
+rm -rf "$CHROOT_DIR/boot"
+
+# --- Build initramfs ---
+INITRAMFS_OUT="$BUILD_DIR/initramfs.cpio.gz"
+echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT"
+(
+    cd "$CHROOT_DIR"
+    find . | cpio --quiet -o -H newc | gzip -9 > "$INITRAMFS_OUT"
+)
+
+# --- Log artifact sizes ---
+echo ""
+echo "[build-microvm-image] artifacts:"
+ls -lh "$VMLINUZ_OUT" "$INITRAMFS_OUT"
+echo ""
+
+# --- Phase 2 artifact size gate (50 MiB total) ---
+ARTIFACT_MAX_MiB="${MICROVM_ARTIFACT_MAX_MiB:-50}"
+total_bytes=$(( $(stat -c %s "$VMLINUZ_OUT") + $(stat -c %s "$INITRAMFS_OUT") ))
+max_bytes=$(( ARTIFACT_MAX_MiB * 1024 * 1024 ))
+total_mib=$(( (total_bytes + 1048575) / 1048576 ))
+if [ "$total_bytes" -gt "$max_bytes" ]; then
+    echo "ERROR: artifact size ${total_mib} MiB exceeds ${ARTIFACT_MAX_MiB} MiB gate" >&2
+    echo "       Set MICROVM_ARTIFACT_MAX_MiB=N to override during development." >&2
+    exit 1
+fi
+echo "[build-microvm-image] artifact size gate: PASS (${total_mib} MiB / ${ARTIFACT_MAX_MiB} MiB)"
+echo ""
+echo "[build-microvm-image] done."
@@ -82,7 +82,7 @@ case "$DISTRO" in
            exit 1
        fi
        SOURCE_IMAGE="ubuntu-${UBUNTU_VERSION}-minimal-cloudimg-amd64.img"
-        SOURCE_URL="https://cloud-images.ubuntu.com/minimal/releases/noble/release/${SOURCE_IMAGE}"
+        SOURCE_URL="https://cloud-images.ubuntu.com/minimal/releases/${UBUNTU_CODENAME:-noble}/release/${SOURCE_IMAGE}"
        OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-ubuntu.qcow2"
        OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-ubuntu.raw"
        DISTRO_VERSION="${UBUNTU_VERSION}"
@@ -188,11 +188,12 @@ if [[ -f "$OUTPUT_RAW" ]] && [[ $(( $(date +%s) - $(stat -c %Y "$OUTPUT_RAW") ))

    if [[ "$DO_IMPORT" == true ]]; then
        echo "Importing as AMI..."
+        rm -f "$OUTPUT_IMAGE"
        IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64)
        if [[ -n "${SYSTEM_TAG:-}" ]]; then
            IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
        fi
-        (cd "$PROJECT_DIR" && ./bin/spx admin images import "${IMPORT_ARGS[@]}")
+        spx admin images import "${IMPORT_ARGS[@]}"
    fi
    exit 0
 fi
@@ -253,9 +254,13 @@ if [[ "$DISTRO" == "ubuntu" ]]; then
    done
    echo "Resizing partition..."
    # After qemu-img resize the GPT backup header sits at the old end of disk.
-    # Move it to the new end before parted tries to resize, otherwise parted
-    # refuses with "Unable to satisfy all constraints on the partition."
-    sudo sgdisk --move-second-header "${NBD_DEV}" 2>/dev/null || true
+    # Relocate it to the new end (sfdisk --relocate is util-linux built-in,
+    # no extra packages needed), then re-read the partition table before parted
+    # runs — without this, parted sees stale GPT geometry and fails with
+    # "Unable to satisfy all constraints on the partition."
+    sudo sgdisk --move-second-header "${NBD_DEV}"
+    sudo partprobe "${NBD_DEV}" 2>/dev/null || true
+    sleep 1
    sudo parted --script "${NBD_DEV}" resizepart 1 100%
 fi

@@ -417,14 +422,15 @@ echo ""

 if [[ "$DO_IMPORT" == true ]]; then
    echo "Importing as AMI..."
+    rm -f "$OUTPUT_IMAGE"
    IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64)
    if [[ -n "${SYSTEM_TAG:-}" ]]; then
        IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
    fi
-    (cd "$PROJECT_DIR" && sudo -u spinifex-storage ./bin/spx admin images import "${IMPORT_ARGS[@]}")
+    spx admin images import "${IMPORT_ARGS[@]}"
 else
    echo "To import as AMI, run:"
-    echo "  cd $PROJECT_DIR && ./bin/spx admin images import \\"
+    echo "  spx admin images import \\"
    echo "    --file $OUTPUT_RAW \\"
    if [[ -n "${SYSTEM_TAG:-}" ]]; then
        echo "    --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 \\"
@@ -5,7 +5,7 @@ set -euo pipefail
 # Usage: scripts/check-coverage.sh <coverprofile> [quiet]

 PROFILE="${1:?Usage: check-coverage.sh <coverprofile> [quiet]}"
-MIN=65.0
+MIN=70.0
 QUIET="${2:-}"

 if [[ ! -s "$PROFILE" ]]; then
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Dashboard server for the Spinifex GPU demo.
+
+Proxies streams from the GPU instances and serves the web UI.
+
+Environment variables:
+  YOLO_HOST     Base URL of the YOLO instance,   e.g. http://10.0.0.3:8080
+  OLLAMA_HOST   Base URL of the Ollama instance, e.g. http://10.0.0.2:11434
+  OLLAMA_MODEL  Model name to use for chat       (default: llama3.1:70b)
+  PORT          Server port                       (default: 8000)
+"""
+
+import asyncio
+import json
+import os
+
+import httpx
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.responses import HTMLResponse, StreamingResponse
+from fastapi.staticfiles import StaticFiles
+
+YOLO_HOST = os.environ.get("YOLO_HOST", "http://localhost:8080")
+OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
+OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "llama3.1:70b")
+PORT = int(os.environ.get("PORT", "8000"))
+
+app = FastAPI()
+app.mount("/static", StaticFiles(directory="static"), name="static")
+
+
+@app.get("/", response_class=HTMLResponse)
+async def index():
+    with open("static/index.html") as f:
+        return f.read()
+
+
+@app.get("/proxy/video")
+async def proxy_video():
+    async def stream():
+        while True:
+            try:
+                async with httpx.AsyncClient(timeout=None) as client:
+                    async with client.stream("GET", f"{YOLO_HOST}/video") as resp:
+                        async for chunk in resp.aiter_bytes(8192):
+                            yield chunk
+            except Exception:
+                await asyncio.sleep(5)
+
+    return StreamingResponse(stream(), media_type="multipart/x-mixed-replace; boundary=frame")
+
+
+@app.get("/proxy/descriptions")
+async def proxy_descriptions():
+    async def stream():
+        while True:
+            try:
+                async with httpx.AsyncClient(timeout=httpx.Timeout(connect=10.0, read=None, write=None, pool=None)) as client:
+                    async with client.stream("GET", f"{YOLO_HOST}/descriptions") as resp:
+                        async for line in resp.aiter_lines():
+                            yield line + "\n"
+            except Exception:
+                yield 'data: {"type":"error","text":"YOLO stream offline, retrying..."}\n\n'
+                await asyncio.sleep(5)
+
+    return StreamingResponse(stream(), media_type="text/event-stream")
+
+
+@app.post("/chat")
+async def chat(request: Request):
+    body = await request.json()
+    prompt = body.get("prompt", "").strip()
+    if not prompt:
+        return {"error": "empty prompt"}
+
+    async def stream():
+        async with httpx.AsyncClient(timeout=120) as client:
+            async with client.stream(
+                "POST",
+                f"{OLLAMA_HOST}/v1/chat/completions",
+                json={
+                    "model": OLLAMA_MODEL,
+                    "messages": [{"role": "user", "content": prompt}],
+                    "stream": True,
+                },
+            ) as resp:
+                async for line in resp.aiter_lines():
+                    if not line.startswith("data: "):
+                        continue
+                    data_str = line[6:]
+                    if data_str.strip() == "[DONE]":
+                        yield f"data: {json.dumps({'token': '', 'done': True})}\n\n"
+                        break
+                    try:
+                        data = json.loads(data_str)
+                        token = data["choices"][0]["delta"].get("content", "")
+                        yield f"data: {json.dumps({'token': token, 'done': False})}\n\n"
+                    except (json.JSONDecodeError, KeyError, IndexError):
+                        pass
+
+    return StreamingResponse(stream(), media_type="text/event-stream")
+
+
+@app.get("/status")
+async def status():
+    results: dict[str, str] = {}
+    async with httpx.AsyncClient(timeout=3) as client:
+        for name, url in [
+            ("yolo", f"{YOLO_HOST}/health"),
+            ("ollama", f"{OLLAMA_HOST}/v1/models"),
+        ]:
+            try:
+                r = await client.get(url)
+                results[name] = "ok" if r.status_code < 500 else "error"
+            except Exception:
+                results[name] = "offline"
+    return results
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=PORT)
@@ -0,0 +1,116 @@
+#!/bin/bash
+# deploy.sh — Deploy and start the Spinifex GPU demo across four instances.
+#
+# Usage:
+#   DASHBOARD_IP=<ip> YOLO_IP=<ip> OLLAMA_IP=<ip> QWEN_IP=<ip> \
+#     VIDEO_PATH=/path/to/video.mp4 \
+#     scripts/demo/deploy.sh
+#
+# Required env vars:
+#   DASHBOARD_IP   4x-1: serves the web dashboard
+#   YOLO_IP        4x-3: runs YOLO inference
+#   OLLAMA_IP      4x-2: runs Ollama + llama3.1:70b
+#   QWEN_IP        12x:  runs Ollama + qwen3-vl:235b
+#   VIDEO_PATH     Local path to the .mp4 file to run YOLO on
+#
+# Optional:
+#   SSH_KEY        (default: ~/.ssh/spinifex-key)
+#   SSH_USER       (default: ec2-user)
+
+set -euo pipefail
+
+DASHBOARD_IP="${DASHBOARD_IP:?set DASHBOARD_IP}"
+YOLO_IP="${YOLO_IP:?set YOLO_IP}"
+OLLAMA_IP="${OLLAMA_IP:?set OLLAMA_IP}"
+QWEN_IP="${QWEN_IP:?set QWEN_IP}"
+VIDEO_PATH="${VIDEO_PATH:?set VIDEO_PATH}"
+
+SSH_KEY="${SSH_KEY:-$HOME/.ssh/spinifex-key}"
+SSH_USER="${SSH_USER:-ec2-user}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+_ssh() {
+    local ip="$1"; shift
+    ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        -o ConnectTimeout=10 -o BatchMode=yes \
+        -i "$SSH_KEY" "${SSH_USER}@${ip}" "$@"
+}
+
+_scp() {
+    scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        -i "$SSH_KEY" -r "$@"
+}
+
+# ── YOLO instance (4x-3) ────────────────────────────────────────────────────
+echo "==> [YOLO] Deploying to $YOLO_IP..."
+_scp "$SCRIPT_DIR/yolo_stream.py" "$SCRIPT_DIR/requirements-yolo.txt" \
+    "${SSH_USER}@${YOLO_IP}:~/"
+_scp "$VIDEO_PATH" "${SSH_USER}@${YOLO_IP}:~/demo_video.mp4"
+
+echo "   Installing Python deps..."
+_ssh "$YOLO_IP" '
+    . ~/yolo-rocm/bin/activate
+    pip install --quiet -r ~/requirements-yolo.txt
+'
+
+echo "   Starting yolo_stream.py..."
+_ssh "$YOLO_IP" "
+    pkill -f yolo_stream.py || true
+    sleep 1
+    . ~/yolo-rocm/bin/activate
+    nohup env \
+        VIDEO_PATH=~/demo_video.mp4 \
+        QWEN_HOST=http://${QWEN_IP}:11434 \
+        YOLO_DEVICE=0 \
+        PORT=8080 \
+        python ~/yolo_stream.py > ~/yolo_stream.log 2>&1 &
+    echo \"   PID \$!\"
+"
+echo "   YOLO stream: http://${YOLO_IP}:8080/video"
+
+# ── Dashboard instance (4x-1) ───────────────────────────────────────────────
+echo ""
+echo "==> [Dashboard] Deploying to $DASHBOARD_IP..."
+_ssh "$DASHBOARD_IP" 'mkdir -p ~/demo/static'
+_scp "$SCRIPT_DIR/dashboard_server.py" "$SCRIPT_DIR/requirements-dashboard.txt" \
+    "${SSH_USER}@${DASHBOARD_IP}:~/demo/"
+_scp "$SCRIPT_DIR/static/index.html" \
+    "${SSH_USER}@${DASHBOARD_IP}:~/demo/static/"
+
+echo "   Installing Python deps..."
+_ssh "$DASHBOARD_IP" '
+    python3 -m venv ~/demo-venv 2>/dev/null || true
+    . ~/demo-venv/bin/activate
+    pip install --quiet -r ~/demo/requirements-dashboard.txt
+'
+
+echo "   Starting dashboard_server.py..."
+_ssh "$DASHBOARD_IP" "
+    pkill -f dashboard_server.py || true
+    sleep 1
+    . ~/demo-venv/bin/activate
+    cd ~/demo
+    nohup env \
+        YOLO_HOST=http://${YOLO_IP}:8080 \
+        OLLAMA_HOST=http://${OLLAMA_IP}:11434 \
+        OLLAMA_MODEL=llama3.1:70b \
+        PORT=8000 \
+        python dashboard_server.py > ~/dashboard.log 2>&1 &
+    echo \"   PID \$!\"
+"
+
+echo ""
+echo "✅ Demo deployed"
+echo ""
+echo "   Dashboard:  http://${DASHBOARD_IP}:8000"
+echo "   YOLO feed:  http://${YOLO_IP}:8080/video"
+echo "   Ollama:     http://${OLLAMA_IP}:11434"
+echo "   Qwen3-VL:   http://${QWEN_IP}:11434"
+echo ""
+echo "   Logs:"
+echo "     ssh -i ${SSH_KEY} ${SSH_USER}@${YOLO_IP}      tail -f ~/yolo_stream.log"
+echo "     ssh -i ${SSH_KEY} ${SSH_USER}@${DASHBOARD_IP} tail -f ~/dashboard.log"
+echo ""
+echo "   NOTE: Ollama and qwen3-vl must already be running on their instances."
+echo "   If not: _ssh \$IP 'sudo systemctl start ollama && ollama run <model>'"
@@ -0,0 +1,4 @@
+fastapi
+uvicorn[standard]
+httpx
+python-multipart
@@ -0,0 +1,5 @@
+fastapi
+uvicorn[standard]
+httpx
+opencv-python-headless
+ultralytics
@@ -0,0 +1,498 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Spinifex — GPU Compute Demo</title>
+<style>
+* { margin: 0; padding: 0; box-sizing: border-box; }
+
+:root {
+  --bg:       #07070d;
+  --surface:  #0e0e18;
+  --border:   #1a1a2e;
+  --accent:   #6366f1;
+  --accent2:  #22d3ee;
+  --text:     #d4d4d8;
+  --muted:    #52525b;
+  --green:    #22c55e;
+  --red:      #ef4444;
+}
+
+body {
+  background: var(--bg);
+  color: var(--text);
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+  height: 100vh;
+  display: flex;
+  flex-direction: column;
+  overflow: hidden;
+}
+
+/* ── Header ── */
+header {
+  background: var(--surface);
+  border-bottom: 1px solid var(--border);
+  padding: 0 24px;
+  height: 52px;
+  display: flex;
+  align-items: center;
+  gap: 20px;
+  flex-shrink: 0;
+  z-index: 10;
+}
+
+.logo {
+  font-size: 17px;
+  font-weight: 700;
+  letter-spacing: -0.3px;
+  color: #fff;
+  white-space: nowrap;
+}
+.logo em { color: var(--accent); font-style: normal; }
+
+.divider { width: 1px; height: 24px; background: var(--border); }
+
+.subtitle {
+  font-size: 12px;
+  color: var(--muted);
+  letter-spacing: 0.3px;
+}
+
+.instances {
+  margin-left: auto;
+  display: flex;
+  gap: 20px;
+}
+
+.inst {
+  display: flex;
+  align-items: center;
+  gap: 6px;
+  font-size: 11px;
+  color: var(--muted);
+}
+
+.inst-dot {
+  width: 7px; height: 7px;
+  border-radius: 50%;
+  background: var(--green);
+}
+.inst-dot.offline { background: var(--red); }
+.inst-dot.live {
+  background: var(--green);
+  box-shadow: 0 0 0 0 var(--green);
+  animation: ring 2s infinite;
+}
+
+@keyframes ring {
+  0%   { box-shadow: 0 0 0 0 rgba(34,197,94,.6); }
+  70%  { box-shadow: 0 0 0 6px rgba(34,197,94,0); }
+  100% { box-shadow: 0 0 0 0 rgba(34,197,94,0); }
+}
+
+/* ── Layout ── */
+main {
+  flex: 1;
+  display: grid;
+  grid-template-columns: 1fr 400px;
+  grid-template-rows: 1fr 1fr;
+  gap: 10px;
+  padding: 10px;
+  overflow: hidden;
+}
+
+/* ── Panel ── */
+.panel {
+  background: var(--surface);
+  border: 1px solid var(--border);
+  border-radius: 10px;
+  display: flex;
+  flex-direction: column;
+  overflow: hidden;
+  min-height: 0;
+}
+
+.panel-head {
+  padding: 9px 14px;
+  background: rgba(0,0,0,.3);
+  border-bottom: 1px solid var(--border);
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  flex-shrink: 0;
+}
+
+.panel-head h2 {
+  font-size: 11px;
+  font-weight: 600;
+  text-transform: uppercase;
+  letter-spacing: 1px;
+  color: var(--muted);
+}
+
+.chip {
+  margin-left: auto;
+  font-size: 10px;
+  font-weight: 500;
+  padding: 2px 8px;
+  border-radius: 99px;
+  border: 1px solid var(--border);
+  color: var(--muted);
+  white-space: nowrap;
+}
+.chip.accent { border-color: rgba(99,102,241,.4); color: #818cf8; }
+.chip.cyan   { border-color: rgba(34,211,238,.3); color: var(--accent2); }
+
+/* ── Video panel ── */
+.video-panel { grid-row: 1 / 3; position: relative; }
+
+.video-wrap {
+  flex: 1;
+  background: #000;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  overflow: hidden;
+  min-height: 0;
+}
+
+.video-wrap img {
+  width: 100%;
+  height: 100%;
+  object-fit: contain;
+  display: block;
+}
+
+.video-footer {
+  padding: 8px 14px;
+  font-size: 10px;
+  color: var(--muted);
+  border-top: 1px solid var(--border);
+  flex-shrink: 0;
+  display: flex;
+  gap: 16px;
+}
+
+/* ── Description panel ── */
+.desc-content {
+  flex: 1;
+  overflow-y: auto;
+  padding: 14px;
+  display: flex;
+  flex-direction: column;
+  gap: 14px;
+  scroll-behavior: smooth;
+  min-height: 0;
+}
+
+.desc-entry {
+  padding-bottom: 14px;
+  border-bottom: 1px solid var(--border);
+}
+.desc-entry:last-child { border-bottom: none; padding-bottom: 0; }
+
+.desc-time {
+  font-size: 10px;
+  color: var(--muted);
+  margin-bottom: 5px;
+  font-variant-numeric: tabular-nums;
+}
+
+.desc-text {
+  font-size: 13px;
+  line-height: 1.65;
+  color: var(--text);
+}
+
+.cursor {
+  display: inline-block;
+  width: 2px; height: 13px;
+  background: var(--accent);
+  vertical-align: text-bottom;
+  margin-left: 1px;
+  animation: blink 1s step-end infinite;
+}
+@keyframes blink { 0%,100%{opacity:1} 50%{opacity:0} }
+
+.desc-placeholder {
+  font-size: 12px;
+  color: var(--muted);
+  font-style: italic;
+}
+
+/* ── Chat panel ── */
+.chat-messages {
+  flex: 1;
+  overflow-y: auto;
+  padding: 14px;
+  display: flex;
+  flex-direction: column;
+  gap: 10px;
+  min-height: 0;
+}
+
+.msg {
+  max-width: 88%;
+  padding: 9px 13px;
+  border-radius: 10px;
+  font-size: 13px;
+  line-height: 1.55;
+  word-break: break-word;
+}
+.msg.user {
+  background: #1e1e42;
+  align-self: flex-end;
+  color: #c7d2fe;
+}
+.msg.assistant {
+  background: #111120;
+  align-self: flex-start;
+  color: var(--text);
+  border: 1px solid var(--border);
+}
+
+.chat-foot {
+  border-top: 1px solid var(--border);
+  padding: 10px;
+  display: flex;
+  gap: 8px;
+  flex-shrink: 0;
+}
+
+.chat-foot input {
+  flex: 1;
+  background: var(--bg);
+  border: 1px solid var(--border);
+  border-radius: 7px;
+  padding: 8px 12px;
+  color: var(--text);
+  font-size: 13px;
+  outline: none;
+  transition: border-color .15s;
+}
+.chat-foot input:focus { border-color: var(--accent); }
+.chat-foot input::placeholder { color: var(--muted); }
+
+.send-btn {
+  background: var(--accent);
+  color: #fff;
+  border: none;
+  border-radius: 7px;
+  padding: 8px 16px;
+  font-size: 13px;
+  font-weight: 600;
+  cursor: pointer;
+  transition: background .15s, opacity .15s;
+  white-space: nowrap;
+}
+.send-btn:hover { background: #4f52d1; }
+.send-btn:disabled { opacity: .4; cursor: not-allowed; }
+
+/* scrollbar */
+::-webkit-scrollbar { width: 4px; }
+::-webkit-scrollbar-track { background: transparent; }
+::-webkit-scrollbar-thumb { background: var(--border); border-radius: 2px; }
+</style>
+</head>
+<body>
+
+<header>
+  <div class="logo">Spini<em>fex</em></div>
+  <div class="divider"></div>
+  <div class="subtitle">GPU Compute Platform &mdash; Live Demo</div>
+  <div class="instances">
+    <div class="inst">
+      <div class="inst-dot live" id="dot-yolo"></div>
+      <span>YOLO&nbsp;&middot;&nbsp;g7e.4xlarge</span>
+    </div>
+    <div class="inst">
+      <div class="inst-dot live" id="dot-qwen"></div>
+      <span>Qwen3-VL 235B&nbsp;&middot;&nbsp;g7e.12xlarge</span>
+    </div>
+    <div class="inst">
+      <div class="inst-dot live" id="dot-llama"></div>
+      <span>Llama&nbsp;3.1 70B&nbsp;&middot;&nbsp;g7e.4xlarge</span>
+    </div>
+  </div>
+</header>
+
+<main>
+
+  <!-- Left: YOLO video, full height -->
+  <div class="panel video-panel">
+    <div class="panel-head">
+      <h2>Live Object Detection</h2>
+      <span class="chip accent">YOLO11 &middot; AMD MI350X</span>
+    </div>
+    <div class="video-wrap">
+      <img id="video-feed" src="/proxy/video" alt="YOLO stream loading...">
+    </div>
+    <div class="video-footer">
+      <span>Real-time inference &mdash; AMD MI350X GPU</span>
+      <span id="fps-counter"></span>
+    </div>
+  </div>
+
+  <!-- Top right: qwen3-vl scene analysis -->
+  <div class="panel">
+    <div class="panel-head">
+      <h2>Scene Analysis</h2>
+      <span class="chip cyan">Qwen3-VL 235B &middot; MI350X</span>
+    </div>
+    <div class="desc-content" id="desc-content">
+      <p class="desc-placeholder">Waiting for first frame analysis&hellip;</p>
+    </div>
+  </div>
+
+  <!-- Bottom right: Llama3.1 chat -->
+  <div class="panel">
+    <div class="panel-head">
+      <h2>Chat</h2>
+      <span class="chip accent">Llama 3.1 70B &middot; MI350X</span>
+    </div>
+    <div class="chat-messages" id="chat-messages">
+      <div class="msg assistant">
+        Hello! I&rsquo;m Llama&nbsp;3.1&nbsp;70B running on an AMD MI350X GPU, orchestrated by Spinifex. Ask me anything.
+      </div>
+    </div>
+    <div class="chat-foot">
+      <input type="text" id="chat-input" placeholder="Ask Llama 3.1…" autocomplete="off">
+      <button class="send-btn" id="send-btn" onclick="sendChat()">Send</button>
+    </div>
+  </div>
+
+</main>
+
+<script>
+// ── Scene description stream ─────────────────────────────────────────────────
+const descContent = document.getElementById('desc-content');
+let activeEntry = null;
+let activeText  = '';
+
+function initDescriptions() {
+  const es = new EventSource('/proxy/descriptions');
+
+  es.onmessage = (e) => {
+    let data;
+    try { data = JSON.parse(e.data); } catch { return; }
+
+    if (data.type === 'token' && data.text) {
+      if (!activeEntry) {
+        // Remove placeholder if present
+        descContent.querySelector('.desc-placeholder')?.remove();
+
+        activeEntry = document.createElement('div');
+        activeEntry.className = 'desc-entry';
+        activeEntry.innerHTML = `<div class="desc-time">${new Date().toLocaleTimeString()}</div><div class="desc-text"></div>`;
+        descContent.appendChild(activeEntry);
+        activeText = '';
+      }
+      activeText += data.text;
+      const textEl = activeEntry.querySelector('.desc-text');
+      textEl.innerHTML = escHtml(activeText) + '<span class="cursor"></span>';
+      descContent.scrollTop = descContent.scrollHeight;
+    }
+
+    if (data.type === 'end' && activeEntry) {
+      activeEntry.querySelector('.desc-text').textContent = activeText;
+      activeEntry = null;
+      activeText  = '';
+      // Keep only the last 4 entries
+      const entries = descContent.querySelectorAll('.desc-entry');
+      entries.forEach((el, i) => { if (i < entries.length - 4) el.remove(); });
+    }
+  };
+
+  es.onerror = () => { setTimeout(initDescriptions, 3000); };
+}
+
+// ── Chat ─────────────────────────────────────────────────────────────────────
+const chatMessages = document.getElementById('chat-messages');
+const chatInput    = document.getElementById('chat-input');
+const sendBtn      = document.getElementById('send-btn');
+
+async function sendChat() {
+  const prompt = chatInput.value.trim();
+  if (!prompt || sendBtn.disabled) return;
+
+  chatInput.value = '';
+  sendBtn.disabled = true;
+
+  appendMsg('user', prompt);
+
+  const assistantEl = appendMsg('assistant', '');
+  let text = '';
+
+  try {
+    const resp = await fetch('/chat', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ prompt }),
+    });
+
+    const reader  = resp.body.getReader();
+    const decoder = new TextDecoder();
+    let buf = '';
+
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buf += decoder.decode(value, { stream: true });
+      const lines = buf.split('\n');
+      buf = lines.pop();
+      for (const line of lines) {
+        if (!line.startsWith('data: ')) continue;
+        let d;
+        try { d = JSON.parse(line.slice(6)); } catch { continue; }
+        text += d.token;
+        assistantEl.innerHTML = escHtml(text) + (d.done ? '' : '<span class="cursor"></span>');
+        chatMessages.scrollTop = chatMessages.scrollHeight;
+      }
+    }
+  } catch (err) {
+    assistantEl.textContent = '(connection error)';
+  }
+
+  sendBtn.disabled = false;
+  chatInput.focus();
+}
+
+function appendMsg(role, text) {
+  const el = document.createElement('div');
+  el.className = `msg ${role}`;
+  el.textContent = text;
+  chatMessages.appendChild(el);
+  chatMessages.scrollTop = chatMessages.scrollHeight;
+  return el;
+}
+
+chatInput.addEventListener('keydown', (e) => { if (e.key === 'Enter') sendChat(); });
+
+// ── Status polling ────────────────────────────────────────────────────────────
+async function pollStatus() {
+  try {
+    const data = await fetch('/status').then(r => r.json());
+    setDot('dot-yolo',  data.yolo   === 'ok');
+    setDot('dot-llama', data.ollama === 'ok');
+  } catch {}
+  setTimeout(pollStatus, 5000);
+}
+
+function setDot(id, ok) {
+  const el = document.getElementById(id);
+  if (!el) return;
+  el.className = 'inst-dot' + (ok ? ' live' : ' offline');
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+function escHtml(str) {
+  return str.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
+}
+
+// ── Boot ──────────────────────────────────────────────────────────────────────
+initDescriptions();
+pollStatus();
+</script>
+</body>
+</html>
@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+"""YOLO inference server for the Spinifex demo.
+
+Reads a video file, runs YOLO detection, and serves:
+  GET /video        — MJPEG stream of annotated frames
+  GET /descriptions — SSE stream of qwen3-vl scene descriptions
+  GET /health       — {"ok": true}
+
+Environment variables:
+  VIDEO_PATH        Path to input .mp4 (required)
+  QWEN_HOST         Base URL of the qwen3-vl Ollama instance, e.g. http://10.0.0.5:11434
+  YOLO_MODEL        Model name (default: yolo11n.pt)
+  YOLO_DEVICE       GPU device index (default: 0)
+  DESCRIBE_INTERVAL Seconds between qwen3-vl calls (default: 5.0)
+  PORT              Server port (default: 8080)
+"""
+
+import asyncio
+import base64
+import json
+import os
+import queue
+import threading
+import time
+
+import cv2
+import httpx
+import uvicorn
+from fastapi import FastAPI
+from fastapi.responses import StreamingResponse
+from ultralytics import YOLO
+
+VIDEO_PATH = os.environ["VIDEO_PATH"]
+QWEN_HOST = os.environ.get("QWEN_HOST", "")
+YOLO_MODEL = os.environ.get("YOLO_MODEL", "yolo11x.pt")
+YOLO_DEVICE = os.environ.get("YOLO_DEVICE", "0")
+DESCRIBE_INTERVAL = float(os.environ.get("DESCRIBE_INTERVAL", "5.0"))
+PORT = int(os.environ.get("PORT", "8080"))
+
+app = FastAPI()
+
+_frame_lock = threading.Lock()
+_latest_frame: bytes | None = None
+_desc_queue: queue.Queue = queue.Queue(maxsize=100)
+
+
+def _encode(frame, quality: int = 80) -> bytes:
+    _, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, quality])
+    return buf.tobytes()
+
+
+def _describe(img_b64: str) -> None:
+    if not QWEN_HOST:
+        return
+    try:
+        with httpx.Client(timeout=180) as client:
+            with client.stream(
+                "POST",
+                f"{QWEN_HOST}/api/generate",
+                json={
+                    "model": "qwen3-vl:235b",
+                    "prompt": "Briefly describe what is happening in this scene. Focus on the objects detected and any notable activity.",
+                    "images": [img_b64],
+                    "stream": True,
+                },
+            ) as resp:
+                for line in resp.iter_lines():
+                    if not line:
+                        continue
+                    try:
+                        data = json.loads(line)
+                        token = data.get("response", "")
+                        if token and not _desc_queue.full():
+                            _desc_queue.put(("token", token))
+                        if data.get("done"):
+                            _desc_queue.put(("end", ""))
+                    except json.JSONDecodeError:
+                        pass
+    except Exception as exc:
+        _desc_queue.put(("error", str(exc)))
+
+
+def inference_loop() -> None:
+    model = YOLO(YOLO_MODEL)
+    cap = cv2.VideoCapture(VIDEO_PATH)
+    if not cap.isOpened():
+        raise RuntimeError(f"Cannot open video: {VIDEO_PATH}")
+
+    fps = cap.get(cv2.CAP_PROP_FPS) or 25.0
+    frame_delay = 1.0 / fps
+    last_describe = 0.0
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
+            continue
+
+        results = model(frame, device=YOLO_DEVICE, verbose=False)
+        annotated = results[0].plot()
+
+        with _frame_lock:
+            global _latest_frame
+            _latest_frame = _encode(annotated)
+
+        now = time.monotonic()
+        if QWEN_HOST and now - last_describe >= DESCRIBE_INTERVAL:
+            last_describe = now
+            img_b64 = base64.b64encode(_encode(annotated, quality=60)).decode()
+            threading.Thread(target=_describe, args=(img_b64,), daemon=True).start()
+
+        time.sleep(frame_delay)
+
+
+@app.on_event("startup")
+async def startup() -> None:
+    threading.Thread(target=inference_loop, daemon=True).start()
+
+
+@app.get("/health")
+async def health():
+    return {"ok": True}
+
+
+@app.get("/video")
+async def video():
+    async def generate():
+        while True:
+            with _frame_lock:
+                frame = _latest_frame
+            if frame:
+                yield b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" + frame + b"\r\n"
+            await asyncio.sleep(0.04)
+
+    return StreamingResponse(
+        generate(),
+        media_type="multipart/x-mixed-replace; boundary=frame",
+    )
+
+
+@app.get("/descriptions")
+async def descriptions():
+    async def generate():
+        yield 'data: {"type":"ready"}\n\n'
+        loop = asyncio.get_event_loop()
+        while True:
+            try:
+                kind, text = await loop.run_in_executor(
+                    None, lambda: _desc_queue.get(timeout=30)
+                )
+                yield f"data: {json.dumps({'type': kind, 'text': text})}\n\n"
+            except queue.Empty:
+                yield 'data: {"type":"ping"}\n\n'
+
+    return StreamingResponse(generate(), media_type="text/event-stream")
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=PORT)
@@ -91,9 +91,8 @@ if ! id -Gn 2>/dev/null | grep -qw spinifex; then
    newgrp spinifex
 fi

-echo "=== Building and importing LB image ==="
-cd "$PROJECT_ROOT" && make build-lb-agent
-"$PROJECT_ROOT/scripts/build-system-image.sh" "$PROJECT_ROOT/scripts/images/lb.conf" --import --quiet
+echo "=== Building and installing microVM artifacts ==="
+cd "$PROJECT_ROOT" && make build-lb-agent install-microvm

 echo "=== Done ==="
 echo "Services: sudo systemctl status spinifex.target"
@@ -13,7 +13,7 @@ set -euo pipefail

 PROFILE=""
 BASE_REF=""
-THRESHOLD=65
+THRESHOLD=60
 QUIET="${QUIET:-}"

 while [[ $# -gt 0 ]]; do
@@ -0,0 +1,62 @@
+#!/bin/bash
+set -euo pipefail
+
+# ubuntu-gpu-amd-setup.sh — Chroot setup for an AMD GPU-capable guest image.
+#
+# Installs linux-firmware (AMD firmware blobs required by the amdgpu kernel
+# module), ROCm CLI tools, Python toolchain, and common utilities. Rebuilds
+# initramfs so AMD firmware is available at early boot inside the guest.
+#
+# Host-side VFIO setup (driver blacklisting, IOMMU, initramfs) is handled
+# by `spx admin gpu setup` — this script is for the guest VM image only.
+#
+# Must run inside an Ubuntu 26.04 chroot with /proc /sys /dev bind-mounted.
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update -qq
+apt-get install -y --no-install-recommends curl gnupg ca-certificates
+
+# AMD ROCm apt repository — rocminfo and rocm-smi-lib are not in the Ubuntu
+# default repos, so we add AMD's signed repo before the main package install.
+UBUNTU_CODENAME=$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME}}")
+mkdir -p /etc/apt/keyrings
+curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key \
+    | gpg --dearmor -o /etc/apt/keyrings/rocm.gpg
+echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.3 ${UBUNTU_CODENAME} main" \
+    > /etc/apt/sources.list.d/rocm.list
+apt-get update -qq
+
+# linux-firmware carries the AMD GPU firmware blobs loaded by the amdgpu
+# kernel module at boot. Without them the GPU is invisible to guest userland.
+apt-get install -y -o Acquire::Retries=3 --no-install-recommends \
+    linux-image-generic \
+    linux-headers-generic \
+    linux-firmware \
+    initramfs-tools \
+    pciutils \
+    rocminfo rocm-smi-lib \
+    python3 python3-venv python3-pip \
+    git curl wget htop tmux \
+    ffmpeg libgl1 libglib2.0-0
+
+KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||')
+if [ -z "${KVER}" ]; then
+    echo "ERROR: No kernel found under /boot"
+    exit 1
+fi
+echo "Rebuilding initramfs for kernel: ${KVER}"
+update-initramfs -u -k "${KVER}"
+
+# Prevent unattended-upgrades from replacing linux-firmware mid-lifecycle;
+# a firmware change without a matching initramfs rebuild can make the GPU invisible.
+mkdir -p /etc/apt/apt.conf.d
+cat > /etc/apt/apt.conf.d/99-gpu-ami <<'EOF'
+Unattended-Upgrade::Package-Blacklist {
+    "linux-firmware";
+    "linux-image-";
+    "linux-headers-";
+};
+EOF
+
+echo "AMD GPU image setup complete: kernel=${KVER}, linux-firmware and ROCm userland installed"
@@ -0,0 +1,12 @@
+IMAGE_NAME="ubuntu-amd-gpu"
+IMAGE_DESCRIPTION="Ubuntu 26.04 AMD GPU guest image — linux-firmware, ROCm CLI, Python toolchain"
+DISTRO="ubuntu"
+UBUNTU_VERSION="26.04"
+UBUNTU_CODENAME="resolute"
+
+# 12G provides headroom for kernel headers, linux-firmware, and ROCm packages
+IMAGE_SIZE="12G"
+
+SETUP_SCRIPT="scripts/images/ubuntu-gpu-amd-setup.sh"
+
+SYSTEM_TAG="gpu-vendor=amd"
@@ -0,0 +1,107 @@
+#!/bin/bash
+set -euo pipefail
+
+# ubuntu-gpu-nvidia-setup.sh — Chroot setup for an NVIDIA GPU-capable guest image.
+#
+# Pre-builds the NVIDIA headless server driver via DKMS against a pinned kernel
+# so guests can use the GPU immediately after launch. Also installs Python
+# toolchain and common utilities.
+#
+# nouveau is blacklisted inside the guest image so the NVIDIA kernel module can
+# bind — this is distinct from the host-side blacklist that `spx admin gpu setup`
+# configures for VFIO passthrough.
+#
+# Host-side VFIO setup (IOMMU, vfio-pci binding, host initramfs) is handled
+# by `spx admin gpu setup` — this script is for the guest VM image only.
+#
+# Must run inside an Ubuntu 26.04 chroot with /proc /sys /dev bind-mounted
+# (required by DKMS during the nvidia package post-install hook).
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update -qq
+
+# Detect the cloud image's existing kernel BEFORE installing anything.
+# Installing linux-image-generic would upgrade to a newer kernel version;
+# DKMS would then build for that version but the VM boots the original
+# cloud image kernel, causing a version mismatch at modprobe time.
+KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||')
+if [[ -z "${KVER}" ]]; then
+    echo "ERROR: No kernel found under /boot"
+    exit 1
+fi
+echo "Target kernel: ${KVER}"
+
+# Install headers for the exact kernel already present, plus initramfs-tools.
+apt-get install -y --no-install-recommends \
+    "linux-headers-${KVER}" \
+    initramfs-tools
+
+# Install the NVIDIA server DKMS driver. Use the distro default version (no
+# hardcoded suffix) so Ubuntu 26.04's 580 driver is used instead of 550.
+# nvidia-dkms-server is a meta package that resolves to the current recommended
+# version; nvidia-utils-server brings nvidia-smi and related tools.
+apt-get install -y --no-install-recommends \
+    nvidia-dkms-server \
+    nvidia-utils-server
+
+# Detect the installed NVIDIA DKMS module name + version for explicit build.
+NVIDIA_DKMS_NAME=$(dkms status 2>/dev/null | awk -F'[,/ ]+' '/nvidia/{print $1; exit}')
+NVIDIA_DKMS_VER=$(dkms status 2>/dev/null  | awk -F'[,/ ]+' '/nvidia/{print $2; exit}')
+
+if [[ -n "${NVIDIA_DKMS_NAME}" && -n "${NVIDIA_DKMS_VER}" ]]; then
+    echo "Building NVIDIA DKMS module ${NVIDIA_DKMS_NAME}/${NVIDIA_DKMS_VER} for kernel ${KVER}..."
+    dkms install -m "${NVIDIA_DKMS_NAME}" -v "${NVIDIA_DKMS_VER}" --kernelver "${KVER}" --force 2>&1
+else
+    echo "WARNING: nvidia DKMS module not found in dkms status — driver install may have failed"
+fi
+
+# Verify the module was actually built.
+NVIDIA_KO=$(find "/lib/modules/${KVER}/updates" -name "nvidia.ko*" 2>/dev/null | head -1)
+if [[ -z "${NVIDIA_KO}" ]]; then
+    echo "ERROR: nvidia.ko not found under /lib/modules/${KVER}/updates after DKMS build"
+    exit 1
+fi
+echo "NVIDIA kernel module: ${NVIDIA_KO}"
+
+# Pin the exact kernel + headers so unattended-upgrades cannot replace the
+# kernel the DKMS module was built for.
+apt-mark hold \
+    "linux-image-${KVER}" \
+    "linux-headers-${KVER}"
+
+NVIDIA_PKGS=$(dpkg --get-selections 'nvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
+LIBNVIDIA_PKGS=$(dpkg --get-selections 'libnvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
+# shellcheck disable=SC2086
+[[ -n "${NVIDIA_PKGS}" ]]    && apt-mark hold ${NVIDIA_PKGS}
+# shellcheck disable=SC2086
+[[ -n "${LIBNVIDIA_PKGS}" ]] && apt-mark hold ${LIBNVIDIA_PKGS}
+
+# Blacklist nouveau inside the guest — nvidia.ko and nouveau conflict, and
+# without this blacklist the NVIDIA driver will not bind after boot.
+cat > /etc/modprobe.d/blacklist-nouveau.conf <<'EOF'
+blacklist nouveau
+options nouveau modeset=0
+EOF
+
+# Common tooling matching the AMD image.
+apt-get install -y -o Acquire::Retries=3 \
+    pciutils \
+    python3 python3-venv python3-pip \
+    git curl wget htop tmux \
+    ffmpeg libgl1 libglib2.0-0
+
+mkdir -p /etc/apt/apt.conf.d
+cat > /etc/apt/apt.conf.d/99-gpu-ami <<'EOF'
+Unattended-Upgrade::Package-Blacklist {
+    "linux-";
+    "nvidia-";
+    "libnvidia-";
+};
+EOF
+
+# Rebuild initramfs with nouveau blacklisted and NVIDIA module included.
+update-initramfs -u -k "${KVER}"
+
+
+echo "NVIDIA GPU image setup complete: kernel=${KVER}, NVIDIA driver + DKMS pre-built"
@@ -0,0 +1,12 @@
+IMAGE_NAME="ubuntu-nvidia-gpu"
+IMAGE_DESCRIPTION="Ubuntu 26.04 NVIDIA GPU guest image — NVIDIA server driver (DKMS pre-built), Python toolchain"
+DISTRO="ubuntu"
+UBUNTU_VERSION="26.04"
+UBUNTU_CODENAME="resolute"
+
+# 12G provides headroom for kernel headers, NVIDIA driver, and DKMS build artefacts
+IMAGE_SIZE="12G"
+
+SETUP_SCRIPT="scripts/images/ubuntu-gpu-nvidia-setup.sh"
+
+SYSTEM_TAG="gpu-vendor=nvidia"
@@ -1,82 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-# ubuntu-gpu-setup.sh — Chroot setup for Ubuntu 24.04 GPU AMI
-#
-# Installs NVIDIA headless server driver with DKMS module pre-built against
-# a pinned kernel. Blacklists nouveau. Prevents unattended-upgrades from
-# invalidating the pre-built module.
-#
-# Must run inside an Ubuntu 24.04 chroot with /proc /sys /dev bind-mounted
-# (required by DKMS during the nvidia package post-install hook).
-
-export DEBIAN_FRONTEND=noninteractive
-
-apt-get update
-
-# Install a pinned kernel and its headers so DKMS has a stable build target.
-# linux-image-generic and linux-headers-generic pull in the current HWE stack.
-apt-get install -y --no-install-recommends \
-    linux-image-generic \
-    linux-headers-generic
-
-# Capture the installed kernel version before adding the driver.
-KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||')
-if [[ -z "${KVER}" ]]; then
-    echo "ERROR: No kernel found under /boot after linux-image-generic install"
-    exit 1
-fi
-echo "Target kernel: ${KVER}"
-
-# Install the NVIDIA headless server driver.
-# The nvidia-dkms-550-server post-install hook runs `dkms autoinstall`, which
-# builds the module against every kernel found under /lib/modules/ — including
-# ${KVER} whose headers we just installed above.
-apt-get install -y --no-install-recommends \
-    nvidia-headless-550-server \
-    nvidia-utils-550-server
-
-# Verify the DKMS module was registered and built.
-if dkms status 2>/dev/null | grep -q "nvidia"; then
-    echo "NVIDIA DKMS module status: $(dkms status 2>/dev/null | grep nvidia | head -1)"
-else
-    echo "WARNING: nvidia DKMS module not found in dkms status — manual verification required"
-fi
-
-# Pin the kernel so unattended-upgrades cannot replace it and invalidate
-# the pre-built DKMS module.
-apt-mark hold \
-    linux-image-generic \
-    linux-headers-generic \
-    "linux-image-${KVER}" \
-    "linux-headers-${KVER}"
-
-# Pin all installed NVIDIA packages for the same reason.
-NVIDIA_PKGS=$(dpkg --get-selections 'nvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
-LIBNVIDIA_PKGS=$(dpkg --get-selections 'libnvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
-# shellcheck disable=SC2086
-[[ -n "${NVIDIA_PKGS}" ]]    && apt-mark hold ${NVIDIA_PKGS}
-# shellcheck disable=SC2086
-[[ -n "${LIBNVIDIA_PKGS}" ]] && apt-mark hold ${LIBNVIDIA_PKGS}
-
-# Blacklist nouveau so it does not conflict with the NVIDIA driver on boot.
-cat > /etc/modprobe.d/blacklist-nouveau.conf << 'EOF'
-blacklist nouveau
-options nouveau modeset=0
-EOF
-
-# Prevent unattended-upgrades from updating kernel or NVIDIA packages,
-# which would break the pre-built DKMS module.
-mkdir -p /etc/apt/apt.conf.d
-cat > /etc/apt/apt.conf.d/99-gpu-ami << 'EOF'
-Unattended-Upgrade::Package-Blacklist {
-    "linux-";
-    "nvidia-";
-    "libnvidia-";
-};
-EOF
-
-# Rebuild initramfs with nouveau blacklisted and NVIDIA module included.
-update-initramfs -u -k "${KVER}"
-
-echo "GPU AMI setup complete: kernel=${KVER}, NVIDIA 550 driver installed, DKMS module pre-built"
@@ -1,12 +0,0 @@
-IMAGE_NAME="ubuntu-gpu"
-IMAGE_DESCRIPTION="Ubuntu 24.04 GPU-ready AMI (NVIDIA A10G / G5 family)"
-DISTRO="ubuntu"
-UBUNTU_VERSION="24.04"
-
-# 12G provides headroom for kernel headers, NVIDIA driver, and DKMS build artefacts
-IMAGE_SIZE="12G"
-
-# NVIDIA driver install and DKMS pre-build are handled in the setup script
-SETUP_SCRIPT="scripts/images/ubuntu-gpu-setup.sh"
-
-SYSTEM_TAG="gpu"
@@ -0,0 +1,248 @@
+#!/bin/bash
+set -euo pipefail
+
+export AWS_PROFILE=spinifex
+
+# Resolve the actual user's home — $HOME is /root when run under sudo.
+_USER_HOME=$(getent passwd "${SUDO_USER:-$(id -un)}" | cut -d: -f6)
+SSH_KEY="${SSH_KEY:-${_USER_HOME}/.ssh/spinifex-key}"
+SSH_USER="${SSH_USER:-ec2-user}"
+SSH_TIMEOUT="${SSH_TIMEOUT:-300}"
+
+usage() {
+    cat <<'EOF'
+launch-gpu.sh — Launch a GPU instance from the Spinifex AMD or NVIDIA GPU AMI.
+
+Detects the GPU brand from the specified instance type and selects the matching
+pre-built AMI automatically. Use --amd or --nvidia to override.
+
+Usage:
+  scripts/launch-gpu.sh --type <instance-type> [--disk GB] [--amd|--nvidia]
+
+Options:
+  --type TYPE    EC2 instance type to launch (required)
+  --disk GB      Root disk size in GB (default: 50)
+  --ami  NAME    Override the AMI name to use (default: ubuntu-amd-gpu or ubuntu-nvidia-gpu)
+  --amd          Force AMD GPU AMI  (ubuntu-amd-gpu)
+  --nvidia       Force NVIDIA GPU AMI (ubuntu-nvidia-gpu)
+
+Env overrides:
+  SSH_KEY      Path to SSH private key (default: ~/.ssh/spinifex-key)
+  SSH_USER     SSH user inside guest  (default: ec2-user)
+  SSH_TIMEOUT  Seconds to wait for initial SSH (default: 300)
+EOF
+}
+
+INSTANCE_TYPE=""
+DISK_GB=50
+GPU_BRAND=""
+AMI_OVERRIDE=""
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --type)   INSTANCE_TYPE="$2"; shift 2 ;;
+        --disk)   DISK_GB="$2";        shift 2 ;;
+        --ami)    AMI_OVERRIDE="$2";   shift 2 ;;
+        --amd)    GPU_BRAND="amd";     shift ;;
+        --nvidia) GPU_BRAND="nvidia";  shift ;;
+        -h|--help) usage; exit 0 ;;
+        *) echo "Unknown option: $1"; usage; exit 1 ;;
+    esac
+done
+
+list_gpu_types() {
+    aws ec2 describe-instance-types \
+        --query 'InstanceTypes[?GpuInfo].[InstanceType,GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Manufacturer,GpuInfo.Gpus[0].Name]' \
+        --output table 2>/dev/null || true
+}
+
+if [ -z "$INSTANCE_TYPE" ]; then
+    echo "Error: --type is required."
+    echo ""
+    echo "Available GPU instance types on this node:"
+    list_gpu_types
+    exit 1
+fi
+
+# --- Validate instance type and fetch GPU info in one call ---
+echo "==> Checking instance type: ${INSTANCE_TYPE}"
+INSTANCE_INFO=$(aws ec2 describe-instance-types \
+    --query "InstanceTypes[?InstanceType=='${INSTANCE_TYPE}' && GpuInfo].[GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Manufacturer,GpuInfo.Gpus[0].Name]" \
+    --output text 2>/dev/null || true)
+
+if [ -z "$INSTANCE_INFO" ] || [ "$INSTANCE_INFO" = "None" ]; then
+    echo "'${INSTANCE_TYPE}' is not available or is not a GPU type on this node."
+    echo ""
+    echo "Available GPU instance types:"
+    list_gpu_types
+    exit 1
+fi
+
+IFS=$'\t' read -r GPU_COUNT GPU_MANUFACTURER GPU_NAME <<< "$INSTANCE_INFO"
+echo "   ${GPU_COUNT}× ${GPU_NAME} (${GPU_MANUFACTURER})"
+
+# --- Resolve GPU brand ---
+if [ -z "$GPU_BRAND" ]; then
+    case "$GPU_MANUFACTURER" in
+        AMD|amd)       GPU_BRAND="amd" ;;
+        NVIDIA|nvidia) GPU_BRAND="nvidia" ;;
+        *)
+            echo "Cannot auto-detect GPU brand from manufacturer '${GPU_MANUFACTURER}'."
+            echo "Use --amd or --nvidia to specify explicitly."
+            exit 1
+            ;;
+    esac
+fi
+
+case "$GPU_BRAND" in
+    amd)    AMI_NAME="ubuntu-amd-gpu" ;;
+    nvidia) AMI_NAME="ubuntu-nvidia-gpu" ;;
+esac
+
+# --- Resolve AMI ---
+AMI_LOOKUP="${AMI_OVERRIDE:-${AMI_NAME}}"
+echo "==> Resolving AMI: ${AMI_LOOKUP}"
+AMI_ID=$(aws ec2 describe-images \
+    --query "Images[?Name=='${AMI_LOOKUP}'].ImageId | [0]" --output text 2>/dev/null || true)
+if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
+    echo "No AMI found with name '${AMI_LOOKUP}'."
+    echo "Build and import it first, or specify the AMI name with --ami <name>"
+    echo "  scripts/build-system-image.sh scripts/images/ubuntu-gpu-${GPU_BRAND}.conf --import"
+    exit 1
+fi
+echo "   $AMI_ID"
+
+# --- Resolve subnet ---
+echo "==> Resolving subnet"
+SUBNET_ID=$(aws ec2 describe-subnets \
+    --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId | [0]' --output text 2>/dev/null || true)
+if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
+    echo "No public subnet found"
+    exit 1
+fi
+echo "   $SUBNET_ID"
+
+VPC_ID=$(aws ec2 describe-subnets --subnet-ids "$SUBNET_ID" \
+    --query 'Subnets[0].VpcId' --output text 2>/dev/null || true)
+if [ -z "$VPC_ID" ] || [ "$VPC_ID" = "None" ]; then
+    echo "Could not resolve VPC for subnet $SUBNET_ID"
+    exit 1
+fi
+
+# --- Ensure SSH security group ---
+# The default VPC SG only allows intra-SG traffic; we need a group that
+# permits port 22 from anywhere so the SSH wait below can connect.
+SSH_SG_NAME="spinifex-gpu-ssh"
+echo "==> Ensuring SSH security group (${SSH_SG_NAME})"
+SG_ID=$(aws ec2 describe-security-groups \
+    --filters "Name=group-name,Values=${SSH_SG_NAME}" "Name=vpc-id,Values=${VPC_ID}" \
+    --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || true)
+if [ -z "$SG_ID" ] || [ "$SG_ID" = "None" ]; then
+    SG_ID=$(aws ec2 create-security-group \
+        --group-name "$SSH_SG_NAME" \
+        --description "Allow SSH access to GPU instances" \
+        --vpc-id "$VPC_ID" \
+        --query 'GroupId' --output text)
+    aws ec2 authorize-security-group-ingress \
+        --group-id "$SG_ID" \
+        --protocol tcp \
+        --port 22 \
+        --cidr 0.0.0.0/0 >/dev/null
+    echo "   Created $SG_ID"
+else
+    echo "   Reusing $SG_ID"
+fi
+
+# --- Ensure SSH key ---
+if [ ! -f "$SSH_KEY" ]; then
+    echo "==> Generating SSH key"
+    sudo -u "${SUDO_USER:-$(id -un)}" mkdir -p "$(dirname "$SSH_KEY")"
+    sudo -u "${SUDO_USER:-$(id -un)}" ssh-keygen -t ed25519 -f "$SSH_KEY" -N ""
+fi
+# Delete before import — stale key pair material causes import to fail
+# silently, leaving the instance unreachable via SSH.
+echo "==> Importing SSH key"
+aws ec2 delete-key-pair --key-name spinifex-key >/dev/null 2>&1 || true
+aws ec2 import-key-pair --key-name spinifex-key \
+    --public-key-material "fileb://${SSH_KEY}.pub" >/dev/null 2>&1 || true
+
+# --- Launch ---
+echo "==> Launching ${INSTANCE_TYPE} (${DISK_GB} GB)..."
+ID=$(aws ec2 run-instances \
+    --image-id "$AMI_ID" \
+    --instance-type "$INSTANCE_TYPE" \
+    --key-name spinifex-key \
+    --subnet-id "$SUBNET_ID" \
+    --security-group-ids "$SG_ID" \
+    --count 1 \
+    --block-device-mappings "[{\"DeviceName\":\"/dev/sda1\",\"Ebs\":{\"VolumeSize\":${DISK_GB},\"VolumeType\":\"gp3\",\"DeleteOnTermination\":true}}]" \
+    --query 'Instances[0].InstanceId' --output text)
+if [ -z "$ID" ] || [ "$ID" = "None" ] || [ "$ID" = "null" ]; then
+    echo "run-instances returned no InstanceId"
+    exit 1
+fi
+echo "   $ID"
+
+# --- Wait for running ---
+echo "==> Waiting for running state..."
+COUNT=0; STATE="unknown"
+while [ $COUNT -lt 120 ]; do
+    STATE=$(aws ec2 describe-instances --instance-ids "$ID" \
+        --query "Reservations[0].Instances[0].State.Name" --output text 2>/dev/null || echo "unknown")
+    [ "$STATE" = "running" ] && break
+    [ "$STATE" = "terminated" ] && { echo "Instance terminated unexpectedly"; exit 1; }
+    sleep 2; COUNT=$((COUNT + 2))
+done
+[ "$STATE" = "running" ] || { echo "Instance did not reach running state (last: $STATE)"; exit 1; }
+
+# --- Wait for public IP ---
+echo "==> Waiting for public IP..."
+IP=""
+for _i in $(seq 1 120); do
+    IP=$(aws ec2 describe-instances --instance-ids "$ID" \
+        --query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null || true)
+    [ -n "$IP" ] && [ "$IP" != "None" ] && [ "$IP" != "null" ] && break
+    sleep 1
+done
+[ -n "$IP" ] && [ "$IP" != "None" ] || { echo "No public IP assigned after 120s"; exit 1; }
+echo "   $IP"
+
+# Clear stale known_hosts entry — instances reuse IPs across runs.
+KNOWN_HOSTS="$HOME/.ssh/known_hosts"
+[ -f "$KNOWN_HOSTS" ] && ssh-keygen -f "$KNOWN_HOSTS" -R "$IP" >/dev/null 2>&1 || true
+
+# --- Wait for SSH ---
+echo "==> Waiting for SSH..."
+_ssh() {
+    ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        -o ConnectTimeout=5 -o BatchMode=yes \
+        -i "$SSH_KEY" "${SSH_USER}@${IP}" "$@"
+}
+
+elapsed=0
+while [ "$elapsed" -lt "$SSH_TIMEOUT" ]; do
+    _ssh 'echo ready' >/dev/null 2>&1 && break
+    sleep 5; elapsed=$((elapsed + 5))
+done
+[ "$elapsed" -lt "$SSH_TIMEOUT" ] || { echo "SSH timeout after ${SSH_TIMEOUT}s"; exit 1; }
+
+# --- GPU visibility check ---
+echo "==> Checking GPU visibility..."
+lspci_out=$(_ssh 'lspci -nn 2>/dev/null' || true)
+if printf '%s\n' "$lspci_out" | grep -qiE "1002:|10de:|Instinct|Aqua|Processing accelerator|NVIDIA"; then
+    printf '%s\n' "$lspci_out" \
+        | grep -iE "1002:|10de:|Instinct|Aqua|Processing accelerator|NVIDIA" \
+        | sed 's/^/   ✓ /'
+else
+    echo "   WARNING: GPU not visible in lspci — check host-side passthrough assignment"
+    printf '%s\n' "$lspci_out" | head -20 | sed 's/^/   /'
+fi
+
+# --- Done ---
+echo ""
+echo "✅ ${INSTANCE_TYPE} ready — ${ID} (${IP})"
+echo ""
+echo "   ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no ${SSH_USER}@${IP}"
+echo ""
+echo "   Terminate when done:"
+echo "   AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids ${ID}"
@@ -268,10 +268,9 @@ sudo systemctl start spinifex.target
 # Wait for services to start
 sleep 5

-# --- Build + import LB image (needs services running) ---
-echo "==> Building and importing LB image"
-cd "$PROJECT_ROOT" && make build-lb-agent
-"$PROJECT_ROOT/scripts/build-system-image.sh" "$PROJECT_ROOT/scripts/images/lb.conf" --import --quiet
+# --- Build + install microVM artifacts (kernel + initramfs for direct-boot LBs) ---
+echo "==> Building and installing microVM artifacts"
+cd "$PROJECT_ROOT" && make build-lb-agent install-microvm

 # --- Smoke test ---
 sudo -u "$INVOKING_USER" "$SCRIPT_DIR/smoke-test.sh"
@@ -326,7 +326,7 @@ if [ -n "$WAN_BRIDGE" ]; then
                   /etc/systemd/network/15-spinifex-veth-wan.network \
                   /etc/systemd/network/16-spinifex-veth-wan-ovs.network
        sudo networkctl reload 2>/dev/null || true
-        sudo ovs-vsctl --if-exists del-port "$WAN_BRIDGE" veth-wan-ovs
+        sudo ovs-vsctl --if-exists del-port veth-wan-ovs 2>/dev/null || true
        sudo ip link del veth-wan-br 2>/dev/null || true
    fi

@@ -346,6 +346,13 @@ install_files() {
        $SUDO install -m 0755 "$EXTRACT_DIR/setup.sh" /usr/local/share/spinifex/setup.sh
        info "  /usr/local/share/spinifex/setup.sh"
    fi
+
+    # microVM kernel + initramfs
+    $SUDO install -d /usr/share/spinifex/microvm
+    if [ -d "$EXTRACT_DIR/microvm" ]; then
+        $SUDO cp "$EXTRACT_DIR/microvm/"* /usr/share/spinifex/microvm/
+        info "  /usr/share/spinifex/microvm/*"
+    fi
 }

 # --- Create directories ---
@@ -380,11 +387,14 @@ create_directories() {
    # /run is tmpfs — direct mkdir doesn't survive reboot, and units have
    # ReadWritePaths= on these paths which fails namespace setup with ENOENT
    # if the dirs are absent at service start.
-    $SUDO install -m 0644 /dev/stdin /etc/tmpfiles.d/spinifex.conf <<'TMPEOF'
+    _tmpf=$(mktemp)
+    cat > "$_tmpf" <<'TMPEOF'
 # Type  Path               Mode  User                 Group                Age
 d       /run/spinifex      0770  root                 spinifex             -
 d       /run/spinifex/nbd  0770  spinifex-viperblock  spinifex-viperblock  -
 TMPEOF
+    $SUDO install -m 0644 "$_tmpf" /etc/tmpfiles.d/spinifex.conf
+    rm -f "$_tmpf"
    if [ "${ISO_BUILD:-0}" != "1" ]; then
        # Live systems: materialise the runtime dirs immediately so a re-run of
        # setup.sh on an existing host has correct /run state without a reboot.
@@ -25,6 +25,12 @@ GPU_FAMILY="${GPU_FAMILY:-g5}"

 export AWS_PROFILE=spinifex

+# AWS commands run as the invoking user so their ~/.aws/config (spinifex profile
+# + CA bundle) is used when the script is called via sudo.
+INVOKING_USER="${SUDO_USER:-$(id -un)}"
+INVOKING_HOME=$(getent passwd "$INVOKING_USER" | cut -d: -f6)
+aws_as_user() { sudo -u "$INVOKING_USER" env HOME="$INVOKING_HOME" AWS_PROFILE=spinifex aws "$@"; }
+
 # --- Wait for EC2 daemon to subscribe to NATS ---
 # Port 3000 (UI) becomes ready before the daemon finishes initialising.
 # Poll describe-key-pairs until it doesn't return InternalError.
@@ -32,7 +38,7 @@ echo "==> Waiting for EC2 daemon to be ready"
 DAEMON_TIMEOUT=60
 DAEMON_ELAPSED=0
 while [ $DAEMON_ELAPSED -lt $DAEMON_TIMEOUT ]; do
-    if aws ec2 describe-key-pairs --output text >/dev/null 2>&1; then
+    if aws_as_user ec2 describe-key-pairs --output text >/dev/null 2>&1; then
        break
    fi
    sleep 2
@@ -53,9 +59,9 @@ if [ ! -f "$SSH_KEY.pub" ]; then
 fi

 echo "==> Importing SSH key"
-! aws ec2 import-key-pair --key-name spinifex-key \
-    --public-key-material "fileb://$SSH_KEY.pub"
-aws ec2 describe-key-pairs
+aws_as_user ec2 import-key-pair --key-name spinifex-key \
+    --public-key-material "fileb://$SSH_KEY.pub" || true
+aws_as_user ec2 describe-key-pairs

 # --- GPU pre-flight (TEST_GPU=1 only) ---
 if [[ "$TEST_GPU" == "1" ]]; then
@@ -82,7 +88,7 @@ if [[ "$TEST_GPU" == "1" ]]; then
    echo "   GPU PCI ID: ${GPU_VENDOR_ID}:${GPU_DEVICE_ID}"

    echo "==> [GPU] Verifying ${GPU_FAMILY} instance types advertised"
-    G5_TYPES=$(aws ec2 describe-instance-types \
+    G5_TYPES=$(aws_as_user ec2 describe-instance-types \
        --query "InstanceTypes[?starts_with(InstanceType, \`${GPU_FAMILY}\`)].InstanceType" \
        --output text 2>/dev/null || true)
    if [[ -z "$G5_TYPES" ]]; then
@@ -110,7 +116,7 @@ esac

 AMI_NAME="ami-${IMAGE_NAME}"

-EXISTING_AMI_ID=$(aws ec2 describe-images \
+EXISTING_AMI_ID=$(aws_as_user ec2 describe-images \
    --query "Images[?Name=='${AMI_NAME}'] | [0].ImageId" \
    --output text)

@@ -137,7 +143,7 @@ else
    INSTANCE_TYPE="t3.small"
 fi

-AMI_ID=$(aws ec2 describe-images \
+AMI_ID=$(aws_as_user ec2 describe-images \
    --query "Images[?Name=='${AMI_NAME}'] | [0].ImageId" \
    --output text)

@@ -146,7 +152,7 @@ if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
    exit 1
 fi

-SUBNET_ID=$(aws ec2 describe-subnets --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId | [0]' --output text)
+SUBNET_ID=$(aws_as_user ec2 describe-subnets --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId | [0]' --output text)
 if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
    echo "❌ No subnet found"
    exit 1
@@ -154,7 +160,7 @@ fi

 echo "  AMI: $AMI_ID  type: $INSTANCE_TYPE  subnet: $SUBNET_ID"

-INSTANCE_ID=$(aws ec2 run-instances \
+INSTANCE_ID=$(aws_as_user ec2 run-instances \
    --image-id "$AMI_ID" \
    --instance-type "$INSTANCE_TYPE" \
    --key-name spinifex-key \
@@ -173,7 +179,7 @@ echo "==> Waiting for instance to reach running state"
 COUNT=0
 STATE="unknown"
 while [ $COUNT -lt 60 ]; do
-    DESCRIBE=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" 2>/dev/null) || {
+    DESCRIBE=$(aws_as_user ec2 describe-instances --instance-ids "$INSTANCE_ID" 2>/dev/null) || {
        sleep 2; COUNT=$((COUNT + 1)); continue
    }
    STATE=$(echo "$DESCRIBE" | jq -r '.Reservations[0].Instances[0].State.Name // "not-found"')
@@ -196,7 +202,7 @@ echo "==> Waiting for public IP assignment (up to 300s)"
 SSH_INST_PORT=22
 SSH_INST_HOST=""
 for _i in $(seq 1 300); do
-    _ip=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" \
+    _ip=$(aws_as_user ec2 describe-instances --instance-ids "$INSTANCE_ID" \
        --query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null || true)
    if [ -n "$_ip" ] && [ "$_ip" != "None" ] && [ "$_ip" != "null" ]; then
        SSH_INST_HOST="$_ip"
@@ -256,14 +262,14 @@ if [[ "$TEST_GPU" == "1" ]]; then
               -o ConnectTimeout=5 -o BatchMode=yes \
               -p "$SSH_INST_PORT" -i "$SSH_KEY" \
               ec2-user@"$SSH_INST_HOST" \
-               'lspci 2>/dev/null | grep -i nvidia' 2>/dev/null; then
+               'lspci 2>/dev/null | grep -iE "nvidia|amdgpu|instinct"' 2>/dev/null; then
            GPU_VISIBLE=1
            break
        fi
        sleep 5
    done
    if [[ $GPU_VISIBLE -eq 0 ]]; then
-        echo "❌ NVIDIA GPU not visible in guest via lspci"
+        echo "❌ GPU not visible in guest via lspci"
        exit 1
    fi
    echo "   GPU visible in guest"
@@ -886,8 +886,8 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP s
 // multi-node Predastore cluster. Each node gets its own DB entry (port 6660)
 // and shard entry (port 9991) on a distinct IP. Node ID 1 is the bootstrap leader.
 func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region, natsToken, configDir, bindIP string) (string, error) {
-	if len(nodes) < 3 {
-		return "", fmt.Errorf("multi-node predastore requires at least 3 nodes, got %d", len(nodes))
+	if len(nodes) < 2 {
+		return "", fmt.Errorf("multi-node predastore requires at least 2 nodes, got %d", len(nodes))
 	}

 	data := struct {
@@ -738,10 +738,9 @@ func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) {

 	_, err := GenerateMultiNodePredastoreConfig(tmpl, []PredastoreNodeConfig{
 		{ID: 1, Host: "10.0.0.1"},
-		{ID: 2, Host: "10.0.0.2"},
 	}, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1")
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "at least 3 nodes")
+	assert.Contains(t, err.Error(), "at least 2 nodes")
 }

 func TestGenerateMultiNodePredastoreConfig_InvalidTemplate(t *testing.T) {
@@ -2,473 +2,47 @@ package awserrors

 import "testing"

-func TestErrorLookup(t *testing.T) {
-	expected := []struct {
-		code    string
-		http    int
-		message string
-	}{
-		{code: "AccountDisabled", http: 400, message: "The functionality you have requested has been administratively disabled for this account."},
-		{code: "ActiveVpcPeeringConnectionPerVpcLimitExceeded", http: 400, message: "You've reached the limit on the number of active VPC peering connections you can have for the specified VPC."},
-		{code: "AddressLimitExceeded", http: 400, message: "You've reached the limit on the number of Elastic IP addresses that you can allocate. For more information, see Elastic IP address limit."},
-		{code: "AsnConflict", http: 400, message: "The Autonomous System Numbers (ASNs) of the specified customer gateway and the specified virtual private gateway are the same."},
-		{code: "AttachmentLimitExceeded", http: 400, message: "You've reached the limit on the number of Amazon EBS volumes or network interfaces that can be attached to a single instance. The number of Amazon EBS volumes that you can attach to an instance depends on the instance type. For more information, see Amazon EBS volume limits for Amazon EC2 instances in the Amazon EC2 User Guide."},
-		{code: "AuthFailure", http: 403, message: "The provided credentials could not be validated. You might not be authorized to carry out the request; for example, trying to associate an Elastic IP address that is not yours, or trying to use an AMI for which you do not have permissions. Ensure that your account is authorized to use Amazon EC2, that your credit card details are correct, and that you are using the correct credentials."},
-		{code: "BandwidthLimitExceeded", http: 500, message: "You've reached the limit on the network bandwidth that is available to an Amazon EC2 instance. For more information, see Amazon EC2 instance network bandwidth."},
-		{code: "Blocked", http: 403, message: "Your account is currently blocked. Contact Support if you have questions."},
-		{code: "BootForVolumeTypeUnsupported", http: 400, message: "The specified volume type cannot be used as a boot volume. For more information, see Amazon EBS volume types."},
-		{code: "BundlingInProgress", http: 400, message: "The specified instance already has a bundling task in progress."},
-		{code: "CannotDelete", http: 400, message: "You cannot delete the 'default' security group in your VPC, but you can change its rules. For more information, see Amazon EC2 security groups."},
-		{code: "CapacityBlockDescribeLimitExceeded", http: 400, message: "You've reached the limit for this account. The returned message provides details."},
-		{code: "ClientInvalidParameterValue", http: 400, message: "A parameter specified in a request is not valid, is unsupported, or cannot be used. The returned message provides an explanation of the error value. For example, if you are launching an instance, you can't specify a security group and subnet that are in different VPCs."},
-		{code: "ClientVpnAuthorizationRuleLimitExceeded", http: 400, message: "You've reached the limit on the number of authorization rules that can be added to a single Client VPN endpoint."},
-		{code: "ClientVpnCertificateRevocationListLimitExceeded", http: 400, message: "You've reached the limit on the number of client certificate revocation lists that can be added to a single Client VPN endpoint."},
-		{code: "ClientVpnEndpointAssociationExists", http: 409, message: "The specified target network is already associated with the Client VPN endpoint."},
-		{code: "ClientVpnEndpointLimitExceeded", http: 400, message: "You've reached the limit on the number of Client VPN endpoints that you can create."},
-		{code: "ClientVpnRouteLimitExceeded", http: 400, message: "You've reached the limit on the number of routes that can be added to a single Client VPN endpoint."},
-		{code: "ClientVpnTerminateConnectionsLimitExceeded", http: 400, message: "The number of client connections you're attempting to terminate exceeds the limit."},
-		{code: "ConcurrentCreateImageNoRebootLimitExceeded", http: 400, message: "The maximum number of concurrent CreateImage requests for the instance has been reached. Wait for the current CreateImage requests to complete, and then retry your request."},
-		{code: "ConcurrentSnapshotLimitExceeded", http: 409, message: "You've reached the limit on the number of concurrent snapshots you can create on the specified volume. Wait until the 'pending' requests have completed, and check that you do not have snapshots that are in an incomplete state, such as 'error', which count against your concurrent snapshot limit."},
-		{code: "ConcurrentTagAccess", http: 400, message: "You can't run simultaneous commands to modify a tag for a specific resource. Allow sufficient wait time for the previous request to complete, then retry your request."},
-		{code: "CreditSpecificationUpdateInProgress", http: 400, message: "The default credit specification for the instance family is currently being updated. It takes about five minutes to complete. For more information, see Set the default credit specification for the account."},
-		{code: "CustomerGatewayLimitExceeded", http: 400, message: "You've reached the limit on the number of customer gateways you can create for the AWS Region. For more information, see Amazon VPC quotas."},
-		{code: "CustomerKeyHasBeenRevoked", http: 400, message: "The KMS key cannot be accessed. For more information, see Amazon EBS encryption."},
-		{code: "DeclarativePoliciesAccessDeniedException", http: 404, message: "You do not have sufficient access to perform this action, or the specified TargetId does not exist, or the specified TargetId is not in your organization. To generate an account status report for declarative policies, the caller must be the management account or a delegated administrator for the organization and the specified TargetId must belong to your organization."},
-		{code: "DeclarativePoliciesNotEnabledException", http: 400, message: "Trusted access is not enabled. Trusted access must be enabled for the service for which the declarative policy will enforce a baseline configuration. The API uses the following service principal to identify the EC2 service: ec2.amazonaws.com. For more information on how to enable trusted access with the AWS CLI and AWS SDKs, see Using Organizations with other AWS services."},
-		{code: "DefaultSubnetAlreadyExistsInAvailabilityZone", http: 409, message: "A default subnet already exists in the specified Availability Zone. You can have only one default subnet per Availability Zone."},
-		{code: "DefaultVpcAlreadyExists", http: 409, message: "A default VPC already exists in the AWS Region. You can only have one default VPC per Region."},
-		{code: "DefaultVpcDoesNotExist", http: 400, message: "There is no default VPC in which to carry out the request. If you've deleted your default VPC, you can create a new one. For more information, see Create a default VPC."},
-		{code: "DeleteConversionTaskError", http: 400, message: "The conversion task cannot be canceled."},
-		{code: "DependencyViolation", http: 400, message: "The specified object has dependent resources. A number of resources in a VPC may have dependent resources, which prevent you from deleting or detaching them. Remove the dependencies first, then retry your request. For example, this error occurs if you try to delete a security group in a VPC that is in use by another security group."},
-		{code: "DiskImageSizeTooLarge", http: 400, message: "The disk image exceeds the allowed limit (for instance or volume import)."},
-		{code: "DryRunOperation", http: 412, message: "The user has the required permissions, so the request would have succeeded, but the DryRun parameter was used."},
-		{code: "DuplicateSubnetsInSameZone", http: 400, message: "For an interface VPC endpoint, you can specify only one subnet per Availability Zone."},
-		{code: "EncryptedVolumesNotSupported", http: 400, message: "Encrypted Amazon EBS volumes may only be attached to instances that support Amazon EBS encryption. For more information, see Amazon EBS encryption."},
-		{code: "ExistingVpcEndpointConnections", http: 400, message: "You cannot delete a VPC endpoint service configuration or change the load balancers for the endpoint service if there are endpoints attached to the service."},
-		{code: "FilterLimitExceeded", http: 400, message: "The request uses too many filters or too many filter values."},
-		{code: "FleetNotInModifiableState", http: 400, message: "The Spot Fleet request must be in the active state in order to modify it. For more information, see Spot Fleet request types."},
-		{code: "FlowLogAlreadyExists", http: 409, message: "A flow log with the specified configuration already exists."},
-		{code: "FlowLogsLimitExceeded", http: 400, message: "You've reached the limit on the number of flow logs you can create. For more information, see Amazon VPC quotas."},
-		{code: "Gateway.NotAttached", http: 400, message: "An internet gateway is not attached to a VPC. If you are trying to detach an internet gateway, ensure that you specify the correct VPC. If you are trying to associate an Elastic IP address with a network interface or an instance, ensure that an internet gateway is attached to the relevant VPC."},
-		{code: "HostAlreadyCoveredByReservation", http: 400, message: "The specified Dedicated Host is already covered by a reservation."},
-		{code: "HostLimitExceeded", http: 400, message: "You've reached the limit on the number of Dedicated Hosts that you can allocate. For more information, see Dedicated Hosts."},
-		{code: "IdempotentInstanceTerminated", http: 400, message: "The request to launch an instance uses the same client token as a previous request for which the instance has been terminated."},
-		{code: "IdempotentParameterMismatch", http: 400, message: "The request uses the same client token as a previous, but non-identical request. Do not reuse a client token with different requests, unless the requests are identical."},
-		{code: "InaccessibleStorageLocation", http: 400, message: "The specified Amazon S3 URL cannot be accessed. Check the access permissions for the URL."},
-		{code: "InaccessibleStorageLocationException", http: 400, message: "The specified Amazon S3 bucket can't be accessed. An S3 bucket must be available before generating an account status report for declarative policies (you can create a new one or use an existing one), you must own the bucket, it must be in the same Region in which the request was made, and it must have an appropriate bucket policy. For a sample S3 policy, see Sample Amazon S3 policy under Examples."},
-		{code: "IncompatibleHostRequirements", http: 400, message: "There are no available or compatible Dedicated Hosts available on which to launch or start the instance."},
-		{code: "IncompleteSignature", http: 400, message: "The request signature does not conform to AWS standards."},
-		{code: "IncorrectInstanceState", http: 409, message: "The instance is in an incorrect state for the requested action. For example, some instance attributes, such as user data, can only be modified if the instance is in a 'stopped' state. If you are associating an Elastic IP address with a network interface, ensure that the instance that the interface is attached to is not in the 'pending' state."},
-		{code: "IncorrectModificationState", http: 400, message: "A new modification action on an EBS Elastic Volume cannot occur because the volume is currently being modified."},
-		{code: "IncorrectSpotRequestState", http: 400, message: "The Spot Instance request is in an incorrect state for the request. Spot request status information can help you track your Amazon EC2 Spot Instance requests. For more information, see Spot request status."},
-		{code: "IncorrectState", http: 400, message: "The resource is in an incorrect state for the request. This error can occur if you are trying to attach a volume that is still being created or detach a volume that is not in the 'available' state. Verify that the volume is in the 'available' state. If you are creating a snapshot, ensure that the previous request to create a snapshot on the same volume has completed. If you are deleting a virtual private gateway, ensure that it's detached from the VPC."},
-		{code: "IncorrectStateException", http: 400, message: "The resource is in an incorrect state for the request. This error can occur if you're trying to cancel the generation of an account status report for declarative policies that already has the complete, cancelled, or error status. Cancelation is only possible for reports with a running status. This error can also occur when you're trying to get a summary for a report that has not yet reached the completed status. This error can also occur if you start report generation while another report is being generated. Only one report per organization can be generated at a time. For more information, see Generating the account status report for declarative policies."},
-		{code: "InstanceCreditSpecification.NotSupported", http: 400, message: "The specified instance does not use CPU credits for CPU usage; only T2 instances use CPU credits for CPU usage."},
-		{code: "InstanceEventStartTimeCannotChange", http: 400, message: "The specified scheduled event start time does not meet the requirements for rescheduling a scheduled event. For more information, see Limitations."},
-		{code: "InstanceLimitExceeded", http: 400, message: "You've reached the limit on the number of instances you can run concurrently. This error can occur if you are launching an instance or if you are creating a Capacity Reservation. Capacity Reservations count towards your On-Demand Instance limits. If your request fails due to limit constraints, increase your On-Demand Instance limit for the required instance type and try again. For more information, see EC2 On-Demand instance limits."},
-		{code: "InstanceTpmEkPubNotFound", http: 400, message: "The public Trusted Platform Module (TPM) Endorsement Key (EK) cannot be found."},
-		{code: "InsufficientAddressCapacity", http: 503, message: "Not enough available addresses to satisfy your minimum request. Reduce the number of addresses you are requesting or wait for additional capacity to become available."},
-		{code: "InsufficientCapacity", http: 503, message: "There is not enough capacity to fulfill your import instance request. You can wait for additional capacity to become available."},
-		{code: "InsufficientCapacityOnHost", http: 400, message: "There is not enough capacity on the Dedicated Host to launch or start the instance."},
-		{code: "InsufficientFreeAddressesInSubnet", http: 503, message: "The specified subnet does not contain enough free private IP addresses to fulfill your request. Use the DescribeSubnets request to view how many IP addresses are available (unused) in your subnet. IP addresses associated with stopped instances are considered unavailable."},
-		{code: "InsufficientHostCapacity", http: 503, message: "There is not enough capacity to fulfill your Dedicated Host request. Reduce the number of Dedicated Hosts in your request, or wait for additional capacity to become available."},
-		{code: "InsufficientInstanceCapacity", http: 400, message: "There is not enough capacity to fulfill your request. This error can occur if you launch a new instance, restart a stopped instance, create a new Capacity Reservation, or modify an existing Capacity Reservation. Reduce the number of instances in your request, or wait for additional capacity to become available. You can also try launching an instance by selecting different instance types (which you can resize at a later stage). The returned message might also give specific guidance about how to solve the problem."},
-		{code: "InsufficientReservedInstanceCapacity", http: 503, message: "Not enough available Reserved Instances to satisfy your minimum request. Reduce the number of Reserved Instances in your request or wait for additional capacity to become available."},
-		{code: "InsufficientReservedInstancesCapacity", http: 400, message: "There is insufficient capacity for the requested Reserved Instances."},
-		{code: "InsufficientVolumeCapacity", http: 503, message: "There is not enough capacity to fulfill your EBS volume provision request. You can try to provision a different volume type, EBS volume in a different availability zone, or you can wait for additional capacity to become available."},
-		{code: "InterfaceInUseByTrafficMirrorSession", http: 409, message: "The Traffic Mirror source that you are trying to create uses an interface that is already associated with a session. An interface can only be associated with a session, or with a target, but not both."},
-		{code: "InterfaceInUseByTrafficMirrorTarget", http: 409, message: "The Traffic Mirror source that you are trying to create uses an interface that is already associated with a target. An interface can only be associated with a session, or with a target, but not both. If the interface is associated with a target, it cannot be associated with another target."},
-		{code: "InternalError", http: 500, message: "An internal error has occurred. Retry your request, but if the problem persists, contact us with details by posting a message on AWS re:Post."},
-		{code: "InternalFailure", http: 404, message: "The request processing has failed because of an unknown error, exception, or failure."},
-		{code: "InternetGatewayLimitExceeded", http: 400, message: "You've reached the limit on the number of internet gateways that you can create. For more information, see Amazon VPC quotas."},
-		{code: "InvalidAMIAttributeItemValue", http: 400, message: "The value of an item added to, or removed from, an image attribute is not valid. If you are specifying a userId, check that it is in the form of an AWS account ID, without hyphens."},
-		{code: "InvalidAMIID.Malformed", http: 400, message: "The specified AMI ID is malformed. Ensure that you provide the full AMI ID, in the form ami-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidAMIID.NotFound", http: 400, message: "The specified AMI doesn't exist. Check the AMI ID, and ensure that you specify the AWS Region in which the AMI is located, if it's not in the default Region. This error might also occur if you specified an incorrect kernel ID when launching an instance. This error might also occur if the AMI doesn't meet your Allowed AMIs criteria. To use this AMI, you must change the Allowed AMIs criteria. For more information, see Control the discovery and use of AMIs in Amazon EC2 with Allowed AMIs."},
-		{code: "InvalidAMIID.Unavailable", http: 400, message: "The specified AMI has been deregistered and is no longer available, or is not in a state from which you can launch an instance or modify attributes."},
-		{code: "InvalidAMIName.Duplicate", http: 409, message: "The specified AMI name is already in use by another AMI. If you have recently deregistered an AMI with the same name, allow enough time for the change to propagate through the system, and retry your request."},
-		{code: "InvalidAMIName.Malformed", http: 400, message: "AMI names must be between 3 and 128 characters long, and may only contain letters, numbers, and the following special characters: '-', '_', '.', '/', '(', and ')'."},
-		{code: "InvalidAction", http: 400, message: "The action or operation requested is not valid. Verify that the action is typed correctly."},
-		{code: "InvalidAddress.Locked", http: 400, message: "The specified Elastic IP address cannot be released from your account. A reverse DNS record may be associated with the Elastic IP address. To unlock the address, contact Support."},
-		{code: "InvalidAddress.Malformed", http: 400, message: "The specified IP address is not valid. Ensure that you provide the address in the form xx.xx.xx.xx; for example, 55.123.45.67"},
-		{code: "InvalidAddress.NotFound", http: 400, message: "The specified Elastic IP address that you are describing cannot be found. Ensure that you specify the AWS Region in which the IP address is located, if it's not in the default Region."},
-		{code: "InvalidAddressID.NotFound", http: 400, message: "The specified allocation ID for the Elastic IP address you are trying to release cannot be found. Ensure that you specify the AWS Region in which the IP address is located, if it's not in the default Region."},
-		{code: "InvalidAffinity", http: 400, message: "The specified affinity value is not valid."},
-		{code: "InvalidAllocationID.NotFound", http: 404, message: "The specified allocation ID you are trying to describe or associate does not exist. Ensure that you specify the AWS Region in which the IP address is located, if it's not in the default Region."},
-		{code: "InvalidAssociationID.NotFound", http: 404, message: "The specified association ID (for an Elastic IP address, a route table, or network ACL) does not exist. Ensure that you specify the AWS Region in which the association ID is located, if it's not in the default Region."},
-		{code: "InvalidAttachment.NotFound", http: 400, message: "Indicates an attempt to detach a volume from an instance to which it is not attached."},
-		{code: "InvalidAttachmentID.NotFound", http: 404, message: "The specified network interface attachment does not exist."},
-		{code: "InvalidAutoPlacement", http: 400, message: "The specified value for auto-placement is not valid."},
-		{code: "InvalidAvailabilityZone", http: 400, message: "The specified Availability Zone is not valid."},
-		{code: "InvalidBlockDeviceMapping", http: 400, message: "A block device mapping parameter is not valid. The returned message indicates the incorrect value."},
-		{code: "InvalidBundleID.NotFound", http: 400, message: "The specified bundle task ID cannot be found. Ensure that you specify the AWS Region in which the bundle task is located, if it's not in the default Region."},
-		{code: "InvalidCapacityBlockOfferingIdExpired", http: 400, message: "The Capacity Block offering ID is no longer available."},
-		{code: "InvalidCapacityBlockOfferingIdMalformed", http: 400, message: "The Capacity Block offering ID is malformed."},
-		{code: "InvalidCapacityBlockOfferingIdNotFound", http: 400, message: "The Capacity Block offering ID cannot be found for this account."},
-		{code: "InvalidCapacityReservationId.Malformed", http: 400, message: "The ID for the Capacity Reservation is malformed. Ensure that you specify the Capacity Reservation ID in the form cr-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidCapacityReservationId.NotFound", http: 404, message: "The specified Capacity Reservation ID does not exist."},
-		{code: "InvalidCapacityReservationState.PendingActivation", http: 400, message: "Your Capacity Block is not active yet."},
-		{code: "InvalidCarrierGatewayID.NotFound", http: 400, message: "The specified carrier gateway ID cannot be found. Ensure that you specify the AWS Region in which the carrier gateway is located, if it's not in the default Region."},
-		{code: "InvalidCharacter", http: 400, message: "A specified character is invalid."},
-		{code: "InvalidCidr.InUse", http: 409, message: "The specified inside tunnel CIDR is already in use by another VPN tunnel for the virtual private gateway."},
-		{code: "InvalidClientToken", http: 400, message: "The specified client token is not valid. For more information, see Ensuring idempotency."},
-		{code: "InvalidClientTokenId", http: 403, message: "The X.509 certificate or credentials provided do not exist in our records."},
-		{code: "InvalidClientVpnActiveAssociationNotFound", http: 409, message: "You cannot perform this action on the Client VPN endpoint while it is in the pending-association state."},
-		{code: "InvalidClientVpnAssociationIdNotFound", http: 400, message: "The specified target network association cannot be found."},
-		{code: "InvalidClientVpnConnection.IdNotFound", http: 400, message: "The specified Client VPN endpoint cannot be found."},
-		{code: "InvalidClientVpnConnection.UserNotFound", http: 400, message: "The specified user does not have an active connection to the specified Client VPN endpoint."},
-		{code: "InvalidClientVpnDuplicateAssociationException", http: 400, message: "The specified target network has already been associated with the Client VPN endpoint."},
-		{code: "InvalidClientVpnDuplicateAuthorizationRule", http: 400, message: "The specified authorization has already been added to the Client VPN endpoint."},
-		{code: "InvalidClientVpnDuplicateRoute", http: 400, message: "The specified route has already been added to the Client VPN endpoint."},
-		{code: "InvalidClientVpnEndpointAuthorizationRuleNotFound", http: 400, message: "The specified authorization rule cannot be found."},
-		{code: "InvalidClientVpnEndpointId.NotFound", http: 400, message: "The specified Client VPN Endpoint cannot be found."},
-		{code: "InvalidClientVpnRouteNotFound", http: 400, message: "The specified route cannot be found."},
-		{code: "InvalidClientVpnSubnetId.DifferentAccount", http: 400, message: "The specified subnet belongs to a different account."},
-		{code: "InvalidClientVpnSubnetId.DuplicateAz", http: 409, message: "You have already associated a subnet from this Availability Zone with the Client VPN endpoint."},
-		{code: "InvalidClientVpnSubnetId.NotFound", http: 400, message: "The specified subnet cannot be found in the VPN with which the Client VPN endpoint is associated."},
-		{code: "InvalidClientVpnSubnetId.OverlappingCidr", http: 400, message: "The specified target network's CIDR range overlaps with the Client VPN endpoint's client CIDR range."},
-		{code: "InvalidConversionTaskId", http: 400, message: "The specified conversion task ID (for instance or volume import) is not valid."},
-		{code: "InvalidConversionTaskId.Malformed", http: 400, message: "The specified conversion task ID (for instance or volume import) is malformed. Ensure that you've specified the ID in the form import-i-xxxxxxxx."},
-		{code: "InvalidCpuCredits", http: 400, message: "The specified CpuCredit value is invalid. Valid values are standard and unlimited."},
-		{code: "InvalidCpuCredits.Malformed", http: 400, message: "The specified CpuCredit value is invalid. Valid values are standard and unlimited."},
-		{code: "InvalidCustomerGateway.DuplicateIpAddress", http: 409, message: "There is a conflict among the specified gateway IP addresses. Each VPN connection in an AWS Region must be created with a unique customer gateway IP address (across all AWS accounts). For more information, see Your customer gateway device in the AWS Site-to-Site VPN User Guide."},
-		{code: "InvalidCustomerGatewayID.NotFound", http: 400, message: "The specified customer gateway ID cannot be found. Ensure that you specify the AWS Region in which the customer gateway is located, if it's not in the default Region."},
-		{code: "InvalidCustomerGatewayId.Malformed", http: 400, message: "The specified customer gateway ID is malformed, or cannot be found. Specify the ID in the form cgw-xxxxxxxx, and ensure that you specify the AWS Region in which the customer gateway is located, if it's not in the default Region."},
-		{code: "InvalidCustomerGatewayState", http: 400, message: "The customer gateway is not in the available state, and therefore cannot be used."},
-		{code: "InvalidDeclarativePoliciesReportId.Malformed", http: 400, message: "The specified account status report ID for declarative policies is malformed. Ensure that you specify the ID in the form p-xxxxxxxxxx. For more information, see Generating the account status report for declarative policies."},
-		{code: "InvalidDhcpOptionID.NotFound", http: 404, message: "The specified DHCP options set does not exist. Ensure that you specify the AWS Region in which the DHCP options set is located, if it's not in the default Region."},
-		{code: "InvalidDhcpOptionsID.NotFound", http: 404, message: "The specified DHCP options set does not exist. Ensure that you specify the AWS Region in which the DHCP options set is located, if it's not in the default Region."},
-		{code: "InvalidDhcpOptionsId.Malformed", http: 400, message: "The specified DHCP options set ID is malformed. Ensure that you provide the full DHCP options set ID in the request, in the form dopt-xxxxxxxx."},
-		{code: "InvalidEgressOnlyInternetGatewayId.Malformed", http: 400, message: "The specified egress-only internet gateway ID is malformed. Ensure that you specify the ID in the form eigw-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidEgressOnlyInternetGatewayId.NotFound", http: 404, message: "The specified egress-only internet gateway does not exist."},
-		{code: "InvalidElasticGpuID.Malformed", http: 400, message: "The specified Elastic GPU ID is malformed. Ensure that you specify the ID in the form egpu-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidElasticGpuID.NotFound", http: 404, message: "The specified Elastic GPU does not exist."},
-		{code: "InvalidExportTaskID.Malformed", http: 400, message: "The specified export task ID cannot be found."},
-		{code: "InvalidExportTaskID.NotFound", http: 400, message: "The specified export task ID is malformed. Ensure that you specify the ID in the form export-ami-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidFilter", http: 400, message: "The specified filter is not valid."},
-		{code: "InvalidFlowLogId.NotFound", http: 404, message: "The specified flow log does not exist."},
-		{code: "InvalidFormat", http: 400, message: "The specified disk format (for the instance or volume import) is not valid."},
-		{code: "InvalidFpgaImageID.Malformed", http: 400, message: "The specified Amazon FPGA image (AFI) ID is malformed. Ensure that you provide the full AFI ID in the request, in the form afi-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidFpgaImageID.NotFound", http: 404, message: "The specified Amazon FPGA image (AFI) ID does not exist. Ensure that you specify the AWS Region in which the AFI is located, if it's not in the default Region."},
-		{code: "InvalidGatewayID.NotFound", http: 404, message: "The specified gateway does not exist."},
-		{code: "InvalidGroup.Duplicate", http: 400, message: "You cannot create a security group with the same name as an existing security group in the same VPC."},
-		{code: "InvalidGroup.InUse", http: 409, message: "The specified security group can't be deleted because it's in use by another security group. You can remove dependencies by modifying or deleting rules in the affected security groups."},
-		{code: "InvalidGroup.NotFound", http: 404, message: "The specified security group does not exist. This error can occur because the ID of a recently created security group has not propagated through the system. For more information, see Ensuring idempotency. You can't specify a security group that is in a different AWS Region or VPC than the request."},
-		{code: "InvalidGroup.Reserved", http: 400, message: "The name 'default' is reserved, and cannot be used to create a new security group."},
-		{code: "InvalidGroupId.Malformed", http: 400, message: "The specified security group ID is malformed. Ensure that you provide the full security group ID in the request, in the form sg-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidHostConfiguration", http: 400, message: "The specified Dedicated Host configuration is not supported."},
-		{code: "InvalidHostID.Malformed", http: 400, message: "The specified Dedicated Host ID is not formed correctly. Ensure that you provide the full ID in the form h-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidHostID.NotFound", http: 404, message: "The specified Dedicated Host ID does not exist. Ensure that you specify the AWS Region in which the Dedicated Host is located, if it's not in the default Region."},
-		{code: "InvalidHostId", http: 400, message: "The specified Dedicated Host ID is not valid."},
-		{code: "InvalidHostId.Malformed", http: 400, message: "The specified Dedicated Host ID is not formed correctly. Ensure that you provide the full ID in the form h-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidHostId.NotFound", http: 404, message: "The specified Dedicated Host ID does not exist. Ensure that you specify the region in which the Dedicated Host is located, if it's not in the default region."},
-		{code: "InvalidHostReservationId.Malformed", http: 400, message: "The specified Dedicated Host Reservation ID is not formed correctly. Ensure that you provide the full ID in the form hr-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidHostReservationOfferingId.Malformed", http: 400, message: "The specified Dedicated Host Reservation offering is not formed correctly. Ensure that you provide the full ID in the form hro-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidHostState", http: 400, message: "The Dedicated Host must be in the available state to complete the operation."},
-		{code: "InvalidID", http: 400, message: "The specified ID for the resource you are trying to tag is not valid. Ensure that you provide the full resource ID; for example, ami-2bb65342 for an AMI. If you're using the command line tools on a Windows system, you might need to use quotation marks for the key-value pair; for example, \"Name=TestTag\"."},
-		{code: "InvalidIPAddress.InUse", http: 409, message: "The specified IP address is already in use. If you are trying to release an address, you must first disassociate it from the instance."},
-		{code: "InvalidIamInstanceProfileArn.Malformed", http: 400, message: "The specified IAM instance profile ARN is not valid. For more information about valid ARN formats, see Amazon Resource Names (ARNs)."},
-		{code: "InvalidInput", http: 400, message: "An input parameter in the request is not valid. For example, you may have specified an incorrect Reserved Instance listing ID in the request or the Reserved Instance you tried to list cannot be sold in the Reserved Instances Marketplace (for example, if it has a scope of Region, or is a Convertible Reserved Instance)."},
-		{code: "InvalidInstanceAttributeValue", http: 400, message: "The specified instance attribute value is not valid. This error is most commonly encountered when trying to set the InstanceType/--instance-type attribute to an unrecognized value."},
-		{code: "InvalidInstanceConnectEndpointId.Malformed", http: 400, message: "The specified EC2 Instance Connect Endpoint ID is malformed. Ensure that you specify the ID in the form eice-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidInstanceConnectEndpointId.NotFound", http: 404, message: "The specified EC2 Instance Connect Endpoint does not exist."},
-		{code: "InvalidInstanceCreditSpecification", http: 409, message: "If you are modifying the credit option for CPU usage for T2 instances, the request may not contain duplicate instance IDs."},
-		{code: "InvalidInstanceCreditSpecification.DuplicateInstanceId", http: 409, message: "If you are modifying the credit option for CPU usage for T2 instances, the request may not contain duplicate instance IDs."},
-		{code: "InvalidInstanceEventIDNotFound", http: 400, message: "The specified ID of the event whose date and time you are modifying cannot be found. Verify the ID of the event and try your request again."},
-		{code: "InvalidInstanceEventStartTime", http: 400, message: "The specified scheduled event start time does not meet the requirements for rescheduling a scheduled event. For more information, see Limitations."},
-		{code: "InvalidInstanceFamily", http: 400, message: "The instance family is not supported for this request. For example, the instance family for the Dedicated Host Reservation offering is different from the instance family of the Dedicated Hosts. Or, you can only modify the default credit specification for burstable performance instance families (T2, T3, and T3a). For more information, see Set the default credit specification for the account."},
-		{code: "InvalidInstanceID", http: 400, message: "This error can occur when trying to perform an operation on an instance that has multiple network interfaces. A network interface can have individual attributes; therefore, you may need to specify the network interface ID as part of the request, or use a different request. For example, each network interface in an instance can have a source/destination check flag. To modify this attribute, modify the network interface attribute, and not the instance attribute. To create a route in a route table, provide a specific network interface ID as part of the request."},
-		{code: "InvalidInstanceID.Malformed", http: 400, message: "The specified instance ID is malformed. Ensure that you provide the full instance ID in the request, in the form i-xxxxxxxx or i-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidInstanceID.NotFound", http: 404, message: "The specified instance does not exist. This error might occur because the ID of a recently created instance has not propagated through the system. For more information, see Ensuring idempotency."},
-		{code: "InvalidInstanceID.NotLinkable", http: 400, message: "The specified instance cannot be linked to the specified VPC. This error may also occur if the instance was recently launched, and its ID has not yet propagated through the system. Wait a few minutes, or wait until the instance is in the running state, and then try again."},
-		{code: "InvalidInstanceState", http: 400, message: "The instance is not in an appropriate state to complete the request. If you're modifying the instance placement, the instance must be in the stopped state."},
-		{code: "InvalidInstanceType", http: 400, message: "The instance type is not supported for this request. For example, you can only bundle instance store-backed Windows instances."},
-		{code: "InvalidInterface.IpAddressLimitExceeded", http: 400, message: "The number of private IP addresses for a specified network interface exceeds the limit for the type of instance you are trying to launch. For more information, see IP addresses per network interface per instance type."},
-		{code: "InvalidInternetGatewayID.NotFound", http: 404, message: "The specified internet gateway does not exist. Ensure that you specify the AWS Region in which the internet gateway is located, if it's not in the default Region."},
-		{code: "InvalidInternetGatewayId.Malformed", http: 400, message: "The specified internet gateway ID is malformed. Ensure that you provide the full ID in the request, in the form igw-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidKernelId.Malformed", http: 400, message: "The specified kernel ID is not valid. Ensure that you specify the kernel ID in the form aki-xxxxxxxx."},
-		{code: "InvalidKey.Format", http: 400, message: "The key pair is not specified in a valid OpenSSH public key format."},
-		{code: "InvalidKeyPair.Duplicate", http: 409, message: "The key pair name already exists in that AWS Region. If you are creating or importing a key pair, ensure that you use a unique name."},
-		{code: "InvalidKeyPair.Format", http: 400, message: "The format of the public key you are attempting to import is not valid."},
-		{code: "InvalidKeyPair.NotFound", http: 404, message: "The specified key pair name does not exist. Ensure that you specify the AWS Region in which the key pair is located, if it's not in the default Region."},
-		{code: "InvalidLaunchTargets", http: 400, message: "One or more specified targets are invalid. Verify the capacity for the Capacity Reservation selected or verify the ID."},
-		{code: "InvalidLaunchTemplateId.Malformed", http: 400, message: "The ID for the launch template is malformed. Ensure that you specify the launch template ID in the form lt-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidLaunchTemplateId.NotFound", http: 404, message: "The specified launch template ID does not exist. Ensure that you specify the AWS Region in which the launch template is located."},
-		{code: "InvalidLaunchTemplateId.VersionNotFound", http: 404, message: "The specified launch template version does not exist."},
-		{code: "InvalidLaunchTemplateName.AlreadyExistsException", http: 409, message: "The specified launch template name is already in use."},
-		{code: "InvalidLaunchTemplateName.MalformedException", http: 400, message: "The specified launch template name is invalid. A launch template name must be between 3 and 128 characters, and may contain letters, numbers, and the following characters: '-', '_', '.', '/', '(', and ')'."},
-		{code: "InvalidLaunchTemplateName.NotFoundException", http: 404, message: "The specified launch template name does not exist. Check the spelling of the name and ensure that you specify the AWS Region in which the launch template is located. Launch template names are case-sensitive."},
-		{code: "InvalidManifest", http: 400, message: "The specified AMI has an unparsable manifest, or you may not have access to the location of the manifest file in Amazon S3."},
-		{code: "InvalidMaxResults", http: 400, message: "The specified value for MaxResults is not valid."},
-		{code: "InvalidNatGatewayID.NotFound", http: 404, message: "The specified NAT gateway ID does not exist. Ensure that you specify the AWS Region in which the NAT gateway is located, if it's not in the default Region."},
-		{code: "InvalidNetworkAclEntry.NotFound", http: 404, message: "The specified network ACL entry does not exist."},
-		{code: "InvalidNetworkAclID.NotFound", http: 404, message: "The specified network ACL does not exist. Ensure that you specify the AWS Region in which the network ACL is located, if it's not in the default Region."},
-		{code: "InvalidNetworkAclId.Malformed", http: 400, message: "The specified network ACL ID is malformed. Ensure that you provide the ID in the form acl-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidNetworkInterface.InUse", http: 409, message: "The specified interface is currently in use and cannot be deleted or attached to another instance. Ensure that you have detached the network interface first. If a network interface is in use, you may also receive the InvalidParameterValue error."},
-		{code: "InvalidNetworkInterface.NotFound", http: 404, message: "The specified network interface does not exist."},
-		{code: "InvalidNetworkInterfaceAttachmentId.Malformed", http: 400, message: "The ID for the network interface attachment is malformed. Ensure that you use the attachment ID rather than the network interface ID, in the form eni-attach-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidNetworkInterfaceID.NotFound", http: 404, message: "The specified network interface does not exist. Ensure that you specify the AWS Region in which the network interface is located, if it's not in the default Region."},
-		{code: "InvalidNetworkInterfaceId.Malformed", http: 400, message: "The specified network interface ID is malformed. Ensure that you specify the network interface ID in the form eni-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidNetworkLoadBalancerArn.Malformed", http: 400, message: "The specified Network Load Balancer ARN is malformed. Ensure that you specify the ARN in the form arn:aws:elasticloadbalancing:region:account-id:loadbalancer/net/load-balancer-name/load-balancer-id."},
-		{code: "InvalidNetworkLoadBalancerArn.NotFound", http: 404, message: "The specified Network Load Balancer ARN does not exist."},
-		{code: "InvalidNextToken", http: 400, message: "The specified NextToken is not valid."},
-		{code: "InvalidOption.Conflict", http: 409, message: "A VPN connection between the virtual private gateway and the customer gateway already exists."},
-		{code: "InvalidPaginationToken", http: 403, message: "The specified pagination token is not valid or is expired."},
-		{code: "InvalidParameter", http: 400, message: "A parameter specified in a request is not valid, is unsupported, or cannot be used. The returned message provides an explanation of the error value. For example, if you are launching an instance, you can't specify a security group and subnet that are in different VPCs."},
-		{code: "InvalidParameterCombination", http: 400, message: "Indicates an incorrect combination of parameters, or a missing parameter. For example, trying to terminate an instance without specifying the instance ID."},
-		{code: "InvalidParameterDependency", http: 400, message: "Indicates an incorrect combination of parameters, or a missing parameter. For example, trying to terminate an instance without specifying the instance ID."},
-		{code: "InvalidParameterValue", http: 400, message: "A value specified in a parameter is not valid, is unsupported, or cannot be used. Ensure that you specify a resource by using its full ID. The returned message provides an explanation of the error value."},
-		{code: "InvalidPermission.Duplicate", http: 409, message: "The specified inbound or outbound rule already exists for that security group."},
-		{code: "InvalidPermission.Malformed", http: 400, message: "The specified security group rule is malformed. If you are specifying an IP address range, ensure that you use CIDR notation; for example, 203.0.113.0/24."},
-		{code: "InvalidPermission.NotFound", http: 404, message: "The specified rule does not exist in this security group."},
-		{code: "InvalidPlacementGroup.Duplicate", http: 409, message: "The specified placement group already exists in that AWS Region."},
-		{code: "InvalidPlacementGroup.InUse", http: 409, message: "The specified placement group is in use. If you are trying to delete a placement group, ensure that its instances have been terminated."},
-		{code: "InvalidPlacementGroup.Unknown", http: 400, message: "The specified placement group cannot be found. Ensure that you specify the AWS Region in which the placement group is located, if it's not in the default Region."},
-		{code: "InvalidPlacementGroupId.Malformed", http: 400, message: "The specified placement group ID is malformed. Ensure that you specify the ID in the form pg-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidPolicyDocument", http: 400, message: "The specified policy document is not a valid JSON policy document."},
-		{code: "InvalidPrefixListID.NotFound", http: 404, message: "The specified prefix list ID does not exist."},
-		{code: "InvalidPrefixListId.Malformed", http: 400, message: "The specified prefix list ID is malformed. Ensure that you provide the ID in the form pl-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidProductInfo", http: 400, message: "(AWS Marketplace) The product code is not valid."},
-		{code: "InvalidPurchaseToken.Expired", http: 403, message: "The specified purchase token has expired."},
-		{code: "InvalidPurchaseToken.Malformed", http: 400, message: "The specified purchase token is not valid."},
-		{code: "InvalidQuantity", http: 400, message: "The specified quantity of Dedicated Hosts is not valid."},
-		{code: "InvalidQueryParameter", http: 400, message: "The AWS query string is malformed or does not adhere to AWS standards."},
-		{code: "InvalidRamDiskId.Malformed", http: 400, message: "The specified RAM disk ID is not valid. Ensure that you specify the RAM disk ID in the form ari-xxxxxxxx."},
-		{code: "InvalidRegion", http: 400, message: "The specified AWS Region is not valid. For copying a snapshot or image, specify the source Region using its Region code, for example, us-west-2."},
-		{code: "InvalidRequest", http: 400, message: "The request is not valid. The returned message provides details about the nature of the error."},
-		{code: "InvalidReservationID.Malformed", http: 400, message: "The specified reservation ID is not valid."},
-		{code: "InvalidReservationID.NotFound", http: 404, message: "The specified reservation does not exist."},
-		{code: "InvalidReservedInstancesId", http: 404, message: "The specified Reserved Instance does not exist."},
-		{code: "InvalidReservedInstancesOfferingId", http: 404, message: "The specified Reserved Instances offering does not exist."},
-		{code: "InvalidResourceConfigurationArn.Malformed", http: 400, message: "The specified resource configuration ARN is malformed."},
-		{code: "InvalidResourceConfigurationArn.NotFound", http: 404, message: "The specified resource configuration ARN does not exist."},
-		{code: "InvalidResourceType.Unknown", http: 400, message: "The specified resource type is not supported or is not valid. To view resource types that support longer IDs, use DescribeIdFormat."},
-		{code: "InvalidRoute.InvalidState", http: 400, message: "The specified route is not valid."},
-		{code: "InvalidRoute.Malformed", http: 400, message: "The specified route is not valid. If you are deleting a route in a VPN connection, ensure that you've entered the value for the CIDR block correctly."},
-		{code: "InvalidRoute.NotFound", http: 404, message: "The specified route does not exist in the specified route table. Ensure that you indicate the exact CIDR range for the route in the request. This error can also occur if you've specified a route table ID in the request that does not exist."},
-		{code: "InvalidRouteTableID.NotFound", http: 404, message: "The specified route table does not exist. Ensure that you specify the AWS Region in which the route table is located, if it's not in the default Region."},
-		{code: "InvalidRouteTableId.Malformed", http: 400, message: "The specified route table ID is malformed. Ensure that you specify the route table ID in the form rtb-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidScheduledInstance", http: 404, message: "The specified Scheduled Instance does not exist."},
-		{code: "InvalidSecurity.RequestHasExpired", http: 400, message: "The difference between the request timestamp and the AWS server time is greater than 5 minutes. Ensure that your system clock is accurate and configured to use the correct time zone."},
-		{code: "InvalidSecurityGroupID.NotFound", http: 404, message: "The specified security group does not exist."},
-		{code: "InvalidSecurityGroupId.Malformed", http: 400, message: "The specified security group ID is not valid. Ensure that you specify the security group ID in the form sg-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidSecurityGroupRuleId.Malformed", http: 400, message: "The specified security group rule ID is not valid. Ensure that you specify the security group rule ID in the form sgr-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidSecurityGroupRuleId.NotFound", http: 404, message: "The specified security group rule does not exist."},
-		{code: "InvalidServiceName", http: 400, message: "The name of the service is not valid. To get a list of available service names, use DescribeVpcEndpointServices."},
-		{code: "InvalidSnapshot.InUse", http: 409, message: "The snapshot that you are trying to delete is in use by one or more AMIs."},
-		{code: "InvalidSnapshot.NotFound", http: 404, message: "The specified snapshot does not exist. Ensure that you specify the AWS Region in which the snapshot is located, if it's not in the default Region."},
-		{code: "InvalidSnapshotID.Malformed", http: 400, message: "The snapshot ID is not valid."},
-		{code: "InvalidSpotDatafeed.NotFound", http: 400, message: "You have no data feed for Spot Instances."},
-		{code: "InvalidSpotFleetRequestConfig", http: 400, message: "The Spot Fleet request configuration is not valid. Ensure that you provide valid values for all of the configuration parameters; for example, a valid AMI ID. Limits apply on the target capacity and the number of launch specifications per Spot Fleet request. For more information, see Fleet quotas."},
-		{code: "InvalidSpotFleetRequestId.Malformed", http: 400, message: "The specified Spot Fleet request ID is malformed. Ensure that you specify the Spot Fleet request ID in the form sfr- followed by 36 characters, including hyphens; for example, sfr-123f8fc2-11aa-22bb-33cc-example12710."},
-		{code: "InvalidSpotFleetRequestId.NotFound", http: 404, message: "The specified Spot Fleet request ID does not exist. Ensure that you specify the AWS Region in which the Spot Fleet request is located, if it's not in the default Region."},
-		{code: "InvalidSpotInstanceRequestID.Malformed", http: 400, message: "The specified Spot Instance request ID is not valid. Ensure that you specify the Spot Instance request ID in the form sir-xxxxxxxx."},
-		{code: "InvalidSpotInstanceRequestID.NotFound", http: 404, message: "The specified Spot Instance request ID does not exist. Ensure that you specify the AWS Region in which the Spot Instance request is located, if it's not in the default Region."},
-		{code: "InvalidState", http: 400, message: "The specified resource is not in the correct state for the request; for example, if you are trying to enable monitoring on a recently terminated instance, or if you are trying to create a snapshot when a previous identical request has not yet completed."},
-		{code: "InvalidStateTransition", http: 400, message: "The specified VPC peering connection is not in the correct state for the request. For example, you may be trying to accept a VPC peering request that has failed, or that was rejected."},
-		{code: "InvalidSubnet", http: 404, message: "The specified subnet ID is not valid or does not exist."},
-		{code: "InvalidSubnet.Conflict", http: 409, message: "The specified CIDR block conflicts with that of another subnet in your VPC."},
-		{code: "InvalidSubnet.Range", http: 400, message: "The CIDR block you've specified for the subnet is not valid. The allowed block size is between a /28 netmask and /16 netmask."},
-		{code: "InvalidSubnetID.Malformed", http: 400, message: "The specified subnet ID is malformed. Ensure that you specify the ID in the form subnet-xxxxxxxxxxxxxxxxx"},
-		{code: "InvalidSubnetID.NotFound", http: 404, message: "or InvalidSubnetId.NotFound \tThe specified subnet does not exist."},
-		{code: "InvalidTagKey.Malformed", http: 400, message: "The specified tag key is not valid. Tag keys cannot be empty or null, and cannot start with aws:."},
-		{code: "InvalidTargetArn.Unknown", http: 404, message: "The specified ARN for the specified user or role is not valid or does not exist."},
-		{code: "InvalidTargetException", http: 404, message: "The specified TargetId is not valid, does not exist, or is in another organization. You can only generate an account status report for declarative policies in your own organization. Ensure that you specify the TargetId in one of the following forms: r-xxxx, ou-xxxx-xxxxxxxx, or a 12-digit account ID in the form xxxxxxxxxxxx."},
-		{code: "InvalidTenancy", http: 400, message: "The tenancy of the instance or VPC is not supported for the requested action. For example, you cannot modify the tenancy of an instance or VPC that has a tenancy attribute of default."},
-		{code: "InvalidTime", http: 400, message: "The specified timestamp is not valid."},
-		{code: "InvalidTrafficMirrorFilterNotFound", http: 404, message: "The specified Traffic Mirror filter does not exist."},
-		{code: "InvalidTrafficMirrorFilterRuleNotFound", http: 404, message: "The specified Traffic Mirror filter rule does not exist."},
-		{code: "InvalidTrafficMirrorSessionNotFound", http: 404, message: "The specified Traffic Mirror session does not exist."},
-		{code: "InvalidTrafficMirrorTargetNoFound", http: 404, message: "The specified Traffic Mirror target does not exist."},
-		{code: "InvalidUserID.Malformed", http: 400, message: "The specified user or owner is not valid. If you are performing a DescribeImages request, you must specify a valid value for the owner or executableBy parameters, such as an AWS account ID. If you are performing a DescribeSnapshots request, you must specify a valid value for the owner or restorableBy parameters."},
-		{code: "InvalidVolume.NotFound", http: 404, message: "The specified volume does not exist."},
-		{code: "InvalidVolume.ZoneMismatch", http: 400, message: "The specified volume is not in the same Availability Zone as the specified instance. You can only attach an Amazon EBS volume to an instance if they are in the same Availability Zone."},
-		{code: "InvalidVolumeID.Duplicate", http: 409, message: "The Amazon EBS volume already exists."},
-		{code: "InvalidVolumeID.Malformed", http: 400, message: "The specified volume ID is not valid. Check the letter-number combination carefully."},
-		{code: "InvalidVolumeID.ZoneMismatch", http: 400, message: "The specified volume and instance are in different Availability Zones."},
-		{code: "InvalidVpcEndpoint.NotFound", http: 404, message: "The specified VPC endpoint does not exist. If you are performing a bulk request that is partially successful or unsuccessful, the response includes a list of the unsuccessful items. If the request succeeds, the list is empty."},
-		{code: "InvalidVpcEndpointId.Malformed", http: 400, message: "The specified VPC endpoint ID is malformed. Use the full VPC endpoint ID in the request, in the form vpce-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidVpcEndpointId.NotFound", http: 404, message: "The specified VPC endpoint does not exist. If you are performing a bulk request that is partially successful or unsuccessful, the response includes a list of the unsuccessful items. If the request succeeds, the list is empty."},
-		{code: "InvalidVpcEndpointService.NotFound", http: 404, message: "The specified VPC endpoint service does not exist. If you are performing a bulk request that is partially successful or unsuccessful, the response includes a list of the unsuccessful items. If the request succeeds, the list is empty."},
-		{code: "InvalidVpcEndpointServiceId.NotFound", http: 404, message: "The specified VPC endpoint service does not exist. If you are performing a bulk request that is partially successful or unsuccessful, the response includes a list of the unsuccessful items. If the request succeeds, the list is empty."},
-		{code: "InvalidVpcEndpointServiceIdId.Malformed", http: 400, message: "The specified VPC endpoint service ID is malformed. Ensure that you specify the ID in the form vpc-svc-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidVpcEndpointType", http: 400, message: "The specified VPC endpoint type is not valid."},
-		{code: "InvalidVpcID.Malformed", http: 400, message: "The specified VPC ID is malformed. Ensure that you've specified the ID in the form vpc-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidVpcID.NotFound", http: 404, message: "The specified VPC does not exist."},
-		{code: "InvalidVpcPeeringConnectionID.NotFound", http: 404, message: "The specified VPC peering connection ID does not exist."},
-		{code: "InvalidVpcPeeringConnectionId.Malformed", http: 400, message: "The specified VPC peering connection ID is malformed. Ensure that you provide the ID in the form pcx-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidVpcPeeringConnectionState.DnsHostnamesDisabled", http: 400, message: "To enable DNS hostname resolution for the VPC peering connection, DNS hostname support must be enabled for the VPCs."},
-		{code: "InvalidVpcRange", http: 400, message: "The specified CIDR block range is not valid. The block range must be between a /28 netmask and /16 netmask. For more information, see VPC CIDR blocks."},
-		{code: "InvalidVpcState", http: 400, message: "The specified VPC already has a virtual private gateway attached to it."},
-		{code: "InvalidVpnConnection.InvalidState", http: 400, message: "The VPN connection must be in the available state to complete the request."},
-		{code: "InvalidVpnConnection.InvalidType", http: 400, message: "The specified VPN connection does not support static routes."},
-		{code: "InvalidVpnConnectionID", http: 400, message: "The specified VPN connection ID cannot be found."},
-		{code: "InvalidVpnConnectionID.NotFound", http: 404, message: "The specified VPN connection ID does not exist."},
-		{code: "InvalidVpnGatewayAttachment.NotFound", http: 404, message: "An attachment between the specified virtual private gateway and specified VPC does not exist. This error can also occur if you've specified an incorrect VPC ID in the request."},
-		{code: "InvalidVpnGatewayID.NotFound", http: 404, message: "The specified virtual private gateway does not exist."},
-		{code: "InvalidVpnGatewayState", http: 400, message: "The virtual private gateway is not in an available state."},
-		{code: "InvalidZone.NotFound", http: 404, message: "The specified Availability Zone does not exist, or is not available for you to use. Use the DescribeAvailabilityZones request to list the Availability Zones that are currently available to you. Specify the full name of the Availability Zone: for example, us-east-1a."},
-		{code: "KeyPairLimitExceeded", http: 400, message: "You've reached the limit on the number of key pairs that you can have in this AWS Region. For more information, see Amazon EC2 key pairs."},
-		{code: "LegacySecurityGroup", http: 400, message: "Any VPC created using an API version older than 2011-01-01 may have the 2009-07-15-default security group. You must delete this security group before you can attach an internet gateway to the VPC."},
-		{code: "LimitPriceExceeded", http: 400, message: "The cost of the total order is greater than the specified limit price (instance count * price)."},
-		{code: "LogDestinationNotFound", http: 404, message: "The specified Amazon S3 bucket does not exist. Ensure that you have specified the ARN for an existing Amazon S3 bucket, and that the ARN is in the correct format."},
-		{code: "LogDestinationPermissionIssue", http: 400, message: "You do not have sufficient permissions to publish flow logs to the specific Amazon S3 bucket."},
-		{code: "MalformedQueryString", http: 400, message: "The query string contains a syntax error."},
-		{code: "MaxConfigLimitExceededException", http: 400, message: "You\u2019ve exceeded your maximum allowed Spot placement configurations. You can retry configurations that you used within the last 24 hours, or wait for 24 hours before specifying a new configuration. For more information, see Spot placement score."},
-		{code: "MaxIOPSLimitExceeded", http: 400, message: "You've reached the limit on your IOPS usage for the AWS Region. For more information, see Quotas for Amazon EBS."},
-		{code: "MaxScheduledInstanceCapacityExceeded", http: 400, message: "You've attempted to launch more instances than you purchased."},
-		{code: "MaxSpotFleetRequestCountExceeded", http: 400, message: "You've reached one or both of these limits: the total number of Spot Fleet requests that you can make, or the total number of instances in all Spot Fleets for the AWS Region (the target capacity). For more information, see Fleet quotas."},
-		{code: "MaxSpotInstanceCountExceeded", http: 400, message: "You've reached the limit on the number of Spot Instances that you can launch. The limit depends on the instance type. For more information, see Spot Instance limits."},
-		{code: "MaxTemplateLimitExceeded", http: 400, message: "You've reached the limit on the number of launch templates you can create. For more information, see Launch template restrictions."},
-		{code: "MaxTemplateVersionLimitExceeded", http: 400, message: "You've reached the limit on the number of launch template versions that you can create. For more information, see Launch template restrictions."},
-		{code: "MissingAction", http: 400, message: "The request is missing an action or a required parameter."},
-		{code: "MissingAuthenticationToken", http: 403, message: "The request must contain valid credentials."},
-		{code: "MissingInput", http: 400, message: "An input parameter is missing."},
-		{code: "MissingParameter", http: 400, message: "The request is missing a required parameter. Ensure that you have supplied all the required parameters for the request; for example, the resource ID."},
-		{code: "NatGatewayLimitExceeded", http: 400, message: "You've reached the limit on the number of NAT gateways that you can create. For more information, see Amazon VPC quotas."},
-		{code: "NatGatewayMalformed", http: 400, message: "The specified NAT gateway ID is not formed correctly. Ensure that you specify the NAT gateway ID in the form nat-xxxxxxxxxxxxxxxxx."},
-		{code: "NatGatewayNotFound", http: 404, message: "The specified NAT gateway does not exist."},
-		{code: "NetworkAclEntryAlreadyExists", http: 409, message: "The specified rule number already exists in this network ACL."},
-		{code: "NetworkAclEntryLimitExceeded", http: 400, message: "You've reached the limit on the number of rules that you can add to the network ACL. For more information, see Amazon VPC quotas."},
-		{code: "NetworkAclLimitExceeded", http: 400, message: "You've reached the limit on the number of network ACLs that you can create for the specified VPC. For more information, see Amazon VPC quotas."},
-		{code: "NetworkInterfaceLimitExceeded", http: 400, message: "You've reached the limit on the number of network interfaces that you can create. For more information, see Amazon VPC quotas."},
-		{code: "NetworkInterfaceNotSupported", http: 400, message: "The network interface is not supported for Traffic Mirroring."},
-		{code: "NetworkLoadBalancerNotFoundException", http: 404, message: "The specified Network Load Balancer does not exist."},
-		{code: "NlbInUseByTrafficMirrorTargetException", http: 400, message: "The Network Load Balancer is already configured as a Traffic Mirror target."},
-		{code: "NoSuchVersion", http: 404, message: "The specified API version does not exist."},
-		{code: "NonEBSInstance", http: 400, message: "The specified instance does not support Amazon EBS. Restart the instance and try again, to ensure that the code is run on an instance with updated code."},
-		{code: "NotExportable", http: 400, message: "The specified instance cannot be exported. You can only export certain instances. For more information, see Considerations for instance export."},
-		{code: "OperationNotPermitted", http: 400, message: "The specified operation is not allowed. This error can occur for a number of reasons; for example, you might be trying to terminate an instance that has termination protection enabled, or trying to detach the primary network interface (eth0) from an instance."},
-		{code: "OptInRequired", http: 403, message: "You are not authorized to use the requested service. Ensure that you have subscribed to the service you are trying to use. If you are new to AWS, your account might take some time to be activated while your credit card details are being verified."},
-		{code: "OutstandingVpcPeeringConnectionLimitExceeded", http: 400, message: "You've reached the limit on the number of VPC peering connection requests that you can create for the specified VPC."},
-		{code: "PendingSnapshotLimitExceeded", http: 409, message: "You've reached the limit on the number of Amazon EBS snapshots that you can have in the pending state."},
-		{code: "PendingVerification", http: 409, message: "Your account is pending verification. Until the verification process is complete, you may not be able to carry out requests with this account. If you have questions, contact Support."},
-		{code: "PendingVpcPeeringConnectionLimitExceeded", http: 409, message: "You've reached the limit on the number of pending VPC peering connections that you can have."},
-		{code: "PlacementGroupLimitExceeded", http: 400, message: "You've reached the limit on the number of placement groups that you can have."},
-		{code: "PrivateIpAddressLimitExceeded", http: 400, message: "You've reached the limit on the number of private IP addresses that you can assign to the specified network interface for that type of instance. For more information, see IP addresses per network interface."},
-		{code: "RequestEntityTooLarge", http: 413, message: "Request body exceeds the maximum allowed size."},
-		{code: "RequestExpired", http: 403, message: "The request reached the service more than 15 minutes after the date stamp on the request or more than 15 minutes after the request expiration date (such as for presigned URLs), or the date stamp on the request is more than 15 minutes in the future. If you're using temporary security credentials, this error can also occur if the credentials have expired. For more information, see Temporary security credentials in the IAM User Guide."},
-		{code: "RequestLimitExceeded", http: 503, message: "The maximum request rate permitted by the Amazon EC2 APIs has been exceeded for your account. For best results, use an increasing or variable sleep interval between requests. For more information, see Query API request rate."},
-		{code: "RequestResourceCountExceeded", http: 409, message: "Details in your Spot request exceed the numbers allowed by the Spot service in one of the following ways, depending on the action that generated the error: \u2014If you get this error when you submitted a request for Spot Instances, check the number of Spot Instances specified in your request. The number shouldn't exceed the 3,000 maximum allowed per request. Resend your Spot Instance request and specify a number less than 3,000. If your account's regional Spot request limit is greater than 3,000 instances, you can access these instances by submitting multiple smaller requests. \u2014If you get this error when you sent Describe Spot Instance requests, check the number of requests for Spot Instance data, the amount of data you requested, and how often you sent the request. The frequency with which you requested the data combined with the amount of data exceeds the levels allowed by the Spot service. Try again and submit fewer large Describe requests over longer intervals."},
-		{code: "ReservationCapacityExceeded", http: 400, message: "The targeted Capacity Reservation does not enough available instance capacity to fulfill your request. Either increase the instance capacity for the targeted Capacity Reservation, or target a different Capacity Reservation."},
-		{code: "ReservedInstancesCountExceeded", http: 400, message: "You've reached the limit for the number of Reserved Instances."},
-		{code: "ReservedInstancesLimitExceeded", http: 400, message: "Your current quota does not allow you to purchase the required number of Reserved Instances."},
-		{code: "ReservedInstancesUnavailable", http: 400, message: "The requested Reserved Instances are not available."},
-		{code: "Resource.AlreadyAssigned", http: 400, message: "The specified private IP address is already assigned to a resource. Unassign the private IP first, or use a different private IP address."},
-		{code: "Resource.AlreadyAssociated", http: 409, message: "The specified resource is already in use. For example, in EC2-VPC, you cannot associate an Elastic IP address with an instance if it's already associated with another instance. You also cannot attach an internet gateway to more than one VPC at a time."},
-		{code: "ResourceCountExceeded", http: 400, message: "You have exceeded the number of resources allowed for this request; for example, if you try to launch more instances than AWS allows in a single request. This limit is separate from your individual resource limit. If you get this error, break up your request into smaller requests; for example, if you are launching 15 instances, try launching 5 instances in 3 separate requests."},
-		{code: "ResourceCountLimitExceeded", http: 400, message: "You have exceeded a resource limit for creating routes."},
-		{code: "ResourceLimitExceeded", http: 400, message: "You have exceeded an Amazon EC2 resource limit. For example, you might have too many snapshot copies in progress."},
-		{code: "RetryableError", http: 400, message: "A request submitted by an AWS service on your behalf could not be completed. The requesting service might automatically retry the request."},
-		{code: "RouteAlreadyExists", http: 409, message: "A route for the specified CIDR block already exists in this route table."},
-		{code: "RouteLimitExceeded", http: 400, message: "You've reached the limit on the number of routes that you can add to a route table."},
-		{code: "RouteTableLimitExceeded", http: 400, message: "You've reached the limit on the number of route tables that you can create for the specified VPC. For more information about route table limits, see Amazon VPC quotas."},
-		{code: "RulesPerSecurityGroupLimitExceeded", http: 400, message: "You've reached the limit on the number of rules that you can add to a security group. For more information, see Amazon VPC quotas."},
-		{code: "ScheduledInstanceLimitExceeded", http: 400, message: "You've reached the limit on the number of Scheduled Instances that you can purchase."},
-		{code: "ScheduledInstanceParameterMismatch", http: 400, message: "The launch specification does not match the details for the Scheduled Instance."},
-		{code: "ScheduledInstanceSlotNotOpen", http: 400, message: "You can launch a Scheduled Instance only during its scheduled time periods."},
-		{code: "ScheduledInstanceSlotUnavailable", http: 400, message: "The requested Scheduled Instance is no longer available during this scheduled time period."},
-		{code: "SecurityGroupLimitExceeded", http: 400, message: "You've reached the limit on the number of security groups that you can create, or that you can assign to an instance."},
-		{code: "SecurityGroupsPerInstanceLimitExceeded", http: 400, message: "You've reached the limit on the number of security groups that you can assign to an instance. For more information, see Amazon EC2 security groups."},
-		{code: "SecurityGroupsPerInterfaceLimitExceeded", http: 400, message: "You've reached the limit on the number of security groups you can associate with the specified network interface. For more information, see Amazon VPC quotas."},
-		{code: "SerialConsoleSessionUnavailable", http: 403, message: "The serial console access is not enabled for this account. Use EnableSerialConsoleAccess to enable access."},
-		{code: "ServerInternal", http: 500, message: "An internal error has occurred. Retry your request, but if the problem persists, contact us with details by posting a message on AWS re:Post."},
-		{code: "ServiceUnavailable", http: 503, message: "The request has failed due to a temporary failure of the server."},
-		{code: "SignatureDoesNotMatch", http: 403, message: "The request signature that Amazon has does not match the signature that you provided. Check your AWS credentials and signing method."},
-		{code: "SnapshotCopyUnsupported.InterRegion", http: 400, message: "Inter-region snapshot copy is not supported for this AWS Region."},
-		{code: "SnapshotCreationPerVolumeRateExceeded", http: 400, message: "The rate limit for creating concurrent snapshots of an EBS volume has been exceeded. Wait at least 15 seconds between concurrent volume snapshots."},
-		{code: "SnapshotLimitExceeded", http: 400, message: "You've reached the limit on the number of Amazon EBS snapshots that you can create."},
-		{code: "SpotMaxPriceTooLow", http: 400, message: "The request can't be fulfilled yet because your maximum price is below the Spot price. In this case, no instance is launched and your request remains open."},
-		{code: "SubnetLimitExceeded", http: 400, message: "You've reached the limit on the number of subnets that you can create for the specified VPC. For more information, see Amazon VPC quotas."},
-		{code: "TagLimitExceeded", http: 400, message: "You've reached the limit on the number of tags that you can assign to the specified resource. For more information, see Tag restrictions."},
-		{code: "TagPolicyViolation", http: 400, message: "You attempted to create or update a resource with tags that are not compliant with the tag policy requirements for this account. For more information, see Grant permission to tag resources during creation."},
-		{code: "TargetCapacityLimitExceededException", http: 400, message: "The value for targetCapacity exceeds your limit on the amount of Spot placement target capacity you can explore. Reduce the targetCapacity value, and try again. For more information, see Spot placement score."},
-		{code: "Throttling", http: 503, message: "Rate exceeded."},
-		{code: "TrafficMirrorFilterInUse", http: 400, message: "The Traffic Mirror filter cannot be deleted because a Traffic Mirror session is currently using it."},
-		{code: "TrafficMirrorFilterLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror filters has been exceeded."},
-		{code: "TrafficMirrorFilterRuleAlreadyExists", http: 409, message: "The Traffic Mirror filter rule already exists."},
-		{code: "TrafficMirrorFilterRuleLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror filter rules has been exceeded."},
-		{code: "TrafficMirrorSessionLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror sessions has been exceeded."},
-		{code: "TrafficMirrorSessionsPerInterfaceLimitExceeded", http: 400, message: "The allowed number of Traffic Mirror sessions for the specified network interface has been exceeded."},
-		{code: "TrafficMirrorSessionsPerTargetLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror sessions for the specified Traffic Mirror target has been exceeded."},
-		{code: "TrafficMirrorSourcesPerTargetLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror sources for the specified Traffic Mirror target has been exceeded."},
-		{code: "TrafficMirrorTargetInUseException", http: 400, message: "The Traffic Mirror target cannot be deleted because a Traffic Mirror session is currently using it."},
-		{code: "TrafficMirrorTargetLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror targets has been exceeded."},
-		{code: "UnauthorizedOperation", http: 403, message: "You are not authorized to perform this operation. Check your IAM policies, and ensure that you are using the correct credentials. For more information, see Identity and access management for Amazon EC2. If the returned message is encoded, you can decode it using the DecodeAuthorizationMessage action. For more information, see DecodeAuthorizationMessage in the AWS Security Token Service API Reference."},
-		{code: "Unavailable", http: 503, message: "The server is overloaded and can't handle the request."},
-		{code: "UnavailableHostRequirements", http: 400, message: "There are no valid Dedicated Hosts available on which you can launch an instance."},
-		{code: "UnfulfillableCapacity", http: 400, message: "At this time there isn't enough spare capacity to fulfill your request for Spot Instances. You can wait a few minutes to see whether capacity becomes available for your request. Alternatively, create a more flexible request. For example, include additional instance types, include additional Availability Zones, or use the capacity-optimized allocation strategy."},
-		{code: "UnknownParameter", http: 404, message: "An unknown or unrecognized parameter was supplied. Requests that could cause this error include supplying a misspelled parameter or a parameter that is not supported for the specified API version."},
-		{code: "UnknownPrincipalType.Unsupported", http: 400, message: "The principal type is not supported."},
-		{code: "UnknownVolumeType", http: 400, message: "The specified volume type is unsupported. The supported volume types are gp2, io1, st1, sc1, and standard."},
-		{code: "Unsupported", http: 400, message: "The specified request is unsupported. For example, you might be trying to launch an instance in an Availability Zone that currently has constraints on that instance type. The returned message provides details of the unsupported request."},
-		{code: "UnsupportedException", http: 400, message: "Capacity Reservations are not supported for this Region."},
-		{code: "UnsupportedHibernationConfiguration", http: 400, message: "The instance could not be launched because one or more parameter values do not meet the prerequisites for enabling hibernation. For more information, see Hibernation Prerequisites. Alternatively, the instance could not be hibernated because it is not enabled for hibernation."},
-		{code: "UnsupportedHostConfiguration", http: 400, message: "The specified Dedicated Host configuration is unsupported. For more information about supported configurations, see Dedicated Hosts."},
-		{code: "UnsupportedInstanceAttribute", http: 400, message: "The specified attribute cannot be modified."},
-		{code: "UnsupportedInstanceTypeOnHost", http: 400, message: "The instance type is not supported on the Dedicated Host. For more information about supported instance types, see Amazon EC2 Dedicated Hosts Pricing."},
-		{code: "UnsupportedOperation", http: 400, message: "The specified request includes an unsupported operation. For example, you can't stop an instance that's instance store-backed. Or you might be trying to launch an instance type that is not supported by the specified AMI. The returned message provides details of the unsupported operation."},
-		{code: "UnsupportedProtocol", http: 400, message: "SOAP has been deprecated and is no longer supported."},
-		{code: "UnsupportedTenancy", http: 400, message: "The specified tenancy is unsupported. You can change the tenancy of a VPC to default only."},
-		{code: "UpdateLimitExceeded", http: 400, message: "The default credit specification for an instance family can be modified only once in a rolling 5-minute period, and up to four times in a rolling 24-hour period. For more information, see Set the default credit specification for the account."},
-		{code: "VPCIdNotSpecified", http: 400, message: "You have no default VPC in which to carry out the request. Specify a VPC or subnet ID or, in the case of security groups, specify the ID and not the security group name. If you deleted your default VPC, you can create a new one. For more information, see Create a default VPC."},
-		{code: "VPCResourceNotSpecified", http: 400, message: "The specified resource can be used only in a VPC; for example, T2 instances. Ensure that you have a VPC in your account, and then specify a subnet ID or network interface ID in the request."},
-		{code: "ValidationError", http: 400, message: "The input fails to satisfy the constraints specified by an AWS service."},
-		{code: "VcpuLimitExceeded", http: 400, message: "You've reached the limit on the number of vCPUs (virtual processing units) assigned to the running instances in your account. You are limited to running one or more On-Demand instances in an AWS account, and Amazon EC2 measures usage towards each limit based on the total number of vCPUs that are assigned to the running On-Demand instances in your AWS account. If your request fails due to limit constraints, increase your On-Demand instance limits and try again. For more information, see EC2 On-Demand instance limits."},
-		{code: "VolumeIOPSLimit", http: 400, message: "The maximum IOPS limit for the volume has been reached. For more information, see Amazon EBS volume types."},
-		{code: "VolumeInUse", http: 400, message: "The specified Amazon EBS volume is attached to an instance. Ensure that the specified volume is in an \u2018available\u2019 state."},
-		{code: "VolumeLimitExceeded", http: 400, message: "You've reached the limit on your Amazon EBS volume storage. For more information, see Quotas for Amazon EBS."},
-		{code: "VolumeModificationSizeLimitExceeded", http: 400, message: "You've reached the limit on your Amazon EBS volume modification storage in this Region. For more information, see Quotas for Amazon EBS."},
-		{code: "VolumeTypeNotAvailableInZone", http: 400, message: "The specified Availability Zone does not support Provisioned IOPS SSD volumes. Try launching your instance in a different Availability Zone, or don't specify a zone in the request. If you're creating a volume, try specifying a different Availability Zone in the request."},
-		{code: "VpcEndpointLimitExceeded", http: 400, message: "You've reached the limit on the number of VPC endpoints that you can create in the AWS Region. For more information, see Amazon VPC quotas."},
-		{code: "VpcLimitExceeded", http: 400, message: "You've reached the limit on the number of VPCs that you can create in the AWS Region. For more information about VPC limits, see Amazon VPC quotas."},
-		{code: "VpcPeeringConnectionAlreadyExists", http: 409, message: "A VPC peering connection between the VPCs already exists."},
-		{code: "VpcPeeringConnectionsPerVpcLimitExceeded", http: 400, message: "You've reached the limit on the number of VPC peering connections that you can have per VPC. For more information, see Amazon VPC quotas."},
-		{code: "VpnConnectionLimitExceeded", http: 400, message: "You've reached the limit on the number of VPN connections that you can create. For more information, see Amazon VPC quotas."},
-		{code: "VpnGatewayAttachmentLimitExceeded", http: 400, message: "You've reached the limit on the number of VPCs that can be attached to the specified virtual private gateway."},
-		{code: "VpnGatewayLimitExceeded", http: 400, message: "You've reached the limit on the number of virtual private gateways that you can create. For more information about limits, see Amazon VPC quotas."},
-		{code: "ZonesMismatched", http: 400, message: "The Availability Zone for the instance does not match that of the Dedicated Host"},
-
-		// IAM error codes
-		{code: "NoSuchEntity", http: 404, message: "The request was rejected because it referenced a resource entity that does not exist."},
-		{code: "EntityAlreadyExists", http: 409, message: "The request was rejected because it attempted to create a resource that already exists."},
-		{code: "DeleteConflict", http: 409, message: "The request was rejected because it attempted to delete a resource that has attached subordinate entities."},
-		{code: "LimitExceeded", http: 409, message: "The request was rejected because it attempted to create resources beyond the current AWS account limits."},
-		{code: "MalformedPolicyDocument", http: 400, message: "The policy document is malformed."},
-		{code: "AccessDenied", http: 403, message: "User is not authorized to perform this action."},
-		// ELBv2 error codes
-		{code: "LoadBalancerNotFound", http: 400, message: "One or more load balancers not found."},
-		{code: "TargetGroupNotFound", http: 400, message: "One or more target groups not found."},
-		{code: "ListenerNotFound", http: 400, message: "One or more listeners not found."},
-		{code: "DuplicateLoadBalancerName", http: 400, message: "A load balancer with the same name already exists."},
-		{code: "DuplicateTargetGroupName", http: 400, message: "A target group with the same name already exists."},
-		{code: "DuplicateListener", http: 400, message: "A listener with the specified port already exists on this load balancer."},
-		{code: "TooManyLoadBalancers", http: 400, message: "You've reached the limit on the number of load balancers for your account."},
-		{code: "TooManyTargetGroups", http: 400, message: "You've reached the limit on the number of target groups for your account."},
-		{code: "TooManyListeners", http: 400, message: "You've reached the limit on the number of listeners per load balancer."},
-		{code: "TooManyRegistrationsForTargetId", http: 400, message: "You've reached the limit on the number of times a target can be registered with a target group."},
-		{code: "InvalidTarget", http: 400, message: "The specified target does not exist, is not in the same VPC as the target group, or has an unsupported instance type."},
-		{code: "ResourceInUse", http: 400, message: "The target group is currently in use by a listener or rule."},
-		{code: "InvalidSecurityGroup", http: 400, message: "The specified security group does not exist."},
-		{code: "InvalidScheme", http: 400, message: "The specified scheme is not valid. Specify 'internet-facing' or 'internal'."},
-		{code: "SubnetNotFound", http: 400, message: "The specified subnet does not exist."},
-		{code: "AvailabilityZoneNotSupported", http: 400, message: "The specified Availability Zone is not supported."},
-		{code: "InvalidConfigurationRequest", http: 400, message: "Security groups are not supported for load balancers with type 'network'."},
+// TestErrorLookup_Structure asserts invariants of the ErrorLookup map without
+// duplicating every entry. Catches wholesale-deletion accidents, invalid HTTP
+// codes, empty messages, and regressions of SDK-surfaced codes — the failure
+// modes that actually matter — without locking in a 1:1 mirror that has to be
+// updated in lockstep with every new error code.
+func TestErrorLookup_Structure(t *testing.T) {
+	if len(ErrorLookup) < 400 {
+		t.Fatalf("ErrorLookup unexpectedly small: %d entries", len(ErrorLookup))
 	}

-	if len(ErrorLookup) != len(expected) {
-		t.Fatalf("expected %d errors, got %d", len(expected), len(ErrorLookup))
+	validHTTP := map[int]bool{400: true, 403: true, 404: true, 409: true, 412: true, 413: true, 500: true, 503: true}
+	for code, msg := range ErrorLookup {
+		if !validHTTP[msg.HTTPCode] {
+			t.Errorf("%s has invalid HTTPCode %d", code, msg.HTTPCode)
+		}
+		if msg.Message == "" {
+			t.Errorf("%s has empty Message", code)
+		}
 	}

-	for _, tt := range expected {
-		msg, ok := ErrorLookup[tt.code]
+	// Spot-check business-critical codes that the AWS SDK surfaces by name.
+	critical := map[string]int{
+		ErrorAuthFailure:                  403,
+		ErrorInvalidInstanceIDNotFound:    404,
+		ErrorInvalidAMIIDNotFound:         400,
+		ErrorInvalidKeyPairNotFound:       404,
+		ErrorInvalidVpcIDNotFound:         404,
+		ErrorInvalidGroupNotFound:         404,
+		ErrorInvalidSubnetIDNotFound:      404,
+		ErrorServerInternal:               500,
+		ErrorInsufficientInstanceCapacity: 400,
+		ErrorUnauthorizedOperation:        403,
+	}
+	for code, wantHTTP := range critical {
+		msg, ok := ErrorLookup[code]
 		if !ok {
-			t.Fatalf("missing error code %s", tt.code)
+			t.Errorf("critical code %q missing from ErrorLookup", code)
+			continue
 		}
-		if msg.HTTPCode != tt.http {
-			t.Fatalf("%s HTTPCode = %d, want %d", tt.code, msg.HTTPCode, tt.http)
-		}
-		if msg.Message != tt.message {
-			t.Fatalf("%s message = %q, want %q", tt.code, msg.Message, tt.message)
+		if msg.HTTPCode != wantHTTP {
+			t.Errorf("%s HTTPCode = %d, want %d", code, msg.HTTPCode, wantHTTP)
 		}
 	}
 }
@@ -49,7 +49,6 @@ import (
 	handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
 	"github.com/mulgadc/spinifex/spinifex/instancetypes"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
-	"github.com/mulgadc/spinifex/spinifex/tags"
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/spinifex/spinifex/vm"
@@ -342,12 +341,18 @@ func (d *Daemon) onNATSReconnect(_ *nats.Conn) {
 	go d.reconcileOnHeal("nats-reconnect")
 }

+// execCommand wraps exec.Command so tests can substitute a fake builder.
+// Mirrors the sudoCommand seam in network.go — getSystemMemory shells out to
+// sysctl (darwin) or grep /proc/meminfo (linux), neither of which can be
+// faked without an indirection that the test can swap.
+var execCommand = exec.Command
+
 // getSystemMemory returns the total system memory in GB
 func getSystemMemory() (float64, error) {
 	switch runtime.GOOS {
 	case "darwin":
 		// macOS: use sysctl
-		cmd := exec.Command("sysctl", "-n", "hw.memsize")
+		cmd := execCommand("sysctl", "-n", "hw.memsize")
 		output, err := cmd.Output()
 		if err != nil {
 			return 0, fmt.Errorf("failed to get system memory on macOS: %w", err)
@@ -360,7 +365,7 @@ func getSystemMemory() (float64, error) {

 	case "linux":
 		// Linux: read from /proc/meminfo
-		cmd := exec.Command("grep", "MemTotal", "/proc/meminfo")
+		cmd := execCommand("grep", "MemTotal", "/proc/meminfo")
 		output, err := cmd.Output()
 		if err != nil {
 			return 0, fmt.Errorf("failed to read /proc/meminfo: %w", err)
@@ -489,17 +494,34 @@ func (rm *ResourceManager) GetAvailableInstanceTypeInfos(showCapacity bool) []*e
 			continue
 		}

-		requiresGPU := instancetypes.IsGPUType(it)
-		availGPU := 0
-		if rm.gpuManager != nil && requiresGPU {
-			availGPU = rm.gpuManager.Available()
+		// GPU types are capacity-gated by GPU pool size, not host CPU/memory.
+		// The GPU is the scarce resource; CPU/memory on GPU-class hardware is abundant.
+		if instancetypes.IsGPUType(it) {
+			availGPU := 0
+			if rm.gpuManager != nil {
+				availGPU = rm.gpuManager.Available()
+			}
+			gpusNeeded := instancetypes.GPUCountForType(name)
+			count := 0
+			if gpusNeeded > 0 {
+				count = availGPU / gpusNeeded
+			}
+			if showCapacity {
+				for range count {
+					infos = append(infos, it)
+				}
+			} else if count > 0 {
+				infos = append(infos, it)
+			}
+			continue
 		}
+
 		count := canAllocateCount(
 			rm.hostVCPU-rm.reservedVCPU, rm.allocatedVCPU,
 			rm.hostMemGB-rm.reservedMem, rm.allocatedMem,
 			vCPUs, memMiB,
 			1<<30, // effectively unlimited — let resources be the constraint
-			availGPU, requiresGPU,
+			0, false,
 		)

 		if showCapacity {
@@ -713,6 +735,7 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.CreateSecurityGroup", d.handleEC2CreateSecurityGroup, "spinifex-workers"},
 		{"ec2.DeleteSecurityGroup", d.handleEC2DeleteSecurityGroup, "spinifex-workers"},
 		{"ec2.DescribeSecurityGroups", d.handleEC2DescribeSecurityGroups, "spinifex-workers"},
+		{"ec2.DescribeSecurityGroupRules", d.handleEC2DescribeSecurityGroupRules, "spinifex-workers"},
 		{"ec2.AuthorizeSecurityGroupIngress", d.handleEC2AuthorizeSecurityGroupIngress, "spinifex-workers"},
 		{"ec2.AuthorizeSecurityGroupEgress", d.handleEC2AuthorizeSecurityGroupEgress, "spinifex-workers"},
 		{"ec2.RevokeSecurityGroupIngress", d.handleEC2RevokeSecurityGroupIngress, "spinifex-workers"},
@@ -720,11 +743,13 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.ModifyInstanceAttribute", d.handleEC2ModifyInstanceAttribute, "spinifex-workers"},
 		{"ec2.DescribeInstanceAttribute", d.handleEC2DescribeInstanceAttribute, "spinifex-workers"},
 		{"ec2.start", d.handleEC2StartStoppedInstance, "spinifex-workers"},
+		{fmt.Sprintf("ec2.start.%s", d.node), d.handleEC2StartStoppedInstanceDirect, ""},
 		{"ec2.terminate", d.handleEC2TerminateStoppedInstance, "spinifex-workers"},
 		{"ec2.DescribeStoppedInstances", d.handleEC2DescribeStoppedInstances, "spinifex-workers"},
 		{"ec2.DescribeTerminatedInstances", d.handleEC2DescribeTerminatedInstances, "spinifex-workers"},
-		// these 2 fan out to all nodes and gateway aggregates the results
+		// these 3 fan out to all nodes and gateway aggregates the results
 		{"ec2.DescribeInstances", d.handleEC2DescribeInstances, ""},
+		{"ec2.DescribeInstanceStatus", d.handleEC2DescribeInstanceStatus, ""},
 		{"ec2.DescribeInstanceTypes", d.handleEC2DescribeInstanceTypes, ""},
 		{"ec2.EnableEbsEncryptionByDefault", d.handleEC2EnableEbsEncryptionByDefault, "spinifex-workers"},
 		{"ec2.DisableEbsEncryptionByDefault", d.handleEC2DisableEbsEncryptionByDefault, "spinifex-workers"},
@@ -1160,29 +1185,6 @@ func (d *Daemon) startCluster() error {
 	// Wire system credentials + gateway URL for LB agent SigV4 auth.
 	d.wireLBAgentConfig()

-	// Set up lazy system AMI discovery for LB VMs. The image may not exist
-	// at daemon startup (imported later), so we resolve it at request time.
-	if d.imageService != nil {
-		imgSvc := d.imageService
-		d.elbv2Service.SetSystemAMIFunc(func() (string, error) {
-			imagesOut, imgErr := imgSvc.DescribeImages(&ec2.DescribeImagesInput{
-				Filters: []*ec2.Filter{{
-					Name:   aws.String("tag:" + tags.ManagedByKey),
-					Values: []*string{aws.String(tags.ManagedByELBv2)},
-				}},
-			}, utils.GlobalAccountID)
-			if imgErr != nil {
-				return "", fmt.Errorf("describe LB system images: %w", imgErr)
-			}
-			if len(imagesOut.Images) == 0 {
-				return "", errors.New("LB system image not imported; run: spx admin images import --name lb-alpine-3.21.6-x86_64")
-			}
-			amiID := aws.StringValue(imagesOut.Images[0].ImageId)
-			slog.Info("System AMI resolved for LB VMs", "amiId", amiID, "name", aws.StringValue(imagesOut.Images[0].Name))
-			return amiID, nil
-		})
-	}
-
 	// System VMs (LB, NAT GW) use the dedicated sys.micro instance type.
 	d.elbv2Service.SetSystemInstanceTypeFunc(func() string {
 		return "sys.micro"
@@ -1342,6 +1344,11 @@ func (d *Daemon) upgradeJetStreamReplicas() {
 	}
 }

+// initRetrySleep is the sleep seam used by initServiceWithRetry. Tests
+// override it to drive backoff cadence without the real 35s wall-clock
+// budget that 7 doublings (500ms→1s→2s→4s→8s→10s→10s) would impose.
+var initRetrySleep = time.Sleep
+
 // initServiceWithRetry initializes a service using the provided init function,
 // retrying with exponential backoff (500ms→10s) for up to 5 minutes. During
 // cluster restarts, JetStream KV may be temporarily unavailable while NATS
@@ -1369,7 +1376,7 @@ func initServiceWithRetry[T any](name string, initFn func() (T, error)) (T, erro
 		}

 		slog.Warn("Failed to init "+name, "error", err, "attempt", attempt, "elapsed", elapsed.Round(time.Second))
-		time.Sleep(retryDelay)
+		initRetrySleep(retryDelay)
 		retryDelay = min(retryDelay*2, 10*time.Second)
 	}
 }
@@ -1732,6 +1739,7 @@ func (d *Daemon) applyGPUConfig(enabled bool) {
 		mgr := gpu.NewManager(probe.Devices)
 		d.gpuManager = mgr
 		d.resourceMgr.reloadGPUTypes(models, mgr)
+		d.instanceService.SetGPUClaimer(&daemonGPUClaimer{d: d})
 		slog.Info("GPU passthrough enabled via config reload", "gpus", len(probe.Devices))
 		return
 	}
@@ -1743,6 +1751,7 @@ func (d *Daemon) applyGPUConfig(enabled bool) {
 		return
 	}
 	d.gpuManager = nil
+	d.instanceService.SetGPUClaimer(nil)
 	d.resourceMgr.reloadGPUTypes(nil, nil)
 	slog.Info("GPU passthrough disabled via config reload")
 }
@@ -1848,10 +1857,10 @@ func (rm *ResourceManager) canAllocate(instanceType *ec2.InstanceTypeInfo, count
 	rm.mu.RLock()
 	defer rm.mu.RUnlock()

-	requiresGPU := instancetypes.IsGPUType(instanceType)
-	availGPU := 0
-	if rm.gpuManager != nil && requiresGPU {
-		availGPU = rm.gpuManager.Available()
+	// GPU capacity is managed exclusively by gpuManager.Claim; don't double-gate
+	// on host CPU/memory which is always abundant on GPU-class hardware.
+	if instancetypes.IsGPUType(instanceType) {
+		return count
 	}

 	return canAllocateCount(
@@ -1860,7 +1869,7 @@ func (rm *ResourceManager) canAllocate(instanceType *ec2.InstanceTypeInfo, count
 		instanceTypeVCPUs(instanceType),
 		instanceTypeMemoryMiB(instanceType),
 		count,
-		availGPU, requiresGPU,
+		0, false,
 	)
 }

@@ -2096,21 +2105,34 @@ func (d *Daemon) wireLBAgentConfig() {
 			"awsgwBindIP", awsgwBindIP, "mgmtBridgeIP", d.mgmtBridgeIP, "advertiseIP", advertiseIP)
 	}

-	// Pass mgmt route info so lbVMUserData can add a bootcmd route for
+	// Pass mgmt route info so buildMicrovmNICs can add a host route for
 	// internal LBs that reach the AWSGW via the management NIC.
 	if d.mgmtRouteVia != "" {
 		d.elbv2Service.MgmtRouteGateway = d.mgmtBridgeIP
 		d.elbv2Service.MgmtRouteTarget = d.mgmtRouteVia
 	}

-	// Always expose mgmtBridgeIP and advertiseIP so lbVMUserData can synthesize
-	// a mgmt-NIC fallback route for internal-scheme LBs on single-node setups
-	// (where MgmtRoute{Gateway,Target} stay empty because internet-facing LBs
-	// reach AWSGW via VPC + EIP SNAT). Internal LBs have no EIP, so without
-	// this fallback the agent has no return path and the LB stays in
-	// provisioning forever.
+	// Always expose mgmtBridgeIP and advertiseIP so buildMicrovmNICs can
+	// synthesize a mgmt-NIC fallback route for internal-scheme LBs on
+	// single-node setups (where MgmtRoute{Gateway,Target} stay empty because
+	// internet-facing LBs reach AWSGW via VPC + EIP SNAT). Internal LBs have
+	// no EIP, so without this fallback the agent has no return path and the
+	// LB stays in provisioning forever.
 	d.elbv2Service.MgmtBridgeIP = d.mgmtBridgeIP
 	d.elbv2Service.AdvertiseIP = advertiseIP
+
+	// Read CA cert so direct-boot microvm guests can verify AWSGW TLS.
+	// The NATS CA cert signs the AWSGW server cert (same local CA).
+	if d.config.NATS.CACert != "" {
+		if caBytes, err := os.ReadFile(d.config.NATS.CACert); err == nil {
+			d.elbv2Service.CACert = string(caBytes)
+			slog.Info("CA cert loaded for LB agent TLS", "path", d.config.NATS.CACert)
+		} else {
+			slog.Warn("Failed to read CA cert for LB agent TLS", "path", d.config.NATS.CACert, "err", err)
+		}
+	} else {
+		slog.Warn("NATS CACert not configured — direct-boot LB VMs will not verify AWSGW TLS")
+	}
 }

 // resolveGPUModel maps a discovered GPU to an instance type model.
@@ -2,9 +2,11 @@ package daemon

 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log/slog"
 	"strings"
+	"time"

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -15,6 +17,14 @@ import (
 	"github.com/nats-io/nats.go"
 )

+// startStoppedForwardTimeout bounds the wait for ec2.start.{nodeId} to respond.
+// StartStoppedInstance does volume rehydrate + QEMU launch + QMP handshake,
+// which can take 10-20s on a cold start. Match the awsgw upstream 30s budget
+// so a slow-but-alive target node never trips the fallback path — falling back
+// while the target is still launching causes a duplicate Run that collides on
+// the deterministic tap name (TUNSETIFF: Device or resource busy).
+const startStoppedForwardTimeout = 30 * time.Second
+
 // handleEC2RunInstances orchestrates the RunInstances flow across
 // InstanceService.PrepareRunInstances (validation + ENI creation),
 // daemon-local Insert/WriteState/per-instance subscribe, and
@@ -313,8 +323,86 @@ func (d *Daemon) handleEC2DescribeInstanceTypes(msg *nats.Msg) {
 	handleNATSRequest(msg, d.instanceService.DescribeInstanceTypes)
 }

-// startStoppedInstanceRequest is the payload for ec2.start topic
+// handleEC2DescribeInstanceStatus responds with status entries for VMs on this node visible to the caller.
+func (d *Daemon) handleEC2DescribeInstanceStatus(msg *nats.Msg) {
+	handleNATSRequest(msg, d.instanceService.DescribeInstanceStatus)
+}
+
+// handleEC2StartStoppedInstance handles the generic ec2.start queue-group topic.
+// It reads the stopped instance's LastNode from shared KV and forwards the
+// request to ec2.start.{lastNode} when the instance last ran on a different
+// node. This keeps instances on their original node so the per-node resource
+// manager sees the correct allocation. Local fallback fires only when the
+// targeted node has no active subscriber (ErrNoResponders — node down or not
+// yet recovered) or returns InsufficientInstanceCapacity. A timeout from a
+// reachable-but-slow target is surfaced as ServerInternal so the caller can
+// retry; falling back in that case races the still-running launch on the
+// target and collides on the deterministic tap name.
 func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) {
+	// Peek at the instance ID without full unmarshal — we only need it for the
+	// LastNode lookup. The full unmarshal happens inside StartStoppedInstance.
+	var peek struct {
+		InstanceID string `json:"instance_id"`
+	}
+	if err := json.Unmarshal(msg.Data, &peek); err != nil || peek.InstanceID == "" {
+		// Can't determine target node — fall through to local start which will
+		// return the appropriate error (missing parameter / unmarshal failure).
+		handleNATSRequest(msg, d.instanceService.StartStoppedInstance)
+		return
+	}
+
+	lastNode := d.instanceService.StoppedInstanceNode(peek.InstanceID)
+	if lastNode != "" && lastNode != d.node {
+		targetTopic := fmt.Sprintf("ec2.start.%s", lastNode)
+		forwardMsg := nats.NewMsg(targetTopic)
+		forwardMsg.Data = msg.Data
+		forwardMsg.Header.Set(utils.AccountIDHeader, utils.AccountIDFromMsg(msg))
+
+		slog.Info("ec2.start: forwarding to original node",
+			"instanceId", peek.InstanceID, "lastNode", lastNode)
+		resp, err := d.natsConn.RequestMsg(forwardMsg, startStoppedForwardTimeout)
+		if err == nil {
+			// ValidateErrorPayload returns a non-nil error when the payload IS an
+			// AWS error response; nil means it is a success payload.
+			errPayload, isErrPayload := utils.ValidateErrorPayload(resp.Data)
+			isCapacity := isErrPayload != nil &&
+				errPayload.Code != nil &&
+				*errPayload.Code == awserrors.ErrorInsufficientInstanceCapacity
+
+			if !isCapacity {
+				// Relay success or any non-capacity error as-is.
+				if relayErr := msg.Respond(resp.Data); relayErr != nil {
+					slog.Error("ec2.start: failed to relay response from original node",
+						"instanceId", peek.InstanceID, "lastNode", lastNode, "err", relayErr)
+				}
+				return
+			}
+			slog.Warn("ec2.start: original node at capacity, starting locally",
+				"instanceId", peek.InstanceID, "lastNode", lastNode)
+		} else if errors.Is(err, nats.ErrNoResponders) {
+			// Target node has no active subscription — down or restarting. Fall
+			// back to local start so the instance can resume somewhere.
+			slog.Warn("ec2.start: original node has no subscriber, starting locally",
+				"instanceId", peek.InstanceID, "lastNode", lastNode, "err", err)
+		} else {
+			// Timeout or other transport error from a node whose subscription
+			// did exist. Do NOT fall back — the target may still be launching
+			// the VM, and a duplicate Run here would collide on tap setup.
+			slog.Error("ec2.start: forward to original node failed, not falling back",
+				"instanceId", peek.InstanceID, "lastNode", lastNode, "err", err)
+			respondWithError(msg, awserrors.ErrorServerInternal)
+			return
+		}
+	}
+
+	handleNATSRequest(msg, d.instanceService.StartStoppedInstance)
+}
+
+// handleEC2StartStoppedInstanceDirect handles ec2.start.{nodeId} — the
+// node-targeted topic used by handleEC2StartStoppedInstance to forward start
+// requests back to the original node. Always starts locally; never forwards
+// further, preventing routing loops.
+func (d *Daemon) handleEC2StartStoppedInstanceDirect(msg *nats.Msg) {
 	handleNATSRequest(msg, d.instanceService.StartStoppedInstance)
 }

@@ -438,113 +526,5 @@ func (d *Daemon) handleEC2ModifyInstanceAttribute(msg *nats.Msg) {

 // handleEC2DescribeInstanceAttribute returns a single requested attribute for an instance.
 func (d *Daemon) handleEC2DescribeInstanceAttribute(msg *nats.Msg) {
-	var input ec2.DescribeInstanceAttributeInput
-	if err := json.Unmarshal(msg.Data, &input); err != nil {
-		slog.Error("handleEC2DescribeInstanceAttribute: failed to unmarshal request", "err", err)
-		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
-	}
-
-	if input.InstanceId == nil || *input.InstanceId == "" {
-		slog.Error("handleEC2DescribeInstanceAttribute: missing instance_id")
-		respondWithError(msg, awserrors.ErrorMissingParameter)
-		return
-	}
-	if input.Attribute == nil || *input.Attribute == "" {
-		slog.Error("handleEC2DescribeInstanceAttribute: missing attribute")
-		respondWithError(msg, awserrors.ErrorMissingParameter)
-		return
-	}
-
-	instanceID := *input.InstanceId
-	attribute := *input.Attribute
-	accountID := utils.AccountIDFromMsg(msg)
-
-	// Look up instance: running first, then stopped KV.
-	var instance *vm.VM
-	if running, ok := d.vmMgr.Get(instanceID); ok {
-		instance = running
-	}
-
-	if instance == nil {
-		if d.stateStore == nil {
-			slog.Error("handleEC2DescribeInstanceAttribute: state store not available")
-			respondWithError(msg, awserrors.ErrorServerInternal)
-			return
-		}
-		stopped, err := d.stateStore.LoadStoppedInstance(instanceID)
-		if err != nil {
-			slog.Error("handleEC2DescribeInstanceAttribute: failed to load stopped instance",
-				"instanceId", instanceID, "err", err)
-			respondWithError(msg, awserrors.ErrorServerInternal)
-			return
-		}
-		instance = stopped
-	}
-
-	if instance == nil {
-		slog.Warn("handleEC2DescribeInstanceAttribute: instance not found",
-			"instanceId", instanceID)
-		respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound)
-		return
-	}
-
-	if !checkInstanceOwnership(msg, instanceID, instance.AccountID) {
-		return
-	}
-
-	output := &ec2.DescribeInstanceAttributeOutput{
-		InstanceId: &instanceID,
-	}
-
-	switch attribute {
-	case ec2.InstanceAttributeNameInstanceType:
-		val := instance.InstanceType
-		output.InstanceType = &ec2.AttributeValue{Value: &val}
-
-	case ec2.InstanceAttributeNameUserData:
-		val := instance.UserData
-		output.UserData = &ec2.AttributeValue{Value: &val}
-
-	case ec2.InstanceAttributeNameDisableApiTermination:
-		val := false
-		output.DisableApiTermination = &ec2.AttributeBooleanValue{Value: &val}
-
-	case ec2.InstanceAttributeNameDisableApiStop:
-		val := false
-		output.DisableApiStop = &ec2.AttributeBooleanValue{Value: &val}
-
-	case ec2.InstanceAttributeNameInstanceInitiatedShutdownBehavior:
-		val := ec2.ShutdownBehaviorStop
-		output.InstanceInitiatedShutdownBehavior = &ec2.AttributeValue{Value: &val}
-
-	case ec2.InstanceAttributeNameEbsOptimized:
-		val := false
-		output.EbsOptimized = &ec2.AttributeBooleanValue{Value: &val}
-
-	case ec2.InstanceAttributeNameEnaSupport:
-		val := true
-		output.EnaSupport = &ec2.AttributeBooleanValue{Value: &val}
-
-	case ec2.InstanceAttributeNameSourceDestCheck:
-		val := true
-		output.SourceDestCheck = &ec2.AttributeBooleanValue{Value: &val}
-
-	case ec2.InstanceAttributeNameGroupSet:
-		if instance.Instance != nil && len(instance.Instance.SecurityGroups) > 0 {
-			output.Groups = instance.Instance.SecurityGroups
-		} else {
-			output.Groups = []*ec2.GroupIdentifier{}
-		}
-
-	default:
-		slog.Warn("handleEC2DescribeInstanceAttribute: unsupported attribute",
-			"instanceId", instanceID, "attribute", attribute)
-		respondWithError(msg, awserrors.ErrorInvalidParameterValue)
-		return
-	}
-
-	slog.Info("handleEC2DescribeInstanceAttribute: completed",
-		"instanceId", instanceID, "attribute", attribute, "accountID", accountID)
-	respondWithJSON(msg, output)
+	handleNATSRequest(msg, d.instanceService.DescribeInstanceAttribute)
 }
@@ -14,6 +14,10 @@ func (d *Daemon) handleEC2DescribeSecurityGroups(msg *nats.Msg) {
 	handleNATSRequest(msg, d.vpcService.DescribeSecurityGroups)
 }

+func (d *Daemon) handleEC2DescribeSecurityGroupRules(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.DescribeSecurityGroupRules)
+}
+
 func (d *Daemon) handleEC2AuthorizeSecurityGroupIngress(msg *nats.Msg) {
 	handleNATSRequest(msg, d.vpcService.AuthorizeSecurityGroupIngress)
 }
@@ -1,9 +1,11 @@
 package daemon

 import (
+	"bytes"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"log/slog"
 	"os"
 	"strings"
 	"testing"
@@ -252,91 +254,6 @@ func TestHandleEC2CreateKeyPair_RoundTrip(t *testing.T) {
 	assert.NotEmpty(t, *output.KeyMaterial)
 }

-func TestHandleEC2CreateTags_RoundTrip(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createFullTestDaemon(t, natsURL)
-
-	sub, err := daemon.natsConn.QueueSubscribe("ec2.CreateTags", "spinifex-workers", daemon.handleEC2CreateTags)
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	input := &ec2.CreateTagsInput{
-		Resources: []*string{aws.String("i-12345678")},
-		Tags: []*ec2.Tag{
-			{Key: aws.String("Name"), Value: aws.String("test-instance")},
-		},
-	}
-	reqData, _ := json.Marshal(input)
-	reply, err := daemon.natsConn.Request("ec2.CreateTags", reqData, 5*time.Second)
-	require.NoError(t, err)
-	require.NotNil(t, reply)
-
-	var output ec2.CreateTagsOutput
-	err = json.Unmarshal(reply.Data, &output)
-	require.NoError(t, err)
-}
-
-func TestHandleEC2DescribeImages_RoundTrip(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createFullTestDaemon(t, natsURL)
-
-	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeImages", "spinifex-workers", daemon.handleEC2DescribeImages)
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	input := &ec2.DescribeImagesInput{}
-	reqData, _ := json.Marshal(input)
-	reply, err := daemon.natsConn.Request("ec2.DescribeImages", reqData, 5*time.Second)
-	require.NoError(t, err)
-	require.NotNil(t, reply)
-
-	var output ec2.DescribeImagesOutput
-	err = json.Unmarshal(reply.Data, &output)
-	require.NoError(t, err)
-}
-
-func TestHandleEC2DescribeVolumes_RoundTrip(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createFullTestDaemon(t, natsURL)
-
-	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeVolumes", "spinifex-workers", daemon.handleEC2DescribeVolumes)
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	input := &ec2.DescribeVolumesInput{}
-	reqData, _ := json.Marshal(input)
-	reply, err := daemon.natsConn.Request("ec2.DescribeVolumes", reqData, 5*time.Second)
-	require.NoError(t, err)
-	require.NotNil(t, reply)
-
-	var output ec2.DescribeVolumesOutput
-	err = json.Unmarshal(reply.Data, &output)
-	require.NoError(t, err)
-}
-
-func TestHandleEC2DescribeKeyPairs_RoundTrip(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createFullTestDaemon(t, natsURL)
-
-	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeKeyPairs", "spinifex-workers", daemon.handleEC2DescribeKeyPairs)
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	input := &ec2.DescribeKeyPairsInput{}
-	reqData, _ := json.Marshal(input)
-	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeKeyPairs", reqData, 5*time.Second)
-	require.NoError(t, err)
-	require.NotNil(t, reply)
-
-	var output ec2.DescribeKeyPairsOutput
-	err = json.Unmarshal(reply.Data, &output)
-	require.NoError(t, err)
-}
-
 // --- handleHealthCheck tests ---

 func TestHandleHealthCheck(t *testing.T) {
@@ -447,6 +364,64 @@ func TestHandleEC2RunInstances_InvalidKeyPair(t *testing.T) {
 	assert.Contains(t, string(reply.Data), "InvalidKeyPair.NotFound")
 }

+// TestHandleEC2RunInstances_MessageParsing locks the three rejection paths
+// that don't get past initial input validation: malformed JSON, an
+// unrecognised instance type, and a missing ImageId. Each must surface a
+// specific awserrors code on the NATS reply rather than crashing the handler
+// or falling through to ErrorServerInternal.
+func TestHandleEC2RunInstances_MessageParsing(t *testing.T) {
+	tests := []struct {
+		name    string
+		payload []byte
+		wantErr string
+	}{
+		{
+			name:    "Malformed JSON",
+			payload: []byte(`{"invalid": json}`),
+			wantErr: awserrors.ErrorValidationError,
+		},
+		{
+			name: "Invalid Instance Type",
+			payload: mustMarshal(t, &ec2.RunInstancesInput{
+				ImageId:      aws.String("ami-0abcdef1234567890"),
+				InstanceType: aws.String("invalid.type"),
+				MinCount:     aws.Int64(1),
+				MaxCount:     aws.Int64(1),
+			}),
+			wantErr: awserrors.ErrorInvalidInstanceType,
+		},
+		{
+			name: "Missing ImageId",
+			payload: mustMarshal(t, &ec2.RunInstancesInput{
+				InstanceType: aws.String(getTestInstanceType(t)),
+				MinCount:     aws.Int64(1),
+				MaxCount:     aws.Int64(1),
+			}),
+			wantErr: awserrors.ErrorMissingParameter,
+		},
+	}
+
+	daemon := createFullTestDaemon(t, sharedNATSURL)
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.RunInstances", "spinifex-workers", daemon.handleEC2RunInstances)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			reply, err := natsRequest(daemon.natsConn, "ec2.RunInstances", tt.payload, 5*time.Second)
+			require.NoError(t, err)
+			assert.Contains(t, string(reply.Data), tt.wantErr)
+		})
+	}
+}
+
+func mustMarshal(t *testing.T, v any) []byte {
+	t.Helper()
+	b, err := json.Marshal(v)
+	require.NoError(t, err)
+	return b
+}
+
 func TestHandleEC2RunInstances_ValidKeyPairPassesValidation(t *testing.T) {
 	natsURL := sharedNATSURL

@@ -1009,132 +984,6 @@ func TestHandleEC2ModifyVolume_VolumeNotFound(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, errResp["Code"])
 }

-// --- Account settings handler tests ---
-
-func TestHandleEC2GetEbsEncryptionByDefault(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createFullTestDaemon(t, natsURL)
-
-	sub, err := daemon.natsConn.QueueSubscribe("ec2.GetEbsEncryptionByDefault", "spinifex-workers", daemon.handleEC2GetEbsEncryptionByDefault)
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	input := &ec2.GetEbsEncryptionByDefaultInput{}
-	reqData, _ := json.Marshal(input)
-	reply, err := natsRequest(daemon.natsConn, "ec2.GetEbsEncryptionByDefault", reqData, 5*time.Second)
-	require.NoError(t, err)
-
-	var output ec2.GetEbsEncryptionByDefaultOutput
-	err = json.Unmarshal(reply.Data, &output)
-	require.NoError(t, err)
-	assert.NotNil(t, output.EbsEncryptionByDefault)
-}
-
-func TestHandleEC2GetSerialConsoleAccessStatus(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createFullTestDaemon(t, natsURL)
-
-	sub, err := daemon.natsConn.QueueSubscribe("ec2.GetSerialConsoleAccessStatus", "spinifex-workers", daemon.handleEC2GetSerialConsoleAccessStatus)
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	input := &ec2.GetSerialConsoleAccessStatusInput{}
-	reqData, _ := json.Marshal(input)
-	reply, err := natsRequest(daemon.natsConn, "ec2.GetSerialConsoleAccessStatus", reqData, 5*time.Second)
-	require.NoError(t, err)
-
-	var output ec2.GetSerialConsoleAccessStatusOutput
-	err = json.Unmarshal(reply.Data, &output)
-	require.NoError(t, err)
-	assert.NotNil(t, output.SerialConsoleAccessEnabled)
-}
-
-func TestHandleEC2EnableEbsEncryptionByDefault(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createFullTestDaemon(t, natsURL)
-
-	sub, err := daemon.natsConn.QueueSubscribe("ec2.EnableEbsEncryptionByDefault", "spinifex-workers", daemon.handleEC2EnableEbsEncryptionByDefault)
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	input := &ec2.EnableEbsEncryptionByDefaultInput{}
-	reqData, _ := json.Marshal(input)
-	reply, err := natsRequest(daemon.natsConn, "ec2.EnableEbsEncryptionByDefault", reqData, 5*time.Second)
-	require.NoError(t, err)
-
-	var output ec2.EnableEbsEncryptionByDefaultOutput
-	err = json.Unmarshal(reply.Data, &output)
-	require.NoError(t, err)
-	assert.NotNil(t, output.EbsEncryptionByDefault)
-	assert.True(t, *output.EbsEncryptionByDefault)
-}
-
-func TestHandleEC2DisableEbsEncryptionByDefault(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createFullTestDaemon(t, natsURL)
-
-	sub, err := daemon.natsConn.QueueSubscribe("ec2.DisableEbsEncryptionByDefault", "spinifex-workers", daemon.handleEC2DisableEbsEncryptionByDefault)
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	input := &ec2.DisableEbsEncryptionByDefaultInput{}
-	reqData, _ := json.Marshal(input)
-	reply, err := natsRequest(daemon.natsConn, "ec2.DisableEbsEncryptionByDefault", reqData, 5*time.Second)
-	require.NoError(t, err)
-
-	var output ec2.DisableEbsEncryptionByDefaultOutput
-	err = json.Unmarshal(reply.Data, &output)
-	require.NoError(t, err)
-	assert.NotNil(t, output.EbsEncryptionByDefault)
-	assert.False(t, *output.EbsEncryptionByDefault)
-}
-
-func TestHandleEC2EnableSerialConsoleAccess(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createFullTestDaemon(t, natsURL)
-
-	sub, err := daemon.natsConn.QueueSubscribe("ec2.EnableSerialConsoleAccess", "spinifex-workers", daemon.handleEC2EnableSerialConsoleAccess)
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	input := &ec2.EnableSerialConsoleAccessInput{}
-	reqData, _ := json.Marshal(input)
-	reply, err := natsRequest(daemon.natsConn, "ec2.EnableSerialConsoleAccess", reqData, 5*time.Second)
-	require.NoError(t, err)
-
-	var output ec2.EnableSerialConsoleAccessOutput
-	err = json.Unmarshal(reply.Data, &output)
-	require.NoError(t, err)
-	assert.NotNil(t, output.SerialConsoleAccessEnabled)
-	assert.True(t, *output.SerialConsoleAccessEnabled)
-}
-
-func TestHandleEC2DisableSerialConsoleAccess(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createFullTestDaemon(t, natsURL)
-
-	sub, err := daemon.natsConn.QueueSubscribe("ec2.DisableSerialConsoleAccess", "spinifex-workers", daemon.handleEC2DisableSerialConsoleAccess)
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	input := &ec2.DisableSerialConsoleAccessInput{}
-	reqData, _ := json.Marshal(input)
-	reply, err := natsRequest(daemon.natsConn, "ec2.DisableSerialConsoleAccess", reqData, 5*time.Second)
-	require.NoError(t, err)
-
-	var output ec2.DisableSerialConsoleAccessOutput
-	err = json.Unmarshal(reply.Data, &output)
-	require.NoError(t, err)
-	assert.NotNil(t, output.SerialConsoleAccessEnabled)
-	assert.False(t, *output.SerialConsoleAccessEnabled)
-}
-
 // --- handleEC2CreateImage tests ---

 func TestHandleEC2CreateImage_InstanceNotFound(t *testing.T) {
@@ -1842,6 +1691,9 @@ func TestHandleEC2DescribeInstanceAttribute_DefaultAttribute_DisableApiTerminati
 	require.NoError(t, err)
 	defer sub.Unsubscribe()

+	// Smoke test the daemon→service delegate: write a stopped instance with
+	// the protection flag set and confirm the round-trip surfaces the real
+	// value (not the pre-migration hardcoded false).
 	instanceID := "i-describe-def-001"
 	instance := &vm.VM{
 		ID:           instanceID,
@@ -1851,6 +1703,9 @@ func TestHandleEC2DescribeInstanceAttribute_DefaultAttribute_DisableApiTerminati
 		Instance: &ec2.Instance{
 			InstanceId: aws.String(instanceID),
 		},
+		RunInstancesInput: &ec2.RunInstancesInput{
+			DisableApiTermination: aws.Bool(true),
+		},
 	}
 	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
 	require.NoError(t, err)
@@ -1868,7 +1723,7 @@ func TestHandleEC2DescribeInstanceAttribute_DefaultAttribute_DisableApiTerminati
 	err = json.Unmarshal(reply.Data, &output)
 	require.NoError(t, err)
 	require.NotNil(t, output.DisableApiTermination)
-	assert.Equal(t, false, *output.DisableApiTermination.Value)
+	assert.Equal(t, true, *output.DisableApiTermination.Value)
 }

 func TestHandleEC2DescribeInstanceAttribute_DefaultAttribute_ShutdownBehavior(t *testing.T) {
@@ -2066,7 +1921,8 @@ func TestHandleEC2DescribeInstanceAttribute_InvalidJSON(t *testing.T) {
 	var errResp map[string]any
 	err = json.Unmarshal(reply.Data, &errResp)
 	require.NoError(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, errResp["Code"])
+	// utils.UnmarshalJsonPayload returns ValidationError on parse failure.
+	assert.Equal(t, awserrors.ErrorValidationError, errResp["Code"])
 }

 // --- Delegate handler round-trip tests (table-driven) ---
@@ -3481,6 +3337,62 @@ func TestHandleEC2DescribeInstances_MalformedInstanceID(t *testing.T) {
 	assert.Contains(t, string(reply.Data), "InvalidInstanceID.Malformed")
 }

+// --- handleEC2DescribeInstanceStatus round-trip ---
+
+func TestHandleEC2DescribeInstanceStatus_RoundTrip(t *testing.T) {
+	daemon := createFullTestDaemon(t, sharedNATSURL)
+
+	runningVM := &vm.VM{
+		ID:        "i-status-001",
+		Status:    vm.StateRunning,
+		AccountID: testAccountID,
+		Reservation: &ec2.Reservation{
+			ReservationId: aws.String("r-status-001"),
+			OwnerId:       aws.String(testAccountID),
+		},
+		Instance: &ec2.Instance{InstanceId: aws.String("i-status-001")},
+	}
+	daemon.vmMgr.Insert(runningVM)
+	t.Cleanup(func() { daemon.vmMgr.Delete(runningVM.ID) })
+
+	sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceStatus.rt", daemon.handleEC2DescribeInstanceStatus)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	reqData, _ := json.Marshal(&ec2.DescribeInstanceStatusInput{})
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceStatus.rt", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceStatusOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &output))
+
+	found := false
+	for _, s := range output.InstanceStatuses {
+		if s.InstanceId != nil && *s.InstanceId == "i-status-001" {
+			found = true
+			assert.Equal(t, "running", *s.InstanceState.Name)
+			assert.Equal(t, "ok", *s.InstanceStatus.Status)
+			assert.Equal(t, "ok", *s.SystemStatus.Status)
+		}
+	}
+	assert.True(t, found, "expected running instance in DescribeInstanceStatus output")
+}
+
+func TestHandleEC2DescribeInstanceStatus_MalformedInstanceID(t *testing.T) {
+	daemon := createFullTestDaemon(t, sharedNATSURL)
+
+	sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceStatus.malformed", daemon.handleEC2DescribeInstanceStatus)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	input := &ec2.DescribeInstanceStatusInput{InstanceIds: []*string{aws.String("not-an-id")}}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceStatus.malformed", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	assert.Contains(t, string(reply.Data), "InvalidInstanceID.Malformed")
+}
+
 // --- Terminated instance handler tests ---

 func TestHandleEC2DescribeTerminatedInstances_ReturnsTerminatedInstances(t *testing.T) {
@@ -4160,3 +4072,77 @@ func TestDelegateHandlers_VPCAttributes(t *testing.T) {
 		})
 	}
 }
+
+// --- respondWithJSON tests ---
+
+// captureSlogForTest redirects the default slog logger to a buffer for the
+// duration of the test and restores it via t.Cleanup.
+func captureSlogForTest(t *testing.T) *bytes.Buffer {
+	t.Helper()
+	var buf bytes.Buffer
+	prev := slog.Default()
+	slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})))
+	t.Cleanup(func() { slog.SetDefault(prev) })
+	return &buf
+}
+
+func TestRespondWithJSON_MarshalSuccess(t *testing.T) {
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	want := testOutput{Greeting: "hello world"}
+
+	sub, err := nc.Subscribe("test.respond.ok", func(msg *nats.Msg) {
+		respondWithJSON(msg, want)
+	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	reply, err := nc.Request("test.respond.ok", nil, 5*time.Second)
+	require.NoError(t, err)
+
+	var got testOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &got))
+	assert.Equal(t, want, got)
+}
+
+func TestRespondWithJSON_MarshalFailureReturnsServerInternal(t *testing.T) {
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	buf := captureSlogForTest(t)
+
+	sub, err := nc.Subscribe("test.respond.marshalfail", func(msg *nats.Msg) {
+		// Channels are not marshalable by encoding/json, forcing the
+		// marshal-failure branch.
+		respondWithJSON(msg, make(chan int))
+	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	reply, err := nc.Request("test.respond.marshalfail", nil, 5*time.Second)
+	require.NoError(t, err)
+
+	var errResp map[string]any
+	require.NoError(t, json.Unmarshal(reply.Data, &errResp))
+	assert.Equal(t, awserrors.ErrorServerInternal, errResp["Code"])
+	assert.Contains(t, buf.String(), "Failed to marshal response")
+}
+
+func TestRespondWithJSON_RespondFailureLogs(t *testing.T) {
+	buf := captureSlogForTest(t)
+
+	// A bare *nats.Msg has no Sub binding; Respond returns ErrMsgNotBound
+	// after a successful marshal, exercising the secondary log-only branch.
+	msg := &nats.Msg{Subject: "test.respond.unbound"}
+
+	assert.NotPanics(t, func() {
+		respondWithJSON(msg, testOutput{Greeting: "ignored"})
+	})
+
+	logged := buf.String()
+	assert.Contains(t, logged, "Failed to respond to NATS request")
+	assert.NotContains(t, logged, "Failed to marshal response")
+}
@@ -1,9 +1,11 @@
 package daemon

 import (
-	"encoding/base64"
 	"fmt"
 	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
 	"time"

 	"github.com/aws/aws-sdk-go/aws"
@@ -17,13 +19,11 @@ import (
 // Compile-time check that Daemon implements SystemInstanceLauncher.
 var _ handlers_elbv2.SystemInstanceLauncher = (*Daemon)(nil)

-// LaunchSystemInstance creates and starts a system-managed VM. The VM is owned
-// by the system account (GlobalAccountID) and is not visible to customer
-// DescribeInstances calls.
-//
-// This is the internal equivalent of RunInstances — it reuses the same
-// instance service, resource manager, volume preparation, and QEMU launch
-// path, but skips the NATS request/response envelope and key pair validation.
+// LaunchSystemInstance creates and starts a system-managed VM (ELBv2 LB).
+// The VM is owned by the system account (GlobalAccountID), is not visible to
+// customer DescribeInstances calls, and always boots via direct kernel boot
+// (bundled vmlinuz+initramfs, fw_cfg-delivered config). There is no AMI,
+// volume, or cloud-init path.
 func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) (*handlers_elbv2.SystemInstanceOutput, error) {
 	accountID := utils.GlobalAccountID
 	// ENI account may differ from system account — the ENI is created under
@@ -39,14 +39,6 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 		return nil, fmt.Errorf("unknown instance type: %s", input.InstanceType)
 	}

-	// Validate AMI
-	if d.imageService == nil {
-		return nil, fmt.Errorf("image service not initialized")
-	}
-	if _, err := d.imageService.GetAMIConfig(input.ImageID); err != nil {
-		return nil, fmt.Errorf("AMI %s not found: %w", input.ImageID, err)
-	}
-
 	// Allocate resources
 	if err := d.resourceMgr.allocate(instanceType); err != nil {
 		return nil, fmt.Errorf("insufficient capacity for %s: %w", input.InstanceType, err)
@@ -57,7 +49,6 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 	// customer-facing listings (Nodes page).
 	runInput := &ec2.RunInstancesInput{
 		InstanceType: aws.String(input.InstanceType),
-		ImageId:      aws.String(input.ImageID),
 		MinCount:     aws.Int64(1),
 		MaxCount:     aws.Int64(1),
 		TagSpecifications: []*ec2.TagSpecification{
@@ -72,10 +63,6 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 	if input.SubnetID != "" {
 		runInput.SubnetId = aws.String(input.SubnetID)
 	}
-	if input.UserData != "" {
-		encoded := base64.StdEncoding.EncodeToString([]byte(input.UserData))
-		runInput.UserData = aws.String(encoded)
-	}

 	// Create VM via instance service
 	instance, ec2Instance, err := d.instanceService.RunInstance(runInput)
@@ -276,6 +263,14 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 			instance.MgmtMAC = generateMgmtMAC(instance.ID)
 			instance.MgmtIP = mgmtIP

+			// Inject the allocated MAC and CIDR into NIC[1] so writeFwCfgBlobs
+			// carries real values — the ELBv2 service leaves them blank because
+			// MAC/IP are only known after daemon allocation.
+			if len(input.NICs) > 1 && input.NICs[1].MAC == "" {
+				input.NICs[1].MAC = instance.MgmtMAC
+				input.NICs[1].CIDR = instance.MgmtIP + "/24"
+			}
+
 			tapName := vm.MgmtTapName(instance.ID)
 			tapErr := d.networkPlumber.SetupTap(vm.TapSpec{Name: tapName, Bridge: mgmtBridge})
 			if tapErr != nil {
@@ -302,30 +297,25 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 		}
 	}

-	// Prepare volumes (root volume clone from AMI, cloud-init ISO, etc.)
-	volumeInfos, err := d.instanceService.GenerateVolumes(runInput, instance)
+	// Direct-boot (microvm) path: build the vm.Config directly with microvm
+	// machine settings and fw_cfg blobs for network, lb-agent env, and CA cert.
+	cfg, err := d.buildDirectBootConfig(instance.ID, input)
 	if err != nil {
 		d.cleanupFailedSystemInstance(instance, instanceType)
-		return nil, fmt.Errorf("generate volumes: %w", err)
-	}
-
-	instance.Instance.BlockDeviceMappings = make([]*ec2.InstanceBlockDeviceMapping, 0, len(volumeInfos))
-	for _, vi := range volumeInfos {
-		mapping := &ec2.InstanceBlockDeviceMapping{}
-		mapping.SetDeviceName(vi.DeviceName)
-		mapping.Ebs = &ec2.EbsInstanceBlockDevice{}
-		mapping.Ebs.SetVolumeId(vi.VolumeId)
-		mapping.Ebs.SetAttachTime(vi.AttachTime)
-		mapping.Ebs.SetDeleteOnTermination(vi.DeleteOnTermination)
-		mapping.Ebs.SetStatus("attached")
-		instance.Instance.BlockDeviceMappings = append(instance.Instance.BlockDeviceMappings, mapping)
+		return nil, fmt.Errorf("build direct-boot config: %w", err)
 	}
+	instance.Config = cfg
+	instance.DirectBoot = true

 	// Launch QEMU VM
+	t1 := time.Now()
 	if err := d.vmMgr.Run(instance); err != nil {
 		d.cleanupFailedSystemInstance(instance, instanceType)
 		return nil, fmt.Errorf("launch instance: %w", err)
 	}
+	slog.Info("direct-boot timing",
+		"instanceId", instance.ID,
+		"t1_to_t2_ms", time.Since(t1).Milliseconds())

 	slog.Info("LaunchSystemInstance completed",
 		"instanceId", instance.ID,
@@ -418,3 +408,215 @@ func (d *Daemon) WaitForSystemInstance(instanceID string, timeout time.Duration)
 	}
 	return fmt.Errorf("instance %s did not reach running state within %s", instanceID, timeout)
 }
+
+// buildDirectBootConfig constructs a vm.Config for a direct-boot microVM
+// launch. It does not call buildBaseVMConfig — the microvm machine type
+// requires no PCIe root ports, no UEFI/BIOS, and no drives. Runtime paths
+// (PID file, console log, serial socket) are filled in later by startQEMU.
+//
+// Network topology from input.NICs is serialised to three fw_cfg tmpfiles
+// (netcfg, lb-agent-env, ca-cert) under utils.RuntimeDir().
+func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2.SystemInstanceInput) (vm.Config, error) {
+	it := d.resourceMgr.instanceTypes[input.InstanceType]
+	architecture := "x86_64"
+	if it != nil && it.ProcessorInfo != nil && len(it.ProcessorInfo.SupportedArchitectures) > 0 && it.ProcessorInfo.SupportedArchitectures[0] != nil {
+		architecture = *it.ProcessorInfo.SupportedArchitectures[0]
+	}
+	vcpus := 1
+	memMiB := 128
+	if it != nil {
+		vcpus = int(instanceTypeVCPUs(it))
+		memMiB = int(instanceTypeMemoryMiB(it))
+	}
+
+	const imagePath = "/usr/share/spinifex/microvm"
+
+	fwCfg, err := d.writeFwCfgBlobs(instanceID, input)
+	if err != nil {
+		return vm.Config{}, fmt.Errorf("write fw_cfg blobs: %w", err)
+	}
+
+	// Build kernel cmdline.
+	//
+	// QEMU microvm maps virtio-mmio slots at 0xfeb00000 + n*0x200 with IRQ 5+n.
+	// Without virtio_mmio.device params the kernel never discovers the devices
+	// (auto-kernel-cmdline does not append when -append is specified).
+	//
+	// Boot-time perf flags (Phase 2):
+	//   quiet loglevel=3        — suppress printk to ttyS0 (115200 baud serial
+	//                             writes dominate boot time; warnings+errors only)
+	//   mitigations=off         — skip spectre/meltdown microcode patching at
+	//                             boot. Safe here: KVM guest on trusted host CPU,
+	//                             single-tenant LB workload, no untrusted code in
+	//                             the VM. Saves ~200-500ms.
+	//   tsc=reliable            — skip TSC calibration loop on KVM (host already
+	//                             vouched for invariant TSC)
+	//   no_timer_check          — skip APIC/PIT cross-check; microvm has no PIT
+	//   reboot=t                — triple-fault reboot; skip ACPI/keyboard probes
+	//   i8042.no{pnp,aux,kbd}   — microvm has no PS/2; skip i8042 probe timeouts
+	var sb strings.Builder
+	sb.WriteString("console=ttyS0 quiet loglevel=3 mitigations=off tsc=reliable no_timer_check reboot=t i8042.nopnp i8042.noaux i8042.nokbd")
+	for i := range input.NICs {
+		fmt.Fprintf(&sb, " virtio_mmio.device=0x200@0x%x:%d",
+			0xfeb00000+i*0x200, 5+i)
+	}
+	cmdline := sb.String()
+
+	machineType := microvmMachineType()
+
+	cfg := vm.Config{
+		Name:          instanceID,
+		EnableKVM:     true,
+		NoGraphic:     true,
+		Architecture:  architecture,
+		CPUType:       "host,-pmu",
+		CPUCount:      vcpus,
+		Memory:        memMiB,
+		MachineType:   machineType,
+		KernelImage:   filepath.Join(imagePath, "vmlinuz"),
+		Initrd:        filepath.Join(imagePath, "initramfs.cpio.gz"),
+		KernelCmdline: cmdline,
+		FwCfg:         fwCfg,
+	}
+
+	// Wire netdevs and devices for each NIC in declaration order.
+	// Tap device creation happens later in startQEMU (host-side only here).
+	allNICs := buildNICNetdevs(instanceID, input, machineType)
+	cfg.NetDevs = append(cfg.NetDevs, allNICs.netdevs...)
+	cfg.Devices = append(cfg.Devices, allNICs.devices...)
+
+	return cfg, nil
+}
+
+// nicNetdevResult holds the QEMU netdev and device entries for all NICs.
+type nicNetdevResult struct {
+	netdevs []vm.NetDev
+	devices []vm.Device
+}
+
+// microvmMachineType returns the QEMU -M string for the microvm machine.
+// isa-serial is kept on permanently for per-VM console log access — the
+// boot-time saving from turning it off is not worth losing console output.
+func microvmMachineType() string {
+	return "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on"
+}
+
+// buildNICNetdevs produces QEMU -netdev and -device entries for each NIC in
+// input. The NIC order determines the netdev IDs: net0 = primary ENI,
+// net1 = mgmt (if present), net2+ = extra ENIs. Extra ENIs beyond the primary
+// are only included when corresponding ExtraENIs entries exist.
+func buildNICNetdevs(instanceID string, input *handlers_elbv2.SystemInstanceInput, machineType string) nicNetdevResult {
+	var res nicNetdevResult
+
+	for i, nic := range input.NICs {
+		netID := fmt.Sprintf("net%d", i)
+		// Resolve the tap name from the corresponding ENI.
+		tapName := tapNameForNIC(i, nic, instanceID, input)
+		res.netdevs = append(res.netdevs, vm.NetDev{
+			Value: fmt.Sprintf("tap,id=%s,ifname=%s,script=no,downscript=no", netID, tapName),
+		})
+		res.devices = append(res.devices, vm.NetDevice(machineType, netID, nic.MAC))
+	}
+
+	return res
+}
+
+// tapNameForNIC returns the Linux tap device name for a NIC at the given index.
+// Index 0 → primary ENI tap, index 1 → mgmt tap, index 2+ → extra ENI taps.
+func tapNameForNIC(idx int, _ handlers_elbv2.NICConfig, instanceID string, input *handlers_elbv2.SystemInstanceInput) string {
+	switch idx {
+	case 0:
+		return vm.TapDeviceName(input.ENIID)
+	case 1:
+		return vm.MgmtTapName(instanceID)
+	default:
+		extraIdx := idx - 2
+		if extraIdx < len(input.ExtraENIs) {
+			return vm.TapDeviceName(input.ExtraENIs[extraIdx].ENIID)
+		}
+		return fmt.Sprintf("tap-unknown-%d", idx)
+	}
+}
+
+// writeFwCfgBlobs serialises NIC configuration, lb-agent env, and CA cert to
+// per-VM tmpfiles under utils.RuntimeDir(). Returns the fw_cfg entries for the
+// three blobs and an error if any write fails or the NIC default invariant is
+// violated.
+func (d *Daemon) writeFwCfgBlobs(instanceID string, input *handlers_elbv2.SystemInstanceInput) ([]vm.FwCfgEntry, error) {
+	runtimeDir := utils.RuntimeDir()
+
+	netcfgPath := filepath.Join(runtimeDir, fmt.Sprintf("fwcfg-%s-netcfg.tmp", instanceID))
+	lbenvPath := filepath.Join(runtimeDir, fmt.Sprintf("fwcfg-%s-lbenv.tmp", instanceID))
+	cacertPath := filepath.Join(runtimeDir, fmt.Sprintf("fwcfg-%s-cacert.tmp", instanceID))
+
+	netcfg, err := buildNetcfgBlob(input.NICs)
+	if err != nil {
+		return nil, err
+	}
+	if err := os.WriteFile(netcfgPath, []byte(netcfg), 0600); err != nil {
+		return nil, fmt.Errorf("write netcfg blob: %w", err)
+	}
+
+	if err := os.WriteFile(lbenvPath, []byte(input.LBAgentEnv), 0600); err != nil {
+		_ = os.Remove(netcfgPath)
+		return nil, fmt.Errorf("write lb-agent-env blob: %w", err)
+	}
+
+	if err := os.WriteFile(cacertPath, []byte(input.CACert), 0600); err != nil {
+		_ = os.Remove(netcfgPath)
+		_ = os.Remove(lbenvPath)
+		return nil, fmt.Errorf("write ca-cert blob: %w", err)
+	}
+
+	slog.Debug("wrote fw_cfg blobs",
+		"instanceId", instanceID,
+		"netcfg", netcfgPath,
+		"lbenv", lbenvPath,
+		"cacert", cacertPath,
+	)
+
+	return []vm.FwCfgEntry{
+		{Name: "opt/spinifex/netcfg", File: netcfgPath},
+		{Name: "opt/spinifex/lb-agent-env", File: lbenvPath},
+		{Name: "opt/spinifex/ca-cert", File: cacertPath},
+	}, nil
+}
+
+// buildNetcfgBlob serialises a slice of NICConfig entries to the shell KEY=value
+// format consumed by the initramfs init script. Exactly one NIC must have
+// IsDefault=true; returns an error otherwise.
+func buildNetcfgBlob(nics []handlers_elbv2.NICConfig) (string, error) {
+	defaultCount := 0
+	for _, n := range nics {
+		if n.IsDefault {
+			defaultCount++
+		}
+	}
+	if defaultCount != 1 {
+		return "", fmt.Errorf("netcfg: exactly one NIC must have IsDefault=true, got %d", defaultCount)
+	}
+
+	var sb strings.Builder
+	for i, n := range nics {
+		prefix := fmt.Sprintf("NIC%d", i)
+		fmt.Fprintf(&sb, "%s_MAC=%s\n", prefix, n.MAC)
+		if n.CIDR != "" {
+			fmt.Fprintf(&sb, "%s_CIDR=%s\n", prefix, n.CIDR)
+		}
+		if n.Gateway != "" {
+			fmt.Fprintf(&sb, "%s_GW=%s\n", prefix, n.Gateway)
+		}
+		if n.IsDefault {
+			fmt.Fprintf(&sb, "%s_DEFAULT=1\n", prefix)
+		} else {
+			fmt.Fprintf(&sb, "%s_DEFAULT=0\n", prefix)
+		}
+		if n.RouteDst != "" {
+			fmt.Fprintf(&sb, "%s_ROUTE_DST=%s\n", prefix, n.RouteDst)
+		}
+		if n.RouteVia != "" {
+			fmt.Fprintf(&sb, "%s_ROUTE_VIA=%s\n", prefix, n.RouteVia)
+		}
+	}
+	return sb.String(), nil
+}
@@ -1,10 +1,16 @@
 package daemon

 import (
+	"fmt"
+	"os"
+	"strings"
 	"testing"
 	"time"

+	"github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
 	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )

 func newDaemonWithVMs(vms ...*vm.VM) *Daemon {
@@ -66,3 +72,240 @@ func TestWaitForSystemInstance_Timeout(t *testing.T) {
 		t.Fatal("expected timeout error")
 	}
 }
+
+// ---------------------------------------------------------------------------
+// buildNetcfgBlob
+// ---------------------------------------------------------------------------
+
+func TestBuildNetcfgBlob_NoDefault(t *testing.T) {
+	nics := []handlers_elbv2.NICConfig{
+		{MAC: "02:aa:bb:cc:dd:01", CIDR: "10.0.1.5/24", IsDefault: false},
+	}
+	_, err := buildNetcfgBlob(nics)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "exactly one NIC must have IsDefault=true")
+}
+
+func TestBuildNetcfgBlob_TwoDefaults(t *testing.T) {
+	nics := []handlers_elbv2.NICConfig{
+		{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
+		{MAC: "02:aa:bb:cc:dd:02", IsDefault: true},
+	}
+	_, err := buildNetcfgBlob(nics)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "exactly one NIC must have IsDefault=true, got 2")
+}
+
+func TestBuildNetcfgBlob_SingleNICMinimal(t *testing.T) {
+	nics := []handlers_elbv2.NICConfig{
+		{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
+	}
+	got, err := buildNetcfgBlob(nics)
+	require.NoError(t, err)
+	assert.Contains(t, got, "NIC0_MAC=02:aa:bb:cc:dd:01\n")
+	assert.Contains(t, got, "NIC0_DEFAULT=1\n")
+	// Optional fields absent when empty.
+	assert.NotContains(t, got, "NIC0_CIDR=")
+	assert.NotContains(t, got, "NIC0_GW=")
+}
+
+func TestBuildNetcfgBlob_FullNIC(t *testing.T) {
+	nics := []handlers_elbv2.NICConfig{
+		{
+			MAC:       "02:aa:bb:cc:dd:01",
+			CIDR:      "10.0.1.5/24",
+			Gateway:   "10.0.1.1",
+			IsDefault: true,
+			RouteDst:  "10.20.0.5/32",
+			RouteVia:  "10.0.1.254",
+		},
+	}
+	got, err := buildNetcfgBlob(nics)
+	require.NoError(t, err)
+	assert.Contains(t, got, "NIC0_MAC=02:aa:bb:cc:dd:01\n")
+	assert.Contains(t, got, "NIC0_CIDR=10.0.1.5/24\n")
+	assert.Contains(t, got, "NIC0_GW=10.0.1.1\n")
+	assert.Contains(t, got, "NIC0_DEFAULT=1\n")
+	assert.Contains(t, got, "NIC0_ROUTE_DST=10.20.0.5/32\n")
+	assert.Contains(t, got, "NIC0_ROUTE_VIA=10.0.1.254\n")
+}
+
+func TestBuildNetcfgBlob_MultiNIC_DefaultFlags(t *testing.T) {
+	nics := []handlers_elbv2.NICConfig{
+		{MAC: "02:aa:bb:cc:dd:01", CIDR: "10.0.1.5/24", Gateway: "10.0.1.1", IsDefault: true},
+		{MAC: "02:aa:bb:cc:dd:02", CIDR: "192.168.100.5/24", IsDefault: false},
+	}
+	got, err := buildNetcfgBlob(nics)
+	require.NoError(t, err)
+	assert.Contains(t, got, "NIC0_DEFAULT=1\n")
+	assert.Contains(t, got, "NIC1_DEFAULT=0\n")
+	assert.Contains(t, got, "NIC1_MAC=02:aa:bb:cc:dd:02\n")
+}
+
+// ---------------------------------------------------------------------------
+// tapNameForNIC
+// ---------------------------------------------------------------------------
+
+func TestTapNameForNIC_PrimaryENI(t *testing.T) {
+	input := &handlers_elbv2.SystemInstanceInput{
+		ENIID: "eni-0123456789abcdef0",
+	}
+	got := tapNameForNIC(0, handlers_elbv2.NICConfig{}, "i-abc123", input)
+	// TapDeviceName strips "eni-" prefix → "tap0123456789abcde" (15 chars max)
+	assert.True(t, strings.HasPrefix(got, "tap"), "expected tap prefix, got %q", got)
+	assert.LessOrEqual(t, len(got), 15)
+}
+
+func TestTapNameForNIC_MgmtNIC(t *testing.T) {
+	input := &handlers_elbv2.SystemInstanceInput{ENIID: "eni-aaa"}
+	got := tapNameForNIC(1, handlers_elbv2.NICConfig{}, "i-deadbeef", input)
+	// MgmtTapName strips "i-" prefix → "mgdeadbeef" (10 chars)
+	assert.True(t, strings.HasPrefix(got, "mg"), "expected mg prefix, got %q", got)
+	assert.LessOrEqual(t, len(got), 15)
+}
+
+func TestTapNameForNIC_ExtraENI(t *testing.T) {
+	input := &handlers_elbv2.SystemInstanceInput{
+		ENIID: "eni-primary",
+		ExtraENIs: []handlers_elbv2.ExtraENIInput{
+			{ENIID: "eni-extra0"},
+			{ENIID: "eni-extra1"},
+		},
+	}
+	got := tapNameForNIC(2, handlers_elbv2.NICConfig{}, "i-inst", input)
+	assert.True(t, strings.HasPrefix(got, "tap"), "expected tap prefix for extra ENI, got %q", got)
+}
+
+func TestTapNameForNIC_ExtraENI_OutOfRange(t *testing.T) {
+	input := &handlers_elbv2.SystemInstanceInput{
+		ENIID:     "eni-primary",
+		ExtraENIs: []handlers_elbv2.ExtraENIInput{},
+	}
+	// idx=2 → extraIdx=0 but no ExtraENIs → fallback name
+	got := tapNameForNIC(2, handlers_elbv2.NICConfig{}, "i-inst", input)
+	assert.Equal(t, fmt.Sprintf("tap-unknown-%d", 2), got)
+}
+
+// ---------------------------------------------------------------------------
+// buildNICNetdevs
+// ---------------------------------------------------------------------------
+
+func TestBuildNICNetdevs_SingleNIC(t *testing.T) {
+	input := &handlers_elbv2.SystemInstanceInput{
+		ENIID: "eni-000000000000000a",
+		NICs: []handlers_elbv2.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
+		},
+	}
+	res := buildNICNetdevs("i-test", input, microvmMachineType())
+	require.Len(t, res.netdevs, 1)
+	require.Len(t, res.devices, 1)
+	assert.Contains(t, res.netdevs[0].Value, "tap,id=net0,")
+	// microvm machine type → virtio-net-device transport
+	assert.Contains(t, res.devices[0].Value, "virtio-net-device,netdev=net0")
+	assert.Contains(t, res.devices[0].Value, "02:aa:bb:cc:dd:01")
+}
+
+func TestBuildNICNetdevs_TwoNICs(t *testing.T) {
+	input := &handlers_elbv2.SystemInstanceInput{
+		ENIID: "eni-000000000000000b",
+		NICs: []handlers_elbv2.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
+			{MAC: "02:aa:bb:cc:dd:02", IsDefault: false},
+		},
+	}
+	res := buildNICNetdevs("i-test2", input, microvmMachineType())
+	require.Len(t, res.netdevs, 2)
+	require.Len(t, res.devices, 2)
+	assert.Contains(t, res.netdevs[0].Value, "id=net0,")
+	assert.Contains(t, res.netdevs[1].Value, "id=net1,")
+	assert.Contains(t, res.devices[0].Value, "netdev=net0")
+	assert.Contains(t, res.devices[1].Value, "netdev=net1")
+}
+
+func TestBuildNICNetdevs_EmptyNICs(t *testing.T) {
+	input := &handlers_elbv2.SystemInstanceInput{ENIID: "eni-empty"}
+	res := buildNICNetdevs("i-empty", input, microvmMachineType())
+	assert.Empty(t, res.netdevs)
+	assert.Empty(t, res.devices)
+}
+
+// ---------------------------------------------------------------------------
+// microvmMachineType
+// ---------------------------------------------------------------------------
+
+func TestMicrovmMachineType(t *testing.T) {
+	mt := microvmMachineType()
+	assert.Contains(t, mt, "microvm,")
+	assert.Contains(t, mt, "isa-serial=on")
+}
+
+// ---------------------------------------------------------------------------
+// writeFwCfgBlobs
+// ---------------------------------------------------------------------------
+
+func TestWriteFwCfgBlobs_HappyPath(t *testing.T) {
+	// Use a temp dir as runtime dir so files land somewhere writable.
+	tmpDir := t.TempDir()
+	t.Setenv("XDG_RUNTIME_DIR", tmpDir)
+
+	d := &Daemon{}
+	instanceID := "i-fwcfg-test"
+	input := &handlers_elbv2.SystemInstanceInput{
+		NICs: []handlers_elbv2.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", CIDR: "10.0.1.5/24", Gateway: "10.0.1.1", IsDefault: true},
+		},
+		LBAgentEnv: "LB_BACKEND=10.0.1.10:8080\n",
+		CACert:     "-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n",
+	}
+
+	entries, err := d.writeFwCfgBlobs(instanceID, input)
+	require.NoError(t, err)
+	require.Len(t, entries, 3)
+
+	// Check names are as expected.
+	assert.Equal(t, "opt/spinifex/netcfg", entries[0].Name)
+	assert.Equal(t, "opt/spinifex/lb-agent-env", entries[1].Name)
+	assert.Equal(t, "opt/spinifex/ca-cert", entries[2].Name)
+
+	// Verify files were written and contain expected content.
+	netcfgData, err := os.ReadFile(entries[0].File)
+	require.NoError(t, err)
+	assert.Contains(t, string(netcfgData), "NIC0_MAC=02:aa:bb:cc:dd:01")
+
+	lbenvData, err := os.ReadFile(entries[1].File)
+	require.NoError(t, err)
+	assert.Equal(t, "LB_BACKEND=10.0.1.10:8080\n", string(lbenvData))
+
+	cacertData, err := os.ReadFile(entries[2].File)
+	require.NoError(t, err)
+	assert.Contains(t, string(cacertData), "BEGIN CERTIFICATE")
+
+	// Cleanup: remove the tmpfiles (proves they are real filesystem files).
+	for _, e := range entries {
+		assert.NoError(t, os.Remove(e.File))
+	}
+}
+
+func TestWriteFwCfgBlobs_InvalidNICs_NoDefault(t *testing.T) {
+	tmpDir := t.TempDir()
+	t.Setenv("XDG_RUNTIME_DIR", tmpDir)
+
+	d := &Daemon{}
+	input := &handlers_elbv2.SystemInstanceInput{
+		NICs: []handlers_elbv2.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", IsDefault: false},
+		},
+	}
+
+	_, err := d.writeFwCfgBlobs("i-bad-nics", input)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "IsDefault")
+
+	// No tmpfiles must be left behind.
+	entries, _ := os.ReadDir(tmpDir)
+	for _, e := range entries {
+		assert.False(t, strings.HasPrefix(e.Name(), "fwcfg-i-bad-nics"),
+			"unexpected tmpfile left behind: %s", e.Name())
+	}
+}
@@ -10,11 +10,13 @@ import (
 	"crypto/x509/pkix"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"math/big"
 	"net"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -28,8 +30,19 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/gpu"
+	handlers_ec2_account "github.com/mulgadc/spinifex/spinifex/handlers/ec2/account"
+	handlers_ec2_eigw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eigw"
+	handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip"
+	handlers_ec2_igw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/igw"
+	handlers_ec2_image "github.com/mulgadc/spinifex/spinifex/handlers/ec2/image"
 	handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
+	handlers_ec2_natgw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/natgw"
+	handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
+	handlers_ec2_routetable "github.com/mulgadc/spinifex/spinifex/handlers/ec2/routetable"
+	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	handlers_ec2_volume "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volume"
+	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
+	handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
 	"github.com/mulgadc/spinifex/spinifex/instancetypes"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/qmp"
@@ -48,10 +61,13 @@ import (
 func drainAndClose(t *testing.T, nc *nats.Conn) {
 	t.Helper()
 	done := make(chan struct{})
-	nc.SetClosedHandler(func(*nats.Conn) { close(done) })
+	var once sync.Once
+	nc.SetClosedHandler(func(*nats.Conn) { once.Do(func() { close(done) }) })
 	if err := nc.Drain(); err != nil {
 		nc.Close()
-		return
+		// Do not return early: the handler callback may still be running
+		// (e.g. writing state to the temp dir). Fall through to the wait so
+		// we give it time to finish before t.TempDir() RemoveAll runs.
 	}
 	select {
 	case <-done:
@@ -162,259 +178,6 @@ func seedTestAMI(t *testing.T, store *objectstore.MemoryObjectStore, bucket, ima
 	require.NoError(t, err)
 }

-// TestHandleEC2RunInstances_MessageParsing tests that the handler correctly parses NATS messages
-func TestHandleEC2RunInstances_MessageParsing(t *testing.T) {
-	// Skip integration tests on macOS since viperblock/nbdkit are not available
-	if os.Getenv("SKIP_INTEGRATION") != "" {
-		t.Skip("Skipping integration test - SKIP_INTEGRATION is set")
-	}
-
-	tests := []struct {
-		name           string
-		input          any
-		expectError    bool
-		errorInPayload bool
-		validate       func(t *testing.T, reply *nats.Msg)
-	}{
-		{
-			name:           "Valid RunInstancesInput",
-			input:          createValidRunInstancesInput(t),
-			expectError:    false,
-			errorInPayload: false,
-			validate: func(t *testing.T, reply *nats.Msg) {
-				var reservation ec2.Reservation
-				err := json.Unmarshal(reply.Data, &reservation)
-				require.NoError(t, err, "Failed to unmarshal reservation response")
-
-				assert.NotNil(t, reservation.ReservationId)
-				assert.Len(t, reservation.Instances, 1)
-
-				if len(reservation.Instances) > 0 {
-					instance := reservation.Instances[0]
-					assert.NotNil(t, instance.InstanceId)
-					assert.True(t, len(*instance.InstanceId) > 0)
-					// Instance should be in pending state initially
-					assert.Equal(t, int64(0), *instance.State.Code)
-					assert.Equal(t, "pending", *instance.State.Name)
-				}
-			},
-		},
-		{
-			name: "Invalid Instance Type",
-			input: &ec2.RunInstancesInput{
-				ImageId:      aws.String("ami-0abcdef1234567890"),
-				InstanceType: aws.String("invalid.type"),
-				MinCount:     aws.Int64(1),
-				MaxCount:     aws.Int64(1),
-			},
-			expectError:    false, // No transport error
-			errorInPayload: true,  // But payload contains error
-			validate: func(t *testing.T, reply *nats.Msg) {
-				// Should receive an error response
-				assert.NotNil(t, reply.Data)
-				// The response should contain error information
-				t.Logf("Error response: %s", string(reply.Data))
-			},
-		},
-		{
-			name: "Missing Required ImageId",
-			input: &ec2.RunInstancesInput{
-				InstanceType: aws.String("t3.micro"),
-				MinCount:     aws.Int64(1),
-				MaxCount:     aws.Int64(1),
-			},
-			expectError:    false,
-			errorInPayload: true,
-			validate: func(t *testing.T, reply *nats.Msg) {
-				assert.NotNil(t, reply.Data)
-				t.Logf("Error response: %s", string(reply.Data))
-			},
-		},
-		{
-			name: "Invalid MinCount (zero)",
-			input: &ec2.RunInstancesInput{
-				ImageId:      aws.String("ami-0abcdef1234567890"),
-				InstanceType: aws.String("t3.micro"),
-				MinCount:     aws.Int64(0),
-				MaxCount:     aws.Int64(1),
-			},
-			expectError:    false,
-			errorInPayload: true,
-			validate: func(t *testing.T, reply *nats.Msg) {
-				assert.NotNil(t, reply.Data)
-				t.Logf("Error response: %s", string(reply.Data))
-			},
-		},
-		{
-			name:           "Malformed JSON",
-			input:          []byte(`{"invalid": json}`),
-			expectError:    false,
-			errorInPayload: true,
-			validate: func(t *testing.T, reply *nats.Msg) {
-				assert.NotNil(t, reply.Data)
-				t.Logf("Error response: %s", string(reply.Data))
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			// Skip "Valid RunInstancesInput" test as it requires full infrastructure
-			// (Predastore S3 backend, Viperblock, NBDkit) to actually create volumes
-			if tt.name == "Valid RunInstancesInput" {
-				t.Skip("Skipping valid input test - requires full spinifex infrastructure (viperblock, nbdkit, predastore)")
-			}
-
-			// Start test NATS server
-			natsURL := sharedNATSURL
-
-			// Create test daemon with all services initialized (image, key, etc.)
-			daemon := createFullTestDaemon(t, natsURL)
-
-			// Subscribe to the ec2.launch topic with the handler
-			sub, err := daemon.natsConn.QueueSubscribe("ec2.launch", "spinifex-workers", daemon.handleEC2RunInstances)
-			require.NoError(t, err, "Failed to subscribe to ec2.launch")
-			defer sub.Unsubscribe()
-
-			// Prepare message data
-			var msgData []byte
-			switch v := tt.input.(type) {
-			case []byte:
-				msgData = v
-			default:
-				msgData, err = json.Marshal(tt.input)
-				require.NoError(t, err, "Failed to marshal input")
-			}
-
-			// Send request to NATS and wait for response
-			reply, err := daemon.natsConn.Request("ec2.launch", msgData, 5*time.Second)
-
-			if tt.expectError {
-				assert.Error(t, err, "Expected error but got none")
-				return
-			}
-
-			require.NoError(t, err, "Request failed")
-			require.NotNil(t, reply, "No reply received")
-
-			// Validate response
-			if tt.validate != nil {
-				tt.validate(t, reply)
-			}
-		})
-	}
-}
-
-// TestHandleEC2RunInstances_ResourceManagement tests resource allocation and validation
-// NOTE: This test validates message handling and resource allocation logic without
-// actually launching VMs. Full VM launch requires viperblock, nbdkit, and QEMU/KVM
-// which are not available on macOS.
-func TestHandleEC2RunInstances_ResourceManagement(t *testing.T) {
-	// Skip this test as it requires full infrastructure
-	// The test validates NATS message handling, but daemon.handleEC2RunInstances
-	// attempts to create viperblock volumes which requires:
-	// - S3 backend (predastore) running
-	// - viperblock library with S3 backend
-	// - nbdkit for NBD mounting
-	// - QEMU for VM launch
-	t.Skip("Skipping resource management test - requires full spinifex infrastructure (viperblock, nbdkit, predastore)")
-
-	tests := []struct {
-		name             string
-		instanceType     string
-		expectAllocation bool
-		description      string
-	}{
-		{
-			name:             "Valid t3.micro allocation",
-			instanceType:     "t3.micro",
-			expectAllocation: true,
-			description:      "Should successfully allocate resources for t3.micro",
-		},
-		{
-			name:             "Valid t3.nano allocation",
-			instanceType:     "t3.nano",
-			expectAllocation: true,
-			description:      "Should successfully allocate resources for t3.nano",
-		},
-		{
-			name:             "Invalid instance type",
-			instanceType:     "t99.invalid",
-			expectAllocation: false,
-			description:      "Should fail for invalid instance type",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			natsURL := sharedNATSURL
-
-			daemon := createTestDaemon(t, natsURL)
-
-			sub, err := daemon.natsConn.QueueSubscribe("ec2.launch", "spinifex-workers", daemon.handleEC2RunInstances)
-			require.NoError(t, err)
-			defer sub.Unsubscribe()
-
-			input := &ec2.RunInstancesInput{
-				ImageId:      aws.String("ami-0abcdef1234567890"),
-				InstanceType: aws.String(tt.instanceType),
-				MinCount:     aws.Int64(1),
-				MaxCount:     aws.Int64(1),
-				KeyName:      aws.String("test-key"),
-				SubnetId:     aws.String("subnet-test"),
-				UserData:     aws.String(""), // Empty UserData to bypass cloud-init requirements
-			}
-
-			msgData, err := json.Marshal(input)
-			require.NoError(t, err)
-
-			reply, err := daemon.natsConn.Request("ec2.launch", msgData, 5*time.Second)
-			require.NoError(t, err)
-			require.NotNil(t, reply)
-
-			if tt.expectAllocation {
-				var reservation ec2.Reservation
-				err := json.Unmarshal(reply.Data, &reservation)
-				require.NoError(t, err)
-				assert.Len(t, reservation.Instances, 1)
-			} else {
-				// Should receive error response
-				t.Logf("Expected error response: %s", string(reply.Data))
-			}
-		})
-	}
-}
-
-// TestDaemon_Initialization tests daemon initialization
-func TestDaemon_Initialization(t *testing.T) {
-	tmpDir := t.TempDir()
-
-	// New cluster config
-	clusterCfg := &config.ClusterConfig{
-		Node:  "node-1",
-		Nodes: map[string]config.Config{},
-	}
-
-	cfg := &config.Config{
-		BaseDir: tmpDir,
-		DataDir: tmpDir,
-		WalDir:  tmpDir,
-		NATS: config.NATSConfig{
-			Host: "nats://localhost:4222",
-		},
-	}
-
-	clusterCfg.Nodes["node-1"] = *cfg
-
-	daemon, err := NewDaemon(clusterCfg)
-	require.NoError(t, err)
-
-	assert.NotNil(t, daemon)
-	assert.NotNil(t, daemon.resourceMgr)
-	assert.NotNil(t, daemon.vmMgr)
-	assert.Equal(t, cfg, daemon.config)
-}
-
 // TestResourceManager tests resource manager functionality
 func TestResourceManager(t *testing.T) {
 	rm, err := NewResourceManager(nil, nil)
@@ -1024,20 +787,6 @@ func TestStopInstance_Deallocation(t *testing.T) {
 	assert.Equal(t, 0, daemon.resourceMgr.allocatedVCPU)
 }

-// createValidRunInstancesInput creates a valid RunInstancesInput for testing
-func createValidRunInstancesInput(t *testing.T) *ec2.RunInstancesInput {
-	t.Helper()
-	return &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String(getTestInstanceType(t)),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-		KeyName:      aws.String("test-key"),
-		SubnetId:     aws.String("subnet-test"),
-		UserData:     aws.String(""), // Empty UserData to bypass cloud-init requirements
-	}
-}
-
 // TestCanAllocate_CountEdgeCases tests edge cases for canAllocate with count parameter
 func TestCanAllocate_CountEdgeCases(t *testing.T) {
 	t.Run("MinCount_equals_MaxCount", func(t *testing.T) {
@@ -1718,25 +1467,6 @@ func TestResourceManager_ConcurrentAccess(t *testing.T) {
 	assert.Equal(t, float64(0), rm.allocatedMem, "All memory should be deallocated")
 }

-// TestEBSRequest_DeleteOnTermination_DefaultFalse verifies the default value
-func TestEBSRequest_DeleteOnTermination_DefaultFalse(t *testing.T) {
-	req := types.EBSRequest{
-		Name: "vol-test",
-		Boot: true,
-	}
-	assert.False(t, req.DeleteOnTermination, "DeleteOnTermination should default to false")
-}
-
-// TestEBSRequest_DeleteOnTermination_SetTrue verifies the field can be set
-func TestEBSRequest_DeleteOnTermination_SetTrue(t *testing.T) {
-	req := types.EBSRequest{
-		Name:                "vol-test",
-		Boot:                true,
-		DeleteOnTermination: true,
-	}
-	assert.True(t, req.DeleteOnTermination)
-}
-
 // TestGenerateVolumes_DeleteOnTermination_FromBlockDeviceMapping verifies that
 // the deleteOnTermination flag from RunInstancesInput.BlockDeviceMappings is
 // propagated to the EBSRequest on the instance's volume list.
@@ -2033,29 +1763,6 @@ func TestHandleEC2Events_AttachVolume(t *testing.T) {
 	})
 }

-// TestEBSRequest_JSON_Serialization verifies DeleteOnTermination survives JSON round-trip
-// (important for JetStream state persistence)
-func TestEBSRequest_JSON_Serialization(t *testing.T) {
-	original := types.EBSRequest{
-		Name:                "vol-test-json",
-		Boot:                true,
-		DeleteOnTermination: true,
-		NBDURI:              "nbd:unix:/tmp/test.sock",
-	}
-
-	data, err := json.Marshal(original)
-	require.NoError(t, err)
-
-	var restored types.EBSRequest
-	err = json.Unmarshal(data, &restored)
-	require.NoError(t, err)
-
-	assert.Equal(t, original.Name, restored.Name)
-	assert.Equal(t, original.Boot, restored.Boot)
-	assert.Equal(t, original.DeleteOnTermination, restored.DeleteOnTermination)
-	assert.Equal(t, original.NBDURI, restored.NBDURI)
-}
-
 // TestHandleEC2Events_DetachVolume tests the detach-volume handler in handleEC2Events
 func TestHandleEC2Events_DetachVolume(t *testing.T) {
 	natsURL := sharedNATSURL
@@ -3137,6 +2844,117 @@ func TestGetAvailableInstanceTypeInfos_ShowCapacity(t *testing.T) {
 	assert.Len(t, infos, 1)
 }

+// --- GPU-gated GetAvailableInstanceTypeInfos ---
+
+func makeGPUInstanceType(name string, vcpus int64, memMiB int64) *ec2.InstanceTypeInfo {
+	return &ec2.InstanceTypeInfo{
+		InstanceType: aws.String(name),
+		VCpuInfo:     &ec2.VCpuInfo{DefaultVCpus: aws.Int64(vcpus)},
+		MemoryInfo:   &ec2.MemoryInfo{SizeInMiB: aws.Int64(memMiB)},
+		GpuInfo: &ec2.GpuInfo{
+			Gpus: []*ec2.GpuDeviceInfo{{
+				Count:        aws.Int64(int64(instancetypes.GPUCountForType(name))),
+				Manufacturer: aws.String("AMD"),
+				Name:         aws.String("Instinct MI350X"),
+				MemoryInfo:   &ec2.GpuDeviceMemoryInfo{SizeInMiB: aws.Int64(294896)},
+			}},
+		},
+	}
+}
+
+func makeGPUDevice() gpu.GPUDevice {
+	return gpu.GPUDevice{VendorID: "1002", DeviceID: "75a0", PCIAddress: "0000:03:00.0", Vendor: gpu.VendorAMD}
+}
+
+// GPU types with no gpuManager are hidden — there is no GPU hardware on this node.
+func TestGetAvailableInstanceTypeInfos_GPUType_NoManager_Hidden(t *testing.T) {
+	rm := &ResourceManager{
+		hostVCPU:  128,
+		hostMemGB: 1024.0,
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{
+			"g7e.4xlarge": makeGPUInstanceType("g7e.4xlarge", 16, 128*1024),
+		},
+	}
+	infos := rm.GetAvailableInstanceTypeInfos(false)
+	assert.Empty(t, infos, "GPU type must not appear when gpuManager is nil")
+}
+
+// GPU types gate on pool availability, not host CPU/memory — even a tiny host
+// with a GPU pool exposes the type.
+func TestGetAvailableInstanceTypeInfos_GPUType_GatedByPool(t *testing.T) {
+	mgr := gpu.NewManager([]gpu.GPUDevice{makeGPUDevice(), makeGPUDevice()})
+	rm := &ResourceManager{
+		hostVCPU:  4,   // would normally block a 16-vCPU instance
+		hostMemGB: 8.0, // would normally block a 128 GB instance
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{
+			"g7e.4xlarge": makeGPUInstanceType("g7e.4xlarge", 16, 128*1024),
+		},
+		gpuManager: mgr,
+	}
+	infos := rm.GetAvailableInstanceTypeInfos(false)
+	assert.Len(t, infos, 1, "GPU type must appear when pool has free GPUs regardless of CPU/memory")
+}
+
+// Multi-GPU type (g7e.12xlarge, needs 2) appears only when pool has ≥ 2 free GPUs.
+func TestGetAvailableInstanceTypeInfos_MultiGPUType_RequiresTwoGPUs(t *testing.T) {
+	oneGPU := gpu.NewManager([]gpu.GPUDevice{makeGPUDevice()})
+	rm := &ResourceManager{
+		hostVCPU:  128,
+		hostMemGB: 1024.0,
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{
+			"g7e.12xlarge": makeGPUInstanceType("g7e.12xlarge", 48, 512*1024),
+		},
+		gpuManager: oneGPU,
+	}
+	infos := rm.GetAvailableInstanceTypeInfos(false)
+	assert.Empty(t, infos, "g7e.12xlarge must not appear with only 1 free GPU")
+
+	twoGPUs := gpu.NewManager([]gpu.GPUDevice{makeGPUDevice(), makeGPUDevice()})
+	rm.gpuManager = twoGPUs
+	infos = rm.GetAvailableInstanceTypeInfos(false)
+	assert.Len(t, infos, 1, "g7e.12xlarge must appear when pool has ≥ 2 free GPUs")
+}
+
+// showCapacity=true for GPU types emits pool/gpuCount slots.
+func TestGetAvailableInstanceTypeInfos_GPUType_ShowCapacity(t *testing.T) {
+	mgr := gpu.NewManager([]gpu.GPUDevice{makeGPUDevice(), makeGPUDevice(), makeGPUDevice(), makeGPUDevice()})
+	rm := &ResourceManager{
+		hostVCPU:  128,
+		hostMemGB: 1024.0,
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{
+			"g7e.4xlarge":  makeGPUInstanceType("g7e.4xlarge", 16, 128*1024),
+			"g7e.12xlarge": makeGPUInstanceType("g7e.12xlarge", 48, 512*1024),
+		},
+		gpuManager: mgr,
+	}
+	// 4 GPUs → 4× g7e.4xlarge (1 GPU each) or 2× g7e.12xlarge (2 GPUs each)
+	infos := rm.GetAvailableInstanceTypeInfos(true)
+	count4xl := 0
+	count12xl := 0
+	for _, it := range infos {
+		switch *it.InstanceType {
+		case "g7e.4xlarge":
+			count4xl++
+		case "g7e.12xlarge":
+			count12xl++
+		}
+	}
+	assert.Equal(t, 4, count4xl, "4 GPUs → 4 slots for g7e.4xlarge")
+	assert.Equal(t, 2, count12xl, "4 GPUs → 2 slots for g7e.12xlarge")
+}
+
+// canAllocate for GPU types returns count without CPU/memory gating.
+func TestCanAllocate_GPUType_BypassesCPUMemory(t *testing.T) {
+	rm := &ResourceManager{
+		hostVCPU:      4,   // deliberately tiny
+		hostMemGB:     8.0, // deliberately tiny
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{},
+	}
+	gpuType := makeGPUInstanceType("g7e.4xlarge", 16, 128*1024)
+	assert.Equal(t, 1, rm.canAllocate(gpuType, 1),
+		"GPU type must be allocatable even when CPU/memory would normally block it")
+}
+
 // --- NewDaemon ---

 func TestNewDaemon_WalDirDefaultsToBaseDir(t *testing.T) {
@@ -3708,23 +3526,6 @@ func TestConnectNATS_SucceedsImmediately(t *testing.T) {
 	daemon.natsConn.Close()
 }

-// TestDaemonReadyFlag tests that the ready flag is false by default and can be set.
-func TestDaemonReadyFlag(t *testing.T) {
-	clusterCfg := &config.ClusterConfig{
-		Node:  "node-1",
-		Nodes: map[string]config.Config{},
-	}
-	cfg := config.Config{}
-	clusterCfg.Nodes["node-1"] = cfg
-	daemon, err := NewDaemon(clusterCfg)
-	require.NoError(t, err)
-
-	assert.False(t, daemon.ready.Load(), "daemon should not be ready before Start()")
-
-	daemon.ready.Store(true)
-	assert.True(t, daemon.ready.Load(), "daemon should be ready after setting flag")
-}
-
 // --- ClusterManager TLS ---

 func TestClusterManager_TLSServesHTTPS(t *testing.T) {
@@ -3914,3 +3715,338 @@ func TestResolveGPUModel_OverrideCustomisesConsumerGPU(t *testing.T) {
 	assert.Equal(t, "RTX 3060", m.Name)
 	assert.Equal(t, int64(12288), m.MemoryMiB)
 }
+
+// --- initServiceWithRetry ---
+
+// stubInitRetrySleep replaces the package-level sleep seam with a recorder
+// that captures durations without sleeping. Restored via t.Cleanup.
+func stubInitRetrySleep(t *testing.T) *[]time.Duration {
+	t.Helper()
+	var sleeps []time.Duration
+	prev := initRetrySleep
+	initRetrySleep = func(d time.Duration) { sleeps = append(sleeps, d) }
+	t.Cleanup(func() { initRetrySleep = prev })
+	return &sleeps
+}
+
+func TestInitServiceWithRetry_SuccessFirstAttempt(t *testing.T) {
+	sleeps := stubInitRetrySleep(t)
+	calls := 0
+	got, err := initServiceWithRetry("svc", func() (int, error) {
+		calls++
+		return 42, nil
+	})
+	require.NoError(t, err)
+	assert.Equal(t, 42, got)
+	assert.Equal(t, 1, calls)
+	assert.Empty(t, *sleeps, "no sleep on first-attempt success")
+}
+
+func TestInitServiceWithRetry_SuccessAfterRetries(t *testing.T) {
+	sleeps := stubInitRetrySleep(t)
+	calls := 0
+	got, err := initServiceWithRetry("svc", func() (string, error) {
+		calls++
+		if calls < 3 {
+			return "", errors.New("not ready")
+		}
+		return "ok", nil
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "ok", got)
+	assert.Equal(t, 3, calls)
+	assert.Equal(t, []time.Duration{500 * time.Millisecond, 1 * time.Second}, *sleeps,
+		"two sleeps between three attempts must follow the documented 500ms→1s schedule")
+}
+
+func TestInitServiceWithRetry_DelayCappedAt10s(t *testing.T) {
+	// Drive 7 attempts so backoff doublings hit the cap. Six sleeps separate
+	// the seven attempts: 500ms → 1s → 2s → 4s → 8s → 10s. The 6th sleep
+	// must be 10s (capped), not 16s.
+	sleeps := stubInitRetrySleep(t)
+	calls := 0
+	_, err := initServiceWithRetry("svc", func() (int, error) {
+		calls++
+		if calls >= 7 {
+			return 1, nil
+		}
+		return 0, errors.New("retry")
+	})
+	require.NoError(t, err)
+	want := []time.Duration{
+		500 * time.Millisecond,
+		1 * time.Second,
+		2 * time.Second,
+		4 * time.Second,
+		8 * time.Second,
+		10 * time.Second,
+	}
+	assert.Equal(t, want, *sleeps,
+		"backoff must double then cap at 10s on the 6th sleep")
+}
+
+// --- getSystemMemory ---
+
+func TestGetSystemMemory_Linux_Valid(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("linux-only branch")
+	}
+	orig := execCommand
+	t.Cleanup(func() { execCommand = orig })
+	execCommand = func(name string, args ...string) *exec.Cmd {
+		return exec.Command("printf", "MemTotal:       16777216 kB\n")
+	}
+	mem, err := getSystemMemory()
+	require.NoError(t, err)
+	assert.InDelta(t, 16.0, mem, 0.001)
+}
+
+func TestGetSystemMemory_Linux_Malformed(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("linux-only branch")
+	}
+	orig := execCommand
+	t.Cleanup(func() { execCommand = orig })
+	execCommand = func(name string, args ...string) *exec.Cmd {
+		return exec.Command("printf", "MemTotal:\n")
+	}
+	_, err := getSystemMemory()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "unexpected /proc/meminfo format")
+}
+
+func TestGetSystemMemory_Linux_GrepFails(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("linux-only branch")
+	}
+	orig := execCommand
+	t.Cleanup(func() { execCommand = orig })
+	execCommand = func(name string, args ...string) *exec.Cmd {
+		return exec.Command("/bin/false")
+	}
+	_, err := getSystemMemory()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "/proc/meminfo")
+}
+
+func TestGetSystemMemory_Linux_ParseOverflow(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("linux-only branch")
+	}
+	orig := execCommand
+	t.Cleanup(func() { execCommand = orig })
+	execCommand = func(name string, args ...string) *exec.Cmd {
+		return exec.Command("printf", "MemTotal:       99999999999999999999 kB\n")
+	}
+	_, err := getSystemMemory()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "parse memory size")
+}
+
+func TestGetSystemMemory_Darwin_Valid(t *testing.T) {
+	if runtime.GOOS != "darwin" {
+		t.Skip("darwin-only branch; getSystemMemory dispatches on runtime.GOOS")
+	}
+	orig := execCommand
+	t.Cleanup(func() { execCommand = orig })
+	execCommand = func(name string, args ...string) *exec.Cmd {
+		return exec.Command("printf", "17179869184\n")
+	}
+	mem, err := getSystemMemory()
+	require.NoError(t, err)
+	assert.InDelta(t, 16.0, mem, 0.001)
+}
+
+func TestGetSystemMemory_Darwin_SysctlFails(t *testing.T) {
+	if runtime.GOOS != "darwin" {
+		t.Skip("darwin-only branch; getSystemMemory dispatches on runtime.GOOS")
+	}
+	orig := execCommand
+	t.Cleanup(func() { execCommand = orig })
+	execCommand = func(name string, args ...string) *exec.Cmd {
+		return exec.Command("/usr/bin/false")
+	}
+	_, err := getSystemMemory()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "macOS")
+}
+
+// --- gpuXVGAEnabled ---
+
+func TestGpuXVGAEnabled(t *testing.T) {
+	tests := []struct {
+		name      string
+		dev       *gpu.GPUDevice
+		overrides []config.GPUModelOverride
+		want      bool
+	}{
+		{
+			name: "override present XVGAOff true forces false",
+			dev:  &gpu.GPUDevice{VendorID: "10de", DeviceID: "2236"},
+			overrides: []config.GPUModelOverride{
+				{VendorID: "10de", DeviceID: "2236", XVGAOff: true},
+			},
+			want: false,
+		},
+		{
+			name: "override present XVGAOff false forces true",
+			dev:  &gpu.GPUDevice{VendorID: "10de", DeviceID: "2236"},
+			overrides: []config.GPUModelOverride{
+				{VendorID: "10de", DeviceID: "2236", XVGAOff: false},
+			},
+			want: true,
+		},
+		{
+			name: "no override compute GPU returns false",
+			dev:  &gpu.GPUDevice{VendorID: "10de", DeviceID: "2236"},
+			want: false,
+		},
+		{
+			name: "no override consumer GPU returns true",
+			dev:  &gpu.GPUDevice{VendorID: "10de", DeviceID: "2684"},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, gpuXVGAEnabled(tt.dev, tt.overrides))
+		})
+	}
+}
+
+// --- reloadGPUTypes ---
+
+func TestReloadGPUTypes_EmptyModelsPreservesCPUTypes(t *testing.T) {
+	rm := &ResourceManager{
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{
+			"t3.micro": {InstanceType: aws.String("t3.micro")},
+			"t3.large": {InstanceType: aws.String("t3.large")},
+			"g5.xlarge": {
+				InstanceType: aws.String("g5.xlarge"),
+				GpuInfo: &ec2.GpuInfo{Gpus: []*ec2.GpuDeviceInfo{
+					{Count: aws.Int64(1), Name: aws.String("A10G")},
+				}},
+			},
+			"g5.4xlarge": {
+				InstanceType: aws.String("g5.4xlarge"),
+				GpuInfo: &ec2.GpuInfo{Gpus: []*ec2.GpuDeviceInfo{
+					{Count: aws.Int64(1), Name: aws.String("A10G")},
+				}},
+			},
+		},
+	}
+
+	rm.reloadGPUTypes(nil, nil)
+
+	assert.Contains(t, rm.instanceTypes, "t3.micro")
+	assert.Contains(t, rm.instanceTypes, "t3.large")
+	assert.NotContains(t, rm.instanceTypes, "g5.xlarge")
+	assert.NotContains(t, rm.instanceTypes, "g5.4xlarge")
+	assert.Nil(t, rm.gpuManager)
+}
+
+func TestReloadGPUTypes_NonEmptyReplacesGPUTypes(t *testing.T) {
+	rm := &ResourceManager{
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{
+			"t3.micro": {InstanceType: aws.String("t3.micro")},
+			"g5.xlarge": {
+				InstanceType: aws.String("g5.xlarge"),
+				GpuInfo: &ec2.GpuInfo{Gpus: []*ec2.GpuDeviceInfo{
+					{Count: aws.Int64(1), Name: aws.String("A10G")},
+				}},
+			},
+		},
+	}
+
+	rm.reloadGPUTypes([]instancetypes.GPUModel{instancetypes.NVIDIAt4}, nil)
+
+	assert.Contains(t, rm.instanceTypes, "t3.micro", "CPU type preserved")
+	assert.NotContains(t, rm.instanceTypes, "g5.xlarge", "old GPU type removed")
+
+	hasG4dn := false
+	for name := range rm.instanceTypes {
+		if strings.HasPrefix(name, "g4dn.") {
+			hasG4dn = true
+			break
+		}
+	}
+	assert.True(t, hasG4dn, "new GPU family inserted")
+}
+
+func TestReloadGPUTypes_CallsUpdateInstanceSubscriptions(t *testing.T) {
+	// updateInstanceSubscriptions is called unconditionally at the end of
+	// reloadGPUTypes. Observe the side-effect via a real NATS connection:
+	// an empty rm.instanceSubs becomes populated after the call, proving the
+	// per-type subscribe loop ran exactly once for the new map state.
+	// Requires a non-nil gpuManager because canAllocate gates GPU types on
+	// availGPU > 0.
+	port := freePortForTest(t)
+	startTestNATSOnPortForTest(t, port)
+
+	nc, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", port))
+	require.NoError(t, err)
+	t.Cleanup(func() { drainAndClose(t, nc) })
+
+	gpuMgr := gpu.NewManager([]gpu.GPUDevice{
+		{PCIAddress: "0000:01:00.0", VendorID: "10de", DeviceID: "1eb8"},
+	})
+
+	rm := &ResourceManager{
+		hostVCPU:      16,
+		hostMemGB:     64.0,
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{},
+		natsConn:      nc,
+		nodeID:        "test-node",
+		instanceSubs:  make(map[string]*nats.Subscription),
+		handler:       func(*nats.Msg) {},
+	}
+
+	rm.reloadGPUTypes([]instancetypes.GPUModel{instancetypes.NVIDIAt4}, gpuMgr)
+	subsAfter := len(rm.instanceSubs)
+	require.Greater(t, subsAfter, 0, "subscriptions added after reload — updateInstanceSubscriptions ran")
+
+	// Each g4dn size produces both queue and node-specific topics; record the
+	// count so we can prove a second call does not double-subscribe.
+	rm.reloadGPUTypes([]instancetypes.GPUModel{instancetypes.NVIDIAt4}, gpuMgr)
+	assert.Equal(t, subsAfter, len(rm.instanceSubs),
+		"second reload with identical models must not double-subscribe — proves idempotent invocation")
+}
+
+// --- assertNoClusterServicesInitialised ---
+
+func TestAssertNoClusterServicesInitialised_PerField(t *testing.T) {
+	type fieldCase struct {
+		name    string
+		set     func(d *Daemon)
+		wantMsg string
+	}
+
+	cases := []fieldCase{
+		{name: "natsConn", set: func(d *Daemon) { d.natsConn = &nats.Conn{} }, wantMsg: "natsConn"},
+		{name: "jsManager", set: func(d *Daemon) { d.jsManager = &JetStreamManager{} }, wantMsg: "jsManager"},
+		{name: "instanceService", set: func(d *Daemon) { d.instanceService = &handlers_ec2_instance.InstanceServiceImpl{} }, wantMsg: "instanceService"},
+		{name: "imageService", set: func(d *Daemon) { d.imageService = &handlers_ec2_image.ImageServiceImpl{} }, wantMsg: "imageService"},
+		{name: "snapshotService", set: func(d *Daemon) { d.snapshotService = &handlers_ec2_snapshot.SnapshotServiceImpl{} }, wantMsg: "snapshotService"},
+		{name: "volumeService", set: func(d *Daemon) { d.volumeService = &handlers_ec2_volume.VolumeServiceImpl{} }, wantMsg: "volumeService"},
+		{name: "eigwService", set: func(d *Daemon) { d.eigwService = &handlers_ec2_eigw.EgressOnlyIGWServiceImpl{} }, wantMsg: "eigwService"},
+		{name: "igwService", set: func(d *Daemon) { d.igwService = &handlers_ec2_igw.IGWServiceImpl{} }, wantMsg: "igwService"},
+		{name: "placementGroupService", set: func(d *Daemon) { d.placementGroupService = &handlers_ec2_placementgroup.PlacementGroupServiceImpl{} }, wantMsg: "placementGroupService"},
+		{name: "vpcService", set: func(d *Daemon) { d.vpcService = &handlers_ec2_vpc.VPCServiceImpl{} }, wantMsg: "vpcService"},
+		{name: "routeTableService", set: func(d *Daemon) { d.routeTableService = &handlers_ec2_routetable.RouteTableServiceImpl{} }, wantMsg: "routeTableService"},
+		{name: "natGatewayService", set: func(d *Daemon) { d.natGatewayService = &handlers_ec2_natgw.NatGatewayServiceImpl{} }, wantMsg: "natGatewayService"},
+		{name: "externalIPAM", set: func(d *Daemon) { d.externalIPAM = &handlers_ec2_vpc.ExternalIPAM{} }, wantMsg: "externalIPAM"},
+		{name: "eipService", set: func(d *Daemon) { d.eipService = &handlers_ec2_eip.EIPServiceImpl{} }, wantMsg: "eipService"},
+		{name: "accountService", set: func(d *Daemon) { d.accountService = &handlers_ec2_account.AccountSettingsServiceImpl{} }, wantMsg: "accountService"},
+		{name: "elbv2Service", set: func(d *Daemon) { d.elbv2Service = &handlers_elbv2.ELBv2ServiceImpl{} }, wantMsg: "elbv2Service"},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			d := &Daemon{}
+			tc.set(d)
+			err := d.assertNoClusterServicesInitialised()
+			require.Error(t, err, "%s set non-nil must trip invariant", tc.name)
+			assert.Contains(t, err.Error(), tc.wantMsg)
+		})
+	}
+}
@@ -6,6 +6,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"os/user"
 	"sort"
 	"strings"

@@ -48,7 +49,20 @@ func (p *OVSNetworkPlumber) SetupTap(spec vm.TapSpec) error {
 		}
 	}

-	if out, err := sudoCommand("ip", "tuntap", "add", "dev", spec.Name, "mode", "tap").CombinedOutput(); err != nil {
+	// When the daemon runs non-root (production: User=spinifex-daemon), child
+	// QEMU processes inherit that uid without CAP_NET_ADMIN. The kernel's tun
+	// attach check then requires the tap to be owned by the calling euid
+	// (drivers/net/tun.c: tun_chr_open + TUNSETIFF perm check). Without the
+	// `user` flag the tap defaults to root:root and qemu's TUNSETIFF fails
+	// with EPERM ("could not configure /dev/net/tun: Operation not permitted").
+	// Root daemons skip this — they have CAP_NET_ADMIN ambient.
+	addArgs := []string{"tuntap", "add", "dev", spec.Name, "mode", "tap"}
+	if os.Geteuid() != 0 {
+		if u, err := user.Current(); err == nil {
+			addArgs = append(addArgs, "user", u.Username, "group", u.Gid)
+		}
+	}
+	if out, err := sudoCommand("ip", addArgs...).CombinedOutput(); err != nil {
 		return fmt.Errorf("create tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err)
 	}

@@ -157,8 +171,14 @@ func CheckOVNHealth() OVNHealthStatus {
 		status.BrIntExists = true
 	}

-	// Check ovn-controller is running via ovs-appctl (more reliable than pgrep)
-	if out, err := sudoCommand("ovs-appctl", "-t", "ovn-controller", "version").CombinedOutput(); err == nil && len(out) > 0 {
+	// Check ovn-controller is running via ovs-appctl. Trixie OVN moved the
+	// pid file to /var/run/ovn/; older installs use /var/run/openvswitch/.
+	// Probe at runtime so the same binary works on both layouts.
+	ovnTarget := "ovn-controller"
+	if _, err := os.Stat("/var/run/ovn/ovn-controller.pid"); err == nil {
+		ovnTarget = "/var/run/ovn/ovn-controller"
+	}
+	if out, err := sudoCommand("ovs-appctl", "-t", ovnTarget, "version").CombinedOutput(); err == nil && len(out) > 0 {
 		status.OVNControllerUp = true
 	}

@@ -0,0 +1,62 @@
+package daemon
+
+import (
+	"os"
+	"testing"
+)
+
+// Phase gate constants — changing these is a deliberate decision.
+const (
+	phase1BootGateMs = 500
+	phase1RSSGateMiB = 25
+
+	phase2BootGateMs      = 300
+	phase2ArtifactGateMiB = 50
+)
+
+func TestBootGate_Phase1(t *testing.T) {
+	if phase1BootGateMs != 500 {
+		t.Errorf("Phase 1 boot gate changed: got %d, want 500", phase1BootGateMs)
+	}
+	if phase1RSSGateMiB != 25 {
+		t.Errorf("Phase 1 RSS gate changed: got %d, want 25", phase1RSSGateMiB)
+	}
+}
+
+func TestBootGate_Phase2(t *testing.T) {
+	if phase2BootGateMs != 300 {
+		t.Errorf("Phase 2 boot gate changed: got %d, want 300", phase2BootGateMs)
+	}
+	if phase2ArtifactGateMiB != 50 {
+		t.Errorf("Phase 2 artifact gate changed: got %d, want 50", phase2ArtifactGateMiB)
+	}
+}
+
+func TestMeasureQEMURSSMiB_Self(t *testing.T) {
+	rss, err := measureQEMURSSMiB(os.Getpid())
+	if err != nil {
+		t.Fatalf("measureQEMURSSMiB self: %v", err)
+	}
+	if rss <= 0 {
+		t.Errorf("expected RSS > 0, got %f", rss)
+	}
+	if rss > 1024 {
+		t.Errorf("RSS %f MiB seems unreasonably high for a test process", rss)
+	}
+}
+
+func TestMeasureQEMURSSMiB_InvalidPID(t *testing.T) {
+	for _, pid := range []int{0, -1, -999} {
+		_, err := measureQEMURSSMiB(pid)
+		if err == nil {
+			t.Errorf("expected error for pid=%d, got nil", pid)
+		}
+	}
+}
+
+func TestMeasureQEMURSSMiB_NonExistentPID(t *testing.T) {
+	_, err := measureQEMURSSMiB(999999999)
+	if err == nil {
+		t.Error("expected error for non-existent pid, got nil")
+	}
+}
@@ -0,0 +1,39 @@
+package daemon
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+)
+
+// measureQEMURSSMiB reads /proc/<pid>/status and returns RSS in MiB.
+func measureQEMURSSMiB(pid int) (float64, error) {
+	if pid <= 0 {
+		return 0, fmt.Errorf("invalid pid: %d", pid)
+	}
+	f, err := os.Open(fmt.Sprintf("/proc/%d/status", pid))
+	if err != nil {
+		return 0, fmt.Errorf("open /proc/%d/status: %w", pid, err)
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !strings.HasPrefix(line, "VmRSS:") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			return 0, fmt.Errorf("unexpected VmRSS format: %q", line)
+		}
+		kb, err := strconv.ParseFloat(fields[1], 64)
+		if err != nil {
+			return 0, fmt.Errorf("parse VmRSS kB: %w", err)
+		}
+		return kb / 1024, nil
+	}
+	return 0, fmt.Errorf("VmRSS not found in /proc/%d/status", pid)
+}
@@ -3,9 +3,15 @@ package daemon
 import (
 	"encoding/json"
 	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
 	"testing"
 	"time"

+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/mulgadc/spinifex/spinifex/vm"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -189,3 +195,264 @@ func TestPublishShutdownProgress(t *testing.T) {
 		})
 	}
 }
+
+// configurePidDir overrides BaseDir so pidDir() resolves to an isolated tmp
+// directory and returns that directory after creating it.
+func configurePidDir(t *testing.T, d *Daemon) string {
+	t.Helper()
+	root := t.TempDir()
+	d.config.BaseDir = filepath.Join(root, "spinifex")
+	pidDir := filepath.Join(root, "logs")
+	require.NoError(t, os.MkdirAll(pidDir, 0o750))
+	return pidDir
+}
+
+// startSleepProcess forks a sleep process and arranges for it to be reaped
+// when the test completes. Returns the PID.
+func startSleepProcess(t *testing.T) int {
+	t.Helper()
+	cmd := exec.Command("sleep", "60")
+	require.NoError(t, cmd.Start())
+	var wg sync.WaitGroup
+	wg.Go(func() {
+		_ = cmd.Wait()
+	})
+	t.Cleanup(func() {
+		if cmd.Process != nil {
+			_ = cmd.Process.Kill()
+		}
+		wg.Wait()
+	})
+	return cmd.Process.Pid
+}
+
+// TestHandleShutdownGate covers the GATE phase handler: service stop fan-out,
+// shuttingDown flag, and ACK reply for valid, malformed, and partial-failure inputs.
+func TestHandleShutdownGate(t *testing.T) {
+	t.Run("valid request stops configured services and sets shuttingDown", func(t *testing.T) {
+		daemon := createTestDaemon(t, sharedNATSURL)
+		daemon.config.Services = []string{"awsgw", "ui", "vpcd"}
+		pidDir := configurePidDir(t, daemon)
+
+		pid := startSleepProcess(t)
+		require.NoError(t, utils.WritePidFileTo(pidDir, "awsgw", pid))
+
+		subject := "spinifex.cluster.shutdown.gate"
+		sub, err := daemon.natsConn.Subscribe(subject, daemon.handleShutdownGate)
+		require.NoError(t, err)
+		defer sub.Unsubscribe()
+		require.NoError(t, daemon.natsConn.Flush())
+
+		payload, err := json.Marshal(ShutdownRequest{Phase: "gate"})
+		require.NoError(t, err)
+
+		reply, err := daemon.natsConn.Request(subject, payload, 30*time.Second)
+		require.NoError(t, err)
+
+		var ack ShutdownACK
+		require.NoError(t, json.Unmarshal(reply.Data, &ack))
+
+		assert.Equal(t, "node-1", ack.Node)
+		assert.Equal(t, "gate", ack.Phase)
+		assert.Empty(t, ack.Error)
+		assert.Contains(t, ack.Stopped, "awsgw")
+		assert.True(t, daemon.shuttingDown.Load())
+
+		_, statErr := os.Stat(filepath.Join(pidDir, "awsgw.pid"))
+		assert.True(t, os.IsNotExist(statErr), "awsgw pid file should be removed")
+	})
+
+	t.Run("malformed json returns error ack", func(t *testing.T) {
+		daemon := createTestDaemon(t, sharedNATSURL)
+		daemon.config.Services = []string{}
+		configurePidDir(t, daemon)
+
+		subject := "spinifex.cluster.shutdown.gate.malformed"
+		sub, err := daemon.natsConn.Subscribe(subject, daemon.handleShutdownGate)
+		require.NoError(t, err)
+		defer sub.Unsubscribe()
+		require.NoError(t, daemon.natsConn.Flush())
+
+		reply, err := daemon.natsConn.Request(subject, []byte(`{not valid json}`), 5*time.Second)
+		require.NoError(t, err)
+
+		var ack ShutdownACK
+		require.NoError(t, json.Unmarshal(reply.Data, &ack))
+
+		assert.Equal(t, "gate", ack.Phase)
+		assert.NotEmpty(t, ack.Error)
+		assert.Empty(t, ack.Stopped)
+		assert.False(t, daemon.shuttingDown.Load(), "shuttingDown must not be set on parse failure")
+	})
+
+	t.Run("partial service-stop failure still sends ack", func(t *testing.T) {
+		daemon := createTestDaemon(t, sharedNATSURL)
+		daemon.config.Services = []string{"awsgw", "ui", "vpcd"}
+		pidDir := configurePidDir(t, daemon)
+
+		pid := startSleepProcess(t)
+		require.NoError(t, utils.WritePidFileTo(pidDir, "awsgw", pid))
+
+		subject := "spinifex.cluster.shutdown.gate.partial"
+		sub, err := daemon.natsConn.Subscribe(subject, daemon.handleShutdownGate)
+		require.NoError(t, err)
+		defer sub.Unsubscribe()
+		require.NoError(t, daemon.natsConn.Flush())
+
+		payload, err := json.Marshal(ShutdownRequest{Phase: "gate"})
+		require.NoError(t, err)
+
+		reply, err := daemon.natsConn.Request(subject, payload, 30*time.Second)
+		require.NoError(t, err)
+
+		var ack ShutdownACK
+		require.NoError(t, json.Unmarshal(reply.Data, &ack))
+
+		assert.Equal(t, "gate", ack.Phase)
+		assert.Empty(t, ack.Error)
+		assert.Equal(t, []string{"awsgw"}, ack.Stopped, "only the service with a live pid file should appear in stopped")
+		assert.True(t, daemon.shuttingDown.Load())
+	})
+}
+
+// TestHandleShutdownDrain covers the DRAIN phase handler: graceful StopAll,
+// shutdown marker, state persistence, and progress publishing.
+func TestHandleShutdownDrain(t *testing.T) {
+	t.Run("happy path stops all vms writes marker and persists state", func(t *testing.T) {
+		daemon := createFullTestDaemonWithJetStream(t, sharedJSNATSURL)
+		require.NoError(t, daemon.jsManager.InitClusterStateBucket())
+		t.Cleanup(func() { _ = daemon.jsManager.DeleteShutdownMarker(daemon.node) })
+
+		daemon.vmMgr.Insert(&vm.VM{ID: "i-drain-001"})
+
+		subject := "spinifex.cluster.shutdown.drain.happy"
+		sub, err := daemon.natsConn.Subscribe(subject, daemon.handleShutdownDrain)
+		require.NoError(t, err)
+		defer sub.Unsubscribe()
+		require.NoError(t, daemon.natsConn.Flush())
+
+		payload, err := json.Marshal(ShutdownRequest{Phase: "drain"})
+		require.NoError(t, err)
+
+		reply, err := daemon.natsConn.Request(subject, payload, 30*time.Second)
+		require.NoError(t, err)
+
+		var ack ShutdownACK
+		require.NoError(t, json.Unmarshal(reply.Data, &ack))
+		assert.Equal(t, "drain", ack.Phase)
+		assert.Empty(t, ack.Error)
+
+		marker, err := daemon.jsManager.ReadShutdownMarker(daemon.node)
+		require.NoError(t, err)
+		assert.True(t, marker, "shutdown marker should be written for node")
+
+		statePath := daemon.localStatePath()
+		_, statErr := os.Stat(statePath)
+		assert.NoError(t, statErr, "local state file should exist after DRAIN")
+	})
+
+	t.Run("empty vm map skips StopAll but still writes marker and state", func(t *testing.T) {
+		daemon := createFullTestDaemonWithJetStream(t, sharedJSNATSURL)
+		require.NoError(t, daemon.jsManager.InitClusterStateBucket())
+		t.Cleanup(func() { _ = daemon.jsManager.DeleteShutdownMarker(daemon.node) })
+
+		subject := "spinifex.cluster.shutdown.drain.empty"
+		sub, err := daemon.natsConn.Subscribe(subject, daemon.handleShutdownDrain)
+		require.NoError(t, err)
+		defer sub.Unsubscribe()
+		require.NoError(t, daemon.natsConn.Flush())
+
+		payload, err := json.Marshal(ShutdownRequest{Phase: "drain"})
+		require.NoError(t, err)
+
+		reply, err := daemon.natsConn.Request(subject, payload, 10*time.Second)
+		require.NoError(t, err)
+
+		var ack ShutdownACK
+		require.NoError(t, json.Unmarshal(reply.Data, &ack))
+		assert.Equal(t, "drain", ack.Phase)
+		assert.Empty(t, ack.Error)
+
+		marker, err := daemon.jsManager.ReadShutdownMarker(daemon.node)
+		require.NoError(t, err)
+		assert.True(t, marker)
+	})
+
+	t.Run("malformed json returns error ack with no marker", func(t *testing.T) {
+		daemon := createFullTestDaemonWithJetStream(t, sharedJSNATSURL)
+		require.NoError(t, daemon.jsManager.InitClusterStateBucket())
+		t.Cleanup(func() { _ = daemon.jsManager.DeleteShutdownMarker(daemon.node) })
+
+		subject := "spinifex.cluster.shutdown.drain.malformed"
+		sub, err := daemon.natsConn.Subscribe(subject, daemon.handleShutdownDrain)
+		require.NoError(t, err)
+		defer sub.Unsubscribe()
+		require.NoError(t, daemon.natsConn.Flush())
+
+		reply, err := daemon.natsConn.Request(subject, []byte(`{not json`), 5*time.Second)
+		require.NoError(t, err)
+
+		var ack ShutdownACK
+		require.NoError(t, json.Unmarshal(reply.Data, &ack))
+		assert.Equal(t, "drain", ack.Phase)
+		assert.NotEmpty(t, ack.Error)
+
+		marker, err := daemon.jsManager.ReadShutdownMarker(daemon.node)
+		require.NoError(t, err)
+		assert.False(t, marker, "no marker should be written when parse fails")
+	})
+
+	t.Run("multiple vms publish initial and final progress", func(t *testing.T) {
+		daemon := createFullTestDaemonWithJetStream(t, sharedJSNATSURL)
+		require.NoError(t, daemon.jsManager.InitClusterStateBucket())
+		t.Cleanup(func() { _ = daemon.jsManager.DeleteShutdownMarker(daemon.node) })
+
+		const vmCount = 3
+		for i := range vmCount {
+			daemon.vmMgr.Insert(&vm.VM{ID: fmt.Sprintf("i-drain-%03d", i)})
+		}
+
+		progressSub, err := daemon.natsConn.SubscribeSync("spinifex.cluster.shutdown.progress")
+		require.NoError(t, err)
+		defer progressSub.Unsubscribe()
+
+		subject := "spinifex.cluster.shutdown.drain.multi"
+		sub, err := daemon.natsConn.Subscribe(subject, daemon.handleShutdownDrain)
+		require.NoError(t, err)
+		defer sub.Unsubscribe()
+		require.NoError(t, daemon.natsConn.Flush())
+
+		payload, err := json.Marshal(ShutdownRequest{Phase: "drain"})
+		require.NoError(t, err)
+
+		reply, err := daemon.natsConn.Request(subject, payload, 30*time.Second)
+		require.NoError(t, err)
+
+		var ack ShutdownACK
+		require.NoError(t, json.Unmarshal(reply.Data, &ack))
+		assert.Empty(t, ack.Error)
+
+		var progresses []ShutdownProgress
+		for {
+			msg, err := progressSub.NextMsg(500 * time.Millisecond)
+			if err != nil {
+				break
+			}
+			var p ShutdownProgress
+			require.NoError(t, json.Unmarshal(msg.Data, &p))
+			progresses = append(progresses, p)
+		}
+
+		require.GreaterOrEqual(t, len(progresses), 2, "expected initial and final progress publishes")
+		first := progresses[0]
+		last := progresses[len(progresses)-1]
+
+		assert.Equal(t, "drain", first.Phase)
+		assert.Equal(t, vmCount, first.Total)
+		assert.Equal(t, vmCount, first.Remaining, "initial progress should report all VMs remaining")
+
+		assert.Equal(t, "drain", last.Phase)
+		assert.Equal(t, vmCount, last.Total)
+		assert.Equal(t, 0, last.Remaining, "final progress should report zero remaining")
+	})
+}
@@ -56,6 +56,10 @@ func createDaemonWithJetStream(t *testing.T) *Daemon {
 		ConsumeCleanShutdownMarker: daemon.consumeCleanShutdownMarker(),
 	})

+	// Registered after t.TempDir() so it runs first (LIFO): goroutines
+	// spawned by MarkFailed/MarkRecoveryFailed finish before RemoveAll runs.
+	t.Cleanup(daemon.vmMgr.WaitForBackgroundWork)
+
 	return daemon
 }

@@ -398,16 +398,18 @@ func (d *Daemon) onInstanceUpHook() func(*vm.VM) error {
 		}
 		d.natsSubscriptions[consoleSubKey] = consoleSub

-		// Re-claim GPU after a daemon restart with a still-running QEMU
+		// Re-claim GPU(s) after a daemon restart with a still-running QEMU
 		// process: the manager's reconnect path fires OnInstanceUp without
 		// going through the handler-side Claim, so the GPU pool would
 		// otherwise treat the slot as free. ReclaimByAddress is a no-op
 		// when the same instance already owns the slot, so the launch and
 		// start-stopped paths (which Claim before Run) are unaffected.
-		if d.gpuManager != nil && instance.GPUPCIAddress != "" {
-			if err := d.gpuManager.ReclaimByAddress(instance.GPUPCIAddress, instance.ID); err != nil {
-				slog.Warn("Failed to re-claim GPU on instance up",
-					"gpu", instance.GPUPCIAddress, "instanceId", instance.ID, "err", err)
+		if d.gpuManager != nil {
+			for _, addr := range instance.GPUPCIAddresses {
+				if err := d.gpuManager.ReclaimByAddress(addr, instance.ID); err != nil {
+					slog.Warn("Failed to re-claim GPU on instance up",
+						"gpu", addr, "instanceId", instance.ID, "err", err)
+				}
 			}
 		}
 		return nil
@@ -630,13 +632,13 @@ func (a *instanceCleanerAdapter) RemoveFromPlacementGroup(instance *vm.VM) {
 // original host driver. No-op for instances without a GPU allocation or
 // when GPU passthrough is disabled.
 func (a *instanceCleanerAdapter) ReleaseGPU(instance *vm.VM) {
-	if a.d.gpuManager == nil || instance.GPUPCIAddress == "" {
+	if a.d.gpuManager == nil || len(instance.GPUPCIAddresses) == 0 {
 		return
 	}
 	if err := a.d.gpuManager.Release(instance.ID); err != nil {
 		slog.Error("Failed to release GPU on stop, device may need manual rebind",
-			"gpu", instance.GPUPCIAddress, "instanceId", instance.ID, "err", err)
+			"gpus", instance.GPUPCIAddresses, "instanceId", instance.ID, "err", err)
 		return
 	}
-	slog.Info("GPU released", "gpu", instance.GPUPCIAddress, "instanceId", instance.ID)
+	slog.Info("GPU released", "gpus", instance.GPUPCIAddresses, "instanceId", instance.ID)
 }
@@ -1,9 +1,11 @@
 package daemon

 import (
+	"encoding/json"
 	"testing"

 	"github.com/mulgadc/spinifex/spinifex/gpu"
+	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/vm"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
@@ -97,10 +99,10 @@ func TestOnInstanceDownHook_NoOpWhenAbsent(t *testing.T) {
 }

 // When the daemon's gpuManager is unset, the hook must still register NATS
-// subscriptions and ignore the GPUPCIAddress on the VM.
+// subscriptions and ignore the GPUPCIAddresses on the VM.
 func TestOnInstanceUpHook_NoGPUManager_SkipsReclaim(t *testing.T) {
 	d, _ := newHookTestDaemon(t)
-	instance := &vm.VM{ID: "i-up-nogpu", GPUPCIAddress: "0000:01:00.0"}
+	instance := &vm.VM{ID: "i-up-nogpu", GPUPCIAddresses: []string{"0000:01:00.0"}}

 	require.NoError(t, d.onInstanceUpHook()(instance))
 	require.Contains(t, d.natsSubscriptions, instance.ID)
@@ -116,17 +118,17 @@ func TestOnInstanceUpHook_NoGPUAddress_SkipsReclaim(t *testing.T) {

 	require.NoError(t, d.onInstanceUpHook()(instance))
 	assert.Equal(t, 0, d.gpuManager.AllocatedCount(),
-		"hook must not call Reclaim for instances without a GPUPCIAddress")
+		"hook must not call Reclaim for instances without GPUPCIAddresses")
 }

 // With a gpuManager that has no entries, calling Reclaim for an instance
-// with a GPUPCIAddress will fail inside the manager. The hook logs a warning
+// with a GPUPCIAddresses entry will fail inside the manager. The hook logs a warning
 // and returns nil — the NATS subscriptions must still register so the
 // reconnect path doesn't roll back.
 func TestOnInstanceUpHook_GPUReclaimError_DoesNotPropagate(t *testing.T) {
 	d, _ := newHookTestDaemon(t)
 	d.gpuManager = gpu.NewManager(nil)
-	instance := &vm.VM{ID: "i-up-gpu-missing", GPUPCIAddress: "0000:99:00.0"}
+	instance := &vm.VM{ID: "i-up-gpu-missing", GPUPCIAddresses: []string{"0000:99:00.0"}}

 	require.NoError(t, d.onInstanceUpHook()(instance))
 	require.Contains(t, d.natsSubscriptions, instance.ID)
@@ -149,3 +151,147 @@ func TestOnInstanceDownHook_OnlyRemovesTargetedInstance(t *testing.T) {
 	_, dropPresent := d.natsSubscriptions[drop.ID]
 	assert.False(t, dropPresent)
 }
+
+func TestVolumeMounterAdapter_MountOne(t *testing.T) {
+	tests := []struct {
+		name       string
+		responder  func(t *testing.T, msg *nats.Msg)
+		skipSub    bool
+		wantErr    bool
+		wantErrSub string
+		wantErrIs  error
+		wantNBDURI string
+		initialURI string
+	}{
+		{
+			name: "HappyPath_UpdatesNBDURI",
+			responder: func(t *testing.T, msg *nats.Msg) {
+				resp := types.EBSMountResponse{URI: "nbd://mounted-vol"}
+				data, err := json.Marshal(resp)
+				require.NoError(t, err)
+				require.NoError(t, msg.Respond(data))
+			},
+			wantNBDURI: "nbd://mounted-vol",
+		},
+		{
+			name:       "NATSNoResponders_ReturnsError",
+			skipSub:    true,
+			wantErr:    true,
+			wantErrSub: "ebs.mount NATS request",
+		},
+		{
+			name: "UnmarshalFailure_ReturnsError",
+			responder: func(t *testing.T, msg *nats.Msg) {
+				require.NoError(t, msg.Respond([]byte("not json")))
+			},
+			wantErr:    true,
+			wantErrSub: "unmarshal ebs.mount response",
+		},
+		{
+			name: "ResponseError_IncludedInError",
+			responder: func(t *testing.T, msg *nats.Msg) {
+				resp := types.EBSMountResponse{Error: "boom"}
+				data, err := json.Marshal(resp)
+				require.NoError(t, err)
+				require.NoError(t, msg.Respond(data))
+			},
+			wantErr:    true,
+			wantErrSub: "boom",
+		},
+		{
+			name: "EmptyURI_ReturnsErrMountAmbiguous",
+			responder: func(t *testing.T, msg *nats.Msg) {
+				resp := types.EBSMountResponse{URI: ""}
+				data, err := json.Marshal(resp)
+				require.NoError(t, err)
+				require.NoError(t, msg.Respond(data))
+			},
+			wantErr:   true,
+			wantErrIs: vm.ErrMountAmbiguous,
+		},
+		{
+			name: "EmptyURI_PreservesInitialNBDURIOnFailure",
+			responder: func(t *testing.T, msg *nats.Msg) {
+				resp := types.EBSMountResponse{URI: ""}
+				data, err := json.Marshal(resp)
+				require.NoError(t, err)
+				require.NoError(t, msg.Respond(data))
+			},
+			initialURI: "nbd://stale",
+			wantErr:    true,
+			wantErrIs:  vm.ErrMountAmbiguous,
+			wantNBDURI: "nbd://stale",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			daemon := createTestDaemon(t, sharedNATSURL)
+			adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+
+			if !tt.skipSub {
+				sub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
+					tt.responder(t, msg)
+				})
+				require.NoError(t, err)
+				defer sub.Unsubscribe()
+			}
+
+			req := &types.EBSRequest{
+				Name:       "vol-mountone",
+				DeviceName: "/dev/sdf",
+				NBDURI:     tt.initialURI,
+			}
+
+			err := adapter.MountOne(req)
+			if tt.wantErr {
+				require.Error(t, err)
+				if tt.wantErrSub != "" {
+					assert.Contains(t, err.Error(), tt.wantErrSub)
+				}
+				if tt.wantErrIs != nil {
+					assert.ErrorIs(t, err, tt.wantErrIs)
+				}
+				assert.Equal(t, tt.wantNBDURI, req.NBDURI,
+					"NBDURI must not be overwritten on failure")
+				return
+			}
+
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantNBDURI, req.NBDURI,
+				"happy path must write resolved NBDURI back to req")
+		})
+	}
+}
+
+// --- ReleaseGPU ---
+
+// ReleaseGPU is a no-op when the daemon has no GPU manager.
+func TestReleaseGPU_NoManager_NoOp(t *testing.T) {
+	d := &Daemon{}
+	a := newInstanceCleanerAdapter(d)
+	instance := &vm.VM{ID: "i-nogpu", GPUPCIAddresses: []string{"0000:03:00.0"}}
+	// Must not panic.
+	a.ReleaseGPU(instance)
+}
+
+// ReleaseGPU is a no-op for instances without a GPU allocation.
+func TestReleaseGPU_NoAddresses_NoOp(t *testing.T) {
+	mgr := gpu.NewManager(nil)
+	d := &Daemon{gpuManager: mgr}
+	a := newInstanceCleanerAdapter(d)
+	instance := &vm.VM{ID: "i-nogpu"}
+	// Must not panic or error.
+	a.ReleaseGPU(instance)
+}
+
+// ReleaseGPU logs a warning when the manager returns an error (instance not claimed).
+func TestReleaseGPU_ManagerError_LogsWarning(t *testing.T) {
+	mgr := gpu.NewManager(nil)
+	d := &Daemon{gpuManager: mgr}
+	a := newInstanceCleanerAdapter(d)
+	// GPU address set but no claim registered — Release returns an error.
+	instance := &vm.VM{ID: "i-unclaimed", GPUPCIAddresses: []string{"0000:03:00.0"}}
+	// Must not panic; the error is logged as a warning.
+	a.ReleaseGPU(instance)
+}
@@ -467,60 +467,6 @@ func TestSigV4Auth_NilIAMService(t *testing.T) {
 	}
 }

-func TestSigV4Auth_CallerIdentity(t *testing.T) {
-	encryptedSecret, err := handlers_iam.EncryptSecret(testSecretKey, testMasterKey)
-	if err != nil {
-		t.Fatalf("Failed to encrypt secret: %v", err)
-	}
-
-	mockSvc := &mockIAMService{
-		masterKey: testMasterKey,
-		accessKeys: map[string]*handlers_iam.AccessKey{
-			testAccessKey: {
-				AccessKeyID:     testAccessKey,
-				SecretAccessKey: encryptedSecret,
-				UserName:        "alice",
-				Status:          "Active",
-			},
-		},
-	}
-
-	gw := &GatewayConfig{
-		DisableLogging: true,
-		Region:         testRegion,
-		IAMService:     mockSvc,
-	}
-
-	r := chi.NewRouter()
-	r.Use(gw.SigV4AuthMiddleware())
-	r.HandleFunc("/*", func(w http.ResponseWriter, r *http.Request) {
-		identity := r.Context().Value(ctxIdentity)
-		fmt.Fprintf(w, "identity=%v", identity)
-	})
-
-	authHeader, timestamp := generateTestAuthHeader(
-		"GET", "/", "", "",
-		testAccessKey, testSecretKey, testRegion, testService,
-	)
-
-	req := httptest.NewRequest(http.MethodGet, "/", nil)
-	req.Host = "localhost:9999"
-	req.Header.Set("Authorization", authHeader)
-	req.Header.Set("X-Amz-Date", timestamp)
-
-	resp := doRequest(r, req)
-
-	if resp.StatusCode != http.StatusOK {
-		body, _ := io.ReadAll(resp.Body)
-		t.Fatalf("Expected status 200, got %d, body: %s", resp.StatusCode, string(body))
-	}
-
-	body, _ := io.ReadAll(resp.Body)
-	if string(body) != "identity=alice" {
-		t.Errorf("Expected identity=alice, got: %s", string(body))
-	}
-}
-
 func TestParseAWSQueryArgs_URLDecoding(t *testing.T) {
 	testCases := []struct {
 		name     string
@@ -1351,7 +1297,12 @@ func TestSigV4Auth_EmptyBodyPOST(t *testing.T) {

 // --- computeSignatureWithSecret Direct Unit Tests ---

-func TestComputeSignatureWithSecret_DifferentMethods(t *testing.T) {
+// TestComputeSignatureWithSecret_MethodSensitivity pins the contract that
+// the HTTP method participates in the canonical request: changing the method
+// (with all other inputs fixed) must change the signature. Catches a
+// regression class where method is dropped from the canonical-string
+// composition — signing would silently collapse across methods.
+func TestComputeSignatureWithSecret_MethodSensitivity(t *testing.T) {
 	secret := "test-secret-key"
 	date := "20260305"
 	timestamp := "20260305T120000Z"
@@ -1360,69 +1311,25 @@ func TestComputeSignatureWithSecret_DifferentMethods(t *testing.T) {
 	signedHeaders := "host;x-amz-date"

 	methods := []string{"GET", "POST", "PUT", "PATCH", "DELETE"}
-	signatures := make(map[string]string)
+	signatures := make(map[string]string, len(methods))

 	for _, method := range methods {
 		req := httptest.NewRequest(method, "/", nil)
 		req.Host = "localhost:9999"
 		req.Header.Set("X-Amz-Date", timestamp)

-		sig := computeSignatureWithSecret(req, nil, secret, date, timestamp, region, service, signedHeaders)
-		if sig == "" {
-			t.Errorf("Empty signature for method %s", method)
-		}
-		signatures[method] = sig
+		signatures[method] = computeSignatureWithSecret(req, nil, secret, date, timestamp, region, service, signedHeaders)
 	}

-	// Each method should produce a different signature
 	for i, m1 := range methods {
 		for _, m2 := range methods[i+1:] {
 			if signatures[m1] == signatures[m2] {
-				t.Errorf("Methods %s and %s produced same signature", m1, m2)
+				t.Errorf("methods %s and %s produced the same signature (method missing from canonical request?)", m1, m2)
 			}
 		}
 	}
 }

-func TestComputeSignatureWithSecret_SpecialURIPaths(t *testing.T) {
-	secret := "test-secret-key"
-	date := "20260305"
-	timestamp := "20260305T120000Z"
-	region := "us-east-1"
-	service := "ec2"
-	signedHeaders := "host;x-amz-date"
-
-	testCases := []struct {
-		name string
-		path string
-	}{
-		{"root path", "/"},
-		{"nested path", "/a/b/c"},
-		{"path with encoded space", "/my%20resource"},
-		{"trailing slash", "/path/"},
-		{"empty path defaults to /", ""},
-	}
-
-	signatures := make(map[string]string)
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			reqPath := tc.path
-			if reqPath == "" {
-				reqPath = "/"
-			}
-			req := httptest.NewRequest(http.MethodGet, reqPath, nil)
-			req.Host = "localhost:9999"
-			req.Header.Set("X-Amz-Date", timestamp)
-
-			sig := computeSignatureWithSecret(req, nil, secret, date, timestamp, region, service, signedHeaders)
-			if sig == "" {
-				t.Errorf("Empty signature for path %q", tc.path)
-			}
-			signatures[tc.name] = sig
-		})
-	}
-}
-
 func TestComputeSignatureWithSecret_Determinism(t *testing.T) {
 	secret := "test-secret-key"
 	date := "20260305"
@@ -80,6 +80,9 @@ var ec2Actions = map[string]EC2Handler{
 	"DescribeInstanceTypes": ec2Handler(func(input *ec2.DescribeInstanceTypesInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_instance.DescribeInstanceTypes(input, gw.NATSConn, gw.ExpectedNodes)
 	}),
+	"DescribeInstanceStatus": ec2Handler(func(input *ec2.DescribeInstanceStatusInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_instance.DescribeInstanceStatus(input, gw.NATSConn, gw.DiscoverActiveNodes(), accountID, gw.AZ)
+	}),
 	"GetConsoleOutput": ec2Handler(func(input *ec2.GetConsoleOutputInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_instance.GetConsoleOutput(input, gw.NATSConn, accountID)
 	}),
@@ -314,6 +317,9 @@ var ec2Actions = map[string]EC2Handler{
 	"DescribeSecurityGroups": ec2Handler(func(input *ec2.DescribeSecurityGroupsInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_vpc.DescribeSecurityGroups(input, gw.NATSConn, accountID)
 	}),
+	"DescribeSecurityGroupRules": ec2Handler(func(input *ec2.DescribeSecurityGroupRulesInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_vpc.DescribeSecurityGroupRules(input, gw.NATSConn, accountID)
+	}),
 	"AuthorizeSecurityGroupIngress": ec2Handler(func(input *ec2.AuthorizeSecurityGroupIngressInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_vpc.AuthorizeSecurityGroupIngress(input, gw.NATSConn, accountID)
 	}),
@@ -0,0 +1,276 @@
+package gateway_ec2_instance
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strconv"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/nats-io/nats.go"
+)
+
+const (
+	notApplicable          = "not-applicable"
+	reachabilityDetailName = "reachability"
+)
+
+// DescribeInstanceStatus fans out to every node and, with IncludeAllInstances,
+// also augments from the stopped-instance KV bucket. The aggregator propagates
+// the first deterministic 4xx only when no data was collected, matching
+// DescribeInstances.
+func DescribeInstanceStatus(input *ec2.DescribeInstanceStatusInput, natsConn *nats.Conn, expectedNodes int, accountID, az string) (*ec2.DescribeInstanceStatusOutput, error) {
+	if input == nil {
+		input = &ec2.DescribeInstanceStatusInput{}
+	}
+
+	jsonData, err := json.Marshal(input)
+	if err != nil {
+		slog.Error("DescribeInstanceStatus: Failed to marshal input", "err", err)
+		return nil, fmt.Errorf("failed to marshal input: %w", err)
+	}
+
+	inbox := nats.NewInbox()
+	sub, err := natsConn.SubscribeSync(inbox)
+	if err != nil {
+		slog.Error("DescribeInstanceStatus: Failed to create inbox subscription", "err", err)
+		return nil, fmt.Errorf("failed to create inbox: %w", err)
+	}
+	defer sub.Unsubscribe()
+
+	pubMsg := nats.NewMsg("ec2.DescribeInstanceStatus")
+	pubMsg.Reply = inbox
+	pubMsg.Data = jsonData
+	pubMsg.Header.Set(utils.AccountIDHeader, accountID)
+	if err := natsConn.PublishMsg(pubMsg); err != nil {
+		slog.Error("DescribeInstanceStatus: Failed to publish request", "err", err)
+		return nil, fmt.Errorf("failed to publish request: %w", err)
+	}
+
+	timeout := 3 * time.Second
+	deadline := time.Now().Add(timeout)
+
+	var allStatuses []*ec2.InstanceStatus
+	var clientError string
+	responsesReceived := 0
+
+	if expectedNodes <= 0 {
+		expectedNodes = -1
+		slog.Warn("DescribeInstanceStatus: ExpectedNodes not configured, using timeout-only collection")
+	}
+
+	for time.Now().Before(deadline) {
+		if expectedNodes > 0 && responsesReceived >= expectedNodes {
+			slog.Info("DescribeInstanceStatus: Received responses from all expected nodes", "expected", expectedNodes, "received", responsesReceived)
+			break
+		}
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			break
+		}
+		msg, err := sub.NextMsg(remaining)
+		if err != nil {
+			if err == nats.ErrTimeout {
+				break
+			}
+			slog.Error("DescribeInstanceStatus: Error receiving message", "err", err)
+			break
+		}
+		responsesReceived++
+
+		if responseError, perr := utils.ValidateErrorPayload(msg.Data); perr != nil {
+			code := ""
+			if responseError.Code != nil {
+				code = *responseError.Code
+			}
+			if clientError == "" && code != "" {
+				if info, known := awserrors.ErrorLookup[code]; known && info.HTTPCode >= 400 && info.HTTPCode < 500 {
+					clientError = code
+				}
+			}
+			slog.Warn("DescribeInstanceStatus: Received error from node", "code", code, "responses_received", responsesReceived)
+			continue
+		}
+
+		var nodeOutput ec2.DescribeInstanceStatusOutput
+		if err := json.Unmarshal(msg.Data, &nodeOutput); err != nil {
+			slog.Error("DescribeInstanceStatus: Failed to unmarshal node response", "err", err)
+			continue
+		}
+		if len(nodeOutput.InstanceStatuses) > 0 {
+			allStatuses = append(allStatuses, nodeOutput.InstanceStatuses...)
+			slog.Info("DescribeInstanceStatus: Collected statuses from node", "count", len(nodeOutput.InstanceStatuses), "responses_received", responsesReceived)
+		}
+	}
+
+	if aws.BoolValue(input.IncludeAllInstances) {
+		if stopped := queryStoppedInstancesForStatus(natsConn, input, accountID, az); len(stopped) > 0 {
+			allStatuses = append(allStatuses, stopped...)
+		}
+	}
+
+	finalStatuses := dedupStatuses(allStatuses)
+
+	if clientError != "" && len(finalStatuses) == 0 {
+		return nil, errors.New(clientError)
+	}
+
+	slog.Info("DescribeInstanceStatus: Aggregated response", "total_statuses", len(finalStatuses))
+	return &ec2.DescribeInstanceStatusOutput{InstanceStatuses: finalStatuses}, nil
+}
+
+// queryStoppedInstancesForStatus projects the input to DescribeInstancesInput
+// before forwarding to the stopped-instance KV handler. The handler unmarshals
+// with DisallowUnknownFields, so IncludeAllInstances must be dropped. Filters
+// only valid for DescribeInstanceStatus (availability-zone, instance-state-code)
+// are stripped because the stopped-instance handler would reject them via its
+// own valid-filter map; these are applied gateway-side after transform.
+func queryStoppedInstancesForStatus(natsConn *nats.Conn, input *ec2.DescribeInstanceStatusInput, accountID, az string) []*ec2.InstanceStatus {
+	projected := &ec2.DescribeInstancesInput{
+		InstanceIds: input.InstanceIds,
+		Filters:     stoppedCompatibleFilters(input.Filters),
+	}
+	jsonData, err := json.Marshal(projected)
+	if err != nil {
+		slog.Error("DescribeInstanceStatus: failed to marshal stopped query", "err", err)
+		return nil
+	}
+	reservations := queryInstanceBucket(natsConn, "ec2.DescribeStoppedInstances", jsonData, accountID)
+	var statuses []*ec2.InstanceStatus
+	for _, res := range reservations {
+		if res == nil {
+			continue
+		}
+		for _, inst := range res.Instances {
+			if inst == nil || inst.InstanceId == nil {
+				continue
+			}
+			statuses = append(statuses, buildInstanceStatusFromInstance(inst, az))
+		}
+	}
+	return filterStoppedStatuses(statuses, input.Filters, az)
+}
+
+// stoppedCompatibleFilters strips filters the stopped-instance handler would
+// reject (its valid-filter map differs from DescribeInstanceStatus's).
+func stoppedCompatibleFilters(filters []*ec2.Filter) []*ec2.Filter {
+	if len(filters) == 0 {
+		return nil
+	}
+	out := make([]*ec2.Filter, 0, len(filters))
+	for _, f := range filters {
+		if f == nil || f.Name == nil {
+			continue
+		}
+		switch *f.Name {
+		case "availability-zone", "instance-state-code":
+			continue
+		}
+		out = append(out, f)
+	}
+	return out
+}
+
+// filterStoppedStatuses applies DescribeInstanceStatus-specific filters that
+// could not be forwarded to the stopped handler.
+func filterStoppedStatuses(in []*ec2.InstanceStatus, filters []*ec2.Filter, az string) []*ec2.InstanceStatus {
+	if len(in) == 0 || len(filters) == 0 {
+		return in
+	}
+	out := in[:0]
+	for _, s := range in {
+		keep := true
+		for _, f := range filters {
+			if f == nil || f.Name == nil {
+				continue
+			}
+			switch *f.Name {
+			case "availability-zone":
+				if !filterValueMatches(f.Values, az) {
+					keep = false
+				}
+			case "instance-state-code":
+				code := ""
+				if s.InstanceState != nil && s.InstanceState.Code != nil {
+					code = strconv.FormatInt(*s.InstanceState.Code, 10)
+				}
+				if !filterValueMatches(f.Values, code) {
+					keep = false
+				}
+			}
+			if !keep {
+				break
+			}
+		}
+		if keep {
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
+func filterValueMatches(values []*string, field string) bool {
+	for _, v := range values {
+		if v != nil && *v == field {
+			return true
+		}
+	}
+	return false
+}
+
+func buildInstanceStatusFromInstance(inst *ec2.Instance, az string) *ec2.InstanceStatus {
+	state := &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}
+	if inst.State != nil {
+		if inst.State.Name != nil {
+			state.Name = inst.State.Name
+		}
+		state.Code = inst.State.Code
+	}
+
+	return &ec2.InstanceStatus{
+		AvailabilityZone: aws.String(az),
+		InstanceId:       inst.InstanceId,
+		InstanceState:    state,
+		InstanceStatus: &ec2.InstanceStatusSummary{
+			Status: aws.String(notApplicable),
+			Details: []*ec2.InstanceStatusDetails{{
+				Name:   aws.String(reachabilityDetailName),
+				Status: aws.String(notApplicable),
+			}},
+		},
+		SystemStatus: &ec2.InstanceStatusSummary{
+			Status: aws.String(notApplicable),
+			Details: []*ec2.InstanceStatusDetails{{
+				Name:   aws.String(reachabilityDetailName),
+				Status: aws.String(notApplicable),
+			}},
+		},
+	}
+}
+
+// First-writer-wins by InstanceId, so a running fan-out entry naturally beats a
+// stale stopped-KV entry during stop/start race windows.
+func dedupStatuses(in []*ec2.InstanceStatus) []*ec2.InstanceStatus {
+	if len(in) == 0 {
+		return nil
+	}
+	seen := make(map[string]bool, len(in))
+	out := make([]*ec2.InstanceStatus, 0, len(in))
+	for _, s := range in {
+		if s == nil || s.InstanceId == nil {
+			continue
+		}
+		id := *s.InstanceId
+		if seen[id] {
+			continue
+		}
+		seen[id] = true
+		out = append(out, s)
+	}
+	return out
+}
@@ -0,0 +1,468 @@
+package gateway_ec2_instance
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func runningStatus(id, az string) *ec2.InstanceStatus {
+	return &ec2.InstanceStatus{
+		AvailabilityZone: aws.String(az),
+		InstanceId:       aws.String(id),
+		InstanceState:    &ec2.InstanceState{Code: aws.Int64(16), Name: aws.String("running")},
+		InstanceStatus: &ec2.InstanceStatusSummary{
+			Status: aws.String("ok"),
+			Details: []*ec2.InstanceStatusDetails{{
+				Name: aws.String("reachability"), Status: aws.String("passed"),
+			}},
+		},
+		SystemStatus: &ec2.InstanceStatusSummary{
+			Status: aws.String("ok"),
+			Details: []*ec2.InstanceStatusDetails{{
+				Name: aws.String("reachability"), Status: aws.String("passed"),
+			}},
+		},
+	}
+}
+
+func respondJSON(t *testing.T, msg *nats.Msg, payload any) {
+	t.Helper()
+	data, err := json.Marshal(payload)
+	require.NoError(t, err)
+	require.NoError(t, msg.Respond(data))
+}
+
+func TestDescribeInstanceStatus_SingleNode(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	_, err := nc.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstanceStatusOutput{
+			InstanceStatuses: []*ec2.InstanceStatus{
+				runningStatus("i-001", "us-east-1a"),
+				runningStatus("i-002", "us-east-1a"),
+			},
+		})
+	})
+	require.NoError(t, err)
+
+	out, err := DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{}, nc, 1, "123456789012", "us-east-1a")
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 2)
+}
+
+func TestDescribeInstanceStatus_TwoNodesDedup(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	// First node returns i-001 (running)
+	_, err := nc.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstanceStatusOutput{
+			InstanceStatuses: []*ec2.InstanceStatus{runningStatus("i-001", "us-east-1a")},
+		})
+	})
+	require.NoError(t, err)
+
+	// Second node also reports i-001 — should dedup
+	nc2, err := nats.Connect(nc.ConnectedUrl())
+	require.NoError(t, err)
+	defer nc2.Close()
+
+	_, err = nc2.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstanceStatusOutput{
+			InstanceStatuses: []*ec2.InstanceStatus{runningStatus("i-001", "us-east-1a")},
+		})
+	})
+	require.NoError(t, err)
+
+	require.NoError(t, nc.Flush())
+	require.NoError(t, nc2.Flush())
+
+	out, err := DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{}, nc, 2, "123456789012", "us-east-1a")
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+	assert.Equal(t, "i-001", *out.InstanceStatuses[0].InstanceId)
+}
+
+func TestDescribeInstanceStatus_OneNodeErrorOthersData(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	// Node 1: data
+	_, err := nc.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstanceStatusOutput{
+			InstanceStatuses: []*ec2.InstanceStatus{runningStatus("i-good", "az-a")},
+		})
+	})
+	require.NoError(t, err)
+
+	// Node 2: error
+	nc2, err := nats.Connect(nc.ConnectedUrl())
+	require.NoError(t, err)
+	defer nc2.Close()
+	_, err = nc2.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		require.NoError(t, msg.Respond(utils.GenerateErrorPayload("InvalidParameterValue")))
+	})
+	require.NoError(t, err)
+
+	require.NoError(t, nc.Flush())
+	require.NoError(t, nc2.Flush())
+
+	out, err := DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{}, nc, 2, "123456789012", "az-a")
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+	assert.Equal(t, "i-good", *out.InstanceStatuses[0].InstanceId)
+}
+
+func TestDescribeInstanceStatus_AllNodesError(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	_, err := nc.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		require.NoError(t, msg.Respond(utils.GenerateErrorPayload("InvalidParameterValue")))
+	})
+	require.NoError(t, err)
+
+	_, err = DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{}, nc, 1, "123456789012", "az-a")
+	require.Error(t, err)
+	assert.Equal(t, "InvalidParameterValue", err.Error())
+}
+
+func TestDescribeInstanceStatus_AllNodesTimeoutReturnsEmpty(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	out, err := DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{}, nc, 0, "123456789012", "az-a")
+	require.NoError(t, err)
+	require.NotNil(t, out)
+	assert.Empty(t, out.InstanceStatuses)
+}
+
+func TestDescribeInstanceStatus_IncludeAllAddsStoppedAsNotApplicable(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	// Fan-out responder: a running instance
+	_, err := nc.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstanceStatusOutput{
+			InstanceStatuses: []*ec2.InstanceStatus{runningStatus("i-running", "az-a")},
+		})
+	})
+	require.NoError(t, err)
+
+	// Stopped KV responder
+	_, err = nc.QueueSubscribe("ec2.DescribeStoppedInstances", "spinifex-workers", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{{
+				ReservationId: aws.String("r-stop"),
+				Instances: []*ec2.Instance{{
+					InstanceId: aws.String("i-stopped"),
+					State:      &ec2.InstanceState{Code: aws.Int64(80), Name: aws.String("stopped")},
+				}},
+			}},
+		})
+	})
+	require.NoError(t, err)
+
+	input := &ec2.DescribeInstanceStatusInput{IncludeAllInstances: aws.Bool(true)}
+	out, err := DescribeInstanceStatus(input, nc, 1, "123456789012", "az-a")
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 2)
+
+	statusByID := make(map[string]*ec2.InstanceStatus)
+	for _, s := range out.InstanceStatuses {
+		statusByID[*s.InstanceId] = s
+	}
+
+	running := statusByID["i-running"]
+	require.NotNil(t, running)
+	assert.Equal(t, "ok", *running.InstanceStatus.Status)
+	assert.Equal(t, "passed", *running.InstanceStatus.Details[0].Status)
+
+	stopped := statusByID["i-stopped"]
+	require.NotNil(t, stopped)
+	assert.Equal(t, "stopped", *stopped.InstanceState.Name)
+	assert.Equal(t, "not-applicable", *stopped.InstanceStatus.Status)
+	assert.Equal(t, "not-applicable", *stopped.SystemStatus.Status)
+	assert.Equal(t, "not-applicable", *stopped.InstanceStatus.Details[0].Status)
+	assert.Equal(t, "az-a", *stopped.AvailabilityZone)
+}
+
+func TestDescribeInstanceStatus_RunningWinsOverStoppedDuringRace(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	// Fan-out: i-001 is running
+	_, err := nc.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstanceStatusOutput{
+			InstanceStatuses: []*ec2.InstanceStatus{runningStatus("i-001", "az-a")},
+		})
+	})
+	require.NoError(t, err)
+
+	// Stopped KV also has i-001 (race: stop transition hasn't fully cleared the KV)
+	_, err = nc.QueueSubscribe("ec2.DescribeStoppedInstances", "spinifex-workers", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{{
+				ReservationId: aws.String("r-1"),
+				Instances: []*ec2.Instance{{
+					InstanceId: aws.String("i-001"),
+					State:      &ec2.InstanceState{Code: aws.Int64(80), Name: aws.String("stopped")},
+				}},
+			}},
+		})
+	})
+	require.NoError(t, err)
+
+	input := &ec2.DescribeInstanceStatusInput{IncludeAllInstances: aws.Bool(true)}
+	out, err := DescribeInstanceStatus(input, nc, 1, "123456789012", "az-a")
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+	assert.Equal(t, "running", *out.InstanceStatuses[0].InstanceState.Name)
+	assert.Equal(t, "ok", *out.InstanceStatuses[0].InstanceStatus.Status)
+}
+
+func TestDescribeInstanceStatus_NilInput(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	_, err := nc.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstanceStatusOutput{})
+	})
+	require.NoError(t, err)
+
+	out, err := DescribeInstanceStatus(nil, nc, 1, "123456789012", "az-a")
+	require.NoError(t, err)
+	require.NotNil(t, out)
+	assert.Empty(t, out.InstanceStatuses)
+}
+
+func TestDescribeInstanceStatus_ClosedConnection(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	closed, err := nats.Connect(nc.ConnectedUrl())
+	require.NoError(t, err)
+	closed.Close()
+
+	_, err = DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{}, closed, 1, "123456789012", "az-a")
+	require.Error(t, err)
+}
+
+func TestBuildInstanceStatusFromInstance_PopulatesState(t *testing.T) {
+	in := &ec2.Instance{
+		InstanceId: aws.String("i-x"),
+		State:      &ec2.InstanceState{Code: aws.Int64(80), Name: aws.String("stopped")},
+	}
+	got := buildInstanceStatusFromInstance(in, "az-a")
+	require.NotNil(t, got)
+	assert.Equal(t, "i-x", *got.InstanceId)
+	assert.Equal(t, int64(80), *got.InstanceState.Code)
+	assert.Equal(t, "stopped", *got.InstanceState.Name)
+	assert.Equal(t, "az-a", *got.AvailabilityZone)
+	assert.Equal(t, "not-applicable", *got.InstanceStatus.Status)
+	assert.Equal(t, "not-applicable", *got.SystemStatus.Status)
+}
+
+func TestBuildInstanceStatusFromInstance_FallbackStateName(t *testing.T) {
+	in := &ec2.Instance{InstanceId: aws.String("i-x")}
+	got := buildInstanceStatusFromInstance(in, "az-a")
+	assert.Equal(t, "stopped", *got.InstanceState.Name)
+}
+
+// Regression: the stopped-instance handler unmarshals with DisallowUnknownFields,
+// so forwarding the raw DescribeInstanceStatusInput JSON failed with ValidationError
+// because of the IncludeAllInstances field. The gateway must project to
+// DescribeInstancesInput before publishing.
+func TestDescribeInstanceStatus_IncludeAllProjectsInputForStoppedHandler(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	_, err := nc.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstanceStatusOutput{})
+	})
+	require.NoError(t, err)
+
+	var capturedReq ec2.DescribeInstancesInput
+	var unmarshalErr error
+	_, err = nc.QueueSubscribe("ec2.DescribeStoppedInstances", "spinifex-workers", func(msg *nats.Msg) {
+		dec := json.NewDecoder(bytes.NewReader(msg.Data))
+		dec.DisallowUnknownFields()
+		unmarshalErr = dec.Decode(&capturedReq)
+		respondJSON(t, msg, &ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{{
+				Instances: []*ec2.Instance{{
+					InstanceId: aws.String("i-stop"),
+					State:      &ec2.InstanceState{Code: aws.Int64(80), Name: aws.String("stopped")},
+				}},
+			}},
+		})
+	})
+	require.NoError(t, err)
+
+	input := &ec2.DescribeInstanceStatusInput{
+		IncludeAllInstances: aws.Bool(true),
+		InstanceIds:         []*string{aws.String("i-stop")},
+	}
+	out, err := DescribeInstanceStatus(input, nc, 1, "123456789012", "az-a")
+	require.NoError(t, err)
+	require.NoError(t, unmarshalErr, "projected input must decode cleanly under DisallowUnknownFields")
+	require.Len(t, out.InstanceStatuses, 1)
+	require.NotNil(t, capturedReq.InstanceIds)
+	require.Len(t, capturedReq.InstanceIds, 1)
+	assert.Equal(t, "i-stop", *capturedReq.InstanceIds[0])
+}
+
+func TestStoppedCompatibleFilters_StripsStatusOnlyNames(t *testing.T) {
+	filters := []*ec2.Filter{
+		{Name: aws.String("availability-zone"), Values: []*string{aws.String("az-a")}},
+		{Name: aws.String("instance-state-code"), Values: []*string{aws.String("80")}},
+		{Name: aws.String("instance-state-name"), Values: []*string{aws.String("stopped")}},
+		{Name: aws.String("tag:Name"), Values: []*string{aws.String("foo")}},
+	}
+	got := stoppedCompatibleFilters(filters)
+	require.Len(t, got, 2)
+	names := []string{*got[0].Name, *got[1].Name}
+	assert.ElementsMatch(t, []string{"instance-state-name", "tag:Name"}, names)
+}
+
+func TestStoppedCompatibleFilters_Empty(t *testing.T) {
+	assert.Nil(t, stoppedCompatibleFilters(nil))
+	assert.Nil(t, stoppedCompatibleFilters([]*ec2.Filter{}))
+}
+
+func TestFilterStoppedStatuses_AvailabilityZoneMatch(t *testing.T) {
+	in := []*ec2.InstanceStatus{
+		{InstanceId: aws.String("i-1"), AvailabilityZone: aws.String("az-a"),
+			InstanceState: &ec2.InstanceState{Code: aws.Int64(80)}},
+	}
+	filters := []*ec2.Filter{
+		{Name: aws.String("availability-zone"), Values: []*string{aws.String("az-a")}},
+	}
+	got := filterStoppedStatuses(in, filters, "az-a")
+	require.Len(t, got, 1)
+}
+
+func TestFilterStoppedStatuses_AvailabilityZoneMiss(t *testing.T) {
+	in := []*ec2.InstanceStatus{
+		{InstanceId: aws.String("i-1"), AvailabilityZone: aws.String("az-a"),
+			InstanceState: &ec2.InstanceState{Code: aws.Int64(80)}},
+	}
+	filters := []*ec2.Filter{
+		{Name: aws.String("availability-zone"), Values: []*string{aws.String("az-b")}},
+	}
+	got := filterStoppedStatuses(in, filters, "az-a")
+	assert.Empty(t, got)
+}
+
+func TestFilterStoppedStatuses_StateCodeMatch(t *testing.T) {
+	in := []*ec2.InstanceStatus{
+		{InstanceId: aws.String("i-1"), AvailabilityZone: aws.String("az-a"),
+			InstanceState: &ec2.InstanceState{Code: aws.Int64(80)}},
+	}
+	filters := []*ec2.Filter{
+		{Name: aws.String("instance-state-code"), Values: []*string{aws.String("80")}},
+	}
+	got := filterStoppedStatuses(in, filters, "az-a")
+	require.Len(t, got, 1)
+}
+
+func TestFilterStoppedStatuses_StateCodeMiss(t *testing.T) {
+	in := []*ec2.InstanceStatus{
+		{InstanceId: aws.String("i-1"), AvailabilityZone: aws.String("az-a"),
+			InstanceState: &ec2.InstanceState{Code: aws.Int64(80)}},
+	}
+	filters := []*ec2.Filter{
+		{Name: aws.String("instance-state-code"), Values: []*string{aws.String("16")}},
+	}
+	got := filterStoppedStatuses(in, filters, "az-a")
+	assert.Empty(t, got)
+}
+
+func TestFilterStoppedStatuses_NoFiltersPassthrough(t *testing.T) {
+	in := []*ec2.InstanceStatus{{InstanceId: aws.String("i-1")}}
+	got := filterStoppedStatuses(in, nil, "az-a")
+	assert.Equal(t, in, got)
+}
+
+func TestFilterStoppedStatuses_UnknownFilterIgnored(t *testing.T) {
+	in := []*ec2.InstanceStatus{
+		{InstanceId: aws.String("i-1"), AvailabilityZone: aws.String("az-a"),
+			InstanceState: &ec2.InstanceState{Code: aws.Int64(80)}},
+	}
+	// Filter that the stopped-bucket already applies (instance-state-name)
+	// should not also be enforced gateway-side — pass-through.
+	filters := []*ec2.Filter{
+		{Name: aws.String("instance-state-name"), Values: []*string{aws.String("running")}},
+	}
+	got := filterStoppedStatuses(in, filters, "az-a")
+	require.Len(t, got, 1)
+}
+
+func TestDescribeInstanceStatus_IncludeAllWithAZFilterMatches(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	_, err := nc.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstanceStatusOutput{})
+	})
+	require.NoError(t, err)
+
+	_, err = nc.QueueSubscribe("ec2.DescribeStoppedInstances", "spinifex-workers", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{{
+				InstanceId: aws.String("i-stop"),
+				State:      &ec2.InstanceState{Code: aws.Int64(80), Name: aws.String("stopped")},
+			}}}},
+		})
+	})
+	require.NoError(t, err)
+
+	input := &ec2.DescribeInstanceStatusInput{
+		IncludeAllInstances: aws.Bool(true),
+		Filters: []*ec2.Filter{
+			{Name: aws.String("availability-zone"), Values: []*string{aws.String("az-a")}},
+		},
+	}
+	out, err := DescribeInstanceStatus(input, nc, 1, "123456789012", "az-a")
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+	assert.Equal(t, "i-stop", *out.InstanceStatuses[0].InstanceId)
+}
+
+func TestDescribeInstanceStatus_IncludeAllWithAZFilterMisses(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	_, err := nc.Subscribe("ec2.DescribeInstanceStatus", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstanceStatusOutput{})
+	})
+	require.NoError(t, err)
+
+	_, err = nc.QueueSubscribe("ec2.DescribeStoppedInstances", "spinifex-workers", func(msg *nats.Msg) {
+		respondJSON(t, msg, &ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{{
+				InstanceId: aws.String("i-stop"),
+				State:      &ec2.InstanceState{Code: aws.Int64(80), Name: aws.String("stopped")},
+			}}}},
+		})
+	})
+	require.NoError(t, err)
+
+	input := &ec2.DescribeInstanceStatusInput{
+		IncludeAllInstances: aws.Bool(true),
+		Filters: []*ec2.Filter{
+			{Name: aws.String("availability-zone"), Values: []*string{aws.String("az-other")}},
+		},
+	}
+	out, err := DescribeInstanceStatus(input, nc, 1, "123456789012", "az-a")
+	require.NoError(t, err)
+	assert.Empty(t, out.InstanceStatuses)
+}
+
+func TestDedupStatuses_FirstWins(t *testing.T) {
+	first := runningStatus("i-001", "az-a")
+	second := &ec2.InstanceStatus{
+		InstanceId:    aws.String("i-001"),
+		InstanceState: &ec2.InstanceState{Code: aws.Int64(80), Name: aws.String("stopped")},
+	}
+	got := dedupStatuses([]*ec2.InstanceStatus{first, second})
+	require.Len(t, got, 1)
+	assert.Equal(t, "running", *got[0].InstanceState.Name)
+}
@@ -38,6 +38,9 @@ func ValidateModifyInstanceAttributeInput(input *ec2.ModifyInstanceAttributeInpu
 	if input.SourceDestCheck != nil {
 		count++
 	}
+	if input.DisableApiTermination != nil {
+		count++
+	}
 	if count != 1 {
 		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}
@@ -95,6 +95,24 @@ func TestValidateModifyInstanceAttributeInput_ValidUserData(t *testing.T) {
 	assert.NoError(t, err)
 }

+func TestValidateModifyInstanceAttributeInput_ValidDisableApiTermination(t *testing.T) {
+	err := ValidateModifyInstanceAttributeInput(&ec2.ModifyInstanceAttributeInput{
+		InstanceId:            aws.String("i-abc123"),
+		DisableApiTermination: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
+	})
+	assert.NoError(t, err)
+}
+
+func TestValidateModifyInstanceAttributeInput_DisableApiTerminationWithOther(t *testing.T) {
+	err := ValidateModifyInstanceAttributeInput(&ec2.ModifyInstanceAttributeInput{
+		InstanceId:            aws.String("i-abc123"),
+		DisableApiTermination: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
+		InstanceType:          &ec2.AttributeValue{Value: aws.String("t3.micro")},
+	})
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+}
+
 func TestValidateModifyInstanceAttributeInput_ValidSourceDestCheck(t *testing.T) {
 	err := ValidateModifyInstanceAttributeInput(&ec2.ModifyInstanceAttributeInput{
 		InstanceId:      aws.String("i-abc123"),
@@ -0,0 +1,30 @@
+package gateway_ec2_vpc
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
+	"github.com/nats-io/nats.go"
+)
+
+// Nil input is valid and treated as "all rules in the account"; supplied
+// SecurityGroupRuleIds are validated against the sgr-<17 hex> shape before the
+// request crosses NATS so a malformed ID never hits the handler.
+func DescribeSecurityGroupRules(input *ec2.DescribeSecurityGroupRulesInput, natsConn *nats.Conn, accountID string) (ec2.DescribeSecurityGroupRulesOutput, error) {
+	var output ec2.DescribeSecurityGroupRulesOutput
+	if input != nil {
+		for _, id := range input.SecurityGroupRuleIds {
+			if id == nil || !handlers_ec2_vpc.SGRuleIDRegex.MatchString(*id) {
+				return output, errors.New(awserrors.ErrorInvalidSecurityGroupRuleIdMalformed)
+			}
+		}
+	}
+	svc := handlers_ec2_vpc.NewNATSVPCService(natsConn)
+	result, err := svc.DescribeSecurityGroupRules(input, accountID)
+	if err != nil {
+		return output, err
+	}
+	return *result, nil
+}
@@ -0,0 +1,56 @@
+package gateway_ec2_vpc
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDescribeSecurityGroupRules_NilInput_PassesValidation(t *testing.T) {
+	// Nil input means "all rules in account"; the gateway must not reject it.
+	// The call dives into the NATS service (nil conn) and fails there, so the
+	// assertion is that the error is not the gateway's malformed-ID error.
+	_, err := DescribeSecurityGroupRules(nil, nil, testAccountID)
+	require.Error(t, err)
+	assert.NotEqual(t, awserrors.ErrorInvalidSecurityGroupRuleIdMalformed, err.Error())
+}
+
+func TestDescribeSecurityGroupRules_MalformedRuleID(t *testing.T) {
+	cases := []string{
+		"sgr-toolong0123456789abcdef",
+		"sgr-XYZ",
+		"sg-0123456789abcdef0",
+		"",
+		"sgr-0123456789abcdefX",
+	}
+	for _, bad := range cases {
+		input := &ec2.DescribeSecurityGroupRulesInput{
+			SecurityGroupRuleIds: []*string{aws.String(bad)},
+		}
+		_, err := DescribeSecurityGroupRules(input, nil, testAccountID)
+		assert.EqualError(t, err, awserrors.ErrorInvalidSecurityGroupRuleIdMalformed, "expected InvalidSecurityGroupRuleId.Malformed for %q", bad)
+	}
+}
+
+func TestDescribeSecurityGroupRules_NilRuleIDEntry(t *testing.T) {
+	input := &ec2.DescribeSecurityGroupRulesInput{
+		SecurityGroupRuleIds: []*string{nil},
+	}
+	_, err := DescribeSecurityGroupRules(input, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorInvalidSecurityGroupRuleIdMalformed)
+}
+
+func TestDescribeSecurityGroupRules_ValidRuleIDPassesValidation(t *testing.T) {
+	// A well-formed sgr- ID passes gateway validation; the call then dives
+	// into the NATS service (nil conn) and fails there.
+	input := &ec2.DescribeSecurityGroupRulesInput{
+		SecurityGroupRuleIds: []*string{aws.String("sgr-0123456789abcdef0")},
+	}
+	_, err := DescribeSecurityGroupRules(input, nil, testAccountID)
+	require.Error(t, err)
+	assert.NotEqual(t, awserrors.ErrorInvalidSecurityGroupRuleIdMalformed, err.Error())
+}
@@ -357,16 +357,6 @@ func TestDiscoverActiveNodes_DuplicateNodes(t *testing.T) {
 	assert.Equal(t, 1, result)
 }

-func TestSupportedServices(t *testing.T) {
-	assert.True(t, supportedServices["ec2"])
-	assert.True(t, supportedServices["iam"])
-	assert.True(t, supportedServices["account"])
-	assert.True(t, supportedServices["elasticloadbalancing"])
-	assert.False(t, supportedServices["s3"])
-	assert.False(t, supportedServices["dynamodb"])
-	assert.False(t, supportedServices[""])
-}
-
 func TestParseAWSQueryArgs(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -622,19 +612,6 @@ func connectedNATS(t *testing.T) *nats.Conn {
 	return nc
 }

-func TestRequest_AccountReturns200(t *testing.T) {
-	gw := &GatewayConfig{DisableLogging: true}
-	req := httptest.NewRequest(http.MethodPost, "/", nil)
-	ctx := context.WithValue(req.Context(), ctxService, "account")
-	req = req.WithContext(ctx)
-	w := httptest.NewRecorder()
-
-	gw.Request(w, req)
-
-	resp := w.Result()
-	assert.Equal(t, 200, resp.StatusCode)
-}
-
 // setupEC2Request creates an http.Request with EC2 service context and optional account ID.
 func setupEC2Request(body string, accountID string) *http.Request {
 	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
@@ -950,98 +927,6 @@ func TestIsNATSTransient(t *testing.T) {
 	assert.False(t, isNATSTransient(errors.New("some other error")))
 }

-func TestSlogRequestLogger_CallsNext(t *testing.T) {
-	nextCalled := false
-	next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		nextCalled = true
-	})
-
-	handler := slogRequestLogger(next)
-	req := httptest.NewRequest(http.MethodGet, "/test", nil)
-	w := httptest.NewRecorder()
-	handler.ServeHTTP(w, req)
-
-	assert.True(t, nextCalled)
-	assert.Equal(t, 200, w.Code)
-}
-
-func TestSlogRequestLogger_CapturesStatusCode(t *testing.T) {
-	next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusNotFound)
-	})
-
-	handler := slogRequestLogger(next)
-	req := httptest.NewRequest(http.MethodGet, "/missing", nil)
-	w := httptest.NewRecorder()
-	handler.ServeHTTP(w, req)
-
-	assert.Equal(t, 404, w.Code)
-}
-
-func TestEC2ActionMapCompleteness(t *testing.T) {
-	expectedActions := []string{
-		"DescribeInstances", "RunInstances", "StartInstances", "StopInstances",
-		"TerminateInstances", "RebootInstances", "DescribeInstanceTypes", "GetConsoleOutput",
-		"ModifyInstanceAttribute", "DescribeInstanceAttribute",
-		"CreateKeyPair", "DeleteKeyPair", "DescribeKeyPairs", "ImportKeyPair",
-		"DescribeImages", "CreateImage", "DeregisterImage", "RegisterImage", "CopyImage",
-		"DescribeImageAttribute", "ModifyImageAttribute", "ResetImageAttribute",
-		"DescribeRegions", "DescribeAvailabilityZones",
-		"DescribeVolumes", "ModifyVolume", "CreateVolume", "DeleteVolume",
-		"AttachVolume", "DescribeVolumeStatus", "DescribeVolumesModifications", "DetachVolume",
-		"DescribeAccountAttributes", "EnableEbsEncryptionByDefault",
-		"DisableEbsEncryptionByDefault", "GetEbsEncryptionByDefault",
-		"GetSerialConsoleAccessStatus", "EnableSerialConsoleAccess",
-		"DisableSerialConsoleAccess",
-		"CreateTags", "DeleteTags", "DescribeTags",
-		"CreateSnapshot", "DeleteSnapshot", "DescribeSnapshots", "CopySnapshot",
-		"CreateInternetGateway", "DeleteInternetGateway",
-		"DescribeInternetGateways", "AttachInternetGateway", "DetachInternetGateway",
-		"CreateEgressOnlyInternetGateway", "DeleteEgressOnlyInternetGateway",
-		"DescribeEgressOnlyInternetGateways",
-		"CreatePlacementGroup", "DeletePlacementGroup", "DescribePlacementGroups",
-		"CreateVpc", "DeleteVpc", "DescribeVpcs", "ModifyVpcAttribute", "DescribeVpcAttribute",
-		"CreateSubnet", "DeleteSubnet", "DescribeSubnets", "ModifySubnetAttribute",
-		"CreateNetworkInterface", "DeleteNetworkInterface", "DescribeNetworkInterfaces", "ModifyNetworkInterfaceAttribute",
-		"CreateSecurityGroup", "DeleteSecurityGroup", "DescribeSecurityGroups",
-		"AuthorizeSecurityGroupIngress", "AuthorizeSecurityGroupEgress",
-		"RevokeSecurityGroupIngress", "RevokeSecurityGroupEgress",
-		"DescribeInstanceCreditSpecifications",
-		"AllocateAddress", "ReleaseAddress", "AssociateAddress", "DisassociateAddress", "DescribeAddresses", "DescribeAddressesAttribute",
-		"CreateRouteTable", "DeleteRouteTable", "DescribeRouteTables",
-		"CreateRoute", "DeleteRoute", "ReplaceRoute",
-		"AssociateRouteTable", "DisassociateRouteTable", "ReplaceRouteTableAssociation",
-		"CreateNatGateway", "DeleteNatGateway", "DescribeNatGateways",
-	}
-
-	for _, action := range expectedActions {
-		assert.Contains(t, ec2Actions, action, "ec2Actions missing %s", action)
-	}
-	assert.Len(t, ec2Actions, len(expectedActions), "ec2Actions has unexpected entries")
-}
-
-func TestEC2LocalActionsCompleteness(t *testing.T) {
-	expected := []string{"DescribeRegions", "DescribeAvailabilityZones", "DescribeAccountAttributes"}
-	for _, action := range expected {
-		assert.True(t, ec2LocalActions[action], "ec2LocalActions missing %s", action)
-	}
-	assert.Len(t, ec2LocalActions, len(expected), "ec2LocalActions has unexpected entries")
-}
-
-func TestIAMActionMapCompleteness(t *testing.T) {
-	expectedActions := []string{
-		"CreateUser", "GetUser", "ListUsers", "DeleteUser",
-		"CreateAccessKey", "ListAccessKeys", "DeleteAccessKey", "UpdateAccessKey",
-		"CreatePolicy", "GetPolicy", "GetPolicyVersion", "ListPolicies", "DeletePolicy",
-		"AttachUserPolicy", "DetachUserPolicy", "ListAttachedUserPolicies",
-	}
-
-	for _, action := range expectedActions {
-		assert.Contains(t, iamActions, action, "iamActions missing %s", action)
-	}
-	assert.Len(t, iamActions, len(expectedActions), "iamActions has unexpected entries")
-}
-
 func TestImportKeyPair_Base64PaddingWorkaround(t *testing.T) {
 	// The ImportKeyPair handler has a workaround that decodes URL-encoded
 	// Base64 padding (%3D%3D → ==) before passing to the generic handler.
@@ -2,6 +2,7 @@ package gpu

 import (
 	"fmt"
+	"log/slog"
 	"os"
 	"path/filepath"
 )
@@ -70,6 +71,15 @@ func unbindVFIO(sysfsRoot, addr, originalDriver string) error {
 		return fmt.Errorf("unbind %s from vfio-pci: %w", addr, err)
 	}

+	// AMD CDNA/RDNA GPUs are left in a corrupted hardware state after QEMU exits
+	// without a proper reset. Issue a PCI reset now so the original driver sees
+	// a clean device. The linux-vendor-reset module (gnif/vendor-reset), if
+	// loaded, intercepts this write and applies the AMD-specific reset sequence.
+	if err := resetPCI(sysfsRoot, addr); err != nil {
+		slog.Warn("PCI reset failed — GPU may require manual recovery",
+			"pci", addr, "err", err)
+	}
+
 	if err := writeSysfs(filepath.Join(devPath, "driver_override"), ""); err != nil {
 		return fmt.Errorf("clear driver_override for %s: %w", addr, err)
 	}
@@ -29,7 +29,7 @@ func discover(sysfsRoot string) ([]GPUDevice, error) {
 		devPath := filepath.Join(devicesPath, entry.Name())

 		class, err := readSysfsString(filepath.Join(devPath, "class"))
-		if err != nil || !isDisplayClass(class) {
+		if err != nil || !isPassthroughClass(class) {
 			continue
 		}

@@ -144,12 +144,13 @@ func enrichNVIDIA(gpu *GPUDevice) {
 	gpu.MemoryMiB = memMiB
 }

-// isDisplayClass reports whether a PCI class string (e.g. "0x030200") is a
-// display controller (class byte 0x03).
-func isDisplayClass(class string) bool {
+// isPassthroughClass reports whether a PCI class string identifies a GPU
+// eligible for VFIO passthrough: display controllers (0x03xx) or processing
+// accelerators (0x12xx) such as AMD Instinct compute GPUs.
+func isPassthroughClass(class string) bool {
 	s := strings.ToLower(strings.TrimSpace(class))
 	s = strings.TrimPrefix(s, "0x")
-	return len(s) >= 2 && s[:2] == "03"
+	return len(s) >= 2 && (s[:2] == "03" || s[:2] == "12")
 }

 // readIOMMUGroup reads the IOMMU group number from the iommu_group symlink.
@@ -209,7 +209,7 @@ func TestDiscover_SkipsIntelGPU(t *testing.T) {
 	}
 }

-func TestIsDisplayClass(t *testing.T) {
+func TestIsPassthroughClass(t *testing.T) {
 	cases := []struct {
 		class string
 		want  bool
@@ -217,13 +217,14 @@ func TestIsDisplayClass(t *testing.T) {
 		{"0x030200", true},  // 3D controller
 		{"0x030000", true},  // VGA
 		{"0x038000", true},  // display controller
+		{"0x120000", true},  // processing accelerator (AMD Instinct)
 		{"0x010802", false}, // NVMe
 		{"0x0c0330", false}, // USB
 		{"0x020000", false}, // Ethernet
 	}
 	for _, c := range cases {
-		if got := isDisplayClass(c.class); got != c.want {
-			t.Errorf("isDisplayClass(%q) = %v, want %v", c.class, got, c.want)
+		if got := isPassthroughClass(c.class); got != c.want {
+			t.Errorf("isPassthroughClass(%q) = %v, want %v", c.class, got, c.want)
 		}
 	}
 }
@@ -108,52 +108,69 @@ func (m *Manager) Release(instanceID string) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()

-	idx := -1
-	for i := range m.pool {
-		if m.pool[i].InstanceID == instanceID {
-			idx = i
-			break
-		}
-	}
-	if idx == -1 {
-		return fmt.Errorf("no GPU claimed by instance %s", instanceID)
-	}
-
-	entry := &m.pool[idx]
 	var firstErr error
+	released := false

-	for _, member := range entry.groupMembers {
-		orig := entry.memberDrivers[member.PCIAddress]
-		if orig == "vfio-pci" {
-			// GPU was already bound to vfio-pci before this instance (e.g. by
-			// spx-test-gpu at node setup). QEMU released the group fd on exit;
-			// the device stays vfio-pci bound and is immediately available for
-			// the next claim. Skipping unbind avoids a needless round-trip and
-			// eliminates a brief window where the device is unbound.
+	for i := range m.pool {
+		if m.pool[i].InstanceID != instanceID {
 			continue
 		}
-		if err := unbindVFIO(m.sysfsRoot, member.PCIAddress, orig); err != nil {
-			slog.Error("GPU release failed for IOMMU group member",
-				"instance", instanceID, "pci", member.PCIAddress, "err", err)
-			if firstErr == nil {
-				firstErr = err
+		released = true
+		entry := &m.pool[i]
+
+		for _, member := range entry.groupMembers {
+			orig := entry.memberDrivers[member.PCIAddress]
+			if orig == "vfio-pci" {
+				// GPU was already bound to vfio-pci before this instance (e.g. by
+				// spx-test-gpu at node setup). QEMU released the group fd on exit;
+				// the device stays vfio-pci bound and is immediately available for
+				// the next claim. Skipping unbind avoids a needless round-trip and
+				// eliminates a brief window where the device is unbound.
+				continue
 			}
+			if err := unbindVFIO(m.sysfsRoot, member.PCIAddress, orig); err != nil {
+				slog.Error("GPU release failed for IOMMU group member",
+					"instance", instanceID, "pci", member.PCIAddress, "err", err)
+				if firstErr == nil {
+					firstErr = err
+				}
+			}
+		}
+
+		entry.InstanceID = ""
+		entry.groupMembers = nil
+		entry.memberDrivers = nil
+
+		if firstErr != nil {
+			entry.Available = false
+			slog.Error("GPU marked unavailable after failed release — operator action required",
+				"gpu", entry.Device.PCIAddress)
+		} else {
+			slog.Info("GPU released", "instance", instanceID, "gpu", entry.Device.PCIAddress)
 		}
 	}

-	entry.InstanceID = ""
-	entry.groupMembers = nil
-	entry.memberDrivers = nil
-
-	if firstErr != nil {
-		entry.Available = false
-		slog.Error("GPU marked unavailable after failed release — operator action required",
-			"gpu", entry.Device.PCIAddress)
-		return firstErr
+	if !released {
+		return fmt.Errorf("no GPU claimed by instance %s", instanceID)
 	}
+	return firstErr
+}

-	slog.Info("GPU released", "instance", instanceID, "gpu", entry.Device.PCIAddress)
-	return nil
+// MarkFailed marks a GPU as permanently unavailable after a VFIO/QEMU launch
+// error. The entry is skipped by Claim until the daemon restarts. Call this
+// after releasing a GPU whose QEMU crashed during startup so the same broken
+// device is not immediately re-assigned to the next instance.
+func (m *Manager) MarkFailed(pciAddress string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	for i := range m.pool {
+		if m.pool[i].Device.PCIAddress == pciAddress && m.pool[i].InstanceID == "" {
+			m.pool[i].Available = false
+			slog.Warn("GPU marked unavailable after launch failure — restart daemon to retry",
+				"pci", pciAddress)
+			return
+		}
+	}
 }

 // Available returns the count of GPUs that can be claimed right now.
@@ -292,6 +292,38 @@ func TestManagerCounts_MultiGPU(t *testing.T) {
 	}
 }

+func TestManagerMarkFailed_MakesUnavailable(t *testing.T) {
+	root, gpu := buildManagerSysfs(t)
+	m := newManagerForTest([]GPUDevice{gpu}, root)
+
+	if m.Available() != 1 {
+		t.Fatalf("want Available=1 before MarkFailed, got %d", m.Available())
+	}
+	m.MarkFailed(gpu.PCIAddress)
+	if m.Available() != 0 {
+		t.Errorf("want Available=0 after MarkFailed, got %d", m.Available())
+	}
+	if m.TotalCount() != 1 {
+		t.Errorf("want TotalCount=1 after MarkFailed (entry retained), got %d", m.TotalCount())
+	}
+	// MarkFailed on an unknown address must be a no-op (not panic).
+	m.MarkFailed("ffff:ff:ff.f")
+}
+
+func TestManagerMarkFailed_ClaimedEntryIgnored(t *testing.T) {
+	root, gpu := buildManagerSysfs(t)
+	m := newManagerForTest([]GPUDevice{gpu}, root)
+
+	if _, err := m.Claim("i-running"); err != nil {
+		t.Fatalf("Claim: %v", err)
+	}
+	// MarkFailed on a claimed (non-empty InstanceID) entry must not mark it unavailable.
+	m.MarkFailed(gpu.PCIAddress)
+	if m.AllocatedCount() != 1 {
+		t.Errorf("claimed GPU must still be allocated after MarkFailed, AllocatedCount=%d", m.AllocatedCount())
+	}
+}
+
 func TestManagerReclaim_Success(t *testing.T) {
 	root, gpu := buildManagerSysfs(t)
 	m := newManagerForTest([]GPUDevice{gpu}, root)
@@ -25,6 +25,7 @@ var knownModels = map[string]modelInfo{
 	"10de:2782": {"NVIDIA GeForce RTX 4070", 12288},
 	"10de:2204": {"NVIDIA GeForce RTX 3090", 24576},
 	// AMD datacenter
+	"1002:75a0": {"AMD Instinct MI350X", 294896},
 	"1002:7448": {"AMD Instinct MI300X", 196608},
 	"1002:740c": {"AMD Instinct MI250X", 65536},
 	// AMD consumer
@@ -49,6 +50,7 @@ var computeModels = map[string]bool{
 	"10de:26b5": true, // L40
 	"10de:26b9": true, // L40S
 	// AMD datacenter
+	"1002:75a0": true, // MI350X
 	"1002:7448": true, // MI300X
 	"1002:740c": true, // MI250X
 }
@@ -0,0 +1,46 @@
+package gpu
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// resetPCI issues a PCI Function-Level Reset for addr via sysfs. It is a no-op
+// for non-AMD devices: NVIDIA and Intel GPUs handle the unbind/rebind cycle
+// without requiring an explicit reset.
+//
+// AMD Instinct (CDNA) data-centre GPUs expose a reset sysfs node and support
+// FLR correctly — writing "1" to it clears the hardware state left behind by
+// QEMU, allowing the next Claim to bind a clean device. Consumer AMD GPUs
+// (Polaris/Vega/Navi) have a firmware reset bug that FLR does not fully fix;
+// those architectures are not the target here.
+//
+// Returns nil when the reset attribute is absent so callers treat this as
+// best-effort.
+func resetPCI(sysfsRoot, addr string) error {
+	vendorBytes, _ := os.ReadFile(filepath.Join(sysfsRoot, "bus/pci/devices", addr, "vendor"))
+	if strings.TrimSpace(string(vendorBytes)) != "0x1002" {
+		return nil // not AMD (or unreadable vendor); unbind/rebind is sufficient
+	}
+
+	resetPath := filepath.Join(sysfsRoot, "bus/pci/devices", addr, "reset")
+	if _, err := os.Stat(resetPath); err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			slog.Warn("AMD GPU does not expose PCI FLR node — GPU may not reset cleanly",
+				"pci", addr)
+			return nil
+		}
+		return fmt.Errorf("stat reset node for %s: %w", addr, err)
+	}
+
+	if err := os.WriteFile(resetPath, []byte("1"), 0o600); err != nil {
+		return fmt.Errorf("PCI reset for %s: %w", addr, err)
+	}
+
+	slog.Info("PCI reset issued", "pci", addr)
+	return nil
+}
@@ -0,0 +1,111 @@
+package gpu
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// makeResetNode pre-creates the sysfs reset attribute for a device.
+func makeResetNode(t *testing.T, root, pciAddr string) {
+	t.Helper()
+	devPath := filepath.Join(root, "bus/pci/devices", pciAddr)
+	must(t, os.WriteFile(filepath.Join(devPath, "reset"), []byte(""), 0o600))
+}
+
+func TestResetPCI_AMDSupported(t *testing.T) {
+	root := t.TempDir()
+	// AMD vendor 0x1002
+	buildSysfsDevice(t, root, "0000:05:00.0", "0x030200", "0x1002", "0x744c", "vfio-pci", 3)
+	makeResetNode(t, root, "0000:05:00.0")
+
+	if err := resetPCI(root, "0000:05:00.0"); err != nil {
+		t.Fatalf("resetPCI: %v", err)
+	}
+
+	got := readSysfsTestFile(t, filepath.Join(root, "bus/pci/devices/0000:05:00.0/reset"))
+	if got != "1" {
+		t.Errorf("reset node = %q, want \"1\"", got)
+	}
+}
+
+func TestResetPCI_AMDNoFLR(t *testing.T) {
+	root := t.TempDir()
+	buildSysfsDevice(t, root, "0000:05:00.0", "0x030200", "0x1002", "0x744c", "vfio-pci", 3)
+	// No reset attribute — AMD device does not expose FLR.
+
+	if err := resetPCI(root, "0000:05:00.0"); err != nil {
+		t.Fatalf("resetPCI should return nil when FLR node is absent, got: %v", err)
+	}
+}
+
+func TestResetPCI_NVIDIASkipped(t *testing.T) {
+	root := t.TempDir()
+	// NVIDIA vendor 0x10de — reset should be skipped entirely.
+	buildSysfsDevice(t, root, "0000:03:00.0", "0x030200", "0x10de", "0x2236", "vfio-pci", 7)
+	makeResetNode(t, root, "0000:03:00.0")
+
+	if err := resetPCI(root, "0000:03:00.0"); err != nil {
+		t.Fatalf("resetPCI: %v", err)
+	}
+
+	// Reset node must NOT have been written for a non-AMD device.
+	got := readSysfsTestFile(t, filepath.Join(root, "bus/pci/devices/0000:03:00.0/reset"))
+	if got == "1" {
+		t.Error("reset node was written for NVIDIA device — should be skipped")
+	}
+}
+
+func TestResetPCI_WriteFails(t *testing.T) {
+	root := t.TempDir()
+	buildSysfsDevice(t, root, "0000:05:00.0", "0x030200", "0x1002", "0x744c", "vfio-pci", 3)
+	devPath := filepath.Join(root, "bus/pci/devices/0000:05:00.0")
+	must(t, os.WriteFile(filepath.Join(devPath, "reset"), []byte(""), 0o444)) // read-only
+
+	if err := resetPCI(root, "0000:05:00.0"); err == nil {
+		t.Fatal("resetPCI should return error when reset write fails, got nil")
+	}
+}
+
+// TestUnbindVFIO_AMDResetIssued verifies the reset node is written when an AMD
+// device is unbound from vfio-pci.
+func TestUnbindVFIO_AMDResetIssued(t *testing.T) {
+	root := t.TempDir()
+	buildSysfsDevice(t, root, "0000:05:00.0", "0x030200", "0x1002", "0x744c", "vfio-pci", 3)
+	makeSysfsDriverDir(t, root, "vfio-pci")
+	makeSysfsDriverDir(t, root, "amdgpu")
+	makeResetNode(t, root, "0000:05:00.0")
+
+	if err := unbindVFIO(root, "0000:05:00.0", "amdgpu"); err != nil {
+		t.Fatalf("unbindVFIO: %v", err)
+	}
+
+	if got := readSysfsTestFile(t, filepath.Join(root, "bus/pci/devices/0000:05:00.0/reset")); got != "1" {
+		t.Errorf("reset node = %q, want \"1\" after unbindVFIO for AMD device", got)
+	}
+	if got := readSysfsTestFile(t, filepath.Join(root, "bus/pci/drivers/amdgpu/bind")); got != "0000:05:00.0" {
+		t.Errorf("amdgpu/bind = %q, want 0000:05:00.0", got)
+	}
+}
+
+// TestUnbindVFIO_NVIDIAResetSkipped verifies no reset is issued for NVIDIA.
+func TestUnbindVFIO_NVIDIAResetSkipped(t *testing.T) {
+	root := t.TempDir()
+	buildSysfsDevice(t, root, "0000:03:00.0", "0x030200", "0x10de", "0x2236", "vfio-pci", 7)
+	makeSysfsDriverDir(t, root, "vfio-pci")
+	makeSysfsDriverDir(t, root, "nvidia")
+	makeResetNode(t, root, "0000:03:00.0")
+
+	if err := unbindVFIO(root, "0000:03:00.0", "nvidia"); err != nil {
+		t.Fatalf("unbindVFIO: %v", err)
+	}
+
+	// Reset node must not have been written for NVIDIA.
+	got := readSysfsTestFile(t, filepath.Join(root, "bus/pci/devices/0000:03:00.0/reset"))
+	if got == "1" {
+		t.Error("reset node was written for NVIDIA device during unbindVFIO — should be skipped")
+	}
+	if got := readSysfsTestFile(t, filepath.Join(root, "bus/pci/drivers/nvidia/bind")); got != "0000:03:00.0" {
+		t.Errorf("nvidia/bind = %q, want 0000:03:00.0", got)
+	}
+}
@@ -67,5 +67,5 @@ func TestGenerateNetworkConfig_MgmtIPWithoutMAC(t *testing.T) {
 	assert.NotContains(t, cfg, "mgmt0:", "mgmt NIC should not appear without MAC")
 }

-// Route for multi-node is handled via bootcmd in lbVMUserData, not
-// in the network-config (Alpine cloud-init doesn't support v2 routes).
+// Route for multi-node LB mgmt traffic is handled via the fw_cfg netcfg
+// blob delivered to the LB microVM, not in the network-config.
@@ -10,6 +10,7 @@ import (
 type InstanceService interface {
 	RunInstances(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error)
 	DescribeInstances(input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error)
+	DescribeInstanceStatus(input *ec2.DescribeInstanceStatusInput, accountID string) (*ec2.DescribeInstanceStatusOutput, error)
 	DescribeInstanceTypes(input *ec2.DescribeInstanceTypesInput, accountID string) (*ec2.DescribeInstanceTypesOutput, error)
 	DescribeInstanceAttribute(input *ec2.DescribeInstanceAttributeInput, accountID string) (*ec2.DescribeInstanceAttributeOutput, error)
 	DescribeStoppedInstances(input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error)
@@ -10,6 +10,7 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"text/template"
 	"time"
@@ -148,8 +149,8 @@ func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs
      addresses:
        - "%s/24"
 `, mgmtMAC, mgmtIP)
-		// Route for multi-node is added via bootcmd in lbVMUserData (Alpine
-		// cloud-init does not support v2 routes under ethernets).
+		// Route for multi-node LB mgmt traffic is delivered to the LB microVM
+		// via the fw_cfg netcfg blob, not here.
 	}

 	return cfg
@@ -609,14 +610,14 @@ func (s *InstanceServiceImpl) LaunchRunInstances(instances []*vm.VM, input *ec2.
 				s.vmMgr.MarkFailed(instance, "gpu_claim_failed")
 				continue
 			}
-			instance.GPUPCIAddress = pciAddr
+			instance.GPUPCIAddresses = []string{pciAddr}
 			instance.GPUXVGAEnabled = xvga
 			slog.Info("LaunchRunInstances: GPU claimed for instance", "instanceId", instance.ID, "gpu", pciAddr, "xvga", xvga)
 		}

 		if err := s.vmMgr.Run(instance); err != nil {
 			slog.Error("LaunchRunInstances: vmMgr.Run failed", "instanceId", instance.ID, "err", err)
-			if instance.GPUPCIAddress != "" && s.gpuClaimer != nil {
+			if len(instance.GPUPCIAddresses) > 0 && s.gpuClaimer != nil {
 				if releaseErr := s.gpuClaimer.Release(instance.ID); releaseErr != nil {
 					slog.Error("LaunchRunInstances: GPU release failed after launch failure",
 						"instanceId", instance.ID, "err", releaseErr)
@@ -730,29 +731,54 @@ func (s *InstanceServiceImpl) StopOrTerminateInstance(instance *vm.VM, command s

 	slog.Info("StopOrTerminateInstance: "+action, "id", command.ID)

-	currentState := s.vmMgr.Status(instance)
-
-	// Idempotent: a concurrent terminate goroutine is already cleaning up.
-	if isTerminate && currentState == vm.StateShuttingDown {
+	// Fold the termination-protection check, idempotency check, transition
+	// validation, and attribute stamp into one lock acquisition so a racing
+	// ModifyInstanceAttribute can't clear protection between the gates. The
+	// async lifecycle goroutine below persists on its own state transitions,
+	// so no synchronous persist is needed here.
+	var (
+		protected, raced, stateMismatch bool
+		currentState                    vm.InstanceState
+	)
+	ok := s.vmMgr.UpdateState(instance.ID, func(v *vm.VM) {
+		currentState = v.Status
+		if isTerminate && v.IsTerminationProtected() {
+			protected = true
+			return
+		}
+		if isTerminate && v.Status == vm.StateShuttingDown {
+			raced = true
+			return
+		}
+		if !vm.IsValidTransition(v.Status, initialState) {
+			stateMismatch = true
+			return
+		}
+		v.Attributes = command.Attributes
+	})
+	if !ok {
+		slog.Warn("StopOrTerminateInstance: instance no longer in running map",
+			"instanceId", instance.ID)
+		return errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+	}
+	if protected {
+		slog.Warn("StopOrTerminateInstance: instance has termination protection",
+			"instanceId", instance.ID)
+		return errors.New(awserrors.ErrorOperationNotPermitted)
+	}
+	if raced {
+		// Idempotent: a concurrent terminate goroutine is already cleaning up.
 		slog.Info("StopOrTerminateInstance: instance already shutting down, terminate is idempotent", "instanceId", instance.ID)
 		return nil
 	}
-
-	// Validate the transition synchronously before dispatching so the AWS
-	// SDK sees IncorrectInstanceState (400) instead of a stale 200. The
-	// async Stop/Terminate path re-validates and surfaces vm.ErrInvalidTransition
-	// on a racing transition; we map that into the same AWS error below.
-	if !vm.IsValidTransition(currentState, initialState) {
+	if stateMismatch {
+		// Surface IncorrectInstanceState synchronously so the AWS SDK sees
+		// 400 instead of a stale 200. The async path re-validates.
 		slog.Warn("StopOrTerminateInstance: instance in incorrect state for "+strings.ToLower(action),
 			"instanceId", instance.ID, "currentState", string(currentState))
 		return errors.New(awserrors.ErrorIncorrectInstanceState)
 	}

-	// Stamp command attributes onto the VM before dispatch so the persisted
-	// state reflects the user-stop / user-terminate intent (e.g. for the
-	// recovery path that distinguishes user-stopped from crash-stopped).
-	s.vmMgr.UpdateState(instance.ID, func(v *vm.VM) { v.Attributes = command.Attributes })
-
 	go func(id string) {
 		var err error
 		if isTerminate {
@@ -851,7 +877,20 @@ func (s *InstanceServiceImpl) newViperblock(volumeName string, size int, volumeC
 		VolumeConfig: volumeConfig,
 	}

-	return viperblock.New(&vbconfig, "s3", cfg)
+	vb, err := viperblock.New(&vbconfig, "s3", cfg)
+	restoreSlogDefault()
+	return vb, err
+}
+
+// restoreSlogDefault re-installs the daemon's Info-level slog handler after
+// viperblock.New mutates the global slog default via its SetDebug method
+// (see viperblock.go SetDebug — it calls slog.SetDefault with LevelError,
+// silencing every Info/Warn in the entire process). Tracked for proper fix
+// in viperblock as mulga-siv-70.
+func restoreSlogDefault() {
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+		Level: slog.LevelInfo,
+	})))
 }

 // prepareRootVolume handles creation/cloning of the root volume
@@ -862,8 +901,6 @@ func (s *InstanceServiceImpl) prepareRootVolume(input *ec2.RunInstancesInput, im
 		return errors.New(awserrors.ErrorServerInternal)
 	}

-	vb.SetDebug(false)
-
 	// Initialize the backend
 	err = vb.Backend.Init()
 	if err != nil {
@@ -970,8 +1007,6 @@ func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig vipe
 		return errors.New(awserrors.ErrorServerInternal)
 	}

-	efiVb.SetDebug(false)
-
 	// Initialize the backend
 	slog.Debug("Initializing EFI Viperblock store backend")
 	err = efiVb.Backend.Init()
@@ -1057,8 +1092,6 @@ func (s *InstanceServiceImpl) prepareCloudInitVolume(input *ec2.RunInstancesInpu
 		return errors.New(awserrors.ErrorServerInternal)
 	}

-	cloudInitVb.SetDebug(false)
-
 	// Initialize the backend
 	slog.Debug("Initializing cloud-init Viperblock store backend")
 	err = cloudInitVb.Backend.Init()
@@ -1343,6 +1376,16 @@ var DescribeInstancesValidFilters = map[string]bool{
 	"tag-value":           true,
 }

+// DescribeInstanceStatusValidFilters lists the filter names accepted by
+// DescribeInstanceStatus. event.*, instance-status.* and system-status.*
+// are rejected: Mulga's static-payload health model has no events and a
+// single value per status field, so those filters can't usefully narrow.
+var DescribeInstanceStatusValidFilters = map[string]bool{
+	"availability-zone":   true,
+	"instance-state-code": true,
+	"instance-state-name": true,
+}
+
 // IsInstanceVisible reports whether the caller can see this instance.
 // Pre-Phase4 instances (empty AccountID) are only visible to root (GlobalAccountID).
 func IsInstanceVisible(callerAccountID, ownerAccountID string) bool {
@@ -1630,11 +1673,11 @@ func (s *InstanceServiceImpl) describeInstancesFromKV(input *ec2.DescribeInstanc
 	return &ec2.DescribeInstancesOutput{Reservations: reservations}, nil
 }

-// ModifyInstanceAttribute applies a single attribute change to a stopped instance.
-// All supported attributes (InstanceType, UserData) require the instance to be
-// stopped. SourceDestCheck is a networking concept that does not apply to
-// bare-metal VMs; it is accepted as a no-op on any instance state so Terraform
-// and the AWS CLI do not error out.
+// ModifyInstanceAttribute applies a single attribute change. InstanceType and
+// UserData require the instance to be stopped. DisableApiTermination works on
+// both running and stopped instances. SourceDestCheck is a networking concept
+// that does not apply to bare-metal VMs; it is accepted as a no-op on any
+// instance state so Terraform and the AWS CLI do not error out.
 func (s *InstanceServiceImpl) ModifyInstanceAttribute(input *ec2.ModifyInstanceAttributeInput, accountID string) (*ec2.ModifyInstanceAttributeOutput, error) {
 	if input.InstanceId == nil || *input.InstanceId == "" {
 		return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -1646,6 +1689,41 @@ func (s *InstanceServiceImpl) ModifyInstanceAttribute(input *ec2.ModifyInstanceA
 		return &ec2.ModifyInstanceAttributeOutput{}, nil
 	}

+	// DisableApiTermination applies to both running and stopped instances.
+	// Try the running map first; fall through to the stopped-store path only
+	// if the instance isn't currently running on this node.
+	if input.DisableApiTermination != nil && input.DisableApiTermination.Value != nil {
+		newVal := input.DisableApiTermination.Value
+		var notVisible bool
+		updated, persistErr := s.vmMgr.UpdateAndPersist(instanceID, func(v *vm.VM) bool {
+			if !IsInstanceVisible(accountID, v.AccountID) {
+				notVisible = true
+				return false
+			}
+			if v.RunInstancesInput == nil {
+				v.RunInstancesInput = &ec2.RunInstancesInput{}
+			}
+			v.RunInstancesInput.DisableApiTermination = newVal
+			return true
+		})
+		if notVisible {
+			slog.Warn("ModifyInstanceAttribute: instance not visible",
+				"instanceId", instanceID, "callerAccount", accountID)
+			return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+		}
+		if updated {
+			if persistErr != nil {
+				slog.Error("ModifyInstanceAttribute: persist failed",
+					"instanceId", instanceID, "err", persistErr)
+				return nil, errors.New(awserrors.ErrorServerInternal)
+			}
+			slog.Info("ModifyInstanceAttribute: updated DisableApiTermination on running instance",
+				"instanceId", instanceID, "value", *newVal)
+			return &ec2.ModifyInstanceAttributeOutput{}, nil
+		}
+		// Not in running map — fall through to stopped-store path.
+	}
+
 	if s.stoppedStore == nil {
 		slog.Error("ModifyInstanceAttribute: stopped store not available")
 		return nil, errors.New(awserrors.ErrorServerInternal)
@@ -1700,6 +1778,15 @@ func (s *InstanceServiceImpl) ModifyInstanceAttribute(input *ec2.ModifyInstanceA
 		}
 	}

+	if input.DisableApiTermination != nil && input.DisableApiTermination.Value != nil {
+		slog.Info("ModifyInstanceAttribute: changing DisableApiTermination on stopped instance",
+			"instanceId", instanceID, "value", *input.DisableApiTermination.Value)
+		if instance.RunInstancesInput == nil {
+			instance.RunInstancesInput = &ec2.RunInstancesInput{}
+		}
+		instance.RunInstancesInput.DisableApiTermination = input.DisableApiTermination.Value
+	}
+
 	if err := s.stoppedStore.WriteStoppedInstance(instanceID, instance); err != nil {
 		slog.Error("ModifyInstanceAttribute: failed to write modified instance to KV",
 			"instanceId", instanceID, "err", err)
@@ -1710,6 +1797,21 @@ func (s *InstanceServiceImpl) ModifyInstanceAttribute(input *ec2.ModifyInstanceA
 	return &ec2.ModifyInstanceAttributeOutput{}, nil
 }

+// StoppedInstanceNode returns the LastNode field stored in the shared KV for a
+// stopped instance. Returns "" when the store is unavailable, the instance is
+// not found, or the entry has no LastNode set. Used by the daemon's ec2.start
+// handler to route start requests back to the original node when possible.
+func (s *InstanceServiceImpl) StoppedInstanceNode(instanceID string) string {
+	if s.stoppedStore == nil {
+		return ""
+	}
+	instance, err := s.stoppedStore.LoadStoppedInstance(instanceID)
+	if err != nil || instance == nil {
+		return ""
+	}
+	return instance.LastNode
+}
+
 // StartStoppedInstance picks up a stopped instance from shared KV, re-launches
 // it on this daemon node, then removes it from shared KV.
 func (s *InstanceServiceImpl) StartStoppedInstance(input *StartStoppedInstanceInput, accountID string) (*StartStoppedInstanceOutput, error) {
@@ -1776,7 +1878,7 @@ func (s *InstanceServiceImpl) StartStoppedInstance(input *StartStoppedInstanceIn
 			s.vmMgr.Delete(instance.ID)
 			return nil, errors.New(awserrors.ErrorInsufficientInstanceCapacity)
 		}
-		instance.GPUPCIAddress = pciAddr
+		instance.GPUPCIAddresses = []string{pciAddr}
 		instance.GPUXVGAEnabled = xvga
 		gpuClaimed = true
 		slog.Info("GPU claimed for instance", "instanceId", input.InstanceID, "gpu", pciAddr, "xvga", xvga)
@@ -1845,6 +1947,12 @@ func (s *InstanceServiceImpl) TerminateStoppedInstance(input *TerminateStoppedIn
 		return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
 	}

+	if instance.IsTerminationProtected() {
+		slog.Warn("TerminateStoppedInstance: instance has termination protection",
+			"instanceId", input.InstanceID)
+		return nil, errors.New(awserrors.ErrorOperationNotPermitted)
+	}
+
 	s.deleteInstanceVolumes(instance, input.InstanceID)
 	s.releaseInstancePublicIP(instance, input.InstanceID)
 	s.deleteInstanceENI(instance, input.InstanceID)
@@ -2007,7 +2115,14 @@ func (s *InstanceServiceImpl) DescribeInstanceAttribute(input *ec2.DescribeInsta
 		output.UserData = &ec2.AttributeValue{Value: &val}

 	case ec2.InstanceAttributeNameDisableApiTermination:
-		val := false
+		// Read under the manager lock so we serialise with a concurrent
+		// ModifyInstanceAttribute writer. Inspect is a no-op race-wise for
+		// stopped instances (no concurrent writers) but keeps the call site
+		// uniform.
+		var val bool
+		s.vmMgr.Inspect(instance, func(v *vm.VM) {
+			val = v.IsTerminationProtected()
+		})
 		output.DisableApiTermination = &ec2.AttributeBooleanValue{Value: &val}

 	case ec2.InstanceAttributeNameDisableApiStop:
@@ -2047,3 +2162,149 @@ func (s *InstanceServiceImpl) DescribeInstanceAttribute(input *ec2.DescribeInsta
 		"instanceId", instanceID, "attribute", attribute, "accountID", accountID)
 	return output, nil
 }
+
+// Terminated and error states are deliberately never surfaced: terminated
+// matches AWS (drops from DescribeInstanceStatus shortly after termination);
+// error is a Spinifex-internal state whose name is not a valid AWS state label.
+var (
+	describeInstanceStatusRunningOnly = map[vm.InstanceState]bool{vm.StateRunning: true}
+	describeInstanceStatusAllIncluded = map[vm.InstanceState]bool{
+		vm.StateRunning:      true,
+		vm.StatePending:      true,
+		vm.StateProvisioning: true,
+		vm.StateStopping:     true,
+		vm.StateStopped:      true,
+		vm.StateShuttingDown: true,
+	}
+)
+
+func (s *InstanceServiceImpl) buildInstanceStatus(v *vm.VM) *ec2.InstanceStatus {
+	state := &ec2.InstanceState{}
+	if info, ok := vm.EC2StateCodes[v.Status]; ok {
+		state.SetCode(info.Code)
+		state.SetName(info.Name)
+	} else {
+		state.SetCode(vm.EC2StateCodes[vm.StatePending].Code)
+		state.SetName(vm.EC2StateCodes[vm.StatePending].Name)
+	}
+
+	status := instanceStatusNotApplicable
+	reachability := instanceStatusNotApplicable
+	if v.Status == vm.StateRunning {
+		status = instanceStatusOK
+		reachability = instanceStatusPassed
+	}
+
+	return &ec2.InstanceStatus{
+		AvailabilityZone: aws.String(s.config.AZ),
+		InstanceId:       aws.String(v.ID),
+		InstanceState:    state,
+		InstanceStatus: &ec2.InstanceStatusSummary{
+			Status: aws.String(status),
+			Details: []*ec2.InstanceStatusDetails{{
+				Name:   aws.String(reachabilityDetailName),
+				Status: aws.String(reachability),
+			}},
+		},
+		SystemStatus: &ec2.InstanceStatusSummary{
+			Status: aws.String(status),
+			Details: []*ec2.InstanceStatusDetails{{
+				Name:   aws.String(reachabilityDetailName),
+				Status: aws.String(reachability),
+			}},
+		},
+	}
+}
+
+const (
+	instanceStatusOK            = "ok"
+	instanceStatusPassed        = "passed"
+	instanceStatusNotApplicable = "not-applicable"
+	reachabilityDetailName      = "reachability"
+)
+
+func instanceStatusMatchesFilters(v *vm.VM, is *ec2.InstanceStatus, filters map[string][]string) bool {
+	for name, values := range filters {
+		if strings.HasPrefix(name, "tag:") {
+			continue
+		}
+		var field string
+		switch name {
+		case "availability-zone":
+			if is.AvailabilityZone != nil {
+				field = *is.AvailabilityZone
+			}
+		case "instance-state-name":
+			if is.InstanceState != nil && is.InstanceState.Name != nil {
+				field = *is.InstanceState.Name
+			}
+		case "instance-state-code":
+			if is.InstanceState != nil && is.InstanceState.Code != nil {
+				field = strconv.FormatInt(*is.InstanceState.Code, 10)
+			}
+		default:
+			return false
+		}
+		if !filterutil.MatchesAny(values, field) {
+			return false
+		}
+	}
+
+	if v.Instance != nil {
+		tags := filterutil.EC2TagsToMap(v.Instance.Tags)
+		return filterutil.MatchesTags(filters, tags)
+	}
+	return filterutil.MatchesTags(filters, nil)
+}
+
+// DescribeInstanceStatus returns per-VM status entries for VMs on this node
+// visible to the caller. Stopped instances come from the gateway's KV query,
+// not this handler.
+func (s *InstanceServiceImpl) DescribeInstanceStatus(input *ec2.DescribeInstanceStatusInput, accountID string) (*ec2.DescribeInstanceStatusOutput, error) {
+	slog.Info("Processing DescribeInstanceStatus request from this node", "accountID", accountID)
+
+	instanceIDFilter := make(map[string]bool)
+	for _, id := range input.InstanceIds {
+		if id == nil || *id == "" {
+			continue
+		}
+		if !strings.HasPrefix(*id, "i-") {
+			return nil, errors.New(awserrors.ErrorInvalidInstanceIDMalformed)
+		}
+		instanceIDFilter[*id] = true
+	}
+
+	parsedFilters, err := filterutil.ParseFilters(input.Filters, DescribeInstanceStatusValidFilters)
+	if err != nil {
+		slog.Warn("DescribeInstanceStatus: invalid filter", "err", err)
+		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+
+	includedStates := describeInstanceStatusRunningOnly
+	if aws.BoolValue(input.IncludeAllInstances) {
+		includedStates = describeInstanceStatusAllIncluded
+	}
+
+	var statuses []*ec2.InstanceStatus
+	s.vmMgr.View(func(vms map[string]*vm.VM) {
+		for _, v := range vms {
+			if !IsInstanceVisible(accountID, v.AccountID) {
+				continue
+			}
+			if len(instanceIDFilter) > 0 && !instanceIDFilter[v.ID] {
+				continue
+			}
+			if !includedStates[v.Status] {
+				continue
+			}
+			is := s.buildInstanceStatus(v)
+			if len(parsedFilters) > 0 && !instanceStatusMatchesFilters(v, is, parsedFilters) {
+				continue
+			}
+			statuses = append(statuses, is)
+		}
+	})
+
+	slog.Info("DescribeInstanceStatus completed", "count", len(statuses))
+	return &ec2.DescribeInstanceStatusOutput{InstanceStatuses: statuses}, nil
+}
@@ -14,7 +14,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	"github.com/mulgadc/spinifex/spinifex/config"
-	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	spxtypes "github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/spinifex/spinifex/vm"
@@ -36,24 +35,6 @@ func mgrWith(vms map[string]*vm.VM) *vm.Manager {
 	return m
 }

-func TestNewInstanceServiceImpl(t *testing.T) {
-	cfg := &config.Config{}
-	instanceTypes := map[string]*ec2.InstanceTypeInfo{
-		"t3.micro": {InstanceType: aws.String("t3.micro")},
-	}
-	store := objectstore.NewMemoryObjectStore()
-	mgr := vm.NewManager()
-
-	svc := NewInstanceServiceImpl(cfg, instanceTypes, nil, store, mgr, nil, nil)
-
-	require.NotNil(t, svc)
-	assert.Equal(t, cfg, svc.config)
-	assert.Equal(t, instanceTypes, svc.instanceTypes)
-	assert.Nil(t, svc.natsConn)
-	assert.Equal(t, mgr, svc.vmMgr)
-	assert.Equal(t, store, svc.objectStore)
-}
-
 func TestGenerateHostname(t *testing.T) {
 	tests := []struct {
 		name       string
@@ -423,32 +404,6 @@ func TestParseVolumeParams_PartialEbs(t *testing.T) {
 	assert.True(t, p.deleteOnTermination, "deleteOnTermination should stay at default")
 }

-func TestMockInstanceService(t *testing.T) {
-	svc := NewMockInstanceService()
-	require.NotNil(t, svc)
-
-	input := &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0123456789abcdef0"),
-		InstanceType: aws.String("t3.micro"),
-		KeyName:      aws.String("my-key"),
-		SubnetId:     aws.String("subnet-abc123"),
-	}
-
-	reservation, err := svc.RunInstances(input, "123456789012")
-	require.NoError(t, err)
-	require.NotNil(t, reservation)
-	assert.Equal(t, "123456789012", *reservation.OwnerId)
-	require.Len(t, reservation.Instances, 1)
-
-	inst := reservation.Instances[0]
-	assert.Equal(t, "i-0123456789abcdef0", *inst.InstanceId)
-	assert.Equal(t, "running", *inst.State.Name)
-	assert.Equal(t, "ami-0123456789abcdef0", *inst.ImageId)
-	assert.Equal(t, "t3.micro", *inst.InstanceType)
-	assert.Equal(t, "my-key", *inst.KeyName)
-	assert.Equal(t, "subnet-abc123", *inst.SubnetId)
-}
-
 func TestCloudInitNetworkConfigWildcard(t *testing.T) {
 	// No MACs → wildcard config (non-VPC or VPC without DEV_NETWORKING)
 	cfg := generateNetworkConfig("", "", "", "", nil)
@@ -1032,6 +987,63 @@ func TestDescribeInstanceAttribute_HiddenForOtherAccount(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
 }

+func TestDescribeInstanceAttribute_DisableApiTermination(t *testing.T) {
+	owner := utils.GlobalAccountID
+
+	tests := []struct {
+		name     string
+		instance *vm.VM
+		want     bool
+	}{
+		{
+			name: "flag true on RunInstancesInput",
+			instance: &vm.VM{
+				ID: "i-prot", AccountID: owner,
+				RunInstancesInput: &ec2.RunInstancesInput{
+					DisableApiTermination: aws.Bool(true),
+				},
+			},
+			want: true,
+		},
+		{
+			name: "flag false on RunInstancesInput",
+			instance: &vm.VM{
+				ID: "i-noprot", AccountID: owner,
+				RunInstancesInput: &ec2.RunInstancesInput{
+					DisableApiTermination: aws.Bool(false),
+				},
+			},
+			want: false,
+		},
+		{
+			name: "RunInstancesInput.DisableApiTermination nil",
+			instance: &vm.VM{
+				ID: "i-nilflag", AccountID: owner,
+				RunInstancesInput: &ec2.RunInstancesInput{},
+			},
+			want: false,
+		},
+		{
+			name:     "RunInstancesInput nil (legacy)",
+			instance: &vm.VM{ID: "i-legacy", AccountID: owner},
+			want:     false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{tc.instance.ID: tc.instance})}
+			out, err := svc.DescribeInstanceAttribute(&ec2.DescribeInstanceAttributeInput{
+				InstanceId: aws.String(tc.instance.ID),
+				Attribute:  aws.String(ec2.InstanceAttributeNameDisableApiTermination),
+			}, owner)
+			require.NoError(t, err)
+			require.NotNil(t, out.DisableApiTermination)
+			assert.Equal(t, tc.want, *out.DisableApiTermination.Value)
+		})
+	}
+}
+
 func TestDescribeStoppedInstances_NilStore(t *testing.T) {
 	svc := &InstanceServiceImpl{}
 	_, err := svc.DescribeStoppedInstances(&ec2.DescribeInstancesInput{}, "")
@@ -1302,6 +1314,81 @@ func TestModifyInstanceAttribute_ClearsStateReason(t *testing.T) {
 	assert.Nil(t, updated.Instance.StateReason)
 }

+func TestModifyInstanceAttribute_DisableApiTermination_Running(t *testing.T) {
+	const owner = "acc"
+	tests := []struct {
+		name     string
+		initial  *ec2.RunInstancesInput
+		setTo    bool
+		wantFlag bool
+	}{
+		{name: "set true on empty input", initial: &ec2.RunInstancesInput{}, setTo: true, wantFlag: true},
+		{name: "set true on nil input (legacy)", initial: nil, setTo: true, wantFlag: true},
+		{name: "clear from true to false", initial: &ec2.RunInstancesInput{DisableApiTermination: aws.Bool(true)}, setTo: false, wantFlag: false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			id := "i-run"
+			mgr := mgrWith(map[string]*vm.VM{
+				id: {ID: id, AccountID: owner, Status: vm.StateRunning, RunInstancesInput: tc.initial},
+			})
+			svc := &InstanceServiceImpl{vmMgr: mgr}
+
+			_, err := svc.ModifyInstanceAttribute(&ec2.ModifyInstanceAttributeInput{
+				InstanceId:            aws.String(id),
+				DisableApiTermination: &ec2.AttributeBooleanValue{Value: aws.Bool(tc.setTo)},
+			}, owner)
+			require.NoError(t, err)
+
+			got, _ := mgr.Get(id)
+			require.NotNil(t, got.RunInstancesInput)
+			require.NotNil(t, got.RunInstancesInput.DisableApiTermination)
+			assert.Equal(t, tc.wantFlag, *got.RunInstancesInput.DisableApiTermination)
+		})
+	}
+}
+
+func TestModifyInstanceAttribute_DisableApiTermination_Running_NotVisible(t *testing.T) {
+	id := "i-run-other"
+	mgr := mgrWith(map[string]*vm.VM{
+		id: {ID: id, AccountID: "owner-acc", Status: vm.StateRunning},
+	})
+	svc := &InstanceServiceImpl{vmMgr: mgr}
+
+	_, err := svc.ModifyInstanceAttribute(&ec2.ModifyInstanceAttributeInput{
+		InstanceId:            aws.String(id),
+		DisableApiTermination: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
+	}, "other-acc")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+}
+
+func TestModifyInstanceAttribute_DisableApiTermination_Stopped(t *testing.T) {
+	id := "i-stop-prot"
+	owner := "acc"
+	store := &fakeStoppedStore{loadByID: map[string]*vm.VM{
+		id: {
+			ID: id, Status: vm.StateStopped, AccountID: owner,
+			Instance: &ec2.Instance{InstanceId: aws.String(id)},
+		},
+	}}
+	svc := &InstanceServiceImpl{
+		vmMgr:        mgrWith(map[string]*vm.VM{}),
+		stoppedStore: store,
+	}
+
+	_, err := svc.ModifyInstanceAttribute(&ec2.ModifyInstanceAttributeInput{
+		InstanceId:            aws.String(id),
+		DisableApiTermination: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
+	}, owner)
+	require.NoError(t, err)
+
+	updated := store.wroteStopped[id]
+	require.NotNil(t, updated)
+	require.NotNil(t, updated.RunInstancesInput)
+	assert.True(t, *updated.RunInstancesInput.DisableApiTermination)
+}
+
 func TestModifyInstanceAttribute_WriteError(t *testing.T) {
 	id := "i-werr"
 	store := &fakeStoppedStore{
@@ -1452,6 +1539,30 @@ func TestTerminateStoppedInstance_NotVisible(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
 }

+func TestTerminateStoppedInstance_TerminationProtected(t *testing.T) {
+	id := "i-prot"
+	v := &vm.VM{
+		ID: id, Status: vm.StateStopped, AccountID: "acc",
+		RunInstancesInput: &ec2.RunInstancesInput{
+			DisableApiTermination: aws.Bool(true),
+		},
+	}
+	v.EBSRequests.Requests = []spxtypes.EBSRequest{
+		{Name: "vol-001", DeleteOnTermination: true},
+	}
+	store := &fakeStoppedStore{loadByID: map[string]*vm.VM{id: v}}
+	vd := &fakeVolumeDeleter{}
+	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
+
+	_, err := svc.TerminateStoppedInstance(&TerminateStoppedInstanceInput{InstanceID: id}, "acc")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorOperationNotPermitted, err.Error())
+
+	assert.Empty(t, vd.calls, "volumes must not be deleted when termination protected")
+	assert.Empty(t, store.wroteTerminated, "must not write to terminated bucket when protected")
+	assert.Empty(t, store.deletedStopped, "must not remove from stopped bucket when protected")
+}
+
 func TestTerminateStoppedInstance_HappyPath(t *testing.T) {
 	id := "i-term-001"
 	store := &fakeStoppedStore{loadByID: map[string]*vm.VM{
@@ -1991,6 +2102,40 @@ func TestStopOrTerminateInstance_InvalidTransition(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorIncorrectInstanceState, err.Error())
 }

+func TestStopOrTerminateInstance_TerminationProtection(t *testing.T) {
+	// DisableApiTermination must block Terminate but never Stop.
+	tests := []struct {
+		name    string
+		attrs   spxtypes.EC2CommandAttributes
+		wantErr string
+	}{
+		{name: "terminate blocked", attrs: spxtypes.EC2CommandAttributes{TerminateInstance: true}, wantErr: awserrors.ErrorOperationNotPermitted},
+		{name: "stop allowed", attrs: spxtypes.EC2CommandAttributes{StopInstance: true}, wantErr: ""},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			id := "i-prot"
+			mgr := mgrWith(map[string]*vm.VM{
+				id: {
+					ID: id, Status: vm.StateRunning,
+					RunInstancesInput: &ec2.RunInstancesInput{DisableApiTermination: aws.Bool(true)},
+				},
+			})
+			v, _ := mgr.Get(id)
+			svc := &InstanceServiceImpl{vmMgr: mgr}
+
+			err := svc.StopOrTerminateInstance(v, spxtypes.EC2InstanceCommand{ID: id, Attributes: tc.attrs})
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				return
+			}
+			require.Error(t, err)
+			assert.Equal(t, tc.wantErr, err.Error())
+			assert.Equal(t, vm.StateRunning, v.Status, "VM state must not change when blocked")
+		})
+	}
+}
+
 type fakeENICreator struct {
 	defaultSubnet *SubnetInfo
 	subnet        *SubnetInfo
@@ -2248,27 +2393,6 @@ func TestRebootInstance_NotFound(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
 }

-func TestLaunchRunInstances_EmptyList(t *testing.T) {
-	svc := &InstanceServiceImpl{
-		config: &config.Config{},
-		vmMgr:  vm.NewManager(),
-	}
-	// Must not panic, must not call into nil deps.
-	svc.LaunchRunInstances(nil, &ec2.RunInstancesInput{}, &ec2.InstanceTypeInfo{})
-}
-
-func TestLaunchRunInstances_SkipsStateChanged(t *testing.T) {
-	id := "i-already-running"
-	mgr := mgrWith(map[string]*vm.VM{id: {ID: id, Status: vm.StateRunning}})
-	v, _ := mgr.Get(id)
-	svc := &InstanceServiceImpl{
-		config: &config.Config{},
-		vmMgr:  mgr,
-	}
-	// Skip path returns early without calling GenerateVolumes / vmMgr.Run.
-	svc.LaunchRunInstances([]*vm.VM{v}, &ec2.RunInstancesInput{}, &ec2.InstanceTypeInfo{})
-}
-
 // TestRunInstances_PrepareError covers the InstanceService-level RunInstances
 // (sync convenience method) error propagation when PrepareRunInstances rejects.
 func TestRunInstances_PrepareError(t *testing.T) {
@@ -2277,17 +2401,230 @@ func TestRunInstances_PrepareError(t *testing.T) {
 	require.Error(t, err)
 }

-// TestSetters exercises the dep-wiring setters so they're not 0-cov.
-func TestSetters(t *testing.T) {
-	svc := NewInstanceServiceImpl(
-		&config.Config{},
-		map[string]*ec2.InstanceTypeInfo{},
-		nil, nil, vm.NewManager(),
-		&fakeResourceCapacityProvider{instanceTypes: map[string]*ec2.InstanceTypeInfo{}},
-		nil,
-	)
-	svc.SetTerminationDeps(&fakeVolumeDeleter{}, &fakeENIDeleter{}, &fakePublicIPReleaser{})
-	svc.SetGPUClaimer(&fakeGPUClaimer{})
-	svc.SetRunInstancesDeps(&fakeAMILoader{}, &fakeKeyValidator{}, nil, nil)
-	require.NotNil(t, svc)
+// --- DescribeInstanceStatus ---
+
+func instanceStatusService(t *testing.T, az string, vms map[string]*vm.VM) *InstanceServiceImpl {
+	t.Helper()
+	return &InstanceServiceImpl{
+		config: &config.Config{AZ: az},
+		vmMgr:  mgrWith(vms),
+	}
+}
+
+func runningVM(id, owner string) *vm.VM {
+	return &vm.VM{
+		ID:        id,
+		Status:    vm.StateRunning,
+		AccountID: owner,
+		Reservation: &ec2.Reservation{
+			ReservationId: aws.String("r-" + id),
+			OwnerId:       aws.String(owner),
+		},
+		Instance: &ec2.Instance{InstanceId: aws.String(id)},
+	}
+}
+
+func TestDescribeInstanceStatus_Empty(t *testing.T) {
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{})
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{}, utils.GlobalAccountID)
+	require.NoError(t, err)
+	require.NotNil(t, out)
+	assert.Empty(t, out.InstanceStatuses)
+}
+
+func TestDescribeInstanceStatus_RunningInstance(t *testing.T) {
+	owner := "111122223333"
+	v := runningVM("i-aaa", owner)
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{v.ID: v})
+
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{}, owner)
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+
+	s := out.InstanceStatuses[0]
+	assert.Equal(t, "i-aaa", *s.InstanceId)
+	assert.Equal(t, "az-a", *s.AvailabilityZone)
+	assert.Equal(t, "running", *s.InstanceState.Name)
+	assert.Equal(t, int64(16), *s.InstanceState.Code)
+	assert.Equal(t, "ok", *s.InstanceStatus.Status)
+	assert.Equal(t, "ok", *s.SystemStatus.Status)
+	require.Len(t, s.InstanceStatus.Details, 1)
+	assert.Equal(t, "passed", *s.InstanceStatus.Details[0].Status)
+}
+
+func TestDescribeInstanceStatus_AccountFilteringHidesOtherTenant(t *testing.T) {
+	v := runningVM("i-other", "999988887777")
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{v.ID: v})
+
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{}, "111122223333")
+	require.NoError(t, err)
+	assert.Empty(t, out.InstanceStatuses)
+}
+
+func TestDescribeInstanceStatus_StoppedExcludedByDefault(t *testing.T) {
+	owner := "111122223333"
+	stopped := runningVM("i-stop", owner)
+	stopped.Status = vm.StateStopped
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{stopped.ID: stopped})
+
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{}, owner)
+	require.NoError(t, err)
+	assert.Empty(t, out.InstanceStatuses)
+}
+
+func TestDescribeInstanceStatus_IncludeAllSurfacesPending(t *testing.T) {
+	owner := "111122223333"
+	pend := runningVM("i-pend", owner)
+	pend.Status = vm.StatePending
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{pend.ID: pend})
+
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
+		IncludeAllInstances: aws.Bool(true),
+	}, owner)
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+	assert.Equal(t, "pending", *out.InstanceStatuses[0].InstanceState.Name)
+	assert.Equal(t, "not-applicable", *out.InstanceStatuses[0].InstanceStatus.Status)
+	assert.Equal(t, "not-applicable", *out.InstanceStatuses[0].SystemStatus.Status)
+}
+
+func TestDescribeInstanceStatus_IncludeAllSurfacesStopped(t *testing.T) {
+	owner := "111122223333"
+	stopped := runningVM("i-stop", owner)
+	stopped.Status = vm.StateStopped
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{stopped.ID: stopped})
+
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
+		IncludeAllInstances: aws.Bool(true),
+	}, owner)
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+	assert.Equal(t, "stopped", *out.InstanceStatuses[0].InstanceState.Name)
+	assert.Equal(t, "not-applicable", *out.InstanceStatuses[0].InstanceStatus.Status)
+	assert.Equal(t, "not-applicable", *out.InstanceStatuses[0].SystemStatus.Status)
+}
+
+func TestDescribeInstanceStatus_TerminatedNeverReturned(t *testing.T) {
+	owner := "111122223333"
+	term := runningVM("i-term", owner)
+	term.Status = vm.StateTerminated
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{term.ID: term})
+
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
+		IncludeAllInstances: aws.Bool(true),
+	}, owner)
+	require.NoError(t, err)
+	assert.Empty(t, out.InstanceStatuses)
+}
+
+func TestDescribeInstanceStatus_ErrorStateNeverReturned(t *testing.T) {
+	owner := "111122223333"
+	errVM := runningVM("i-err", owner)
+	errVM.Status = vm.StateError
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{errVM.ID: errVM})
+
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
+		IncludeAllInstances: aws.Bool(true),
+	}, owner)
+	require.NoError(t, err)
+	assert.Empty(t, out.InstanceStatuses)
+}
+
+func TestDescribeInstanceStatus_InstanceIDFilter(t *testing.T) {
+	owner := "111122223333"
+	keep := runningVM("i-keep", owner)
+	drop := runningVM("i-drop", owner)
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{keep.ID: keep, drop.ID: drop})
+
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
+		InstanceIds: []*string{aws.String("i-keep")},
+	}, owner)
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+	assert.Equal(t, "i-keep", *out.InstanceStatuses[0].InstanceId)
+}
+
+func TestDescribeInstanceStatus_MalformedInstanceID(t *testing.T) {
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{})
+	_, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
+		InstanceIds: []*string{aws.String("not-an-id")},
+	}, utils.GlobalAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error())
+}
+
+func TestDescribeInstanceStatus_UnknownFilter(t *testing.T) {
+	owner := "111122223333"
+	v := runningVM("i-aaa", owner)
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{v.ID: v})
+
+	_, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
+		Filters: []*ec2.Filter{{
+			Name:   aws.String("event.code"),
+			Values: []*string{aws.String("system-reboot")},
+		}},
+	}, owner)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+}
+
+func TestDescribeInstanceStatus_StateNameFilter(t *testing.T) {
+	owner := "111122223333"
+	run := runningVM("i-run", owner)
+	stop := runningVM("i-stop", owner)
+	stop.Status = vm.StateStopped
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{run.ID: run, stop.ID: stop})
+
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
+		IncludeAllInstances: aws.Bool(true),
+		Filters: []*ec2.Filter{{
+			Name:   aws.String("instance-state-name"),
+			Values: []*string{aws.String("running")},
+		}},
+	}, owner)
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+	assert.Equal(t, "i-run", *out.InstanceStatuses[0].InstanceId)
+}
+
+func TestDescribeInstanceStatus_AvailabilityZoneFilter(t *testing.T) {
+	owner := "111122223333"
+	v := runningVM("i-aaa", owner)
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{v.ID: v})
+
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
+		Filters: []*ec2.Filter{{
+			Name:   aws.String("availability-zone"),
+			Values: []*string{aws.String("az-a")},
+		}},
+	}, owner)
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+
+	out, err = svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
+		Filters: []*ec2.Filter{{
+			Name:   aws.String("availability-zone"),
+			Values: []*string{aws.String("az-b")},
+		}},
+	}, owner)
+	require.NoError(t, err)
+	assert.Empty(t, out.InstanceStatuses)
+}
+
+func TestDescribeInstanceStatus_TagFilter(t *testing.T) {
+	owner := "111122223333"
+	tagged := runningVM("i-tag", owner)
+	tagged.Instance.Tags = []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("foo")}}
+	plain := runningVM("i-plain", owner)
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{tagged.ID: tagged, plain.ID: plain})
+
+	out, err := svc.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
+		Filters: []*ec2.Filter{{
+			Name:   aws.String("tag:Name"),
+			Values: []*string{aws.String("foo")},
+		}},
+	}, owner)
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+	assert.Equal(t, "i-tag", *out.InstanceStatuses[0].InstanceId)
 }
@@ -44,6 +44,10 @@ func (s *MockInstanceService) DescribeInstanceTypes(_ *ec2.DescribeInstanceTypes
 	return &ec2.DescribeInstanceTypesOutput{}, nil
 }

+func (s *MockInstanceService) DescribeInstanceStatus(_ *ec2.DescribeInstanceStatusInput, _ string) (*ec2.DescribeInstanceStatusOutput, error) {
+	return &ec2.DescribeInstanceStatusOutput{}, nil
+}
+
 func (s *MockInstanceService) DescribeInstanceAttribute(input *ec2.DescribeInstanceAttributeInput, _ string) (*ec2.DescribeInstanceAttributeOutput, error) {
 	return &ec2.DescribeInstanceAttributeOutput{InstanceId: input.InstanceId}, nil
 }
@@ -49,6 +49,12 @@ func (s *NATSInstanceService) DescribeInstanceTypes(_ *ec2.DescribeInstanceTypes
 	return nil, fmt.Errorf("DescribeInstanceTypes is fan-out; use gateway scatter-gather, not NATSInstanceService")
 }

+// DescribeInstanceStatus is a fan-out operation; use the gateway's scatter-gather
+// helper instead of this single-node client.
+func (s *NATSInstanceService) DescribeInstanceStatus(_ *ec2.DescribeInstanceStatusInput, _ string) (*ec2.DescribeInstanceStatusOutput, error) {
+	return nil, fmt.Errorf("DescribeInstanceStatus is fan-out; use gateway scatter-gather, not NATSInstanceService")
+}
+
 func (s *NATSInstanceService) DescribeInstanceAttribute(input *ec2.DescribeInstanceAttributeInput, accountID string) (*ec2.DescribeInstanceAttributeOutput, error) {
 	return utils.NATSRequest[ec2.DescribeInstanceAttributeOutput](s.natsConn, "ec2.DescribeInstanceAttribute", input, 30*time.Second, accountID)
 }
@@ -562,11 +562,17 @@ func TestRemoveInstance_Success(t *testing.T) {
 	assert.Equal(t, []string{"i-bbb"}, record.NodeInstances["node-b"])
 }

+// TestRemoveInstance_NodeNotFound / _GroupNotFound pin the idempotency
+// contract in service_impl.go:453-497 — RemoveInstance must succeed when the
+// node or group is missing, since callers (instance Terminate) may race with
+// group deletion or have already moved the instance off the node. Production
+// marks both paths with //nolint:nilerr // intentional; without these tests a
+// future cleanup could remove the annotations and silently flip the contract.
+
 func TestRemoveInstance_NodeNotFound(t *testing.T) {
 	svc := setupTestService(t)
 	createTestGroup(t, svc, "remove-nonode-group", "spread")

-	// Removing from a non-existent node is a no-op
 	_, err := svc.RemoveInstance(&RemoveInstanceInput{
 		GroupName:  "remove-nonode-group",
 		NodeName:   "ghost-node",
@@ -578,7 +584,6 @@ func TestRemoveInstance_NodeNotFound(t *testing.T) {
 func TestRemoveInstance_GroupNotFound(t *testing.T) {
 	svc := setupTestService(t)

-	// Removing from a non-existent group is a no-op (group may have been deleted)
 	_, err := svc.RemoveInstance(&RemoveInstanceInput{
 		GroupName:  "deleted-group",
 		NodeName:   "node-a",
@@ -401,28 +401,6 @@ func TestExternalIPAM_FindPoolByName_NotFound(t *testing.T) {
 	assert.Error(t, err)
 }

-func TestExternalPoolConfig_IsDHCP(t *testing.T) {
-	p := ExternalPoolConfig{Source: "dhcp"}
-	assert.True(t, p.IsDHCP())
-
-	p2 := ExternalPoolConfig{Source: "static"}
-	assert.False(t, p2.IsDHCP())
-
-	p3 := ExternalPoolConfig{}
-	assert.False(t, p3.IsDHCP())
-}
-
-func TestValidatePoolConfig_DHCPPool(t *testing.T) {
-	// DHCP pools don't need range_start/range_end
-	pool := ExternalPoolConfig{
-		Name:    "dhcp-pool",
-		Source:  "dhcp",
-		Gateway: "192.168.1.1",
-	}
-	err := ValidatePoolConfig(pool)
-	assert.NoError(t, err)
-}
-
 // dhcpStub captures acquire/release requests and replies with a
 // deterministic lease so the handler-side IPAM code can be exercised
 // without running vpcd.
--- a/Show More
+++ b/Show More