release: update go.mod for v1.4.0

feat: iso release flow (#229 )
feat(elbv2): QEMU microvm direct-boot for LB system instances (#228 )
2026-10-24 13:14:23 +00:00 · 2026-05-19 09:37:54 +10:00 · 2026-05-19 09:07:26 +10:00 · 2026-05-18 15:48:04 +10:00 · 2026-05-18 12:32:07 +10:00 · 2026-05-18 10:18:04 +10:00
491 changed files with 41955 additions and 13021 deletions
@@ -8,7 +8,7 @@ updates:
      day: monday
      timezone: Australia/Sydney
    cooldown:
-      default-days: 1
+      default-days: 3
    groups:
      go-dependencies:
        patterns:
@@ -22,7 +22,7 @@ updates:
      day: monday
      timezone: Australia/Sydney
    cooldown:
-      default-days: 1
+      default-days: 3
    ignore:
      - dependency-name: "*"
        update-types:
@@ -41,7 +41,7 @@ updates:
      day: monday
      timezone: Australia/Sydney
    cooldown:
-      default-days: 1
+      default-days: 3
    ignore:
      - dependency-name: "@types/node"
        update-types:
@@ -0,0 +1,117 @@
+name: E2E Bare-Metal
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  statuses: write
+
+env:
+  GIT_BRANCH: ${{ github.ref_name }}
+
+jobs:
+  e2e_baremetal:
+    name: Bare-Metal Multi-Node E2E
+    runs-on: [ self-hosted, ci-baremetal ]
+    timeout-minutes: 120
+    concurrency:
+      group: e2e-baremetal-cluster
+      cancel-in-progress: false
+    env:
+      BM_ISO_CACHE: /var/cache/spinifex-iso-builder
+    steps:
+      - name: Clean Workspace
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/checkout@v6
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v6
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: |
+            scripts/iso-builder
+            scripts/baremetal
+          path: mulga
+
+      - name: Install SSH Key
+        run: |
+          echo "${{ secrets.BM_SSH_PRIVATE_KEY }}" > "${RUNNER_TEMP}/bm-ci"
+          chmod 0600 "${RUNNER_TEMP}/bm-ci"
+          ssh-keygen -y -f "${RUNNER_TEMP}/bm-ci" > "${RUNNER_TEMP}/bm-ci.pub"
+
+      - name: Build Artifacts
+        run: |
+          mulga/scripts/baremetal/bm-build-artifacts.sh \
+            "$GITHUB_WORKSPACE" \
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            "${RUNNER_TEMP}/bm-bins"
+
+      - name: Build PXE Artifacts
+        run: |
+          sudo make -C mulga/scripts/iso-builder \
+            build-rootfs build-squashfs build-initramfs \
+            CACHE="${BM_ISO_CACHE}" \
+            SPINIFEX_BIN="${RUNNER_TEMP}/bm-bins/spx" \
+            INSTALLER_BIN="${RUNNER_TEMP}/bm-bins/spinifex-installer" \
+            SPINIFEX_SRC="${GITHUB_WORKSPACE}" \
+            VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
+            VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always 2>/dev/null || echo dev)"
+
+      - name: Write ePDU credentials
+        run: |
+          install -m 0600 /dev/null "${RUNNER_TEMP}/epdu.env"
+          cat >> "${RUNNER_TEMP}/epdu.env" <<EOF
+          EPDU_HOST=192.168.1.63
+          EPDU_SNMP_USER=ci-bmpower
+          EPDU_SNMP_SEC_LEVEL=authPriv
+          EPDU_SNMP_AUTH_PROTO=MD5
+          EPDU_SNMP_AUTH_PASS=${{ secrets.BM_EPDU_AUTH_PASS }}
+          EPDU_SNMP_PRIV_PROTO=DES
+          EPDU_SNMP_PRIV_PASS=${{ secrets.BM_EPDU_PRIV_PASS }}
+          EOF
+
+      - name: Cache VM images
+        run: |
+          UBUNTU_IMG="${BM_ISO_CACHE}/ubuntu-24.04-cloudimg-amd64.img"
+          if [ ! -f "$UBUNTU_IMG" ]; then
+            echo "Downloading Ubuntu 24.04 cloud image..."
+            sudo curl -fsSL -o "$UBUNTU_IMG" \
+              "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+            echo "  Downloaded $(du -h "$UBUNTU_IMG" | cut -f1)"
+          else
+            echo "Ubuntu 24.04 cloud image cached ($(du -h "$UBUNTU_IMG" | cut -f1))"
+          fi
+
+          LB_IMG="${BM_ISO_CACHE}/lb-alpine-3.21.6-x86_64.raw"
+          if [ ! -f "$LB_IMG" ]; then
+            echo "Downloading lb-alpine image..."
+            sudo curl -fsSL -o "$LB_IMG" \
+              "https://iso.mulgadc.com/system-ami/lb-alpine-3.21.6-x86_64.raw"
+            echo "  Downloaded $(du -h "$LB_IMG" | cut -f1)"
+          else
+            echo "lb-alpine image cached ($(du -h "$LB_IMG" | cut -f1))"
+          fi
+
+      - name: Run Bare-Metal E2E
+        env:
+          BM_CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
+          BM_SSH_KEY: ${{ runner.temp }}/bm-ci
+          BM_EPDU_ENV: ${{ runner.temp }}/epdu.env
+          BM_UBUNTU_IMAGE: ${{ env.BM_ISO_CACHE }}/ubuntu-24.04-cloudimg-amd64.img
+          BM_LB_ALPINE_IMAGE: ${{ env.BM_ISO_CACHE }}/lb-alpine-3.21.6-x86_64.raw
+          SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
+        run: |
+          sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
+            --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
+            --initrd    "${BM_ISO_CACHE}/initrd.img" \
+            --squashfs  "${BM_ISO_CACHE}/spinifex.squashfs" \
+            --tests-tar "${RUNNER_TEMP}/spinifex-tests.tar"
@@ -11,6 +11,9 @@ on:
        required: false
        default: ""

+permissions:
+  contents: read
+
 env:
  GIT_BRANCH: ${{ github.ref_name }}

@@ -27,7 +30,7 @@ jobs:
      TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
    steps:
      - name: Clean Workspace
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -53,6 +56,26 @@ jobs:
          sparse-checkout: scripts/tofu-cluster
          path: mulga

+      - name: Install gotestsum
+        env:
+          GOTESTSUM_VERSION: "1.12.0"
+        run: |
+          # Prebuilt tarball, not `go install`: gotestsum v1.12.0 pulls
+          # x/tools v0.19.0 whose tokeninternal static-assert pegs a
+          # narrow Go window (~1.22). The self-hosted runner's Go falls
+          # outside that window, so source-install fails with
+          # `invalid array length -delta * delta`. A prebuilt binary
+          # sidesteps toolchain coupling entirely.
+          BIN_DIR="${RUNNER_TEMP}/gotestsum-bin"
+          mkdir -p "${BIN_DIR}"
+          if [ ! -x "${BIN_DIR}/gotestsum" ]; then
+            curl -fsSL \
+              "https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/gotestsum_${GOTESTSUM_VERSION}_linux_amd64.tar.gz" \
+              | tar -xz -C "${BIN_DIR}" gotestsum
+          fi
+          echo "${BIN_DIR}" >> "${GITHUB_PATH}"
+          "${BIN_DIR}/gotestsum" --version
+
      - name: Build Install Artifacts
        working-directory: mulga/scripts/tofu-cluster
        run: |
@@ -95,18 +118,73 @@ jobs:
          # tofu-cluster, wan_ip == cluster IP and the runner's outbound
          # IP is distinct — see e2e-ddil-test-harness.md §3 Scenario C.
          NODE_IPS=$(echo "$INVENTORY" | jq -r '[.nodes[].wan_ip] | join(",")')
+          PRIMARY_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          echo "DDIL_NODES=${NODE_IPS}"

+          # Witness VMs talk to the cluster's AWS gateway via aws-sdk-go,
+          # which resolves credentials from the standard chain. spx admin
+          # init writes [spinifex] to ${SSH_USER}'s ~/.aws/credentials on
+          # node1 — pull them over SSH and export as the SDK's env-var
+          # entries so the harness can launch instances without depending
+          # on a cluster-bootstrapped profile being present on the runner.
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${PRIMARY_IP}" 'cat ~/.aws/credentials')
+          AWS_ACCESS_KEY_ID=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
+            echo "::error::Failed to extract spinifex AWS credentials from ${PRIMARY_IP}"
+            exit 1
+          fi
+
+          # Guest SSH key: bootstrap-install.sh imports spinifex-key into EC2;
+          # the matching private material was generated on the primary node at
+          # ~/.ssh/spinifex-key. Pull it to the runner so the harness can SSH
+          # the witness via its MapPublicIpOnLaunch-assigned EIP.
+          GUEST_KEY="${RUNNER_TEMP}/spinifex-key"
+          scp -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${PRIMARY_IP}:~/.ssh/spinifex-key" "${GUEST_KEY}"
+          chmod 600 "${GUEST_KEY}"
+
          mkdir -p "${RUNNER_TEMP}/ddil-results"
          cd "${GITHUB_WORKSPACE}/spinifex"
+          # gotestsum: human-readable PASS/FAIL/SKIP lines in Actions stdout
+          # while still emitting the machine-readable jsonfile artifact.
          DDIL_NODES="${NODE_IPS}" \
          DDIL_SSH_USER="${SSH_USER}" \
          DDIL_SSH_KEY="${SSH_KEY}" \
+          DDIL_GUEST_SSH_KEY="${GUEST_KEY}" \
          DDIL_QUARANTINED="${{ inputs.quarantined }}" \
          AWS_REGION="${REGION}" \
-            go test -tags=e2e -timeout 45m -json \
-              ./tests/e2e/ddil/scenarios/... \
-              | tee "${RUNNER_TEMP}/ddil-results/results.json"
+          AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
+          AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
+            gotestsum --jsonfile "${RUNNER_TEMP}/ddil-results/results.json" \
+                      --format testname \
+                      -- -tags=e2e -timeout 45m ./tests/e2e/ddil/scenarios/...
+
+      - name: Collect Cluster Logs
+        if: failure()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          # Pull spinifex-* journals + QEMU process listings from every
+          # node into the artifact bundle so failures can be triaged
+          # without re-provisioning the cluster. Best-effort — a missing
+          # node should not mask the original test failure.
+          set +e
+          INVENTORY=$(tofu output -json inventory)
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
+          LOG_DIR="${RUNNER_TEMP}/ddil-results/cluster-logs"
+          mkdir -p "${LOG_DIR}"
+
+          mapfile -t NODE_IPS < <(echo "$INVENTORY" | jq -r '.nodes[].wan_ip')
+          for ip in "${NODE_IPS[@]}"; do
+            echo "=== collecting ${ip} ==="
+            SSH="ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SSH_USER}@${ip}"
+            ${SSH} 'sudo journalctl -u "spinifex-*" --no-pager -n 5000' > "${LOG_DIR}/${ip}-spinifex.log" 2>&1
+            ${SSH} 'ps auxw | grep -E "qemu-system|spinifex" | grep -v grep' > "${LOG_DIR}/${ip}-procs.txt" 2>&1
+            ${SSH} 'ls -la /var/lib/spinifex 2>/dev/null; ls -la /var/log/spinifex 2>/dev/null' > "${LOG_DIR}/${ip}-state.txt" 2>&1
+          done

      - name: Upload DDIL Results
        if: always()
@@ -12,8 +12,10 @@ on:
 permissions:
  contents: read

+# Scheduled runs are loaded from the default branch (main), so the schedule
+# trigger must stay in sync on main for cron to fire.
 env:
-  GIT_BRANCH: ${{ github.ref_name }}
+  GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2

 jobs:
@@ -58,7 +60,7 @@ jobs:

      - name: Clean Workspace
        if: steps.gate.outputs.selected == 'true'
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        if: steps.gate.outputs.selected == 'true'
@@ -19,6 +19,7 @@ on:
        required: false

 permissions:
+  contents: read
  statuses: write

 env:
@@ -28,12 +29,12 @@ jobs:
  e2e_single:
    name: Single-Node E2E
    runs-on: [ self-hosted, ci-single ]
-    timeout-minutes: 30
+    timeout-minutes: 35
    env:
      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
    steps:
      - name: Clean Workspace
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -97,12 +98,12 @@ jobs:
  e2e_pseudo_multi:
    name: Pseudo Multi-Node E2E
    runs-on: [ self-hosted, ci-pseudo ]
-    timeout-minutes: 30
+    timeout-minutes: 25
    env:
      TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log
    steps:
      - name: Clean Workspace
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -161,12 +162,12 @@ jobs:
  e2e_real_multi:
    name: Real Multi-Node E2E
    runs-on: [ self-hosted, ci-multi ]
-    timeout-minutes: 35
+    timeout-minutes: 30
    env:
      TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
    steps:
      - name: Clean Workspace
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -31,6 +31,14 @@ jobs:
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.26.3'
+
+      - name: Build microVM artifacts (lb-agent + kernel + initramfs)
+        run: cd spinifex && make build-microvm-image
+
      - name: Build amd64 Tarball
        env:
          VERSION: ${{ github.ref_name }}
@@ -192,3 +200,150 @@ jobs:
          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
+
+  iso:
+    needs: release
+    runs-on: [ self-hosted, ci-single ]
+    timeout-minutes: 90
+    steps:
+      - name: Clean Workspace
+        run: |
+          ROOTFS="${GITHUB_WORKSPACE}/mulga/scripts/iso-builder/.cache/rootfs"
+          # Mirror make clean's umount sequence unconditionally — lazy unmount
+          # detaches immediately even if the mount is busy, so rm -rf is safe.
+          sudo umount -l -R "${ROOTFS}" 2>/dev/null || true
+          sudo umount -l "${ROOTFS}/dev/pts" 2>/dev/null || true
+          sudo umount -l "${ROOTFS}/dev" 2>/dev/null || true
+          sudo umount -l "${ROOTFS}/sys" 2>/dev/null || true
+          sudo umount -l "${ROOTFS}/proc" 2>/dev/null || true
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - name: Checkout Spinifex
+        uses: actions/checkout@v6
+        with:
+          path: spinifex
+          ref: ${{ github.ref }}
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/predastore
+          path: predastore
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/viperblock
+          path: viperblock
+
+      - name: Checkout Mulga iso-builder
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/iso-builder
+          path: mulga
+
+      - name: Clean ISO builder cache
+        run: sudo make -C mulga/scripts/iso-builder clean
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Configure Go workspace
+        run: go work init ./spinifex ./predastore ./viperblock
+
+      - name: Install ISO builder dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y \
+            debootstrap squashfs-tools xorriso grub-pc-bin grub-efi-amd64-bin \
+            grub-common mtools busybox-static ovmf zstd \
+            nbdkit nbdkit-plugin-dev pkg-config gcc
+
+      - name: Build spinifex binaries
+        run: cd spinifex && make build
+
+      - name: Build viperblock
+        run: cd viperblock && make build
+
+      - name: Build release ISO
+        id: build-iso
+        env:
+          VERSION: ${{ github.ref_name }}
+        run: |
+          sudo make -C mulga/scripts/iso-builder release \
+            VERSION="${VERSION}" \
+            SPINIFEX_SRC="${GITHUB_WORKSPACE}/spinifex" \
+            VIPERBLOCK_SRC="${GITHUB_WORKSPACE}/viperblock" \
+            SPINIFEX_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spx" \
+            INSTALLER_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spinifex-installer"
+
+      - name: Wait for verify
+        id: wait-verify
+        if: steps.build-iso.outcome == 'success'
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          echo "Waiting for verify job to complete..."
+          while true; do
+            CONCLUSION=$(gh api \
+              "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \
+              --jq '.jobs[] | select(.name == "verify") | .conclusion // empty')
+            if [ -n "$CONCLUSION" ]; then
+              echo "Verify conclusion: $CONCLUSION"
+              [ "$CONCLUSION" = "success" ] && exit 0
+              exit 1
+            fi
+            echo "Verify still running, retrying in 30s..."
+            sleep 30
+          done
+
+      - name: Ensure AWS CLI is installed
+        if: steps.build-iso.outcome == 'success' && steps.wait-verify.outcome == 'success'
+        run: |
+          if ! command -v aws &>/dev/null; then
+            curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
+            unzip -q /tmp/awscliv2.zip -d /tmp/aws-install
+            sudo /tmp/aws-install/aws/install
+            rm -rf /tmp/awscliv2.zip /tmp/aws-install
+          fi
+
+      - name: Upload ISO to Cloudflare R2
+        id: upload
+        if: steps.build-iso.outcome == 'success' && steps.wait-verify.outcome == 'success'
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
+          VERSION: ${{ github.ref_name }}
+        run: |
+          aws --endpoint-url=https://e80e329f7dbcf75b80e019c1dce640f2.r2.cloudflarestorage.com \
+            s3 cp "mulga/scripts/iso-builder/dist/spinifex-${VERSION}.iso" \
+                  "s3://spinifex-iso/spinifex-${VERSION}.iso"
+          aws --endpoint-url=https://e80e329f7dbcf75b80e019c1dce640f2.r2.cloudflarestorage.com \
+            s3 cp "mulga/scripts/iso-builder/dist/spinifex-${VERSION}.iso.sha256" \
+                  "s3://spinifex-iso/spinifex-${VERSION}.iso.sha256"
+
+      - name: Mark ISO failed
+        if: always() && (steps.build-iso.outcome == 'failure' || steps.upload.outcome == 'failure')
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ github.ref_name }}
+          REPO: ${{ github.repository }}
+        run: |
+          EXISTING_BODY=$(gh release view "$TAG" --repo "$REPO" --json body -q .body)
+          NOTES_FILE=$(mktemp)
+          printf 'ISO build/upload failed — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
+          gh release edit "$TAG" --repo "$REPO" --notes-file "$NOTES_FILE"
+          rm -f "$NOTES_FILE"
@@ -17,6 +17,9 @@ on:
      - dev
  workflow_dispatch:

+permissions:
+  contents: read
+
 jobs:
  build_and_test:
    name: Build and Test
@@ -33,12 +36,14 @@ jobs:
        uses: actions/checkout@v6
        with:
          repository: mulgadc/viperblock
+          ref: dev
          path: viperblock

      - name: Checkout Predastore
        uses: actions/checkout@v6
        with:
          repository: mulgadc/predastore
+          ref: dev
          path: predastore

      - name: Setup Go
@@ -73,12 +78,14 @@ jobs:
        uses: actions/checkout@v6
        with:
          repository: mulgadc/viperblock
+          ref: dev
          path: viperblock

      - name: Checkout Predastore
        uses: actions/checkout@v6
        with:
          repository: mulgadc/predastore
+          ref: dev
          path: predastore

      - name: Setup Go
@@ -107,12 +114,14 @@ jobs:
        uses: actions/checkout@v6
        with:
          repository: mulgadc/viperblock
+          ref: dev
          path: viperblock

      - name: Checkout Predastore
        uses: actions/checkout@v6
        with:
          repository: mulgadc/predastore
+          ref: dev
          path: predastore

      - name: Setup Go
@@ -17,6 +17,9 @@ on:
      - .github/workflows/ui.yml
  workflow_dispatch:

+permissions:
+  contents: read
+
 jobs:
  ui_build_and_test:
    name: UI Build and Test
@@ -46,8 +49,8 @@ jobs:
      - name: Install Dependencies
        run: pnpm install --frozen-lockfile

-      - name: Run Check
-        run: pnpm check
+      - name: Run Lint
+        run: pnpm lint

      - name: Run Tests with Coverage
        run: pnpm test:coverage
@@ -17,6 +17,8 @@ go.work.sum
 # Build outputs
 bin/
 dist/
+build/microvm/vmlinuz
+build/microvm/initramfs.cpio.gz
 !spinifex/services/spinifexui/frontend/dist/
 tmp/
 *.test
@@ -63,15 +63,15 @@ LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSI

 go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
-	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	GOFIPS140=v1.0.0 go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go

 build-installer:
 	@echo -e "\n....Building spinifex-installer"
-	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
+	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go

 build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
-	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
+	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go

 build-system-image: ## Build a system image from manifest (use IMAGE=lb)
 ifndef IMAGE
@@ -79,8 +79,25 @@ ifndef IMAGE
 endif
 	./scripts/build-system-image.sh scripts/images/$(IMAGE).conf

-build-lb-image: ## Build LB Alpine image
-	$(MAKE) build-system-image IMAGE=lb
+MICROVM_OUT_DIR := build/microvm
+MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
+MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
+
+# Grouped target — script writes both files in one run.
+$(MICROVM_ARTIFACTS) &: $(MICROVM_INPUTS)
+	./scripts/build-microvm-image.sh
+
+# Only triggers when bin/lb-agent is missing; preserves the artifact's mtime so
+# build-microvm-image stays correctly stale-aware.
+bin/lb-agent:
+	$(MAKE) build-lb-agent
+
+build-microvm-image: $(MICROVM_ARTIFACTS) ## Build microVM kernel + initramfs (incremental — skips when up to date)
+
+install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
+	sudo install -d /usr/share/spinifex/microvm
+	sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
+	sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz

 go_run:
 	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
@@ -122,9 +139,18 @@ run:
 	$(MAKE) go_build
 	$(MAKE) go_run

-# Fast iteration: build + install binary + restart all services
+# Fast iteration: build + install binary + restart all services.
+# Microvm artifacts are reinstalled when they already exist on disk — the rule's
+# input timestamps drive a rebuild only if anything actually changed. On a fresh
+# checkout (no build/microvm/vmlinuz yet) the install step is skipped; run
+# `make install-microvm` explicitly the first time.
 deploy: build
 	sudo install -m 755 bin/spx /usr/local/bin/spx
+	@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
+		$(MAKE) install-microvm; \
+	else \
+		echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
+	fi
 	sudo systemctl daemon-reload
 	sudo systemctl restart spinifex.target

@@ -149,9 +175,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base

 install-go:
-	@echo -e "\n....Installing Go 1.26.2 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.3 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.2.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -206,6 +232,14 @@ distro-amd64:
 		-f build/Dockerfile.distro \
 		--output type=local,dest=dist/amd64/ \
 		../
+	@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ]; then \
+		echo "[distro-amd64] staging microvm artifacts into tarball"; \
+		mkdir -p dist/amd64/microvm; \
+		cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz dist/amd64/microvm/; \
+	else \
+		echo "[distro-amd64] WARNING: microvm artifacts missing — tarball will not include them"; \
+		echo "[distro-amd64]          run 'make build-microvm-image' before 'make distro-amd64'"; \
+	fi
 	tar -czf dist/spinifex-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
 	sha256sum dist/spinifex-$(VERSION)-linux-amd64.tar.gz > dist/spinifex-$(VERSION)-linux-amd64.tar.gz.sha256

@@ -238,9 +272,12 @@ ansible-dev-install:
 ansible-dev-reset:
 	cd scripts/ansible && ansible-playbook playbooks/dev-reset.yml

-.PHONY: build build-ui build-installer build-lb-agent build-system-image build-lb-image go_build go_run preflight test test-cover test-race diff-coverage bench run \
+ansible-dev-deploy:
+	cd scripts/ansible && ansible-playbook playbooks/dev-deploy.yml
+
+.PHONY: build build-ui build-installer build-lb-agent build-system-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck \
 	distro distro-amd64 distro-arm64 distro-clean \
-	ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset
+	ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset ansible-dev-deploy
@@ -93,9 +93,24 @@ AWS-Compatible Interfaces – Provision infrastructure with awscli, Terraform, o

 Installation requires an Ubuntu / Debian system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex.

+### Bare Metal ISO
+
+The recommended installation is a [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) for bare-metal hardware.
+
+```bash
+curl -fLO https://iso.mulgadc.com/spinifex.iso
+```
+
+Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware. The install guide walks through the full process.
+
+
 ### Single Node Install

-The installation is straightforward to set up and running on a single node for testing purposes.
+The installation is straightforward to set up and running on a single node for testing purposes. Debian 13 is currently supported, additional Linux distributions are on the immediate roadmap.
+
+>*Prerequisite:* Linux bridge for networking.
+
+Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) prerequisites for setup details.

 ```bash
 curl -fsSL https://install.mulgadc.com | bash
@@ -30,13 +30,13 @@ COPY viperblock/ ./viperblock/

 # Build spinifex binary (spx)
 RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \
-    cd spinifex && go build -ldflags "-s -w \
+    cd spinifex && GOFIPS140=v1.0.0 go build -ldflags "-s -w \
    -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=${VERSION} \
    -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=${COMMIT}" \
    -o /out/spx cmd/spinifex/main.go

 # Build nbdkit viperblock plugin (.so)
-RUN cd viperblock && go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-shared nbd/viperblock.go
+RUN cd viperblock && GOFIPS140=v1.0.0 go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-shared nbd/viperblock.go

 # Copy setup scripts and systemd units
 RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
@@ -0,0 +1,56 @@
+#!/bin/sh
+mount -t proc  proc  /proc
+mount -t sysfs sys   /sys
+mount -t devtmpfs dev /dev
+
+# virtio_mmio is built-in (CONFIG_VIRTIO_MMIO=y) so virtio-net devices are
+# already enumerated by the time PID 1 runs — no modprobe needed.
+# qemu_fw_cfg and virtio_net are =m in stock Alpine linux-virt, so load them.
+modprobe qemu_fw_cfg 2>/dev/null || modprobe fw_cfg_sysfs 2>/dev/null || true
+modprobe virtio_net 2>/dev/null || true
+
+for i in 1 2 3 4 5; do
+    [ -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ] && break
+    sleep 0.02
+done
+
+if [ ! -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ]; then
+    echo "[init] FATAL: fw_cfg netcfg not available" >&2
+    exit 1
+fi
+
+. /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
+
+for n in 0 1 2 3 4 5; do
+    eval mac=\$NIC${n}_MAC
+    [ -z "$mac" ] && continue
+
+    iface=""
+    for attempt in 1 2 3 4 5; do
+        iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
+        [ -n "$iface" ] && break
+        sleep 0.02
+    done
+
+    if [ -z "$iface" ]; then
+        echo "[init] WARNING: no interface found for MAC $mac (NIC$n), skipping" >&2
+        continue
+    fi
+
+    eval cidr=\$NIC${n}_CIDR
+    eval gw=\$NIC${n}_GW
+    eval default=\$NIC${n}_DEFAULT
+    eval rdst=\$NIC${n}_ROUTE_DST
+    eval rvia=\$NIC${n}_ROUTE_VIA
+
+    ip link set "$iface" up
+    [ -n "$cidr" ] && ip addr add "$cidr" dev "$iface"
+    [ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface"
+    [ -n "$rdst" ] && [ -n "$rvia" ] && ip route add "$rdst" via "$rvia" dev "$iface"
+done
+
+cp /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/lb-agent-env/raw /etc/conf.d/lb-agent
+mkdir -p /etc/ssl/certs
+cp /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw /etc/ssl/certs/ca-certificates.crt
+
+exec /sbin/init
@@ -0,0 +1,4 @@
+::sysinit:/etc/rc.local
+::respawn:/bin/sh -c 'set -a; . /etc/conf.d/lb-agent; set +a; exec /usr/local/bin/lb-agent'
+::ctrlaltdel:/sbin/reboot
+::shutdown:/bin/umount -a -r
@@ -5,17 +5,20 @@

 CONF=/etc/spinifex/spinifex.toml
 BIND="0.0.0.0:8443"
-NODE_ID="0"
+# -1 = co-located mode (all [[db]] peers in one process). Single-node spinifex
+# templates list 3 DB peers on 127.0.0.1; only -1 can serve that topology.
+# Multi-node spinifex.toml emits node_id = N (N >= 1) which overrides below.
+# predastore rejects NODE_ID=0 since d3a02f0 (strconv.Atoi-of-garbage guard).
+NODE_ID="-1"

 if [ -f "$CONF" ]; then
    H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
    if [ -n "$H" ]; then
        BIND="$H"
-        BACKEND="distributed"
    fi

    N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
-    if [ -n "$N" ] && [ "$N" != "0" ]; then
+    if [ -n "$N" ]; then
        NODE_ID="$N"
    fi
 fi
@@ -0,0 +1,12 @@
+[Unit]
+Description=Regenerate SSH host keys
+ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key
+Before=ssh.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/ssh-keygen -A
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
@@ -2,27 +2,67 @@
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service
-Wants=spinifex-nats.service
+# No Wants=spinifex-nats.service: Wants= reactivates a stopped unit during
+# `systemctl restart spinifex-daemon`, which would mask the standalone-startup
+# path the Tier 1 DDIL Scenario B exercises. spinifex.target Wants= covers
+# cold boot.

 [Service]
 Type=simple
 User=spinifex-daemon
 Group=spinifex
-ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+# DDIL Tier 1 (siv-17): no ExecStartPre wait-for-nats. The daemon starts in
+# standalone mode and reconnects to NATS in the background — process-exiting
+# on a missing NATS would kill the local VM management plane.
+#
+# Opt-in strict-startup (§1d-strict, siv-44): set SPINIFEX_REQUIRE_NATS=1 to
+# restore pre-DDIL fail-fast UX. First NATS connect uses a ~30s bounded retry
+# and the daemon aborts on timeout. Intended for dev/test/single-node deploys
+# where silent NATS misconfig is more confusing than helpful. Do NOT set this
+# in the shipped unit — it re-introduces the SPOF that Tier 1 removed.
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5
+# DDIL Tier 1 (mulga-siv-58): KillMode=process so systemd only signals the
+# daemon's main PID, not the cgroup. QEMU children stay alive across
+# `systemctl restart spinifex-daemon` and the new daemon reattaches to them
+# via the persisted local state file + pidfile + QMP socket. With the
+# default control-group mode, restart would terminate every guest VM —
+# defeating Scenario B's "daemon restart preserves running instances".
+KillMode=process
+# 15s upper bound on the SIGTERM grace window. The new SIGTERM handler
+# skips VM teardown (it now just persists state and exits), so 15s is
+# plenty of slack for the cluster-manager + NATS unsubscribe path.
+TimeoutStopSec=15
 OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
+# DDIL Tier 1 (mulga-siv-58): QEMU pidfiles, QMP sockets, and the
+# shared qemu.log all resolve via utils.pidPath() — which falls back
+# to os.TempDir() (=/tmp) when $HOME/spinifex doesn't exist. With
+# PrivateTmp=yes, each daemon start gets a fresh /tmp namespace, so
+# pidfiles written by QEMU (running in the OLD daemon's namespace)
+# are invisible to the new daemon. The /local/instances handler then
+# reports PID=0 and Scenario B fails its "live QEMU PID" assertion.
+# Pin XDG_RUNTIME_DIR to the host-wide tmpfs at /run/spinifex (created
+# by tmpfiles.d, already in ReadWritePaths) so pidPath() returns a
+# stable, namespace-independent path that both daemon incarnations and
+# QEMU children resolve identically.
+Environment=XDG_RUNTIME_DIR=/run/spinifex

-# Filesystem access — NoNewPrivileges omitted (daemon uses sudo for ip, ovs-vsctl)
+# Filesystem access — NoNewPrivileges omitted (daemon uses sudo for ip, ovs-vsctl, dhcpcd)
+# CAP_SYS_ADMIN: required for GPU passthrough sysfs operations (vfio group management).
+# CAP_DAC_OVERRIDE: sysfs driver bind/unbind paths (/sys/bus/pci/drivers/vfio-pci/bind,
+# unbind) are root:root 0200; non-root processes need CAP_DAC_OVERRIDE to open them.
+AmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE
 ProtectSystem=full
 PrivateDevices=no
 DevicePolicy=closed
 DeviceAllow=/dev/kvm rw
 DeviceAllow=/dev/net/tun rw
+DeviceAllow=char-vfio rw
+DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
 ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd

@@ -50,6 +90,9 @@ RemoveIPC=yes
 LimitCORE=0
 LimitNOFILE=65536
 LimitNPROC=4096
+# VFIO GPU passthrough requires locking all guest RAM into physical memory.
+# Without this, QEMU fails with RLIMIT_MEMLOCK exceeded when pinning guest pages.
+LimitMEMLOCK=infinity

 [Install]
 WantedBy=spinifex.target
@@ -17,6 +17,7 @@ Environment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.
 Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
 Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key
+Environment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key

 # Filesystem access
 NoNewPrivileges=yes
@@ -0,0 +1,5 @@
+# Allow spinifex group to access VFIO devices for GPU passthrough.
+# QEMU (launched by spinifex-daemon, group spinifex) must open /dev/vfio/N
+# to use vfio-pci passthrough devices. The kernel creates these nodes as
+# root:root 0600 by default; this rule grants group-level read-write access.
+SUBSYSTEM=="vfio", GROUP="spinifex", MODE="0660"
@@ -0,0 +1,172 @@
+// dhcptest is a diagnostic tool that runs a DHCP DORA handshake using the
+// same nclient4 library and options that spinifex-vpcd uses for gateway pool
+// leases. Run it on a node to verify DHCP works at the library level before
+// debugging the full spinifex stack.
+//
+// Usage (must run as root — needs AF_PACKET):
+//
+//	sudo dhcptest --iface=br-wan
+//	sudo dhcptest --iface=br-wan --mac=02:00:00:7c:b7:b9 --promisc
+//	sudo dhcptest --iface=br-wan --spinifex-id=gateway-wan1
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/insomniacslk/dhcp/dhcpv4"
+	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
+)
+
+func main() {
+	iface := flag.String("iface", "br-wan", "Interface/bridge to bind the DHCP client to")
+	macStr := flag.String("mac", "", "Hardware address for DORA (chaddr + AF_PACKET filter). If empty, uses the real interface MAC.")
+	spinifexID := flag.String("spinifex-id", "", "Generate a spinifex virtual MAC from this resource ID (e.g. 'gateway-wan1'). Overrides --mac.")
+	clientID := flag.String("client-id", "", "DHCP option 61 client identifier. Defaults to --spinifex-id or --mac value.")
+	hostname := flag.String("hostname", "", "DHCP option 12 hostname")
+	vendorClass := flag.String("vendor-class", "mulga-spinifex-gw", "DHCP option 60 vendor class identifier")
+	promisc := flag.Bool("promisc", false, "Enable promiscuous mode on the interface before the test (mirrors vpcd's setBridgePromisc)")
+	timeout := flag.Duration("timeout", 10*time.Second, "Per-attempt DHCP timeout")
+	retries := flag.Int("retries", 3, "Number of DORA attempts before giving up")
+	verbose := flag.Bool("verbose", false, "Print full DHCP packet contents")
+	flag.Parse()
+
+	// Resolve the hardware address to use.
+	var hwAddr net.HardwareAddr
+	var err error
+
+	switch {
+	case *spinifexID != "":
+		hwAddr, err = net.ParseMAC(generateMAC(*spinifexID))
+		if err != nil {
+			log.Fatalf("generateMAC(%q): %v", *spinifexID, err)
+		}
+		fmt.Printf("spinifex virtual MAC for %q: %s\n", *spinifexID, hwAddr)
+	case *macStr != "":
+		hwAddr, err = net.ParseMAC(*macStr)
+		if err != nil {
+			log.Fatalf("parse --mac %q: %v", *macStr, err)
+		}
+	default:
+		hwAddr, err = ifaceMAC(*iface)
+		if err != nil {
+			log.Fatalf("get MAC for %s: %v", *iface, err)
+		}
+		fmt.Printf("using real interface MAC: %s\n", hwAddr)
+	}
+
+	// Resolve client-id: default to spinifex-id, then MAC string.
+	cid := *clientID
+	if cid == "" {
+		if *spinifexID != "" {
+			cid = *spinifexID
+		} else {
+			cid = hwAddr.String()
+		}
+	}
+
+	fmt.Printf("iface:        %s\n", *iface)
+	fmt.Printf("hw-addr:      %s\n", hwAddr)
+	fmt.Printf("client-id:    %s\n", cid)
+	fmt.Printf("hostname:     %s\n", *hostname)
+	fmt.Printf("vendor-class: %s\n", *vendorClass)
+	fmt.Printf("promisc:      %v\n", *promisc)
+	fmt.Printf("timeout:      %s  retries: %d\n", *timeout, *retries)
+	fmt.Println()
+
+	if *promisc {
+		if err := setPromisc(*iface, true); err != nil {
+			log.Printf("WARNING: failed to set promisc on %s: %v", *iface, err)
+		} else {
+			fmt.Printf("set %s promisc ON\n\n", *iface)
+		}
+	}
+
+	// Build nclient4 options — identical to spinifex/services/vpcd/dhcp/nclient4.go.
+	opts := []nclient4.ClientOpt{
+		nclient4.WithHWAddr(hwAddr),
+		nclient4.WithTimeout(*timeout),
+		nclient4.WithRetry(*retries),
+	}
+	if *verbose {
+		opts = append(opts, nclient4.WithLogger(nclient4.ShortSummaryLogger{Printfer: log.New(os.Stderr, "[nclient4] ", 0)}))
+	}
+
+	client, err := nclient4.New(*iface, opts...)
+	if err != nil {
+		log.Fatalf("nclient4.New(%s): %v", *iface, err)
+	}
+	defer client.Close()
+
+	// Identity modifiers — identical to identityModifiers() in vpcd dhcp package.
+	var mods []dhcpv4.Modifier
+	if cid != "" {
+		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(cid))))
+	}
+	if *hostname != "" {
+		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptHostName(*hostname)))
+	}
+	if *vendorClass != "" {
+		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClassIdentifier(*vendorClass)))
+	}
+
+	fmt.Println("Sending DHCP Discover...")
+	start := time.Now()
+	ctx, cancel := context.WithTimeout(context.Background(), *timeout*time.Duration(*retries+1))
+	defer cancel()
+
+	lease, err := client.Request(ctx, mods...)
+	if err != nil {
+		fmt.Printf("\nFAILED after %s: %v\n", time.Since(start).Round(time.Millisecond), err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("SUCCESS in %s\n\n", time.Since(start).Round(time.Millisecond))
+	ack := lease.ACK
+	fmt.Printf("  Your IP:     %s\n", ack.YourIPAddr)
+	fmt.Printf("  Subnet:      %s\n", ack.SubnetMask())
+	fmt.Printf("  Gateway:     %v\n", ack.Router())
+	fmt.Printf("  DNS:         %v\n", ack.DNS())
+	fmt.Printf("  Server ID:   %s\n", ack.ServerIdentifier())
+	fmt.Printf("  Lease time:  %s\n", ack.IPAddressLeaseTime(24*time.Hour))
+	if *verbose {
+		fmt.Printf("\n--- Full ACK ---\n%s\n", ack.Summary())
+	}
+}
+
+// generateMAC is the same deterministic hash used by vpcd's topology.go.
+// It produces a locally-administered MAC (02:00:00:xx:xx:xx) from a resource ID.
+func generateMAC(resourceID string) string {
+	h := uint32(0)
+	for _, c := range resourceID {
+		h = h*31 + uint32(c)
+	}
+	return fmt.Sprintf("02:00:00:%02x:%02x:%02x", (h>>16)&0xff, (h>>8)&0xff, h&0xff)
+}
+
+func ifaceMAC(name string) (net.HardwareAddr, error) {
+	ifc, err := net.InterfaceByName(name)
+	if err != nil {
+		return nil, err
+	}
+	return ifc.HardwareAddr, nil
+}
+
+func setPromisc(iface string, on bool) error {
+	state := "on"
+	if !on {
+		state = "off"
+	}
+	out, err := exec.Command("ip", "link", "set", iface, "promisc", state).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(out)))
+	}
+	return nil
+}
@@ -134,12 +134,21 @@ func buildConfig() (*install.Config, error) {
 		}
 	}

+	if os.Getenv("SPINIFEX_GPU_PASSTHROUGH") == "1" {
+		cfg.GPUPassthrough = true
+	}
+
 	role := strings.ToLower(os.Getenv("SPINIFEX_ROLE"))
 	if role == "" {
 		role = "init"
 	}
 	cfg.ClusterRole = role
-	if role == "join" {
+	skipFormation := os.Getenv("SPINIFEX_SKIP_FORMATION") == "1"
+	if skipFormation {
+		cfg.SkipFormation = true
+	}
+
+	if role == "join" && !skipFormation {
 		joinAddr := os.Getenv("SPINIFEX_JOIN_ADDR")
 		if joinAddr == "" {
 			return nil, fmt.Errorf("SPINIFEX_JOIN_ADDR required when SPINIFEX_ROLE=join")
@@ -0,0 +1,17 @@
+package autoinstall
+
+import (
+	"os"
+	"testing"
+)
+
+func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
+	os.Unsetenv("SPINIFEX_AUTO")
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if cfg != nil {
+		t.Fatal("expected nil config when SPINIFEX_AUTO is not set")
+	}
+}
@@ -43,6 +43,19 @@ type Config struct {
 	// the headless path. Passed to `spx admin init --email=<value>` when set;
 	// omitted entirely when empty.
 	Email string
+	// GPUPassthrough enables VFIO GPU passthrough by passing --gpu-passthrough
+	// to `spx admin init`, which writes gpu_passthrough = true in the daemon config.
+	GPUPassthrough bool
+	// InstallCallback, when non-empty, is curled once at the end of firstboot
+	// after the success marker is written. Generic phone-home hook for
+	// provisioning controllers (PXE/MAAS-style flows). Best-effort: non-2xx
+	// and network failures are swallowed so the install is not gated on
+	// controller reachability.
+	InstallCallback string
+	// SkipFormation skips spx admin init/join. Used when a provisioning
+	// controller (e.g. bm-bootstrap.sh) owns cluster formation and will call
+	// spx admin init/join itself with the correct multi-node parameters.
+	SkipFormation bool
 }

 // Write drops the firstboot script and systemd unit into root, which should be
@@ -78,6 +91,14 @@ func writeScript(root string, cfg Config) error {
 		setupOVN += fmt.Sprintf(" --encap-ip=%s", cfg.EncapIP)
 	}

+	callbackBlock := ""
+	if cfg.InstallCallback != "" {
+		callbackBlock = fmt.Sprintf(
+			"\ncurl -fsS --max-time 10 --retry 2 --retry-delay 2 -o /dev/null %s || true\n",
+			shellEscapeSingle(cfg.InstallCallback),
+		)
+	}
+
 	script := fmt.Sprintf(`#!/bin/bash
 # Spinifex firstboot — runs once after ISO installation. On success, writes
 # /var/lib/spinifex/.firstboot-done and the systemd unit's
@@ -146,6 +167,18 @@ SETUP_STAGES=fixown /usr/local/share/spinifex/setup.sh
 # setup.sh's create_directories — it resolves to /etc/spinifex, so
 # {BaseDir}/config/master.key automatically points at /etc/spinifex/master.key.

+# Bootstrap the spinifex user's SSH directory with a generated key pair so
+# ~/.ssh exists and the operator can immediately use ssh-keygen / ssh-copy-id
+# without hitting "No such file or directory" errors documented in the setup guide.
+if [ ! -f /home/spinifex/.ssh/id_ed25519 ]; then
+    mkdir -p /home/spinifex/.ssh
+    chmod 700 /home/spinifex/.ssh
+    ssh-keygen -t ed25519 -f /home/spinifex/.ssh/id_ed25519 -N "" -C "spinifex@$(hostname)"
+    chown -R spinifex:spinifex /home/spinifex/.ssh
+    chmod 600 /home/spinifex/.ssh/id_ed25519
+    chmod 644 /home/spinifex/.ssh/id_ed25519.pub
+fi
+
 # Copy AWS credentials to the spinifex user's home directory.
 # spx admin init runs with HOME=/root (set by the systemd unit), so credentials
 # land in /root/.aws/. Copy them to the spinifex user's home so the operator
@@ -176,7 +209,7 @@ systemctl enable spinifex.target spinifex-banner.service
 # firstboot from the top.
 mkdir -p "$(dirname "$DONE_MARKER")"
 touch "$DONE_MARKER"
-
+%s
 systemctl start spinifex.target

 # Wait for the daemon to bring up external networking. When external_mode=pool
@@ -196,13 +229,16 @@ if grep -q 'external_mode.*pool' /etc/spinifex/spinifex.toml 2>/dev/null; then
        echo "[firstboot] warning: br-ext not up after 30s — external networking may be delayed"
    fi
 fi
-`, cfg.Hostname, setupOVN, clusterCmd)
+`, cfg.Hostname, setupOVN, clusterCmd, callbackBlock)

 	path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
 	return os.WriteFile(path, []byte(script), 0o755)
 }

 func buildClusterCmd(cfg Config) string {
+	if cfg.SkipFormation {
+		return `echo "[firstboot] cluster formation skipped — provisioning controller will handle it"`
+	}
 	emailFlag := ""
 	if cfg.Email != "" {
 		// shellEscapeSingle keeps the email safe if it ever contains a
@@ -210,11 +246,15 @@ func buildClusterCmd(cfg Config) string {
 		// validator already rejects whitespace and @-chains.
 		emailFlag = " --email=" + shellEscapeSingle(cfg.Email)
 	}
+	gpuFlag := ""
+	if cfg.GPUPassthrough {
+		gpuFlag = " --gpu-passthrough"
+	}
 	switch cfg.ClusterRole {
 	case "join":
 		return fmt.Sprintf("spx admin join --node %s --host %s%s", cfg.Hostname, cfg.JoinAddr, emailFlag)
 	default:
-		return fmt.Sprintf("spx admin init --node %s --nodes 1%s", cfg.Hostname, emailFlag)
+		return fmt.Sprintf("spx admin init --node %s --nodes 1%s%s", cfg.Hostname, emailFlag, gpuFlag)
 	}
 }

@@ -0,0 +1,100 @@
+package firstboot
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func makeRootDirs(t *testing.T, root string) {
+	t.Helper()
+	for _, d := range []string{
+		"usr/local/bin",
+		"etc/systemd/system",
+		"etc/systemd/system/multi-user.target.wants",
+	} {
+		if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
+			t.Fatalf("mkdir %s: %v", d, err)
+		}
+	}
+}
+
+func TestWriteScriptNoCallbackWhenEmpty(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+
+	script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
+	if err != nil {
+		t.Fatalf("read script: %v", err)
+	}
+	if strings.Contains(string(script), "curl") {
+		t.Error("script should not contain curl when InstallCallback is empty")
+	}
+}
+
+func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
+	cfg := Config{
+		Hostname:        "test-node",
+		ClusterRole:     "init",
+		InstallCallback: callbackURL,
+	}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+
+	script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
+	if err != nil {
+		t.Fatalf("read script: %v", err)
+	}
+	content := string(script)
+
+	if !strings.Contains(content, "curl") {
+		t.Error("script missing curl command")
+	}
+	if !strings.Contains(content, callbackURL) {
+		t.Errorf("script missing callback URL %q", callbackURL)
+	}
+}
+
+func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
+	cfg := Config{
+		Hostname:        "node1",
+		ClusterRole:     "init",
+		InstallCallback: callbackURL,
+	}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+
+	script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
+	if err != nil {
+		t.Fatalf("read script: %v", err)
+	}
+	content := string(script)
+
+	doneIdx := strings.Index(content, "touch \"$DONE_MARKER\"")
+	curlIdx := strings.Index(content, "curl")
+	if doneIdx < 0 {
+		t.Fatal("done marker not found in script")
+	}
+	if curlIdx < 0 {
+		t.Fatal("curl not found in script")
+	}
+	if curlIdx < doneIdx {
+		t.Error("curl must appear after done marker write")
+	}
+}
@@ -55,6 +55,10 @@ type Config struct {
 	// JoinAddr is the primary node address (host:port) when ClusterRole is "join".
 	JoinAddr string

+	// SkipFormation skips spx admin init/join in firstboot; the provisioning
+	// controller (e.g. bm-bootstrap.sh) owns cluster formation instead.
+	SkipFormation bool
+
 	// CA certificate (PEM), optional.
 	HasCACert bool
 	CACert    string
@@ -67,4 +71,9 @@ type Config struct {
 	// Required on interactive installs; may be empty on headless/CI installs
 	// when SPINIFEX_EMAIL is not supplied on the kernel cmdline.
 	Email string
+
+	// GPUPassthrough enables VFIO GPU passthrough on this node.
+	// Set via SPINIFEX_GPU_PASSTHROUGH=1 on headless installs.
+	// Passes --gpu-passthrough to `spx admin init` during firstboot.
+	GPUPassthrough bool
 }
@@ -23,6 +23,7 @@ import (
 	"io"
 	"log/slog"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -77,6 +78,7 @@ func Run(cfg *Config) error {
 	}

 	slog.Info("installation complete")
+	fireInstallCallback()
 	promptRemoveUSB()
 	return reboot()
 }
@@ -172,6 +174,7 @@ func copyRootfs() error {
 		"--exclude=/etc/openvswitch/",
 		"--exclude=/var/lib/openvswitch/",
 		"--exclude=/var/lib/dhcpcd/",
+		"--exclude=/etc/ssh/ssh_host_*",
 		"--exclude=/lost+found",
 		"--exclude=/boot/efi",
 		"/", mountRoot+"/",
@@ -555,6 +558,11 @@ func installBootloader(disk string) error {
 		}
 		return biosErr
 	}
+	// Copy splash image and unicode font from the ISO (mounted at /cdrom) so the
+	// installed GRUB shows the same branded background as the installer GRUB.
+	// The font must be at /boot/grub/fonts/unicode.pf2 so update-grub finds it
+	// there and emits the same loadfont path as the ISO's grub.cfg — GRUB 2.12
+	// (trixie) needs the font in the boot partition, not just /usr/share/grub/.
 	copySplashImage(mountRoot)
 	copyGrubFont(mountRoot)

@@ -644,6 +652,26 @@ func promptRemoveUSB() {
 	}
 }

+// fireInstallCallback notifies the boot controller that installation is done
+// so it clears the PXE install flag before the node reboots. Without this,
+// a PXE-first boot order causes the node to reinstall on every reboot until
+// firstboot fires the callback — which it can never do if it never runs.
+// No-op when SPINIFEX_INSTALL_CALLBACK is not set (ISO/USB installs).
+func fireInstallCallback() {
+	url := strings.TrimSpace(os.Getenv("SPINIFEX_INSTALL_CALLBACK"))
+	if url == "" {
+		return
+	}
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Get(url) //nolint:noctx // installer has no context; best-effort fire-and-forget
+	if err != nil {
+		slog.Warn("fireInstallCallback: request failed", "url", url, "err", err)
+		return
+	}
+	resp.Body.Close()
+	slog.Info("fireInstallCallback: notified boot controller", "url", url, "status", resp.StatusCode)
+}
+
 func reboot() error {
 	// sync filesystems before reboot so nothing is lost.
 	_ = run("sync")
@@ -662,11 +690,13 @@ func (c *Config) toFirstbootConfig() firstboot.Config {
 		encapIP = c.LANAddress
 	}
 	return firstboot.Config{
-		Hostname:    c.Hostname,
-		EncapIP:     encapIP,
-		ClusterRole: c.ClusterRole,
-		JoinAddr:    c.JoinAddr,
-		Email:       c.Email,
+		Hostname:        c.Hostname,
+		EncapIP:         encapIP,
+		ClusterRole:     c.ClusterRole,
+		JoinAddr:        c.JoinAddr,
+		Email:           c.Email,
+		InstallCallback: strings.TrimSpace(os.Getenv("SPINIFEX_INSTALL_CALLBACK")),
+		SkipFormation:   c.SkipFormation,
 	}
 }

@@ -712,9 +742,46 @@ func partitionPaths(disk string) (efi, root string) {
 	return disk + "2", disk + "3"
 }

-// copySplashImage copies the GRUB splash (embedded in the squashfs at build time by
-// inject-bins.sh) into the installed system so the post-install GRUB shows the same
-// branded background as the installer GRUB. Non-fatal — missing source is logged and skipped.
+// copyGrubFont copies the unicode.pf2 GRUB font into the installed system's
+// /boot/grub/fonts/ directory. This ensures update-grub finds the font at
+// /boot/grub/fonts/unicode.pf2 — the same path the ISO's grub.cfg uses —
+// so the generated grub.cfg enables gfxterm and the background image.
+// Without this, grub-mkconfig falls back to /usr/share/grub/unicode.pf2,
+// a path that GRUB 2.12 (trixie) may fail to resolve at boot time.
+// Non-fatal — a missing source is logged and skipped.
+func copyGrubFont(root string) {
+	candidates := []string{
+		"/cdrom/boot/grub/fonts/unicode.pf2", // ISO tree (preferred)
+		"/usr/share/grub/unicode.pf2",        // live system grub-common fallback
+	}
+	for _, src := range candidates {
+		in, err := os.Open(src)
+		if err != nil {
+			continue
+		}
+		defer in.Close()
+		dstDir := filepath.Join(root, "boot/grub/fonts")
+		if err := os.MkdirAll(dstDir, 0o755); err != nil {
+			slog.Warn("copyGrubFont: cannot create fonts dir", "err", err)
+			return
+		}
+		out, err := os.OpenFile(filepath.Join(dstDir, "unicode.pf2"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
+		if err != nil {
+			slog.Warn("copyGrubFont: cannot open destination", "err", err)
+			return
+		}
+		defer out.Close()
+		if _, err := io.Copy(out, in); err != nil {
+			slog.Warn("copyGrubFont: copy failed", "err", err)
+		}
+		return
+	}
+	slog.Warn("copyGrubFont: no unicode.pf2 found, splash may not display")
+}
+
+// copySplashImage copies the GRUB splash from the live ISO (/cdrom/boot/grub/splash.png)
+// into the installed system so the post-install GRUB shows the same branded background
+// as the installer. Non-fatal — a missing or unreadable source is logged and skipped.
 func copySplashImage(root string) {
 	const src = "/usr/share/spinifex/grub-splash.png"
 	in, err := os.Open(src)
@@ -740,34 +807,6 @@ func copySplashImage(root string) {
 	}
 }

-// copyGrubFont copies the unicode font into the installed system's
-// /boot/grub/fonts/ so the loadfont path in 05_spinifex resolves at boot.
-// grub-install does not copy fonts; we mirror what build-iso.sh does.
-func copyGrubFont(root string) {
-	const src = "/usr/share/grub/unicode.pf2"
-	in, err := os.Open(src)
-	if err != nil {
-		slog.Warn("copyGrubFont: font not found, graphical GRUB may not work", "path", src)
-		return
-	}
-	defer in.Close()
-
-	dstDir := filepath.Join(root, "boot/grub/fonts")
-	if err := os.MkdirAll(dstDir, 0o755); err != nil {
-		slog.Warn("copyGrubFont: cannot create fonts dir", "err", err)
-		return
-	}
-	out, err := os.OpenFile(filepath.Join(dstDir, "unicode.pf2"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
-	if err != nil {
-		slog.Warn("copyGrubFont: cannot open destination", "err", err)
-		return
-	}
-	defer out.Close()
-	if _, err := io.Copy(out, in); err != nil {
-		slog.Warn("copyGrubFont: copy failed", "err", err)
-	}
-}
-
 // maskSystemdUnit creates a symlink to /dev/null for the given unit, which is
 // the standard way to permanently disable a unit so systemd will never start it.
 func maskSystemdUnit(root, unit string) error {
@@ -0,0 +1,27 @@
+package install
+
+import (
+	"os"
+	"testing"
+)
+
+func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
+	const url = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
+	t.Setenv("SPINIFEX_INSTALL_CALLBACK", url)
+
+	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	fb := cfg.toFirstbootConfig()
+	if fb.InstallCallback != url {
+		t.Errorf("InstallCallback = %q, want %q", fb.InstallCallback, url)
+	}
+}
+
+func TestToFirstbootConfigEmptyCallbackWhenEnvUnset(t *testing.T) {
+	os.Unsetenv("SPINIFEX_INSTALL_CALLBACK")
+
+	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	fb := cfg.toFirstbootConfig()
+	if fb.InstallCallback != "" {
+		t.Errorf("InstallCallback = %q, want empty", fb.InstallCallback)
+	}
+}
@@ -24,6 +24,7 @@ import (
 	"github.com/mulgadc/spinifex/cmd/installer/autoinstall"
 	"github.com/mulgadc/spinifex/cmd/installer/install"
 	"github.com/mulgadc/spinifex/cmd/installer/ui"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )

 func main() {
@@ -19,6 +19,8 @@ import (
 	"syscall"

 	"github.com/mulgadc/spinifex/spinifex/lbagent"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )

 func main() {
@@ -45,6 +45,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/formation"
+	"github.com/mulgadc/spinifex/spinifex/gpu"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/utils"
@@ -199,13 +200,13 @@ spx admin images list
 - fetches from remote endpoint for common/trusted images to bootstrap environment, or baked in from compile.

 // If --name specified, download
-spx admin images import --name debian-12-x86_64
+spx admin images import --name debian-13-x86_64

 // List available images
 spx admin images list

 // Manually import a path
-spx admin images import --file /path/to/image --distro debian --version 12 --arch x86_64
+spx admin images import --file /path/to/image --distro debian --version 13 --arch x86_64

 -> x <-
 */
@@ -274,6 +275,7 @@ func init() {
 	adminInitCmd.Flags().String("gateway-ip", "", "OVN gateway router's external IP for SNAT (default: pool range_start for pool mode, required for nat mode without DHCP)")
 	adminInitCmd.Flags().Int("external-prefix-len", 24, "External pool subnet prefix length (auto-detected)")
 	adminInitCmd.Flags().Bool("no-external", false, "Disable external networking (overlay-only, no internet for VMs)")
+	adminInitCmd.Flags().Bool("gpu-passthrough", false, "Enable VFIO GPU passthrough (sets gpu_passthrough = true in daemon config)")

 	// Flags for admin join
 	adminJoinCmd.Flags().String("region", "ap-southeast-2", "Region for this node")
@@ -660,6 +662,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	externalPrefixLen, _ := cmd.Flags().GetInt("external-prefix-len")
 	gatewayIP, _ := cmd.Flags().GetString("gateway-ip")
 	noExternal, _ := cmd.Flags().GetBool("no-external")
+	gpuPassthrough, _ := cmd.Flags().GetBool("gpu-passthrough")

 	// Fire telemetry in background (completes during init work, waited at end)
 	noTelemetry, _ := cmd.Flags().GetBool("no-telemetry")
@@ -901,6 +904,16 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 	fmt.Printf("   System creds: %s\n", filepath.Join(configDir, "system-credentials.json"))

+	// Predastore encryption key is per-node and never transmitted; generate
+	// it locally now so the service has it on first start.
+	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error generating predastore encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
+	fmt.Printf("   Key: %s\n", predastoreKeyPath)
+
 	fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
 	fmt.Printf("   Access Key:  %s\n", bootstrapResult.AdminAccessKey)
 	fmt.Printf("   Secret Key:  %s\n", bootstrapResult.AdminSecretKey)
@@ -980,8 +993,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	var predastoreNodeID int
 	if predastoreNodesStr != "" {
 		ips := strings.Split(predastoreNodesStr, ",")
-		if len(ips) < 3 {
-			fmt.Fprintf(os.Stderr, "❌ Error: --predastore-nodes requires at least 3 IPs, got %d\n", len(ips))
+		if len(ips) < 2 {
+			fmt.Fprintf(os.Stderr, "❌ Error: --predastore-nodes requires at least 2 IPs, got %d\n", len(ips))
 			os.Exit(1)
 		}

@@ -1072,6 +1085,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		BootstrapIgwId:      bootstrapIgwId,
 		BootstrapCidr:       handlers_ec2_vpc.DefaultVPCCidr,
 		BootstrapSubnetCidr: handlers_ec2_vpc.DefaultSubnetCidr,
+
+		GPUPassthrough: gpuPassthrough,
 	}

 	// Print external networking summary
@@ -1098,6 +1113,20 @@ func runAdminInit(cmd *cobra.Command, args []string) {

 	finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP, advertiseIP)

+	// Write node.conf so spx admin banner works on source installs (not just ISO).
+	nodeHostname, _ := os.Hostname()
+	if nodeHostname == "" {
+		nodeHostname = node
+	}
+	nodeConfPath := filepath.Join(configDir, "node.conf")
+	if err := writeNodeConf(nodeConfPath, map[string]string{
+		"MANAGEMENT_IP":    advertiseIP,
+		"MANAGEMENT_IFACE": "br-wan",
+		"NODE_HOSTNAME":    nodeHostname,
+	}); err != nil {
+		fmt.Fprintf(os.Stderr, "⚠️  Warning: could not write %s: %v\n", nodeConfPath, err)
+	}
+
 	// Print success message
 	fmt.Println("\n🎉 Spinifex initialization complete!")
 	fmt.Println()
@@ -1153,6 +1182,17 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	fmt.Println("\n🔐 Generated IAM master key")
 	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))

+	// Predastore encryption key is per-node and never distributed via the
+	// formation server. Generate only the leader's own key here; each
+	// joiner generates its own during `spx admin join`.
+	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error generating predastore encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
+	fmt.Printf("   Key: %s\n", predastoreKeyPath)
+
 	// Read CA cert/key for distribution to joining nodes
 	caCertData, err := os.ReadFile(filepath.Join(configDir, "ca.pem"))
 	if err != nil {
@@ -1243,7 +1283,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,

 	// Generate multi-node predastore config
 	var predastoreNodeID int
-	hasPredastoreConfig := len(predastoreNodes) >= 3
+	hasPredastoreConfig := len(predastoreNodes) >= 2
 	if hasPredastoreConfig {
 		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
 		if err != nil {
@@ -1623,6 +1663,16 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	fmt.Println("✅ IAM master key received from leader")
 	fmt.Printf("✅ Bootstrap file written: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))

+	// Predastore encryption key is per-node: generate locally rather than
+	// receiving from the leader. Each node only opens fragments it sealed
+	// itself, so there is no cluster-wide predastore key to share.
+	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error generating predastore encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("✅ Predastore encryption key generated: %s\n", predastoreKeyPath)
+
 	// Generate server cert signed by CA with this node's bind IP
 	if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
@@ -1646,7 +1696,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {

 	// Generate multi-node predastore config
 	var predastoreNodeID int
-	hasPredastoreConfig := len(predastoreNodes) >= 3
+	hasPredastoreConfig := len(predastoreNodes) >= 2

 	if hasPredastoreConfig {
 		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP)
@@ -2094,6 +2144,27 @@ func writeBootstrapFiles(configDir, bootstrapDir string, masterKey []byte, acces
 	}, nil
 }

+// writePredastoreEncryptionKey generates a fresh 32-byte AES-256 master key
+// for this node's predastore data dir and writes it to
+// <configDir>/predastore/encryption.key with mode 0600. The key is per-node:
+// every node generates its own at init/join time and never transmits it,
+// because predastore only opens fragments on the node that sealed them.
+func writePredastoreEncryptionKey(configDir string) (string, error) {
+	predastoreDir := filepath.Join(configDir, "predastore")
+	if err := os.MkdirAll(predastoreDir, 0750); err != nil {
+		return "", fmt.Errorf("create predastore config dir: %w", err)
+	}
+	key, err := handlers_iam.GenerateMasterKey()
+	if err != nil {
+		return "", fmt.Errorf("generate predastore encryption key: %w", err)
+	}
+	keyPath := filepath.Join(predastoreDir, "encryption.key")
+	if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil {
+		return "", fmt.Errorf("save predastore encryption key: %w", err)
+	}
+	return keyPath, nil
+}
+
 // writeSystemCredentials writes the system access key to a plaintext JSON file.
 // The daemon reads this at startup to inject credentials into ALB VM cloud-init
 // for SigV4-authenticated communication with the AWS gateway.
@@ -2304,6 +2375,12 @@ func runAdminBanner(cmd *cobra.Command, _ []string) {
 	}

 	// Resolve current IP from the management interface at runtime.
+	// If node.conf is absent or has no MANAGEMENT_IFACE (source installs before
+	// the first spx admin init writes node.conf), fall back to br-wan which
+	// setup-ovn.sh always creates as the management bridge.
+	if iface == "" {
+		iface = "br-wan"
+	}
 	currentIP := resolveIfaceIP(iface)
 	if currentIP == "" {
 		currentIP = recordedIP // fall back to value recorded at install time
@@ -2331,17 +2408,17 @@ func runAdminBanner(cmd *cobra.Command, _ []string) {
 	}

 	banner := fmt.Sprintf(`
-  +----------------------------------------------------+
-  |         Spinifex  —  Mulga Defense Corporation     |
-  +----------------------------------------------------+
-  |  Node:      %-39s|
-  |  Login:     %-39s|
-  |  Dashboard: %-39s|
-  |  API:       %-39s|
-  |  SSH:       %-39s|
-  +----------------------------------------------------+
-  |  AWS credentials:  cat ~/.aws/credentials          |
-  +----------------------------------------------------+
+  +--------------------------------------------------------------+
+  |          Spinifex  —  Mulga Defense Corporation              |
+  +--------------------------------------------------------------+
+  |  Node:      %-49s|
+  |  Login:     %-49s|
+  |  Dashboard: %-49s|
+  |  API:       %-49s|
+  |  SSH:       %-49s|
+  +--------------------------------------------------------------+
+  |  AWS credentials:  cat ~/.aws/credentials                    |
+  +--------------------------------------------------------------+

 `,
 		hostname,
@@ -2351,6 +2428,8 @@ func runAdminBanner(cmd *cobra.Command, _ []string) {
 		"spinifex@"+currentIP,
 	)

+	banner += gpuBannerSection()
+
 	// Write to /etc/issue — displayed on the console before the login prompt.
 	// Overwrite entirely; this is a purpose-built appliance so we own this file.
 	if err := os.WriteFile("/etc/issue", []byte(banner), 0o644); err != nil {
@@ -2390,6 +2469,81 @@ func appendBannerToMotd(banner string) error {
 	return os.WriteFile(motdPath, []byte(base+sentinel+banner), 0o644)
 }

+// gpuBannerSection returns an optional banner box section describing GPU state.
+// Returns "" when no GPU hardware is detected. Safe to call at boot before the
+// daemon starts — all checks are sysfs/file reads, no NATS required.
+func gpuBannerSection() string {
+	devices, err := gpu.Discover()
+	if err != nil || len(devices) == 0 {
+		return ""
+	}
+
+	iommuEntries, _ := os.ReadDir("/sys/kernel/iommu_groups/")
+	iommuActive := len(iommuEntries) > 0
+
+	_, vfioErr := os.Stat("/sys/module/vfio_pci")
+	vfioPresent := vfioErr == nil
+
+	passthroughEnabled := false
+	cfgPath := DefaultConfigFile()
+	if cfg, err := config.LoadConfig(cfgPath); err == nil {
+		if nodeCfg, ok := cfg.Nodes[cfg.Node]; ok {
+			passthroughEnabled = nodeCfg.Daemon.GPUPassthrough
+		}
+	}
+
+	models := gpuModelSummary(devices)
+
+	var statusLine, hintLine string
+	switch {
+	case passthroughEnabled:
+		statusLine = "Passthrough enabled"
+	case iommuActive && vfioPresent:
+		statusLine = "Ready to enable"
+		hintLine = "sudo spx admin gpu enable"
+	default:
+		statusLine = "Setup required"
+		hintLine = "sudo spx admin gpu setup"
+	}
+
+	const (
+		sep    = "  +--------------------------------------------------------------+\n"
+		maxVal = 55
+	)
+	if len([]rune(models)) > maxVal {
+		models = string([]rune(models)[:maxVal-3]) + "..."
+	}
+
+	section := sep +
+		fmt.Sprintf("  |  GPU: %-55s|\n", models) +
+		fmt.Sprintf("  |       %-55s|\n", statusLine)
+	if hintLine != "" {
+		section += fmt.Sprintf("  |       %-55s|\n", hintLine)
+	}
+	section += sep + "\n"
+	return section
+}
+
+func gpuModelSummary(devices []gpu.GPUDevice) string {
+	counts := make(map[string]int)
+	var order []string
+	for _, d := range devices {
+		if counts[d.Model] == 0 {
+			order = append(order, d.Model)
+		}
+		counts[d.Model]++
+	}
+	var parts []string
+	for _, m := range order {
+		if n := counts[m]; n > 1 {
+			parts = append(parts, fmt.Sprintf("%dx %s", n, m))
+		} else {
+			parts = append(parts, m)
+		}
+	}
+	return strings.Join(parts, ", ")
+}
+
 // parseNodeConf reads a KEY=VALUE shell-format file and returns a map.
 // Lines starting with # and blank lines are ignored.
 func parseNodeConf(path string) map[string]string {
@@ -0,0 +1,486 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/gpu"
+	"github.com/mulgadc/spinifex/spinifex/types"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var adminGpuCmd = &cobra.Command{
+	Use:   "gpu",
+	Short: "Manage GPU passthrough for a node",
+}
+
+var adminGpuStatusCmd = &cobra.Command{
+	Use:   "status",
+	Short: "Show GPU hardware and passthrough state",
+	Run:   runAdminGpuStatus,
+}
+
+var adminGpuEnableCmd = &cobra.Command{
+	Use:   "enable",
+	Short: "Enable GPU passthrough on a node",
+	Run:   runAdminGpuEnable,
+}
+
+var adminGpuDisableCmd = &cobra.Command{
+	Use:   "disable",
+	Short: "Disable GPU passthrough on a node (blocked while GPU instances are running)",
+	Run:   runAdminGpuDisable,
+}
+
+var adminGpuSetupCmd = &cobra.Command{
+	Use:   "setup",
+	Short: "Configure this host for GPU passthrough (IOMMU, vfio-pci, nouveau blacklist)",
+	Long: `Idempotent host setup for GPU passthrough. Run once before a reboot to configure
+GRUB, vfio-pci early binding, and the nouveau blacklist; then run again after the
+reboot to verify bindings and enable GPU passthrough in the daemon.
+
+Must be run as root.`,
+	Run: runAdminGpuSetup,
+}
+
+func init() {
+	adminCmd.AddCommand(adminGpuCmd)
+	adminGpuCmd.AddCommand(adminGpuStatusCmd)
+	adminGpuCmd.AddCommand(adminGpuEnableCmd)
+	adminGpuCmd.AddCommand(adminGpuDisableCmd)
+	adminGpuCmd.AddCommand(adminGpuSetupCmd)
+
+	adminGpuStatusCmd.Flags().String("node", "", "Target node name (default: local node)")
+	// enable, disable, and setup write to the local spinifex.toml and signal the
+	// local daemon — they must be run directly on the target host.
+}
+
+// gpuNodeStatus queries NATS and returns the NodeStatusResponse for the target node.
+func gpuNodeStatus(targetNode string) (*types.NodeStatusResponse, error) {
+	cfg, nc, err := loadConfigAndConnect()
+	if err != nil {
+		return nil, err
+	}
+	defer nc.Close()
+
+	if targetNode == "" {
+		targetNode = cfg.Node
+	}
+
+	responses, err := collectResponses(nc, "spinifex.node.status", 3*time.Second)
+	if err != nil {
+		return nil, fmt.Errorf("collect node status: %w", err)
+	}
+	for _, raw := range responses {
+		var resp types.NodeStatusResponse
+		if err := json.Unmarshal(raw, &resp); err != nil {
+			continue
+		}
+		if resp.Node == targetNode {
+			return &resp, nil
+		}
+	}
+	return nil, fmt.Errorf("node %q not found or not responding", targetNode)
+}
+
+func runAdminGpuStatus(cmd *cobra.Command, _ []string) {
+	targetNode, _ := cmd.Flags().GetString("node")
+	resp, err := gpuNodeStatus(targetNode)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	iommu := "unknown"
+	vfio := "unknown"
+	if resp.GPUCapable || len(resp.GPUModels) > 0 {
+		iommu = "active"
+		vfio = "loaded"
+	}
+
+	fmt.Printf("Node:            %s\n", resp.Node)
+	if len(resp.GPUModels) > 0 {
+		fmt.Printf("GPU hardware:    %s\n", strings.Join(resp.GPUModels, ", "))
+	} else {
+		fmt.Printf("GPU hardware:    none detected\n")
+	}
+	fmt.Printf("IOMMU:           %s\n", iommu)
+	fmt.Printf("vfio-pci:        %s\n", vfio)
+
+	if resp.GPUPassthrough {
+		fmt.Printf("Passthrough:     enabled\n")
+		fmt.Printf("GPU pool:        %d/%d allocated\n", resp.AllocGPUs, resp.TotalGPUs)
+
+		var gpuTypes []string
+		for _, cap := range resp.InstanceTypes {
+			gpuTypes = append(gpuTypes, cap.Name)
+		}
+		if len(gpuTypes) > 0 {
+			fmt.Printf("Instance types:  %s\n", strings.Join(gpuTypes, " "))
+		}
+	} else if resp.GPUCapable {
+		fmt.Printf("Passthrough:     disabled\n")
+		fmt.Printf("Instance types:  (none — run 'spx admin gpu enable' to activate)\n")
+	} else {
+		fmt.Printf("Passthrough:     disabled\n")
+		fmt.Printf("Instance types:  (prerequisites not met)\n")
+	}
+}
+
+// gpuToggle writes the TOML setting, sends SIGHUP, and polls until the daemon confirms the new state.
+// enable/disable always operate on the local node; run the command directly on the target host.
+func gpuToggle(_ *cobra.Command, enable bool) {
+	cfgPath := viper.GetString("config")
+	if cfgPath == "" {
+		cfgPath = DefaultConfigFile()
+	}
+	cfg, err := config.LoadConfig(cfgPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error loading config: %v\n", err)
+		os.Exit(1)
+	}
+	localNode := cfg.Node
+
+	// Check current state first.
+	resp, err := gpuNodeStatus(localNode)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	if enable {
+		if resp.GPUPassthrough {
+			fmt.Println("GPU passthrough is already enabled.")
+			return
+		}
+		if !resp.GPUCapable {
+			fmt.Fprintln(os.Stderr, "Error: prerequisites not met on this node.")
+			fmt.Fprintln(os.Stderr, "  Run 'sudo spx admin gpu setup' to configure the host.")
+			os.Exit(1)
+		}
+	} else {
+		if !resp.GPUPassthrough {
+			fmt.Println("GPU passthrough is already disabled.")
+			return
+		}
+		if resp.AllocGPUs > 0 {
+			fmt.Fprintf(os.Stderr, "Error: %d GPU instance(s) are running. Terminate them first.\n", resp.AllocGPUs)
+			os.Exit(1)
+		}
+	}
+
+	// Write the TOML setting.
+	if err := admin.SetGPUPassthrough(cfgPath, localNode, enable); err != nil {
+		fmt.Fprintf(os.Stderr, "Error writing config: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Signal the daemon to reload.
+	out, err := exec.Command("systemctl", "kill", "-s", "HUP", "spinifex-daemon").CombinedOutput()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error sending SIGHUP: %v\n%s\n", err, out)
+		os.Exit(1)
+	}
+
+	// Poll until the daemon confirms the new state.
+	action := "enable"
+	if !enable {
+		action = "disable"
+	}
+	fmt.Printf("Waiting for daemon to %s GPU passthrough", action)
+	deadline := time.Now().Add(30 * time.Second)
+	for time.Now().Before(deadline) {
+		time.Sleep(1 * time.Second)
+		fmt.Print(".")
+		r, err := gpuNodeStatus(localNode)
+		if err != nil {
+			continue
+		}
+		if r.GPUPassthrough == enable {
+			fmt.Println(" done.")
+			// Refresh the MOTD banner immediately so the login prompt
+			// reflects the new GPU passthrough state without requiring a reboot.
+			_ = exec.Command("spx", "admin", "banner", "--boot-check").Run()
+			runAdminGpuStatus(adminGpuStatusCmd, nil)
+			return
+		}
+	}
+	fmt.Println()
+	fmt.Fprintln(os.Stderr, "Timed out waiting for daemon. Check: journalctl -u spinifex-daemon -n 50")
+	os.Exit(1)
+}
+
+func runAdminGpuEnable(cmd *cobra.Command, _ []string) {
+	gpuToggle(cmd, true)
+}
+
+func runAdminGpuDisable(cmd *cobra.Command, _ []string) {
+	gpuToggle(cmd, false)
+}
+
+func runAdminGpuSetup(_ *cobra.Command, _ []string) {
+	if os.Getuid() != 0 {
+		fmt.Fprintln(os.Stderr, "Error: spx admin gpu setup must be run as root.")
+		os.Exit(1)
+	}
+
+	fmt.Println("==> Detecting GPU")
+	devices, err := gpu.Discover()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "GPU discovery failed: %v\n", err)
+		os.Exit(1)
+	}
+	if len(devices) == 0 {
+		fmt.Fprintln(os.Stderr, "No NVIDIA or AMD GPU found — is the card seated?")
+		os.Exit(1)
+	}
+	for _, d := range devices {
+		fmt.Printf("    %s - %s\n", d.PCIAddress, d.Model)
+	}
+
+	fmt.Println("==> Collecting PCI IDs for vfio-pci")
+	ids := gpuSetupCollectVFIOIDs(devices)
+	idsCSV := strings.Join(ids, ",")
+	fmt.Printf("    IDs: %s\n", idsCSV)
+
+	rebootNeeded := false
+
+	fmt.Println("==> Checking IOMMU")
+	iommuEntries, _ := os.ReadDir("/sys/kernel/iommu_groups/")
+	if len(iommuEntries) > 0 {
+		fmt.Println("    Active")
+	} else {
+		params := gpuSetupIOMMUParams()
+		changed, err := gpuSetupAddGRUBParams(params)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to update GRUB: %v\n", err)
+			os.Exit(1)
+		}
+		if changed {
+			fmt.Printf("    GRUB updated with: %s\n", params)
+		} else {
+			fmt.Println("    GRUB already has IOMMU params — refreshing")
+		}
+		ug := exec.Command("update-grub")
+		ug.Stdout, ug.Stderr = os.Stdout, os.Stderr
+		if err := ug.Run(); err != nil {
+			fmt.Fprintf(os.Stderr, "update-grub failed: %v\n", err)
+			os.Exit(1)
+		}
+		rebootNeeded = true
+	}
+
+	fmt.Println("==> Configuring vfio udev rule")
+	const vfioUdevRule = "/etc/udev/rules.d/99-spinifex-vfio.rules"
+	if _, err := os.Stat(vfioUdevRule); os.IsNotExist(err) {
+		rule := "SUBSYSTEM==\"vfio\", GROUP=\"spinifex\", MODE=\"0660\"\n"
+		if err := os.WriteFile(vfioUdevRule, []byte(rule), 0o644); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to write vfio udev rule: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("    Rule installed")
+		_ = exec.Command("udevadm", "control", "--reload-rules").Run()
+		_ = exec.Command("udevadm", "trigger", "--subsystem-match=vfio").Run()
+	} else {
+		fmt.Println("    Rule already present")
+	}
+
+	fmt.Println("==> Configuring nouveau blacklist")
+	const nouveauConf = "/etc/modprobe.d/blacklist-nouveau.conf"
+	if _, err := os.Stat(nouveauConf); os.IsNotExist(err) {
+		if err := os.WriteFile(nouveauConf, []byte("blacklist nouveau\noptions nouveau modeset=0\n"), 0o644); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to write nouveau blacklist: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("    nouveau blacklisted")
+		rebootNeeded = true
+	} else {
+		fmt.Println("    Already blacklisted")
+	}
+
+	fmt.Println("==> Configuring amdgpu blacklist")
+	const amdgpuConf = "/etc/modprobe.d/blacklist-amdgpu.conf"
+	hasAMD := false
+	for _, d := range devices {
+		if d.Vendor == gpu.VendorAMD {
+			hasAMD = true
+			break
+		}
+	}
+	if hasAMD {
+		if _, err := os.Stat(amdgpuConf); os.IsNotExist(err) {
+			if err := os.WriteFile(amdgpuConf, []byte("blacklist amdgpu\n"), 0o644); err != nil {
+				fmt.Fprintf(os.Stderr, "Failed to write amdgpu blacklist: %v\n", err)
+				os.Exit(1)
+			}
+			fmt.Println("    amdgpu blacklisted")
+			rebootNeeded = true
+		} else {
+			fmt.Println("    Already blacklisted")
+		}
+	} else {
+		fmt.Println("    No AMD GPUs — skipping")
+	}
+
+	fmt.Println("==> Configuring vfio-pci early binding")
+	const vfioPCIConf = "/etc/modprobe.d/vfio-pci.conf"
+	existing, _ := os.ReadFile(vfioPCIConf)
+	if !strings.Contains(string(existing), "ids="+idsCSV) {
+		if err := os.WriteFile(vfioPCIConf, []byte("options vfio-pci ids="+idsCSV+"\n"), 0o644); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to write vfio-pci.conf: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("    vfio-pci.conf written (ids=%s)\n", idsCSV)
+		rebootNeeded = true
+	} else {
+		fmt.Printf("    Already configured for %s\n", idsCSV)
+	}
+
+	const initramfsModules = "/etc/initramfs-tools/modules"
+	modData, _ := os.ReadFile(initramfsModules)
+	if !strings.Contains(string(modData), "vfio_pci") {
+		f, err := os.OpenFile(initramfsModules, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to open initramfs modules: %v\n", err)
+			os.Exit(1)
+		}
+		_, werr := fmt.Fprint(f, "\n# vfio early binding for GPU passthrough\nvfio\nvfio_iommu_type1\nvfio_pci\n")
+		f.Close()
+		if werr != nil {
+			fmt.Fprintf(os.Stderr, "Failed to write initramfs modules: %v\n", werr)
+			os.Exit(1)
+		}
+		fmt.Println("    vfio modules added to initramfs")
+		rebootNeeded = true
+	} else {
+		fmt.Println("    vfio modules already in initramfs")
+	}
+
+	if rebootNeeded {
+		fmt.Println("==> Updating initramfs")
+		ui := exec.Command("update-initramfs", "-u")
+		ui.Stdout, ui.Stderr = os.Stdout, os.Stderr
+		if err := ui.Run(); err != nil {
+			fmt.Fprintf(os.Stderr, "update-initramfs failed: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("\nSetup complete — reboot required.")
+		fmt.Println("  sudo reboot")
+		fmt.Println("  Then run: sudo spx admin gpu enable")
+		return
+	}
+
+	fmt.Println("==> Verifying vfio-pci module")
+	if _, err := os.Stat("/sys/module/vfio_pci"); os.IsNotExist(err) {
+		fmt.Fprintln(os.Stderr, "vfio_pci not loaded — check: journalctl -b | grep vfio")
+		os.Exit(1)
+	}
+	fmt.Println("    Loaded")
+
+	fmt.Println("==> Verifying GPU driver binding")
+	for _, d := range devices {
+		current, err := gpu.CurrentDriver(d.PCIAddress)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Cannot read driver for %s: %v\n", d.PCIAddress, err)
+			os.Exit(1)
+		}
+		if current == "vfio-pci" {
+			fmt.Printf("    %s → vfio-pci\n", d.PCIAddress)
+			continue
+		}
+		if current != "" {
+			// Bound to native driver — blacklist/initramfs changes may not have taken effect.
+			fmt.Fprintf(os.Stderr, "%s is bound to %q, not vfio-pci — reboot may be required\n", d.PCIAddress, current)
+			fmt.Fprintf(os.Stderr, "  Check: ls -la /sys/bus/pci/devices/%s/driver\n", d.PCIAddress)
+			os.Exit(1)
+		}
+		// Device is unbound (ids= did not auto-claim it) — bind explicitly via driver_override.
+		fmt.Printf("    %s unbound — binding to vfio-pci\n", d.PCIAddress)
+		if _, err := gpu.BindVFIO(d.PCIAddress); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to bind %s to vfio-pci: %v\n", d.PCIAddress, err)
+			fmt.Fprintf(os.Stderr, "  Check: ls -la /sys/bus/pci/devices/%s/driver\n", d.PCIAddress)
+			os.Exit(1)
+		}
+		fmt.Printf("    %s → vfio-pci\n", d.PCIAddress)
+	}
+
+	fmt.Println("==> Enabling GPU passthrough")
+	gpuToggle(nil, true)
+}
+
+// gpuSetupCollectVFIOIDs collects vendor:device PCI ID pairs for each GPU and
+// all sibling PCI functions on the same device slot (e.g. GPU + HDMI audio).
+// All siblings must bind to vfio-pci together for IOMMU group isolation.
+func gpuSetupCollectVFIOIDs(devices []gpu.GPUDevice) []string {
+	seen := make(map[string]bool)
+	var ids []string
+	for _, d := range devices {
+		dot := strings.LastIndex(d.PCIAddress, ".")
+		if dot < 0 {
+			continue
+		}
+		prefix := d.PCIAddress[:dot]
+		matches, _ := filepath.Glob("/sys/bus/pci/devices/" + prefix + ".*")
+		for _, m := range matches {
+			vb, err1 := os.ReadFile(m + "/vendor")
+			db, err2 := os.ReadFile(m + "/device")
+			if err1 != nil || err2 != nil {
+				continue
+			}
+			v := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(string(vb))), "0x")
+			dv := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(string(db))), "0x")
+			id := v + ":" + dv
+			if !seen[id] {
+				seen[id] = true
+				ids = append(ids, id)
+			}
+		}
+	}
+	return ids
+}
+
+func gpuSetupIOMMUParams() string {
+	data, _ := os.ReadFile("/proc/cpuinfo")
+	if strings.Contains(string(data), "GenuineIntel") {
+		return "intel_iommu=on iommu=pt"
+	}
+	return "amd_iommu=on iommu=pt"
+}
+
+// gpuSetupAddGRUBParams appends iommu kernel params to GRUB_CMDLINE_LINUX_DEFAULT.
+// Returns (true, nil) if the file was modified, (false, nil) if params were already present.
+func gpuSetupAddGRUBParams(params string) (bool, error) {
+	const grubFile = "/etc/default/grub"
+	data, err := os.ReadFile(grubFile)
+	if err != nil {
+		return false, fmt.Errorf("read %s: %w", grubFile, err)
+	}
+	content := string(data)
+	if strings.Contains(content, "intel_iommu=on") || strings.Contains(content, "amd_iommu=on") {
+		return false, nil
+	}
+	re := regexp.MustCompile(`(GRUB_CMDLINE_LINUX_DEFAULT="[^"]*)"`)
+	updated := re.ReplaceAllStringFunc(content, func(match string) string {
+		groups := re.FindStringSubmatch(match)
+		if len(groups) < 2 {
+			return match
+		}
+		return groups[1] + " " + params + `"`
+	})
+	if updated == content {
+		return false, fmt.Errorf("GRUB_CMDLINE_LINUX_DEFAULT not found in %s", grubFile)
+	}
+	if err := os.WriteFile(grubFile, []byte(updated), 0o644); err != nil {
+		return false, fmt.Errorf("write %s: %w", grubFile, err)
+	}
+	return true, nil
+}
@@ -22,7 +22,7 @@ import (
 func TestPrintChecksumError(t *testing.T) {
 	image := utils.Images{Checksum: "https://example.com/SUMS", ChecksumType: "sha512"}
 	const imageFile = "/var/lib/img.tar.xz"
-	const imageName = "debian-12-x86_64"
+	const imageName = "debian-13-x86_64"

 	errs := []error{
 		fmt.Errorf("%w: expected abc got def", utils.ErrChecksumMismatch),
@@ -262,3 +262,51 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {
 	require.NoError(t, err)
 	assert.Contains(t, content2, `nats_url = "nats://10.11.12.1:4222"`)
 }
+
+// writePredastoreEncryptionKey is called from both `spx admin init` and `spx
+// admin join`; both depend on it producing a 32-byte file at 0600 in the
+// expected layout. Predastore's keyfile loader rejects anything that isn't
+// exactly that, so a regression here would break service startup on every
+// node.
+func TestWritePredastoreEncryptionKey(t *testing.T) {
+	configDir := t.TempDir()
+
+	keyPath, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join(configDir, "predastore", "encryption.key"), keyPath)
+
+	info, err := os.Stat(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, int64(32), info.Size(), "predastore master key must be exactly 32 bytes")
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "predastore master key must be mode 0600")
+
+	// Per-node invariant: two successive calls (on what would be two
+	// different nodes) must produce different key material. If they ever
+	// returned the same bytes we'd have silently lost the per-node
+	// blast-radius property.
+	configDir2 := t.TempDir()
+	keyPath2, err := writePredastoreEncryptionKey(configDir2)
+	require.NoError(t, err)
+
+	key1, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+	key2, err := os.ReadFile(keyPath2)
+	require.NoError(t, err)
+	assert.NotEqual(t, key1, key2, "per-node keys must differ")
+}
+
+// Joiners run `spx admin join` against a configDir that doesn't yet contain
+// a predastore/ subdir. The helper must create it (predastore subdir is
+// owned by spinifex-storage in prod, but the helper just needs the dir to
+// exist).
+func TestWritePredastoreEncryptionKey_CreatesMissingDir(t *testing.T) {
+	configDir := t.TempDir()
+	// Don't pre-create the predastore subdir — simulate a fresh joiner.
+
+	keyPath, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+
+	dirInfo, err := os.Stat(filepath.Dir(keyPath))
+	require.NoError(t, err)
+	assert.True(t, dirInfo.IsDir())
+}
@@ -115,6 +115,13 @@ var predastoreStartCmd = &cobra.Command{
 			return
 		}

+		encryptionKeyFile := viper.GetString("encryption-key-file")
+
+		if encryptionKeyFile == "" {
+			fmt.Println("Encryption key file is not set")
+			return
+		}
+
 		nodeID := viper.GetInt("node-id")
 		pprofEnabled := viper.GetBool("pprof")
 		pprofOutput := viper.GetString("pprof-output")
@@ -128,6 +135,8 @@ var predastoreStartCmd = &cobra.Command{
 			TlsCert:    tlsCert,
 			TlsKey:     tlsKey,

+			EncryptionKeyFile: encryptionKeyFile,
+
 			NodeID: nodeID,

 			PprofEnabled:    pprofEnabled,
@@ -858,13 +867,20 @@ func init() {
 	viper.BindEnv("tls-key", "SPINIFEX_PREDASTORE_TLS_KEY")
 	viper.BindPFlag("tls-key", predastoreCmd.PersistentFlags().Lookup("tls-key"))

+	// Predastore at-rest encryption master key (per node; mode 0600)
+	predastoreCmd.PersistentFlags().String("encryption-key-file", "", "Path to this node's 32-byte AES-256 master key file (required)")
+	viper.BindEnv("encryption-key-file", "SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE")
+	viper.BindPFlag("encryption-key-file", predastoreCmd.PersistentFlags().Lookup("encryption-key-file"))
+
 	// Predastore Backend
 	predastoreCmd.PersistentFlags().String("backend", "distributed", "Predastore (S3) backend")
 	viper.BindEnv("backend", "SPINIFEX_PREDASTORE_BACKEND")
 	viper.BindPFlag("backend", predastoreCmd.PersistentFlags().Lookup("backend"))

-	// Predastore Node ID
-	predastoreCmd.PersistentFlags().Int("node-id", 0, "Predastore (S3) node ID")
+	// Predastore Node ID. Default -1 is dev mode (launch every configured
+	// QUIC node in-process). Production deployments set this to the node's
+	// real ID (>= 1) via SPINIFEX_PREDASTORE_NODE_ID or --node-id.
+	predastoreCmd.PersistentFlags().Int("node-id", -1, "Predastore (S3) node ID (-1 = dev mode, >= 1 = production)")
 	viper.BindEnv("node-id", "SPINIFEX_PREDASTORE_NODE_ID")
 	viper.BindPFlag("node-id", predastoreCmd.PersistentFlags().Lookup("node-id"))

@@ -1,6 +1,6 @@
 # Predastore S3-compatible storage configuration file in TOML format.
 # Generated for multi-node cluster
-version = "3"
+version = "4"
 region = "{{.Region}}"

 host = "0.0.0.0"
@@ -9,8 +9,13 @@ port = 8443
 debug = false

 [rs]
+{{- if eq (len .Nodes) 2}}
+data = 1
+parity = 1
+{{- else}}
 data = 2
 parity = 1
+{{- end}}
 {{- range .Nodes}}

 [[db]]
@@ -39,10 +44,12 @@ region = "{{.Region}}"
 type = "distributed"
 public = false
 encryption = ""
+account_id = "000000000000"

 [[auth]]
 access_key_id = "{{.AccessKey}}"
 secret_access_key = "{{.SecretKey}}"
+account_id = "000000000000"
 policy = [
  { bucket = "predastore", actions = ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
 ]
@@ -1,6 +1,6 @@
 # Predastore S3-compatible storage configuration file in TOML format.
 # MULTI NODE used for testing
-version = "3"
+version = "4"
 region = "{{.Region}}"

 # Optionally configure port and host, or specify via daemon CLI options
@@ -73,10 +73,12 @@ region = "ap-southeast-2"
 type = "distributed"
 public = false
 encryption = ""
+account_id = "000000000000"

 [[auth]]
 access_key_id = "{{.AccessKey}}"
 secret_access_key = "{{.SecretKey}}"
+account_id = "000000000000"
 policy = [
  { bucket = "predastore", actions = ["s3:ListBucket",  "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
 ]
@@ -52,6 +52,7 @@ advertise = "{{.AdvertiseIP}}"
 {{- end}}
 region = "{{.Region}}"
 az = "{{.Az}}"
+data_dir = "{{.DataDir}}"
 {{- if .Services}}
 services = [{{range $i, $s := .Services}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
 {{- end}}
@@ -67,6 +68,9 @@ dev_networking = false
 {{- else}}
 dev_networking = true
 {{- end}}
+{{- if .GPUPassthrough}}
+gpu_passthrough = true
+{{- end}}

 [nodes.{{.Node}}.awsgw]
 host = "{{.BindIP}}:9999"
@@ -88,7 +88,7 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 	sort.Strings(nodeNames)

 	nodeTable := pterm.TableData{
-		{"NAME", "CPU (used/total)", "MEM (used/total)", "VMs"},
+		{"NAME", "CPU (used/total)", "MEM (used/total)", "GPU (used/total)", "VMs"},
 	}

 	// Aggregate instance type capacity across all nodes
@@ -96,10 +96,17 @@ func runTopNodes(cmd *cobra.Command, args []string) {

 	for _, name := range nodeNames {
 		if resp, ok := respondedNodes[name]; ok {
+			gpuCol := "-"
+			if resp.GPUPassthrough {
+				gpuCol = fmt.Sprintf("%d/%d", resp.AllocGPUs, resp.TotalGPUs)
+			} else if resp.GPUCapable {
+				gpuCol = fmt.Sprintf("0/%d*", len(resp.GPUModels))
+			}
 			nodeTable = append(nodeTable, []string{
 				resp.Node,
 				fmt.Sprintf("%d/%d", resp.AllocVCPU, resp.TotalVCPU),
 				fmt.Sprintf("%s/%s", formatMemGB(resp.AllocMemGB), formatMemGB(resp.TotalMemGB)),
+				gpuCol,
 				strconv.Itoa(resp.VMCount),
 			})

@@ -120,6 +127,7 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 				"-",
 				"-",
 				"-",
+				"-",
 			})
 		}
 	}
@@ -16,7 +16,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package main

-import "github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+import (
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)

 func main() {
 	cmd.Execute()
@@ -45,7 +45,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t

 | Command | Flags | Prerequisites | Basic Logic | Output | Status |
 |---------|-------|---------------|-------------|--------|--------|
-| `spx top nodes` | `--timeout` (default: 3s) | Cluster must be running (NATS) | Publishes to `spinifex.node.status` fan-out topic → collects CPU/memory usage per node → aggregates instance type capacity across all nodes → prints two tables: per-node resource usage and cluster-wide instance type availability | Table 1: NAME, CPU (used/total), MEM (used/total), VMs. Table 2: INSTANCE TYPE, AVAILABLE, VCPU, MEMORY | **DONE** |
+| `spx top nodes` | `--timeout` (default: 3s) | Cluster must be running (NATS) | Publishes to `spinifex.node.status` fan-out topic → collects CPU/memory usage per node → aggregates instance type capacity across all nodes → prints two tables: per-node resource usage and cluster-wide instance type availability | Table 1: NAME, CPU (used/total), MEM (used/total), GPU (used/total), VMs — GPU column shows `allocated/total` when passthrough is enabled, `0/count*` when the node is GPU-capable but passthrough is disabled, or `-` when no GPU is present. Table 2: INSTANCE TYPE, AVAILABLE, VCPU, MEMORY | **DONE** |

 ### Cluster Initialization

@@ -123,6 +123,15 @@ served. A mirror compromise that swaps both image and sums file is not
 detected; closing that gap requires GPG signature verification of the sums
 file, deferred to a later phase.

+### GPU Management
+
+| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------|---------------|-------------|------------|--------|
+| `spx admin gpu status` | `--node` (default: local node) | Cluster must be running (NATS) | Queries `spinifex.node.status` fan-out → finds the target node response → prints Node, GPU hardware (model list or "none detected"), IOMMU state, vfio-pci state, passthrough enabled/disabled, and GPU pool allocation (`allocated/total`). Also lists GPU-capable instance types when passthrough is active. | 1. Local node with passthrough enabled (shows pool and instance types)<br>2. Local node capable but passthrough disabled (shows "run enable")<br>3. Node without GPU hardware (shows "none detected")<br>4. `--node` targets a remote node<br>5. Node not found or not responding (error) | **DONE** |
+| `spx admin gpu enable` | — | Cluster must be running (NATS); node must be GPU-capable (IOMMU + vfio-pci configured) | Checks current passthrough state via NATS → errors if already enabled or prerequisites not met (directs to `setup`) → writes `gpu_passthrough = true` to `spinifex.toml` via `admin.SetGPUPassthrough` → sends SIGHUP to `spinifex-daemon` via `systemctl kill -s HUP` → polls node status for up to 30 s until daemon confirms new state → prints final `gpu status` output. Must be run directly on the target host. | 1. Enable on capable node<br>2. Already enabled (no-op)<br>3. Prerequisites not met (error, suggests `setup`)<br>4. Daemon does not confirm within 30 s (timeout error) | **DONE** |
+| `spx admin gpu disable` | — | Cluster must be running (NATS); no GPU instances may be running | Checks current passthrough state via NATS → errors if already disabled or if `AllocGPUs > 0` (must terminate GPU instances first) → writes `gpu_passthrough = false` → sends SIGHUP to `spinifex-daemon` → polls for up to 30 s → prints final `gpu status` output. Must be run directly on the target host. | 1. Disable with no GPU instances running<br>2. Already disabled (no-op)<br>3. GPU instances running (error with count)<br>4. Daemon does not confirm within 30 s (timeout error) | **DONE** |
+| `spx admin gpu setup` | — | Must be run as root; NVIDIA or AMD GPU must be present | Idempotent host configuration for GPU passthrough. Steps (skipped if already applied): detect GPUs via `gpu.Discover` → collect PCI IDs for all IOMMU-group siblings → check/enable IOMMU in GRUB (`intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt`) → write vfio udev rule (`/etc/udev/rules.d/99-spinifex-vfio.rules`) → blacklist nouveau (`/etc/modprobe.d/blacklist-nouveau.conf`) → blacklist amdgpu if AMD GPU present (`/etc/modprobe.d/blacklist-amdgpu.conf`) → write vfio-pci early binding config (`/etc/modprobe.d/vfio-pci.conf`) → add vfio modules to initramfs. If any change requires a reboot: runs `update-initramfs -u` and exits with reboot instructions. After reboot: verifies `vfio_pci` module is loaded → verifies each GPU is bound to `vfio-pci` (binds explicitly via `driver_override` if unbound) → calls `gpu enable` to activate passthrough. | 1. First run on unconfigured host (GRUB updated, reboot required)<br>2. Re-run after reboot (verifies bindings, enables passthrough)<br>3. Already fully configured (all steps skipped, enables passthrough)<br>4. AMD GPU present (blacklists amdgpu in addition to nouveau)<br>5. Not run as root (error)<br>6. No GPU hardware detected (error) | **DONE** |
+
 ## AWS Commands

 ### EC2 - Instance Management
@@ -133,13 +142,14 @@ file, deferred to a later phase.
 | `describe-instances` | `--instance-ids`, `--filters` (instance-state-name, instance-id, instance-type, vpc-id, subnet-id, tag:\*, tag-key, tag-value) | `--max-results`, `--next-token`, `--dry-run` | None | Gateway fans out NATS `ec2.DescribeInstances` to all nodes (no queue group) → each daemon returns local instances → gateway aggregates and returns combined list. Filters applied per-node before aggregation (reduces payload). Also applies to stopped/terminated instances via `describeInstancesFromKV()`. | 1. Describe all instances (no filter)<br>2. Describe by instance ID<br>3. Describe with filters (e.g. instance-state-name)<br>4. Instance not found returns empty set<br>5. Multi-node aggregation returns instances from all nodes<br>6. Filter by tag<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `start-instances` | `--instance-ids` | `--dry-run`, `--force` | `run-instances` (instance must exist in stopped state) | Gateway sends NATS `ec2.cmd.{instance-id}` → daemon restarts stopped QEMU process with same config → state transitions stopped→pending→running | 1. Start a stopped instance<br>2. Start already-running instance (error: IncorrectInstanceState)<br>3. Start with invalid instance ID<br>4. Verify volumes re-mount on start | **DONE** |
 | `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | `run-instances` (instance must be running) | Gateway sends NATS to target node → daemon issues QMP `system_powerdown` for graceful shutdown → monitors heartbeat until QEMU exits → state transitions running→stopping→stopped | 1. Graceful stop of running instance<br>2. Force stop (kills QEMU process)<br>3. Stop already-stopped instance (error)<br>4. Verify ~30s heartbeat detection | **DONE** |
-| `terminate-instances` | `--instance-ids`, `DeleteOnTermination` (per-volume flag, default true) | `--dry-run` | `run-instances` (instance must exist) | Gateway sends NATS to target node → daemon kills QEMU process → cleans up NBD mounts → deletes volumes with `DeleteOnTermination=true` via `volumeService.DeleteVolume()` (S3 cleanup of vol/, vol-efi/, vol-cloudinit/) → internal volumes (EFI, cloud-init) always cleaned up via `ebs.delete` NATS → volumes with `DeleteOnTermination=false` left in available state → state→terminated | 1. Terminate running instance<br>2. Terminate stopped instance<br>3. Terminate with DeleteOnTermination=true deletes volumes<br>4. Terminate with DeleteOnTermination=false preserves volumes<br>5. Terminate already-terminated (idempotent)<br>6. Internal volumes (EFI, cloud-init) always cleaned up<br>7. Invalid instance ID | **DONE** |
+| `terminate-instances` | `--instance-ids`, `DeleteOnTermination` (per-volume flag, default true) | `--dry-run` | `run-instances` (instance must exist) | Gateway sends NATS to target node → daemon checks `DisableApiTermination` first and rejects with `OperationNotPermitted` if set (running or stopped, no side effects) → kills QEMU process → cleans up NBD mounts → deletes volumes with `DeleteOnTermination=true` via `volumeService.DeleteVolume()` (S3 cleanup of vol/, vol-efi/, vol-cloudinit/) → internal volumes (EFI, cloud-init) always cleaned up via `ebs.delete` NATS → volumes with `DeleteOnTermination=false` left in available state → state→terminated | 1. Terminate running instance<br>2. Terminate stopped instance<br>3. Terminate with DeleteOnTermination=true deletes volumes<br>4. Terminate with DeleteOnTermination=false preserves volumes<br>5. Terminate already-terminated (idempotent)<br>6. Internal volumes (EFI, cloud-init) always cleaned up<br>7. Invalid instance ID<br>8. Running protected instance rejected (OperationNotPermitted, state stays running)<br>9. Stopped protected instance rejected (resources intact) | **DONE** |
 | `reboot-instances` | `--instance-ids` | `--dry-run` | `run-instances` (instance must be running) | Gateway validates instance IDs → sends EC2InstanceCommand with `RebootInstance=true` via NATS `ec2.cmd.{instanceId}` → daemon validates instance is in StateRunning (returns IncorrectInstanceState if stopped) → issues QMP `system_reset` → instance reboots without stopping, stays in running state | 1. Reboot running instance<br>2. Reboot multiple instances<br>3. Reboot stopped instance (error: IncorrectInstanceState)<br>4. Instance not found (error: InvalidInstanceID.NotFound)<br>5. Verify instance stays in running state after reboot | **DONE** |
 | `describe-instance-types` | `--filters` (capacity filter only) | `--instance-types`, `--max-results`, `--next-token`, `--dry-run`, all other filters | None | Gateway fans out NATS `ec2.DescribeInstanceTypes` to all nodes → each daemon reports supported types (t3.micro/small/medium/large) with vCPU/memory specs → gateway deduplicates and returns | 1. List all instance types<br>2. Filter by specific type<br>3. Filter with `capacity=true` shows available slots<br>4. Verify vCPU/memory specs match hardware | **DONE** |
-| `modify-instance-attribute` | `--instance-id`, `--instance-type`, `--user-data` | `--disable-api-termination`, `--ebs-optimized`, `--source-dest-check`, `--instance-initiated-shutdown-behavior`, `--block-device-mappings`, `--groups`, `--ena-support`, `--sriov-net-support` | Instance must be stopped (in NATS KV) | Gateway validates input (exactly one attribute per call, instance ID format) → NATS `ec2.ModifyInstanceAttribute` with `spinifex-workers` queue group → daemon loads stopped instance from JetStream KV → applies attribute change → writes back to KV → returns `{}` on success. **InstanceType**: updates vm.InstanceType, Config, and Instance fields; clears StateReason (enables recovery from instance-type-missing bug). **UserData**: stores decoded content in vm.UserData and re-encodes to base64 for RunInstancesInput (cloud-init on next start). No instance type pre-validation (matches AWS — invalid types accepted, fail at StartInstances time). | 1. Change instance type while stopped<br>2. Change user data while stopped<br>3. Modify running instance (error: NotFound — running instances not in KV)<br>4. Instance not found (error: InvalidInstanceID.NotFound)<br>5. Instance not stopped (error: IncorrectInstanceState)<br>6. Invalid instance type accepted (fails on start with InsufficientInstanceCapacity)<br>7. StateReason cleared on type change (recovery from capacity-unavailable)<br>8. Missing/malformed instance ID (error: InvalidInstanceID.Malformed)<br>9. No attribute set (error: InvalidParameterValue)<br>10. Multiple attributes in one call (error: InvalidParameterValue) | **DONE** |
+| `modify-instance-attribute` | `--instance-id`, `--instance-type`, `--user-data`, `--disable-api-termination` | `--ebs-optimized`, `--source-dest-check`, `--instance-initiated-shutdown-behavior`, `--block-device-mappings`, `--groups`, `--ena-support`, `--sriov-net-support` | Instance must exist; `instance-type` and `user-data` require stopped state, `disable-api-termination` works on running or stopped | Gateway validates input (exactly one attribute per call, instance ID format) → NATS `ec2.ModifyInstanceAttribute` with `spinifex-workers` queue group → daemon dispatches by state: stopped instances loaded from JetStream KV, running instances updated via `vm.Manager.UpdateAndPersist` (atomic flag check + persist under one lock to prevent Modify/Terminate race) → returns `{}` on success. **InstanceType**: updates vm.InstanceType, Config, and Instance fields; clears StateReason (enables recovery from instance-type-missing bug). **UserData**: stores decoded content in vm.UserData and re-encodes to base64 for RunInstancesInput (cloud-init on next start). **DisableApiTermination**: stamps `RunInstancesInput.DisableApiTermination`; flag survives stop/start and daemon restarts. No instance type pre-validation (matches AWS — invalid types accepted, fail at StartInstances time). | 1. Change instance type while stopped<br>2. Change user data while stopped<br>3. Toggle disable-api-termination on running instance<br>4. Toggle disable-api-termination on stopped instance<br>5. Modify instance-type on running instance (error: NotFound — running instances not in KV)<br>6. Instance not found (error: InvalidInstanceID.NotFound)<br>7. Instance not stopped for type/user-data change (error: IncorrectInstanceState)<br>8. Invalid instance type accepted (fails on start with InsufficientInstanceCapacity)<br>9. StateReason cleared on type change (recovery from capacity-unavailable)<br>10. Missing/malformed instance ID (error: InvalidInstanceID.Malformed)<br>11. No attribute set (error: InvalidParameterValue)<br>12. Multiple attributes in one call (error: InvalidParameterValue) | **DONE** |
 | `get-console-output` | `--instance-id` | `--latest` (always returns latest), `--dry-run` | Instance must be running on a node | Gateway sends NATS `ec2.{instanceId}.GetConsoleOutput` (per-instance topic, routed to owning node) → daemon reads console log file from disk → returns last 64KB base64-encoded with timestamp. Always available regardless of serial console access setting (matches AWS behavior). | 1. Get output from running instance<br>2. Empty log file returns empty output<br>3. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
-| `describe-instance-attribute` | `--instance-id`, `--attribute` (instanceType, userData, instanceInitiatedShutdownBehavior, disableApiTermination, disableApiStop, ebsOptimized, enaSupport, sourceDestCheck, rootDeviceName, kernel, ramdisk) | `--dry-run` | Instance must exist (running or stopped) | Gateway validates input → NATS `ec2.DescribeInstanceAttribute` with `spinifex-workers` queue group → daemon checks running instances first (`d.Instances.VMS`), then stopped instances in JetStream KV → returns single attribute per call (matches AWS behavior). Stored attributes (`instanceType`, `userData`) return real values; unstored attributes return AWS defaults (`instanceInitiatedShutdownBehavior`=stop, `disableApiTermination`=false, etc.) | 1. Get instanceType from running instance<br>2. Get userData from stopped instance<br>3. Get default disableApiTermination<br>4. Invalid attribute name (error)<br>5. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
+| `describe-instance-attribute` | `--instance-id`, `--attribute` (instanceType, userData, instanceInitiatedShutdownBehavior, disableApiTermination, disableApiStop, ebsOptimized, enaSupport, sourceDestCheck, rootDeviceName, kernel, ramdisk) | `--dry-run` | Instance must exist (running or stopped) | Gateway validates input → NATS `ec2.DescribeInstanceAttribute` with `spinifex-workers` queue group → daemon checks running instances first (`d.Instances.VMS`), then stopped instances in JetStream KV → returns single attribute per call (matches AWS behavior). Stored attributes (`instanceType`, `userData`, `disableApiTermination`) return real values from `RunInstancesInput`; remaining attributes return AWS defaults (`instanceInitiatedShutdownBehavior`=stop, etc.) | 1. Get instanceType from running instance<br>2. Get userData from stopped instance<br>3. Get disableApiTermination (reflects RunInstances flag + Modify updates)<br>4. Invalid attribute name (error)<br>5. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
 | `describe-instance-credit-specifications` | `--instance-ids` | `--filters`, `--max-results`, `--dry-run` | None | Gateway-only stub — returns `CpuCredits: "standard"` for each requested instance ID. No daemon round-trip. T-series credit mode is not persisted. | 1. Get credit spec for T-series instance<br>2. Multiple instance IDs | **DONE** |
+| `describe-instance-status` | `--instance-ids`, `--include-all-instances`, `--filters` (availability-zone, instance-state-code, instance-state-name, tag:\*) | `--max-results`, `--next-token`, `--dry-run` (silently ignored), `event.*` / `instance-status.*` / `system-status.*` filters (rejected with InvalidParameterValue) | None | Gateway fans out NATS `ec2.DescribeInstanceStatus` to all nodes (no queue group). Each daemon validates input (i- prefix, filter names), iterates local VMs and renders InstanceStatus: `Running`→ok/passed, everything-else-included→not-applicable. When `IncludeAllInstances=true`, gateway also queries `ec2.DescribeStoppedInstances` in parallel and transforms reservation entries into InstanceStatus with `not-applicable`. Aggregated results dedup'd by InstanceId (running > stopped). Terminated and error-state instances never returned. No real health probing — static values reflect Mulga's lifecycle state only. | 1. Status of running instance (ok/passed)<br>2. Filter by instance ID<br>3. Default excludes stopped instances<br>4. IncludeAllInstances=true includes stopped (not-applicable)<br>5. Cross-account instance excluded<br>6. Unknown filter returns InvalidParameterValue<br>7. Malformed instance ID rejected<br>8. Tag filter narrows correctly | **DONE** |
 | `monitor-instances` | — | `--instance-ids` | Instance must exist | Enable basic monitoring (CPU, disk, network) → store metrics in NATS KV → return monitoring state | 1. Enable monitoring<br>2. Verify monitoring state in describe-instances | **NOT STARTED** |
 | `unmonitor-instances` | — | `--instance-ids` | Instance must exist | Disable detailed monitoring → revert to basic monitoring → return monitoring state | 1. Disable monitoring<br>2. Verify monitoring state reverts in describe-instances | **NOT STARTED** |

@@ -336,6 +346,7 @@ Requires OVN/OVS (`apt install openvswitch-switch ovn-host`). VPC/Subnet/ENI/SG
 | `authorize-security-group-egress` | `--group-id`, `--ip-permissions` | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.AuthorizeSecurityGroupEgress` → daemon retrieves SG → appends rules to EgressRules → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Allow HTTPS outbound<br>2. Restrict to specific CIDR<br>3. Missing GroupId (MissingParameter)<br>4. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
 | `revoke-security-group-ingress` | `--group-id`, `--ip-permissions` (matching rules to remove) | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.RevokeSecurityGroupIngress` → daemon retrieves SG → removes matching rules from IngressRules (set difference) → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Revoke existing rule<br>2. Missing GroupId (MissingParameter)<br>3. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
 | `revoke-security-group-egress` | `--group-id`, `--ip-permissions` (matching rules to remove) | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.RevokeSecurityGroupEgress` → daemon retrieves SG → removes matching rules from EgressRules (set difference) → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Revoke existing rule<br>2. Missing GroupId (MissingParameter)<br>3. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
+| `describe-security-group-rules` | `--filters` (group-id, security-group-rule-id, tag:\*, tag-key), `--security-group-rule-ids` | `--max-results`, `--next-token`, `--dry-run` | None | Gateway validates `sgr-` format on SecurityGroupRuleIds (nil input = all rules in account) → NATS `ec2.DescribeSecurityGroupRules` → daemon iterates SG records in `spinifex-vpc-security-groups` KV under account → applies filters → emits one `SecurityGroupRule` per ingress (IsEgress=false) and egress (IsEgress=true) rule, each with stable `sgr-` ID, GroupOwnerId=accountID, ReferencedGroupInfo for SG-source rules → returns `InvalidSecurityGroupRuleId.NotFound` if any requested ID missing → v1→v2 KV migration backfills `sgr-` IDs on existing rules at daemon startup; MaxResults/NextToken accepted but ignored | 1. List all rules in account<br>2. Filter by group-id<br>3. Filter by security-group-rule-id<br>4. Non-existent rule ID (InvalidSecurityGroupRuleId.NotFound)<br>5. Malformed sgr- ID at gateway (InvalidParameterValue)<br>6. Tag filters match zero (Tags not yet populated on rules)<br>7. Unknown filter (InvalidParameterValue)<br>8. Migration backfills sgr- IDs on pre-v2 records | **DONE** |
 | `create-internet-gateway` | `--tag-specifications` | `--dry-run` | None | NATS `ec2.CreateInternetGateway` → daemon generates igw-ID → creates IGWRecord in `spinifex-igw` KV bucket with state=available (not attached) → returns InternetGateway with empty Attachments | 1. Create IGW<br>2. Verify in describe-internet-gateways<br>3. Tags applied at creation | **DONE** |
 | `attach-internet-gateway` | `--internet-gateway-id`, `--vpc-id` | `--dry-run` | IGW and VPC must exist | NATS `ec2.AttachInternetGateway` → daemon validates IGW exists and is not already attached (ResourceAlreadyAssociated) → sets VpcId on record → state=attached → publishes `vpc.igw-attach` event to vpcd (creates OVN external switch, gateway port, SNAT rules) | 1. Attach IGW to VPC<br>2. Attach already-attached IGW (ResourceAlreadyAssociated)<br>3. Non-existent IGW (error)<br>4. Missing params (MissingParameter) | **DONE** |
 | `detach-internet-gateway` | `--internet-gateway-id`, `--vpc-id` | `--dry-run` | IGW must be attached to specified VPC | NATS `ec2.DetachInternetGateway` → daemon validates IGW exists and is attached to specified VPC (GatewayNotAttached if wrong VPC) → clears VpcId → state=available → publishes `vpc.igw-detach` event to vpcd (removes OVN resources) | 1. Detach attached IGW<br>2. Detach from wrong VPC (GatewayNotAttached)<br>3. Detach unattached IGW (error) | **DONE** |
@@ -77,7 +77,7 @@ Prints per-node CPU/memory usage and cluster-wide instance type availability.

 ```bash
 spx admin images list
-spx admin images import --name debian-12-arm64
+spx admin images import --name debian-13-arm64
 ```

 Catalog imports verify the image against the catalog-declared SHA-256/SHA-512 digest before extraction. Use `--file` to import operator-supplied media (verification skipped — operator is responsible for integrity), or `--force` to re-download after a checksum mismatch.
@@ -0,0 +1,223 @@
+---
+title: "GPU Passthrough"
+description: "Configure VFIO GPU passthrough on a Spinifex node to expose NVIDIA or AMD GPUs to EC2 instances."
+category: "Compute"
+tags:
+  - gpu
+  - passthrough
+  - vfio
+  - ec2
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS EC2 Accelerated Computing Instance Types"
+    url: "https://docs.aws.amazon.com/ec2/latest/instancetypes/ac.html"
+---
+
+# GPU Passthrough
+
+> Configure VFIO GPU passthrough on a Spinifex node to expose NVIDIA or AMD GPUs to EC2 instances.
+
+> **Note:** GPU passthrough is supported on x86_64 hosts only.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Host Setup](#host-setup)
+- [Enable Passthrough](#enable-passthrough)
+- [Status and Monitoring](#status-and-monitoring)
+- [GPU Instance Types](#gpu-instance-types)
+- [Launching GPU Instances](#launching-gpu-instances)
+- [Disable Passthrough](#disable-passthrough)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Spinifex supports VFIO-based GPU passthrough, binding NVIDIA and AMD GPUs directly to guest VMs via the `vfio-pci` kernel driver. Once configured, the node exposes GPU-enabled EC2 instance types and the GPU is allocated exclusively to individual instances.
+
+At startup, Spinifex always probes for GPU hardware and surfaces the result in the node banner and in `spx top nodes` — regardless of whether passthrough is enabled. This lets operators verify hardware before activating the feature.
+
+The passthrough state is reflected in the login banner:
+
+![GPU not yet enabled](../../../.github/assets/images/gpu-banner1.png)
+
+---
+
+## Prerequisites
+
+- IOMMU must be supported and enabled in the host BIOS/UEFI (`Intel VT-d` or `AMD-Vi`).
+- An NVIDIA or AMD GPU must be physically installed.
+- The Spinifex node must be running on bare metal (not inside a VM).
+
+---
+
+## Host Setup
+
+Run the one-time host configuration as root. This command is idempotent — safe to re-run after a reboot.
+
+```bash
+sudo spx admin gpu setup
+```
+
+What it does:
+
+1. Detects installed GPUs and their PCI IDs.
+2. Enables IOMMU in GRUB (`intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt`).
+3. Writes the vfio udev rule (`/etc/udev/rules.d/99-spinifex-vfio.rules`).
+4. Blacklists `nouveau` (NVIDIA) and `amdgpu` (AMD) so the host kernel does not claim the device.
+5. Configures `vfio-pci` early binding in `/etc/modprobe.d/vfio-pci.conf`.
+6. Adds vfio modules to initramfs.
+
+If any change requires a reboot, the command prints instructions and exits:
+
+```
+Setup complete — reboot required.
+  sudo reboot
+  Then run: sudo spx admin gpu enable
+```
+
+---
+
+## Enable Passthrough
+
+After setup and reboot, enable GPU passthrough:
+
+```bash
+sudo spx admin gpu enable
+```
+
+The command writes the configuration, signals the daemon (`SIGHUP`), and waits up to 30 seconds for the daemon to confirm the new state. On success it prints the current GPU status.
+
+
+![GPU enabled](../../../.github/assets/images/gpu-enabled.png)
+
+---
+
+## Status and Monitoring
+
+### Per-node status
+
+```bash
+spx admin gpu status
+# or for a specific node in the cluster:
+spx admin gpu status --node <node-name>
+```
+
+Output includes hardware detected, IOMMU state, vfio-pci binding, passthrough enabled/disabled, GPU pool allocation (`allocated/total`), and the GPU-capable EC2 instance types available on that node.
+
+### Cluster view
+
+`spx top nodes` includes a `GPU (used/total)` column:
+
+| Value | Meaning |
+|-------|---------|
+| `1/2` | Passthrough active — 1 of 2 GPUs allocated |
+| `0/1*` | Node has a GPU but passthrough is not enabled |
+| `-` | No GPU hardware detected |
+
+```bash
+spx top nodes
+```
+
+---
+
+## GPU Instance Types
+
+GPU instance types are derived from detected hardware.
+
+To list available GPU instance types on the current node:
+
+```bash
+aws ec2 describe-instance-types \
+  --filters Name=instance-type,Values=g*
+```
+
+---
+
+## Launching GPU Instances
+
+Import the GPU-enabled AMI before launching:
+
+```bash
+spx admin images import --name ubuntu-gpu-nvidia   # NVIDIA hosts
+spx admin images import --name ubuntu-gpu-amd      # AMD hosts
+```
+
+Launch a GPU instance:
+
+```bash
+aws ec2 run-instances \
+  --image-id $GPU_AMI \
+  --instance-type $INSTANCE_TYPE \
+  --key-name spinifex-key
+```
+
+To verify the GPU is visible from inside the instance, SSH in and run:
+
+```bash
+# NVIDIA
+nvidia-smi
+
+# AMD
+rocm-smi
+```
+
+---
+
+## Disable Passthrough
+
+Passthrough cannot be disabled while GPU instances are running. Terminate all GPU instances first, then:
+
+```bash
+sudo spx admin gpu disable
+```
+
+The command signals the daemon and waits for confirmation. The GPU is released back to the host kernel on the next reboot (the vfio-pci binding persists until `setup` is re-run without the blacklists).
+
+---
+
+## Troubleshooting
+
+### IOMMU not active after reboot
+
+Verify IOMMU is enabled in BIOS/UEFI (`Intel VT-d` or `AMD-Vi`). Then check GRUB picked up the parameters:
+
+```bash
+cat /proc/cmdline | grep iommu
+ls /sys/kernel/iommu_groups/
+```
+
+If `/sys/kernel/iommu_groups/` is empty, IOMMU is not active. Re-run `sudo spx admin gpu setup` after enabling it in firmware.
+
+### GPU bound to native driver instead of vfio-pci
+
+```bash
+lspci -k | grep -A3 -i "vga\|3d\|display"
+```
+
+If the driver is `amdgpu` or `nvidia` rather than `vfio-pci`, the blacklist or early binding did not take effect. Re-run setup and reboot:
+
+```bash
+sudo spx admin gpu setup
+sudo reboot
+```
+
+### `spx admin gpu enable` fails with "prerequisites not met"
+
+Run `setup` first to configure the host, then retry `enable`:
+
+```bash
+sudo spx admin gpu setup
+sudo spx admin gpu enable
+```
+
+### Daemon does not confirm within 30 seconds
+
+Check the daemon log for errors:
+
+```bash
+journalctl -u spinifex-daemon -n 50
+```
@@ -157,7 +157,7 @@ If the AMI is for a different architecture (e.g. arm64 on an x86_64 host), impor

 ```bash
 spx admin images list
-spx admin images import --name debian-12-x86_64
+spx admin images import --name debian-13-x86_64
 ```

 ### Cannot SSH Into Instance
@@ -573,6 +573,54 @@ external_interface = "br-wan"
 DHCP server when VMs launch and releases them on terminate. Requires `dhclient`
 on the host (`apt install isc-dhcp-client`).

+## Host-Local Subnet (No Upstream Router)
+
+```
+Network: 192.168.10.0/24 — host-local, reachable from the host only
+Host WAN: 198.51.100.10/24 on br-wan (existing address — unchanged)
+Gateway: 192.168.10.1 — second address added to br-wan
+```
+
+Add the VM pool gateway as a second address on `br-wan` alongside the existing WAN
+IP. The host acts as the gateway for the pool — no upstream router or DHCP server
+needed for this range.
+
+```yaml
+# /etc/netplan/…
+bridges:
+  br-wan:
+    addresses:
+      - 192.168.10.1/24      # VM pool gateway — host-local
+      - 198.51.100.10/24     # existing WAN IP — unchanged
+    routes:
+      - to: default
+        via: 198.51.100.1
+```
+
+```toml
+[network]
+external_mode = "pool"
+
+[[network.external_pools]]
+name        = "wan"
+source      = "static"         # required — no upstream DHCP for this range
+range_start = "192.168.10.2"
+range_end   = "192.168.10.100"
+gateway     = "192.168.10.1"   # second address on br-wan
+prefix_len  = 24
+dns_servers = ["8.8.8.8"]
+```
+
+**Setup:** Apply with `sudo netplan apply`. VMs are reachable from the host at
+`192.168.10.x`. For internet access through the host's WAN interface:
+
+```bash
+sysctl -w net.ipv4.ip_forward=1
+iptables -t nat -A POSTROUTING -s 192.168.10.0/24 -o br-wan -j MASQUERADE
+```
+
+Persist via `/etc/sysctl.d/99-ip-forward.conf` and `netfilter-persistent save`.
+
 ## Datacenter / Colo (ISP Block)

 ```
@@ -74,8 +74,8 @@ Mirror the cloud image you intend to run as guest VMs:

 ```bash
 mkdir -p images
-curl -fsSL "https://cdimage.debian.org/cdimage/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2" \
-  -o images/debian-12-amd64.qcow2
+curl -fsSL "https://cloud.debian.org/images/cloud/trixie/20260501-2465/debian-13-genericcloud-amd64-20260501-2465.qcow2" \
+  -o images/debian-13-amd64.qcow2
 ```

 ## Step 3. Assemble Transfer Media
@@ -140,8 +140,8 @@ sudo systemctl start spinifex.target
 Register the pre-staged image with Spinifex:

 ```bash
-sudo spx admin images import --file /mnt/usb/images/debian-12-amd64.qcow2 \
-  --distro debian --version 12 --arch x86_64
+sudo spx admin images import --file /mnt/usb/images/debian-13-amd64.qcow2 \
+  --distro debian --version 13 --arch x86_64
 ```

 ## Step 9. Verify
@@ -22,6 +22,7 @@ resources:
 ## Table of Contents

 - [Overview](#overview)
+- [Prerequisites](#prerequisites)
 - [Instructions](#instructions)
 - [Troubleshooting](#troubleshooting)

@@ -37,6 +38,22 @@ A Spinifex cluster distributes services across multiple servers for high availab
 - UDP port 6081 open between hosts (Geneve tunnels)
 - TCP ports 4222, 4248, 6641, 6642 open between hosts (NATS, OVN)

+## Prerequisites
+
+> [!IMPORTANT]
+> **Prerequisite — WAN bridge required on every node.**
+>
+> Before running the installer on any server, that server's WAN interface **must** already be enslaved to a Linux bridge named `br-wan`. The host IP, default route, and DHCP must all live on the bridge — not on the bare NIC.
+>
+> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in the next release.
+>
+> **Verify on every node before continuing:**
+>
+> - `ip -br link show br-wan` — bridge exists and is `UP`
+> - `ip route` — default route's `dev` is `br-wan`
+>
+> **Setup references:** [VPC Networking → Bridge Setup](/docs/vpc-networking#bridge-setup-physical-network-wiring) for the topology.
+
 ## Instructions

 ## Step 1. Install Spinifex on Each Server
@@ -31,8 +31,8 @@ This guide builds Spinifex from source. For production deployments, the [binary

 **Requirements:**

- Ubuntu 22.04+ / Debian 12+
- Go 1.26.2+
+- Ubuntu 26.04 / Debian 13
+- Go 1.26.3+
 - GCC, make, pkg-config
 - QEMU/KVM, OVN/Open vSwitch, AWS CLI v2, dhcpcd-base

@@ -22,6 +22,7 @@ resources:
 ## Table of Contents

 - [Overview](#overview)
+- [Prerequisites](#prerequisites)
 - [Instructions](#instructions)
 - [Troubleshooting](#troubleshooting)

@@ -35,8 +36,8 @@ This guide installs Spinifex on a single server using the binary installer. For

 **Supported Operating Systems:**

- Ubuntu 22.04 / 24.04 / 25.10
- Debian 12 / 13
+- Ubuntu 26.04
+- Debian 13

 **What Gets Installed:**

@@ -47,6 +48,22 @@ This guide installs Spinifex on a single server using the binary installer. For
 - Viperblock (EBS-compatible block storage)
 - AWS CLI v2

+## Prerequisites
+
+> [!IMPORTANT]
+> **Prerequisite — WAN bridge required.**
+>
+> Before running the installer, the host's WAN interface **must** already be enslaved to a Linux bridge named `br-wan`. The host IP, default route, and DHCP must all live on the bridge — not on the bare NIC.
+>
+> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in the next release.
+>
+> **Verify before continuing:**
+>
+> - `ip -br link show br-wan` — bridge exists and is `UP`
+> - `ip route` — default route's `dev` is `br-wan`
+>
+> **Setup references:** [VPC Networking → Bridge Setup](/docs/vpc-networking#bridge-setup-physical-network-wiring) for the topology.
+
 ## Instructions

 ## Step 1. Install Spinifex
@@ -58,23 +58,22 @@ spx admin images list

 ```
 NAME                    | DISTRO | VERSION | ARCH   | BOOT
-debian-12-arm64         | debian | 12      | arm64  | bios
-debian-12-x86_64        | debian | 12      | x86_64 | bios
-lb-alpine-3.21.6-x86_64 | alpine | 3.21.6  | x86_64 | bios
-ubuntu-24.04-arm64      | ubuntu | 24.04   | arm64  | bios
-ubuntu-24.04-x86_64     | ubuntu | 24.04   | x86_64 | bios
+debian-13-arm64         | debian | 13      | arm64  | bios
+debian-13-x86_64        | debian | 13      | x86_64 | bios
+ubuntu-26.04-arm64      | ubuntu | 26.04   | arm64  | bios
+ubuntu-26.04-x86_64     | ubuntu | 26.04   | x86_64 | bios
 ```

 Import an image:

 ```bash
-spx admin images import --name ubuntu-24.04-x86_64
+spx admin images import --name ubuntu-26.04-x86_64
 ```

 Or import a local image file:

 ```bash
-spx admin images import --file ~/images/ubuntu-24.04.img --distro ubuntu --version 24.04 --arch x86_64
+spx admin images import --file ~/images/ubuntu-26.04.img --distro ubuntu --version 26.04 --arch x86_64
 ```

 Verify the import and note the AMI ID:
@@ -173,7 +172,7 @@ Launch an instance in the public subnet, note this will select an instance type
 > **Note:** Replace `AMI_NAME` with the previously imported image above with the `ami-` prefix.

 ```bash
-AMI_NAME="ami-ubuntu-24.04-x86_64"
+AMI_NAME="ami-ubuntu-26.04-x86_64"

 AMI_ID=$(aws ec2 describe-images \
  --filters "Name=name,Values=$AMI_NAME" \
@@ -105,8 +105,8 @@ Spinifex nodes initiate a small, fixed set of outbound connections.

 | Destination | Purpose | Protocol | Verification |
 |-------------|---------|----------|--------------|
-| `https://cdimage.debian.org/cdimage/cloud/bookworm/latest` | Debian 12 cloud image download | HTTPS | TLS + checksum verification |
-| `https://cloud-images.ubuntu.com/noble/current` | Ubuntu 24.04 LTS cloud image download | HTTPS | TLS + checksum verification |
+| `https://cloud.debian.org/images/cloud/trixie/20260501-2465` | Debian 13 cloud image download | HTTPS | TLS + checksum verification |
+| `https://cloud-images.ubuntu.com/resolute/current` | Ubuntu 26.04 LTS cloud image download | HTTPS | TLS + checksum verification |
 | `https://d2yp8ipz5jfqcw.cloudfront.net` | Alpine image for managed HAProxy load-balancer | HTTPS | TLS + checksum verification |
 | `https://install.mulgadc.com/install` | One-shot install telemetry POST on `spx admin init` / `join`. | HTTPS | TLS |

@@ -56,7 +56,7 @@ Deploy a VPC with public and private subnets where the private subnet has no int
 **Prerequisites:**

 - Spinifex installed and running (see [Installing Spinifex](/docs/install))
- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- An Ubuntu 26.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
 - OpenTofu or Terraform installed

 ## Instructions
@@ -149,7 +149,7 @@ The private route table should have no `0.0.0.0/0` route. Also verify the privat

 ### AMI Not Found

-Ensure you have imported an Ubuntu 24.04 image:
+Ensure you have imported an Ubuntu 26.04 image:

 ```bash
 aws ec2 describe-images --owners 000000000000 --profile spinifex
@@ -58,7 +58,7 @@ Deploy two Nginx web servers behind an internet-facing Application Load Balancer
 | Elastic IP | `nginx-alb-nat-eip` | Public address for the NAT Gateway |
 | NAT Gateway | `nginx-alb-nat` | Outbound internet for the private subnets (cloud-init apt bootstrap) |
 | Security Group | `nginx-alb-sg` | Allows SSH (22) and HTTP (80) inbound |
-| EC2 Instances | `nginx-alb-1`, `nginx-alb-2` | Ubuntu 24.04 with Nginx via cloud-init (private subnets) |
+| EC2 Instances | `nginx-alb-1`, `nginx-alb-2` | Ubuntu 26.04 with Nginx via cloud-init (private subnets) |
 | ALB | `nginx-alb` | Internet-facing Application Load Balancer on port 80 |
 | Target Group | `nginx-alb-tg` | HTTP health-checked group for both instances |
 | Listener | HTTP :80 | Forwards traffic to the target group |
@@ -66,7 +66,7 @@ Deploy two Nginx web servers behind an internet-facing Application Load Balancer
 **Prerequisites:**

 - Spinifex installed and running (see [Installing Spinifex](/docs/install))
- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- An Ubuntu 26.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
 - OpenTofu or Terraform installed

 ## Instructions
@@ -86,21 +86,15 @@ Or create a `main.tf` file and paste the full configuration below.

 <!-- INCLUDE: main.tf lang:hcl -->

-### Step 2. Install Load Balancer AMI
+### Step 2. Install Debian AMI

-Next install the load balancer AMI images, which is used as the AMI disk image for launching the load-balancer and a requirement.
+Install the Debian 13 AMI which is used in the example to host the `nginx` webservers as an EC2 instance.

 ```bash
-spx admin images import --name lb-alpine-3.21.6-x86_64
+spx admin images import --name debian-13-x86_64
 ```

-### Step 2.1 Install Debian AMI
-
-Next, install the Debian 12 AMI which is used in the example to host the `nginx` webservers as an EC2 instance.
-
-```bash
-spx admin images import --name debian-12-x86_64
-```
+> **Note:** The load balancer itself runs as a direct-boot QEMU microvm using the kernel + initramfs bundled with the Spinifex distribution (`/usr/share/spinifex/microvm/`), so no separate LB AMI import is required.

 ### Step 3. Deploy

@@ -184,7 +178,7 @@ tofu destroy

 ### AMI Not Found

-Ensure you have imported an Ubuntu 24.04 image. Check available AMIs:
+Ensure you have imported an Ubuntu 26.04 image. Check available AMIs:

 ```bash
 aws ec2 describe-images --owners 000000000000 --profile spinifex
@@ -193,7 +187,7 @@ aws ec2 describe-images --owners 000000000000 --profile spinifex
 If missing import:

 ```bash
-spx admin images import --name ubuntu-24.04-x86_64
+spx admin images import --name ubuntu-26.04-x86_64
 ```

 ### Provider Connection Refused
@@ -43,7 +43,7 @@ Deploy a complete Nginx web server on Spinifex using Terraform/OpenTofu. This wo
 **Prerequisites:**

 - Spinifex installed and running (see [Installing Spinifex](/docs/install))
- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- An Ubuntu 26.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
 - OpenTofu or Terraform installed

 ## Instructions
@@ -96,7 +96,7 @@ tofu destroy

 ### AMI Not Found

-Ensure you have imported an Ubuntu 24.04 image. Check available AMIs:
+Ensure you have imported an Ubuntu 26.04 image. Check available AMIs:

 ```bash
 aws ec2 describe-images --owners 000000000000 --profile spinifex
@@ -220,7 +220,7 @@ resource "aws_security_group" "web" {
 }

 # ---------------------------------------------------------------------------
-# EC2 Instance — Ubuntu 24.04 with Nginx installed via cloud-init
+# EC2 Instance — Ubuntu 26.04 with Nginx installed via cloud-init
 # ---------------------------------------------------------------------------

 resource "aws_instance" "nginx" {
@@ -53,7 +53,7 @@ Deploy an EC2 instance running a Flask file-sharing web application backed by S3

 - Spinifex installed and running (see [Installing Spinifex](/docs/install))
 - Predastore running (S3 API on port 8443)
- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- An Ubuntu 26.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
 - OpenTofu or Terraform installed
 - The EC2 instance must be able to reach Predastore — use the host's br-wan IP, not localhost

@@ -153,7 +153,7 @@ cat /opt/webapp/.env

 ### AMI Not Found

-Ensure you have imported an Ubuntu 24.04 image:
+Ensure you have imported an Ubuntu 26.04 image:

 ```bash
 aws ec2 describe-images --owners 000000000000 --profile spinifex
@@ -1,6 +1,6 @@
 module github.com/mulgadc/spinifex

-go 1.26.2
+go 1.26.3

 require (
 	github.com/aws/aws-sdk-go v1.55.8
@@ -12,20 +12,19 @@ require (
 	github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mdlayher/packet v1.1.2
-	github.com/mulgadc/predastore v1.2.0
-	github.com/mulgadc/viperblock v1.2.1
-	github.com/nats-io/nats-server/v2 v2.12.7
-	github.com/nats-io/nats.go v1.51.0
+	github.com/mulgadc/predastore v1.4.0
+	github.com/mulgadc/viperblock v1.4.0
+	github.com/nats-io/nats-server/v2 v2.14.0
+	github.com/nats-io/nats.go v1.52.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
-	github.com/pelletier/go-toml/v2 v2.3.0
+	github.com/pelletier/go-toml/v2 v2.3.1
 	github.com/pterm/pterm v0.12.83
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
-	golang.org/x/crypto v0.50.0
-	golang.org/x/sys v0.43.0
-	gopkg.in/ini.v1 v1.67.1
+	go.uber.org/goleak v1.3.0
+	golang.org/x/crypto v0.51.0
+	gopkg.in/ini.v1 v1.67.2
 )

 require (
@@ -80,7 +79,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/klauspost/compress v1.18.5 // indirect
-	github.com/klauspost/reedsolomon v1.13.3 // indirect
+	github.com/klauspost/reedsolomon v1.14.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
@@ -88,6 +87,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.21 // indirect
+	github.com/mdlayher/packet v1.1.2 // indirect
 	github.com/mdlayher/socket v0.4.1 // indirect
 	github.com/minio/crc64nvme v1.1.1 // indirect
 	github.com/minio/highwayhash v1.0.4 // indirect
@@ -104,7 +104,7 @@ require (
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.20.1 // indirect
-	github.com/quic-go/quic-go v0.59.0 // indirect
+	github.com/quic-go/quic-go v0.59.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.12.0 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
@@ -120,15 +120,15 @@ require (
 	go.opentelemetry.io/otel/trace v1.42.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
-	golang.org/x/mod v0.34.0 // indirect
-	golang.org/x/net v0.53.0 // indirect
+	golang.org/x/mod v0.35.0 // indirect
+	golang.org/x/net v0.54.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
-	golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c // indirect
-	golang.org/x/term v0.42.0 // indirect
-	golang.org/x/text v0.36.0 // indirect
+	golang.org/x/sys v0.44.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect
+	golang.org/x/term v0.43.0 // indirect
+	golang.org/x/text v0.37.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
-	golang.org/x/tools v0.43.0 // indirect
+	golang.org/x/tools v0.44.0 // indirect
 	golang.org/x/vuln v1.1.4 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -31,30 +31,30 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.6 h1:1AX0AthnBQzMx1vbmir3Y4WsnJgiydmnJjiLu+LvXOg=
-github.com/aws/aws-sdk-go-v2 v1.41.6/go.mod h1:dy0UzBIfwSeot4grGvY1AqFWN5zgziMmWGzysDnHFcQ=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 h1:adBsCIIpLbLmYnkQU+nAChU5yhVTvu5PerROm+/Kq2A=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9/go.mod h1:uOYhgfgThm/ZyAuJGNQ5YgNyOlYfqnGpTHXvk3cpykg=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.15 h1:fyvgWTszojq8hEnMi8PPBTvZdTtEVmAVyo+NFLHBhH4=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.15/go.mod h1:gJiYyMOjNg8OEdRWOf3CrFQxM2a98qmrtjx1zuiQfB8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22 h1:GmLa5Kw1ESqtFpXsx5MmC84QWa/ZrLZvlJGa2y+4kcQ=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22/go.mod h1:6sW9iWm9DK9YRpRGga/qzrzNLgKpT2cIxb7Vo2eNOp0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22 h1:dY4kWZiSaXIzxnKlj17nHnBcXXBfac6UlsAx2qL6XrU=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22/go.mod h1:KIpEUx0JuRZLO7U6cbV204cWAEco2iC3l061IxlwLtI=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23 h1:FPXsW9+gMuIeKmz7j6ENWcWtBGTe1kH8r9thNt5Uxx4=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23/go.mod h1:7J8iGMdRKk6lw2C+cMIphgAnT8uTwBwNOsGkyOCm80U=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8 h1:HtOTYcbVcGABLOVuPYaIihj6IlkqubBwFj10K5fxRek=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8/go.mod h1:VsK9abqQeGlzPgUr+isNWzPlK2vKe9INMLWnY65f5Xs=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.14 h1:xnvDEnw+pnj5mctWiYuFbigrEzSm35x7k4KS/ZkCANg=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.14/go.mod h1:yS5rNogD8e0Wu9+l3MUwr6eENBzEeGejvINpN5PAYfY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22 h1:PUmZeJU6Y1Lbvt9WFuJ0ugUK2xn6hIWUBBbKuOWF30s=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22/go.mod h1:nO6egFBoAaoXze24a2C0NjQCvdpk8OueRoYimvEB9jo=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22 h1:SE+aQ4DEqG53RRCAIHlCf//B2ycxGH7jFkpnAh/kKPM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22/go.mod h1:ES3ynECd7fYeJIL6+oax+uIEljmfps0S70BaQzbMd/o=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0 h1:7G26Sae6PMKn4kMcU5JzNfrm1YrKwyOhowXPYR2WiWY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0/go.mod h1:Fw9aqhJicIVee1VytBBjH+l+5ov6/PhbtIK/u3rt/ls=
-github.com/aws/smithy-go v1.25.0 h1:Sz/XJ64rwuiKtB6j98nDIPyYrV1nVNJ4YU74gttcl5U=
-github.com/aws/smithy-go v1.25.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
+github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
+github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
+github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -236,8 +236,8 @@ github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuOb
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
-github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
+github.com/klauspost/reedsolomon v1.14.0 h1:5YSZeclzSYg5nl349+GDG/agDtQ6MZiwUYXvVKN1Jx0=
+github.com/klauspost/reedsolomon v1.14.0/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -286,20 +286,20 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/predastore v1.2.0 h1:E4+ehAm2YEi2n5t4DmqguURMUXwTSDrK7792zVzw7JM=
-github.com/mulgadc/predastore v1.2.0/go.mod h1:+NWROaNCtpPE9f36fUxCHwskW8CuHHEIvjsDQrwOB00=
-github.com/mulgadc/viperblock v1.2.1 h1:G3cSgh/yspZd7Wdg3C9QGqkqzYbEfYpusd6uMddSDmg=
-github.com/mulgadc/viperblock v1.2.1/go.mod h1:CisHy13Uam0gUg5dxV0rKryJF2yGbgfDv09FYApRaNk=
+github.com/mulgadc/predastore v1.4.0 h1:txPZCaVT1cDpBotx8zHPXHZD3Mz62+n207zXISkK4uU=
+github.com/mulgadc/predastore v1.4.0/go.mod h1:fXIGz7yCKcHh68ybcw7aigdOdzIfyt19h/0diPF+JCU=
+github.com/mulgadc/viperblock v1.4.0 h1:3Yr6lRDnFgxkdMOWu+Tq1b97qWHQDNwkPWmCp38CIy8=
+github.com/mulgadc/viperblock v1.4.0/go.mod h1:hHdfo02iJgCdNktiPAq+x+VQE0ZNYOLJXw0abJ81LeE=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
 github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
-github.com/nats-io/nats-server/v2 v2.12.7 h1:prQ9cPiWHcnwfT81Wi5lU9LL8TLY+7pxDru6fQYLCQQ=
-github.com/nats-io/nats-server/v2 v2.12.7/go.mod h1:dOnmkprKMluTmTF7/QHZioxlau3sKHUM/LBPy9AiBPw=
-github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI=
-github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
+github.com/nats-io/nats-server/v2 v2.14.0 h1:+8q0HrDFotwLLcGH/legOEOnowunhK+aZ4GYBIWpQlM=
+github.com/nats-io/nats-server/v2 v2.14.0/go.mod h1:ImVUUDvfClJbb6cuJQRc1VmgDCXKM5ds0OoiG9MVOKo=
+github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc=
+github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
 github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
 github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -308,8 +308,8 @@ github.com/ovn-kubernetes/libovsdb v0.8.1 h1:M2J8bcJt5mXCom0HqzfEtuHkT80CTSQRcYG
 github.com/ovn-kubernetes/libovsdb v0.8.1/go.mod h1:ZlnHLzagmLOSvyd9qfxBIZp6wOSOw0IsRsc+6lNUGbU=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM=
-github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc=
+github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -351,8 +351,8 @@ github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5b
 github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s=
 github.com/pterm/pterm v0.12.83 h1:ie+YmGmA727VuhxBlyGr74Ks+7McV6kT99IB8EU80aA=
 github.com/pterm/pterm v0.12.83/go.mod h1:xlgc6bFWyJIMtmLJvGim+L7jhSReilOlOnodeIYe4Tk=
-github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
-github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
+github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic=
+github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -424,14 +424,14 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
-golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
+golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
+golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
-golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
+golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
+golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -442,8 +442,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
-golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
+golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
+golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -484,33 +484,33 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc=
-golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
+golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
+golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4=
+golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
-golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
+golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
+golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
-golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
+golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
+golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
-golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
+golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
+golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
 golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
 golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
 golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM=
@@ -534,8 +534,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k=
-gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
+gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss=
+gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -548,5 +548,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
-pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
+pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc=
+pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk=
@@ -0,0 +1,26 @@
+// Package fipsboot enforces FIPS 140-3 mode at process startup.
+//
+// Blank-import this package from every binary's main:
+//
+//	import _ "github.com/mulgadc/spinifex/internal/fipsboot"
+//
+// Builds with GOFIPS140=v1.0.0 link the Go Cryptographic Module v1.0.0
+// (NIST CMVP cert #5247). At runtime FIPS mode can still be turned off
+// via GODEBUG=fips140=off; init() panics in that case to close the hole.
+package fipsboot
+
+import (
+	"crypto/fips140"
+	"log/slog"
+)
+
+func init() {
+	if !fips140.Enabled() {
+		panic("fipsboot: FIPS 140-3 mode is not enabled at runtime — refusing to start. " +
+			"Build with GOFIPS140=v1.0.0 and do not set GODEBUG=fips140=off.")
+	}
+	slog.Info("FIPS 140-3 mode enabled",
+		"module", "Go Cryptographic Module v1.0.0",
+		"cmvp_cert", "5247",
+	)
+}
@@ -31,6 +31,7 @@ inventory and roles relative to CWD.
 | `playbooks/dev-teardown.yml` | Wipe state only (no reinstall) | `reset-dev-env.sh` §teardown |
 | `playbooks/dev-install.yml` | Build + install + init + smoketest on clean box | `dev-install.sh` |
 | `playbooks/dev-reset.yml` | Capture settings → teardown → build → install → init → smoketest | `reset-dev-env.sh` (full) |
+| `playbooks/dev-deploy.yml` | Rebuild + swap binaries/microvm artifacts + restart (no setup.sh, no smoketest) | `make deploy` |

 Upcoming (not yet implemented):

@@ -44,6 +45,7 @@ ansible-playbook playbooks/dev-preflight.yml
 ansible-playbook playbooks/dev-teardown.yml
 ansible-playbook playbooks/dev-install.yml
 ansible-playbook playbooks/dev-reset.yml
+ansible-playbook playbooks/dev-deploy.yml
 ```

 Or via `make` (from `spinifex/`):
@@ -53,8 +55,16 @@ make ansible-dev-preflight
 make ansible-dev-teardown
 make ansible-dev-install
 make ansible-dev-reset
+make ansible-dev-deploy
 ```

+### When to use which
+
+- First-time / clean box → `ansible-dev-install`
+- Iterate on Go code, microvm initramfs, lb-agent → `ansible-dev-deploy` (fast)
+- Changed systemd units, helper scripts, logrotate, setup.sh → `ansible-dev-reset` (slow, full rebuild)
+- Need a clean slate without reinstall → `ansible-dev-teardown`
+
 ## Variable overrides

 Defaults in `vars/defaults.yml`. Override per run with `-e`:
@@ -74,7 +84,6 @@ Useful overrides:
 | `spinifex_region` | `ap-southeast-2` | Region passed to `spx admin init` |
 | `spinifex_az` | `{{ region }}a` | AZ passed to `spx admin init` |
 | `spinifex_external_mode` | `pool` | `pool` \| `nat` external networking |
-| `spinifex_build_lb_image` | `true` | Build + import LB system image after init |

 `dev-reset.yml` automatically captures `region`, `az`, `external_mode`,
 pool range, gateway, prefix-len, nat gateway_ip from the existing
@@ -0,0 +1,14 @@
+---
+# In-place artifact swap: rebuilds + replaces binaries/microvm artifacts on
+# an already-installed node, then restarts spinifex.target. Use when iterating
+# on code without altering systemd units, users, dirs, or running the full
+# smoketest. For first-time or unit/script changes use dev-install / dev-reset.
+- name: Dev deploy — build + swap binaries + restart (no setup.sh, no smoketest)
+  hosts: localhost
+  gather_facts: true
+  vars_files:
+    - ../vars/defaults.yml
+
+  roles:
+    - build
+    - deploy
@@ -11,6 +11,18 @@
    chdir: "{{ mulga_root }}/viperblock"
  changed_when: true

+- name: build — detect microVM build script (present only on direct-boot branches)
+  ansible.builtin.stat:
+    path: "{{ spinifex_root }}/scripts/build-microvm-image.sh"
+  register: build_microvm_script
+
+- name: build — microVM kernel + initramfs (direct-boot LBs)
+  ansible.builtin.command:
+    cmd: make build-microvm-image
+    chdir: "{{ spinifex_root }}"
+  changed_when: true
+  when: build_microvm_script.stat.exists
+
 - name: build — verify build artifacts present
  ansible.builtin.stat:
    path: "{{ item }}"
@@ -40,9 +52,15 @@
    mkdir -p "$STAGING/scripts"
    cp "{{ spinifex_root }}/build/scripts/"* "$STAGING/scripts/"
    cp "{{ spinifex_root }}/build/logrotate/spinifex" "$STAGING/logrotate-spinifex"
+    # microVM artifacts ship only when the branch built them.
+    if [ -f "{{ spinifex_root }}/build/microvm/vmlinuz" ] && [ -f "{{ spinifex_root }}/build/microvm/initramfs.cpio.gz" ]; then
+        mkdir -p "$STAGING/microvm"
+        cp "{{ spinifex_root }}/build/microvm/vmlinuz" "$STAGING/microvm/"
+        cp "{{ spinifex_root }}/build/microvm/initramfs.cpio.gz" "$STAGING/microvm/"
+    fi
    tar czf "{{ build_tarball_path }}" -C "$STAGING" .
  changed_when: true

 - name: build — summary
  ansible.builtin.debug:
-    msg: "Built spx + nbdkit plugin, tarball at {{ build_tarball_path }}"
+    msg: "Built spx + nbdkit plugin{{ ' + microvm artifacts' if build_microvm_script.stat.exists else '' }}, tarball at {{ build_tarball_path }}"
@@ -0,0 +1,148 @@
+---
+# In-place artifact swap for an already-installed spinifex node. Mirrors
+# `make deploy`: replace binaries + microvm artifacts, restart services.
+# Does NOT run setup.sh (no user/dir/systemd-unit changes) or smoketest.
+
+- name: deploy — fail fast if spinifex isn't already installed
+  ansible.builtin.stat:
+    path: "{{ spinifex_config_file }}"
+  register: deploy_installed_stat
+  failed_when: not deploy_installed_stat.stat.exists
+
+- name: deploy — assert config present
+  ansible.builtin.assert:
+    that: deploy_installed_stat.stat.exists
+    fail_msg: |
+      {{ spinifex_config_file }} missing — this playbook only updates an
+      existing install. Run ansible-dev-install or ansible-dev-reset first.
+
+- name: deploy — verify build artifacts present
+  ansible.builtin.stat:
+    path: "{{ item }}"
+  register: deploy_artifacts
+  loop:
+    - "{{ spinifex_root }}/bin/spx"
+    - "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so"
+
+- name: deploy — assert artifacts exist after build role
+  ansible.builtin.assert:
+    that: item.stat.exists
+    fail_msg: "Build artifact missing: {{ item.item }} — build role should have produced it"
+  loop: "{{ deploy_artifacts.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+
+- name: deploy — detect microvm artifacts (direct-boot branches only)
+  ansible.builtin.stat:
+    path: "{{ spinifex_root }}/build/microvm/{{ item }}"
+  register: deploy_microvm_stat
+  loop:
+    - vmlinuz
+    - initramfs.cpio.gz
+
+- name: deploy — set microvm presence fact
+  ansible.builtin.set_fact:
+    deploy_microvm_present: "{{ deploy_microvm_stat.results | map(attribute='stat.exists') | min }}"
+
+- name: deploy — detect nbdkit plugin directory
+  ansible.builtin.shell: |
+    set -e
+    nbdkit --dump-config | awk -F= '/^plugindir=/{print $2; exit}'
+  register: deploy_plugindir_cmd
+  changed_when: false
+
+- name: deploy — set plugin path fact
+  ansible.builtin.set_fact:
+    deploy_plugindir: "{{ deploy_plugindir_cmd.stdout | trim }}"
+
+- name: deploy — assert plugindir resolved
+  ansible.builtin.assert:
+    that: deploy_plugindir | length > 0
+    fail_msg: "Could not detect nbdkit plugin directory via `nbdkit --dump-config`"
+
+- name: deploy — stop spinifex.target before binary swap
+  become: true
+  ansible.builtin.systemd:
+    name: spinifex.target
+    state: stopped
+  failed_when: false
+
+- name: deploy — install spx binary
+  become: true
+  ansible.builtin.copy:
+    src: "{{ spinifex_root }}/bin/spx"
+    dest: /usr/local/bin/spx
+    mode: "0755"
+    owner: root
+    group: root
+
+- name: deploy — install viperblock nbdkit plugin
+  become: true
+  ansible.builtin.copy:
+    src: "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so"
+    dest: "{{ deploy_plugindir }}/nbdkit-viperblock-plugin.so"
+    mode: "0755"
+    owner: root
+    group: root
+
+- name: deploy — ensure microvm dir
+  become: true
+  ansible.builtin.file:
+    path: /usr/share/spinifex/microvm
+    state: directory
+    mode: "0755"
+    owner: root
+    group: root
+  when: deploy_microvm_present
+
+- name: deploy — install microvm kernel
+  become: true
+  ansible.builtin.copy:
+    src: "{{ spinifex_root }}/build/microvm/vmlinuz"
+    dest: /usr/share/spinifex/microvm/vmlinuz
+    mode: "0644"
+    owner: root
+    group: root
+  when: deploy_microvm_present
+
+- name: deploy — install microvm initramfs
+  become: true
+  ansible.builtin.copy:
+    src: "{{ spinifex_root }}/build/microvm/initramfs.cpio.gz"
+    dest: /usr/share/spinifex/microvm/initramfs.cpio.gz
+    mode: "0644"
+    owner: root
+    group: root
+  when: deploy_microvm_present
+
+- name: deploy — daemon-reload (covers any unit changes shipped via setup.sh later)
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: deploy — start spinifex.target
+  become: true
+  ansible.builtin.systemd:
+    name: spinifex.target
+    state: started
+
+- name: deploy — wait for awsgw to respond
+  become: true
+  become_user: "{{ spinifex_invoking_user }}"
+  environment:
+    AWS_PROFILE: spinifex
+  ansible.builtin.command: aws ec2 describe-regions
+  register: deploy_awsgw
+  retries: 15
+  delay: 2
+  until: deploy_awsgw.rc == 0
+  changed_when: false
+
+- name: deploy — summary
+  ansible.builtin.debug:
+    msg: |
+      Deployed:
+        spx           → /usr/local/bin/spx
+        nbdkit plugin → {{ deploy_plugindir }}/nbdkit-viperblock-plugin.so
+        microvm       → {{ '/usr/share/spinifex/microvm/{vmlinuz,initramfs.cpio.gz}' if deploy_microvm_present else 'skipped (not built on this branch)' }}
+      spinifex.target restarted; awsgw responding.
@@ -74,19 +74,6 @@
  delay: 2
  changed_when: false

- name: init — build LB agent
-  ansible.builtin.command:
-    cmd: make build-lb-agent
-    chdir: "{{ spinifex_root }}"
-  changed_when: true
-  when: spinifex_build_lb_image | bool
-
- name: init — import LB system image
-  ansible.builtin.command:
-    cmd: "{{ spinifex_root }}/scripts/build-system-image.sh {{ spinifex_lb_image_conf }} --import --quiet"
-  changed_when: true
-  when: spinifex_build_lb_image | bool
-
 - name: init — summary
  ansible.builtin.debug:
    msg: |
@@ -95,4 +82,3 @@
        external_mode={{ spinifex_external_mode }}
        spinifex.target: started
        CA: {{ spinifex_ca_trust_path }}
-        LB image: {{ 'imported' if spinifex_build_lb_image else 'skipped' }}
@@ -32,34 +32,34 @@ spinifex_nat_gateway_ip: ""

 # Preflight checks.
 spinifex_ports:
-  - { port: 4222, service: NATS }
-  - { port: 8443, service: Predastore }
-  - { port: 9999, service: "Spinifex Gateway" }
+    - { port: 4222, service: NATS }
+    - { port: 8443, service: Predastore }
+    - { port: 9999, service: "Spinifex Gateway" }

 spinifex_required_commands:
-  - go
-  - make
-  - nbdkit
+    - go
+    - make
+    - nbdkit
 spinifex_optional_commands:
-  - air
+    - air
 spinifex_ovn_commands:
-  - ovs-vsctl
-  - ovn-controller
-  - ovn-nbctl
+    - ovs-vsctl
+    - ovn-controller
+    - ovn-nbctl

 spinifex_components:
-  - viperblock
-  - predastore
+    - viperblock
+    - predastore

 # Process patterns for teardown.
 # Bracket trick ([b]in/spx etc.) prevents the pattern literal from matching
 # our own pgrep/shell cmdline. Exclude filter drops benign log followers and
 # ansible's own wrappers that also mention these service names.
-spinifex_process_pattern: '([b]in/spx|spinifex-[u]i|nats-[s]erver|[p]redastore|[v]iperblock|[v]pcd)'
-spinifex_process_exclude: '(journalctl|ansible|/bin/sh|pgrep|pkill|tail -[fF])'
+spinifex_process_pattern: "([b]in/spx|spinifex-[u]i|nats-[s]erver|[p]redastore|[v]iperblock|[v]pcd)"
+spinifex_process_exclude: "(journalctl|ansible|/bin/sh|pgrep|pkill|tail -[fF])"
 spinifex_qemu_processes:
-  - qemu-system-x86_64
-  - qemu-system-aarch64
+    - qemu-system-x86_64
+    - qemu-system-aarch64

 # Invoking user (when run under sudo).
 spinifex_invoking_user: "{{ lookup('env','SUDO_USER') | default(ansible_facts['user_id'], true) }}"
@@ -79,12 +79,8 @@ mulga_root: "{{ lookup('pipe', 'readlink -f ' ~ playbook_dir ~ '/../../../..') }
 # Build output.
 build_tarball_path: /tmp/spinifex-local.tar.gz

-# LB system image (built + imported at end of init).
-spinifex_build_lb_image: true
-spinifex_lb_image_conf: "{{ spinifex_root }}/scripts/images/lb.conf"
-
 # Smoketest image — parameterised so tofu examples can share the same base.
 spinifex_smoketest_distro: debian
-spinifex_smoketest_version: "12"
+spinifex_smoketest_version: "13"
 # Local image file path (if present, imported directly; otherwise fall back to --name).
 spinifex_smoketest_local_image: "~/images/{{ spinifex_smoketest_distro }}-{{ spinifex_smoketest_version }}.img"
@@ -0,0 +1,261 @@
+#!/bin/bash
+# build-microvm-image.sh — build a minimal Alpine-based microVM kernel + initramfs
+# for use with QEMU microvm machine type and Spinifex lb-agent.
+#
+# Outputs:
+#   $REPO_ROOT/build/microvm/vmlinuz
+#   $REPO_ROOT/build/microvm/initramfs.cpio.gz
+#
+# Rootfs strategy (first match wins):
+#   1. apk available on host → install directly into a temp chroot (Alpine CI/CD)
+#   2. docker available      → run alpine container, export filesystem
+#   3. podman available      → same via podman
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+BUILD_DIR="$REPO_ROOT/build/microvm"
+INIT_SH="$BUILD_DIR/init.sh"
+INITTAB="$BUILD_DIR/inittab"
+
+ALPINE_VERSION="${ALPINE_VERSION:-3.20}"
+ALPINE_MIRROR="${ALPINE_MIRROR:-https://dl-cdn.alpinelinux.org/alpine}"
+ALPINE_PACKAGES="busybox linux-virt haproxy iproute2 ca-certificates"
+
+echo "[build-microvm-image] repo root: $REPO_ROOT"
+echo "[build-microvm-image] build dir: $BUILD_DIR"
+mkdir -p "$BUILD_DIR"
+
+# --- Verify non-negotiable tools ---
+for tool in cpio gzip find; do
+    if ! command -v "$tool" >/dev/null 2>&1; then
+        echo "ERROR: required tool not found: $tool" >&2
+        exit 1
+    fi
+done
+
+# --- Create chroot ---
+CHROOT_DIR=$(mktemp -d)
+cleanup() {
+    echo "[build-microvm-image] cleaning up chroot: $CHROOT_DIR"
+    rm -rf "$CHROOT_DIR"
+}
+trap cleanup EXIT
+
+# --- Populate rootfs ---
+# Strategy 1: native apk (Alpine hosts, CI)
+if command -v apk >/dev/null 2>&1; then
+    echo "[build-microvm-image] rootfs: native apk"
+    mkdir -p \
+        "$CHROOT_DIR/etc/apk" \
+        "$CHROOT_DIR/lib/apk/db" \
+        "$CHROOT_DIR/var/cache/apk"
+    cat > "$CHROOT_DIR/etc/apk/repositories" <<EOF
+${ALPINE_MIRROR}/v${ALPINE_VERSION}/main
+${ALPINE_MIRROR}/v${ALPINE_VERSION}/community
+EOF
+    apk add \
+        --root "$CHROOT_DIR" \
+        --arch x86_64 \
+        --no-cache \
+        --allow-untrusted \
+        --initdb \
+        $ALPINE_PACKAGES
+
+# Strategy 2/3: container runtime (dev machines)
+else
+    CONTAINER_TOOL=""
+    if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
+        CONTAINER_TOOL=docker
+    elif command -v podman >/dev/null 2>&1; then
+        CONTAINER_TOOL=podman
+    else
+        echo "ERROR: no rootfs build tool available." >&2
+        echo "       Install one of: apk (Alpine), docker, podman" >&2
+        exit 1
+    fi
+
+    echo "[build-microvm-image] rootfs: $CONTAINER_TOOL export (alpine:${ALPINE_VERSION})"
+    cid=$($CONTAINER_TOOL run -d \
+        "alpine:${ALPINE_VERSION}" \
+        sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
+
+    exit_code=$($CONTAINER_TOOL wait "$cid")
+    if [ "$exit_code" != "0" ]; then
+        echo "ERROR: package installation failed in container (exit $exit_code)" >&2
+        $CONTAINER_TOOL rm "$cid" >/dev/null 2>&1 || true
+        exit 1
+    fi
+
+    $CONTAINER_TOOL export "$cid" | tar -x -C "$CHROOT_DIR"
+    $CONTAINER_TOOL rm "$cid" >/dev/null 2>&1
+fi
+
+# --- Fix /dev device nodes ---
+# tar extraction without root silently creates 0-byte regular files for device
+# nodes (mknod requires CAP_MKNOD). The kernel opens /dev/console to wire
+# init's stdio to the serial console; a regular file there causes all init
+# output to be silently discarded. Recreate the two nodes needed before
+# devtmpfs mounts (which provides the rest at runtime).
+rm -rf "$CHROOT_DIR/dev"
+mkdir "$CHROOT_DIR/dev"
+sudo mknod -m 600 "$CHROOT_DIR/dev/console" c 5 1
+sudo mknod -m 666 "$CHROOT_DIR/dev/null"    c 1 3
+
+# --- Place init script ---
+echo "[build-microvm-image] installing init.sh as /init..."
+if [ ! -f "$INIT_SH" ]; then
+    echo "ERROR: $INIT_SH not found — cannot build initramfs" >&2
+    exit 1
+fi
+install -m 0755 "$INIT_SH" "$CHROOT_DIR/init"
+
+# --- Place inittab ---
+if [ -f "$INITTAB" ]; then
+    echo "[build-microvm-image] installing inittab..."
+    install -m 0644 "$INITTAB" "$CHROOT_DIR/etc/inittab"
+fi
+
+# --- Copy lb-agent binary ---
+LB_AGENT_BIN="$REPO_ROOT/bin/lb-agent"
+if [ -f "$LB_AGENT_BIN" ]; then
+    echo "[build-microvm-image] copying lb-agent binary..."
+    mkdir -p "$CHROOT_DIR/usr/local/bin"
+    install -m 0755 "$LB_AGENT_BIN" "$CHROOT_DIR/usr/local/bin/lb-agent"
+else
+    echo "[build-microvm-image] WARNING: $LB_AGENT_BIN not found — lb-agent will be absent from initramfs" >&2
+fi
+
+# --- Create required directories for init ---
+mkdir -p \
+    "$CHROOT_DIR/proc" \
+    "$CHROOT_DIR/sys" \
+    "$CHROOT_DIR/dev" \
+    "$CHROOT_DIR/etc/ssl/certs" \
+    "$CHROOT_DIR/etc/conf.d"
+
+# --- Locate kernel ---
+KERNEL_IMG=$(find "$CHROOT_DIR/boot" -name "vmlinuz*" | sort | tail -1)
+if [ -z "$KERNEL_IMG" ]; then
+    echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
+    exit 1
+fi
+echo "[build-microvm-image] kernel image: $KERNEL_IMG"
+
+KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
+KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
+
+# --- Assert fw_cfg module ---
+if ! find "$CHROOT_DIR/lib/modules" \
+        -name "qemu_fw_cfg.ko*" \
+        -o -name "fw_cfg_sysfs.ko*" \
+    | grep -q .; then
+    if ! grep -q "CONFIG_FW_CFG_SYSFS=y" "$KERNEL_CONFIG" 2>/dev/null; then
+        echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
+        exit 1
+    fi
+    echo "[build-microvm-image] fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
+else
+    echo "[build-microvm-image] fw_cfg: module found in lib/modules"
+fi
+
+# --- Strip kernel modules (Phase 2 keep-list approach) ---
+# Only modules needed by a virtio-mmio microvm guest are kept:
+#   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
+#   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
+#   *failover*.ko* — net_failover + failover (virtio_net deps for live migration)
+# All other loadable modules are removed. Drivers compiled into the kernel
+# (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
+echo "[build-microvm-image] stripping kernel modules to virtio+fw_cfg only..."
+for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
+    kver="$(basename "$kver_dir")"
+    kernel_dir="$kver_dir/kernel"
+    [ -d "$kernel_dir" ] || continue
+
+    # Collect keeper modules and their relative paths from the kernel/ tree.
+    tmp_keep=$(mktemp -d)
+    find "$kernel_dir" -name "virtio*.ko.gz" -o -name "virtio*.ko" \
+                       -o -name "*fw_cfg*.ko.gz" -o -name "*fw_cfg*.ko" \
+                       -o -name "*failover*.ko.gz" -o -name "*failover*.ko" \
+        2>/dev/null | while read -r mod; do
+        rel="${mod#"$kernel_dir/"}"
+        dest_dir="$tmp_keep/$(dirname "$rel")"
+        mkdir -p "$dest_dir"
+        cp "$mod" "$dest_dir/"
+    done
+
+    # Replace kernel module tree with the keeper set.
+    rm -rf "$kernel_dir"
+    mkdir -p "$kernel_dir"
+    # Restore each kept file into the same relative path.
+    (cd "$tmp_keep" && find . -name "*.ko*" | while read -r f; do
+        dest_dir="$kernel_dir/$(dirname "${f#./}")"
+        mkdir -p "$dest_dir"
+        cp "$f" "$dest_dir/"
+    done)
+    rm -rf "$tmp_keep"
+
+    # Regenerate module dependency map from the surviving modules.
+    depmod -b "$CHROOT_DIR" "$kver" 2>/dev/null || true
+done
+
+# --- Strip package-manager artifacts (not needed at runtime) ---
+echo "[build-microvm-image] stripping package manager artifacts..."
+rm -rf \
+    "$CHROOT_DIR/lib/apk" \
+    "$CHROOT_DIR/var/cache/apk" \
+    "$CHROOT_DIR/etc/apk" \
+    "$CHROOT_DIR/usr/share/man" \
+    "$CHROOT_DIR/usr/share/doc" \
+    "$CHROOT_DIR/usr/share/apk" \
+    "$CHROOT_DIR/usr/include" \
+    "$CHROOT_DIR/usr/lib/pkgconfig"
+
+# --- Strip debug symbols from binaries ---
+echo "[build-microvm-image] stripping debug symbols from binaries..."
+find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
+     "$CHROOT_DIR/sbin" "$CHROOT_DIR/bin" \
+     "$CHROOT_DIR/usr/local/bin" \
+    -type f 2>/dev/null | while read -r bin; do
+    strip --strip-all "$bin" 2>/dev/null || true
+done
+
+# --- Copy vmlinuz out before stripping /boot from chroot ---
+VMLINUZ_OUT="$BUILD_DIR/vmlinuz"
+echo "[build-microvm-image] copying kernel: $VMLINUZ_OUT"
+cp "$KERNEL_IMG" "$VMLINUZ_OUT"
+
+# Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
+# and kernel config all live here (~25 MiB). None are needed at runtime: the
+# guest already has the kernel loaded by QEMU's -kernel, and Alpine's initramfs
+# is unused because we replace it with our own /init. Leaving /boot in the cpio
+# nearly doubles uncompressed initramfs size and forces a 512 MiB guest just to
+# survive decompression.
+rm -rf "$CHROOT_DIR/boot"
+
+# --- Build initramfs ---
+INITRAMFS_OUT="$BUILD_DIR/initramfs.cpio.gz"
+echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT"
+(
+    cd "$CHROOT_DIR"
+    find . | cpio --quiet -o -H newc | gzip -9 > "$INITRAMFS_OUT"
+)
+
+# --- Log artifact sizes ---
+echo ""
+echo "[build-microvm-image] artifacts:"
+ls -lh "$VMLINUZ_OUT" "$INITRAMFS_OUT"
+echo ""
+
+# --- Phase 2 artifact size gate (50 MiB total) ---
+ARTIFACT_MAX_MiB="${MICROVM_ARTIFACT_MAX_MiB:-50}"
+total_bytes=$(( $(stat -c %s "$VMLINUZ_OUT") + $(stat -c %s "$INITRAMFS_OUT") ))
+max_bytes=$(( ARTIFACT_MAX_MiB * 1024 * 1024 ))
+total_mib=$(( (total_bytes + 1048575) / 1048576 ))
+if [ "$total_bytes" -gt "$max_bytes" ]; then
+    echo "ERROR: artifact size ${total_mib} MiB exceeds ${ARTIFACT_MAX_MiB} MiB gate" >&2
+    echo "       Set MICROVM_ARTIFACT_MAX_MiB=N to override during development." >&2
+    exit 1
+fi
+echo "[build-microvm-image] artifact size gate: PASS (${total_mib} MiB / ${ARTIFACT_MAX_MiB} MiB)"
+echo ""
+echo "[build-microvm-image] done."
@@ -1,10 +1,11 @@
 #!/bin/bash
 set -euo pipefail

-# build-system-image.sh — Build a minimal Alpine AMI from a manifest
+# build-system-image.sh — Build a minimal system AMI from a manifest
 #
-# Creates a pre-baked Alpine Linux image with custom packages, binaries,
-# and setup scripts installed, ready for import as a Spinifex AMI.
+# Supports Alpine Linux and Ubuntu 24.04 as base distros.
+# Creates a pre-baked image with custom packages, binaries, and setup scripts
+# installed, ready for import as a Spinifex AMI.
 #
 # Requirements: qemu-nbd, qemu-img, sudo (for mount/chroot), curl
 # Usage: ./scripts/build-system-image.sh <manifest.conf> [--import]
@@ -46,8 +47,11 @@ done
 # shellcheck source=/dev/null
 source "$MANIFEST"

+# Default distro is alpine for backward compatibility
+DISTRO="${DISTRO:-alpine}"
+
 # Validate required manifest fields
-for field in IMAGE_NAME ALPINE_VERSION IMAGE_SIZE; do
+for field in IMAGE_NAME IMAGE_SIZE; do
    if [[ -z "${!field:-}" ]]; then
        echo "ERROR: Manifest missing required field: $field"
        exit 1
@@ -56,16 +60,48 @@ done

 # Derived paths
 BUILD_DIR="/tmp/${IMAGE_NAME}-image-build"
-ALPINE_IMAGE="generic_alpine-${ALPINE_VERSION}-x86_64-bios-cloudinit-r0.qcow2"
-ALPINE_URL="https://dl-cdn.alpinelinux.org/alpine/v${ALPINE_VERSION%.*}/releases/cloud/${ALPINE_IMAGE}"
-OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-alpine.qcow2"
-OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-alpine.raw"
 NBD_DEV="/dev/nbd0"
 MOUNT_DIR="${BUILD_DIR}/mnt"

+case "$DISTRO" in
+    alpine)
+        if [[ -z "${ALPINE_VERSION:-}" ]]; then
+            echo "ERROR: Manifest missing required field: ALPINE_VERSION"
+            exit 1
+        fi
+        SOURCE_IMAGE="generic_alpine-${ALPINE_VERSION}-x86_64-bios-cloudinit-r0.qcow2"
+        SOURCE_URL="https://dl-cdn.alpinelinux.org/alpine/v${ALPINE_VERSION%.*}/releases/cloud/${SOURCE_IMAGE}"
+        OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-alpine.qcow2"
+        OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-alpine.raw"
+        DISTRO_VERSION="${ALPINE_VERSION}"
+        ROOT_PART="${NBD_DEV}"
+        ;;
+    ubuntu)
+        if [[ -z "${UBUNTU_VERSION:-}" ]]; then
+            echo "ERROR: Manifest missing required field: UBUNTU_VERSION"
+            exit 1
+        fi
+        SOURCE_IMAGE="ubuntu-${UBUNTU_VERSION}-minimal-cloudimg-amd64.img"
+        SOURCE_URL="https://cloud-images.ubuntu.com/minimal/releases/${UBUNTU_CODENAME:-noble}/release/${SOURCE_IMAGE}"
+        OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-ubuntu.qcow2"
+        OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-ubuntu.raw"
+        DISTRO_VERSION="${UBUNTU_VERSION}"
+        ROOT_PART="${NBD_DEV}p1"
+        ;;
+    *)
+        echo "ERROR: Unknown DISTRO: $DISTRO (supported: alpine, ubuntu)"
+        exit 1
+        ;;
+esac
+
 cleanup() {
    echo "Cleaning up..."
-    sudo umount "${MOUNT_DIR}" 2>/dev/null || true
+    # Ubuntu chroot has bind mounts that must be released first
+    sudo umount "${MOUNT_DIR}/dev/pts" 2>/dev/null || true
+    sudo umount "${MOUNT_DIR}/dev"     2>/dev/null || true
+    sudo umount "${MOUNT_DIR}/sys"     2>/dev/null || true
+    sudo umount "${MOUNT_DIR}/proc"    2>/dev/null || true
+    sudo umount "${MOUNT_DIR}"         2>/dev/null || true
    sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
    exec 9>&- 2>/dev/null || true  # release nbd lock if held
    echo "Done."
@@ -80,7 +116,7 @@ fi

 echo "=== System Image Builder ==="
 echo "Image:   ${IMAGE_NAME} — ${IMAGE_DESCRIPTION:-}"
-echo "Alpine:  ${ALPINE_VERSION}"
+echo "Distro:  ${DISTRO} ${DISTRO_VERSION}"
 echo "Size:    ${IMAGE_SIZE}"
 echo "Build:   ${BUILD_DIR}"
 echo ""
@@ -96,6 +132,11 @@ if ! command -v qemu-img &>/dev/null; then
    exit 1
 fi

+if [[ "$DISTRO" == "ubuntu" ]] && ! command -v parted &>/dev/null; then
+    echo "ERROR: parted not found. Install parted."
+    exit 1
+fi
+
 # Build binaries if BUILD_COMMANDS is set
 if [[ -n "${BUILD_COMMANDS:-}" ]]; then
    echo "Building binaries: ${BUILD_COMMANDS}"
@@ -105,7 +146,7 @@ if [[ -n "${BUILD_COMMANDS:-}" ]]; then
    fi
 fi

-# Verify binaries exist and are statically linked
+# Verify binaries exist; Alpine also requires static linking (uses musl)
 if [[ -n "${INSTALL_BINARIES:-}" ]]; then
    IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
    for pair in "${BINARY_PAIRS[@]}"; do
@@ -115,7 +156,7 @@ if [[ -n "${INSTALL_BINARIES:-}" ]]; then
            echo "ERROR: Binary not found: $src_path"
            exit 1
        fi
-        if ! file "$src_path" | grep -q "statically linked"; then
+        if [[ "$DISTRO" == "alpine" ]] && ! file "$src_path" | grep -q "statically linked"; then
            echo "ERROR: $src_path is not statically linked (Alpine uses musl — dynamic glibc binaries will fail)"
            echo "  Rebuild with: CGO_ENABLED=0 go build ..."
            exit 1
@@ -147,31 +188,32 @@ if [[ -f "$OUTPUT_RAW" ]] && [[ $(( $(date +%s) - $(stat -c %Y "$OUTPUT_RAW") ))

    if [[ "$DO_IMPORT" == true ]]; then
        echo "Importing as AMI..."
-        IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro alpine --version "${ALPINE_VERSION}" --arch x86_64)
+        rm -f "$OUTPUT_IMAGE"
+        IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64)
        if [[ -n "${SYSTEM_TAG:-}" ]]; then
            IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
        fi
-        (cd "$PROJECT_DIR" && ./bin/spx admin images import "${IMPORT_ARGS[@]}")
+        spx admin images import "${IMPORT_ARGS[@]}"
    fi
    exit 0
 fi

 mkdir -p "$BUILD_DIR" "$MOUNT_DIR"

-# Step 1: Download Alpine cloud image
-if [[ -f "${BUILD_DIR}/${ALPINE_IMAGE}" ]]; then
-    echo "Alpine image already downloaded."
+# Step 1: Download base cloud image
+if [[ -f "${BUILD_DIR}/${SOURCE_IMAGE}" ]]; then
+    echo "Base image already downloaded."
 else
-    echo "Downloading Alpine ${ALPINE_VERSION} cloud image..."
-    if ! curl --fail -L -o "${BUILD_DIR}/${ALPINE_IMAGE}" "$ALPINE_URL"; then
-        rm -f "${BUILD_DIR}/${ALPINE_IMAGE}"
-        echo "ERROR: Failed to download Alpine image from $ALPINE_URL"
+    echo "Downloading ${DISTRO} ${DISTRO_VERSION} cloud image..."
+    if ! curl --fail -L -o "${BUILD_DIR}/${SOURCE_IMAGE}" "$SOURCE_URL"; then
+        rm -f "${BUILD_DIR}/${SOURCE_IMAGE}"
+        echo "ERROR: Failed to download image from $SOURCE_URL"
        exit 1
    fi
-    # Verify the download is a valid qcow2 image
-    if ! qemu-img info "${BUILD_DIR}/${ALPINE_IMAGE}" &>/dev/null; then
-        rm -f "${BUILD_DIR}/${ALPINE_IMAGE}"
-        echo "ERROR: Downloaded file is not a valid qcow2 image"
+    # Verify the download is a valid qcow2/img
+    if ! qemu-img info "${BUILD_DIR}/${SOURCE_IMAGE}" &>/dev/null; then
+        rm -f "${BUILD_DIR}/${SOURCE_IMAGE}"
+        echo "ERROR: Downloaded file is not a valid disk image"
        exit 1
    fi
 fi
@@ -179,9 +221,9 @@ fi
 # Step 2: Copy image for customization
 rm -f "$OUTPUT_IMAGE"
 echo "Copying image for customization..."
-cp "${BUILD_DIR}/${ALPINE_IMAGE}" "$OUTPUT_IMAGE"
+cp "${BUILD_DIR}/${SOURCE_IMAGE}" "$OUTPUT_IMAGE"

-# Resize the image (Alpine cloud images are ~200MB, need room for packages)
+# Resize the image to provide room for packages
 qemu-img resize "$OUTPUT_IMAGE" "$IMAGE_SIZE"

 # Step 3: Connect via qemu-nbd
@@ -194,7 +236,7 @@ fi
 sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
 sudo qemu-nbd --connect="${NBD_DEV}" "$OUTPUT_IMAGE"

-# Wait for the nbd device to be ready (kernel needs time to scan the block device)
+# Wait for the nbd device to be ready
 for i in $(seq 1 10); do
    if sudo blockdev --getsize64 "${NBD_DEV}" &>/dev/null; then
        break
@@ -203,32 +245,61 @@ for i in $(seq 1 10); do
 done
 sleep 1

+# Ubuntu images have a partition table; wait for partition devices then resize
+# the partition to fill the expanded image before resizing the filesystem.
+if [[ "$DISTRO" == "ubuntu" ]]; then
+    for i in $(seq 1 10); do
+        if [[ -e "${ROOT_PART}" ]]; then break; fi
+        sleep 1
+    done
+    echo "Resizing partition..."
+    # After qemu-img resize the GPT backup header sits at the old end of disk.
+    # Relocate it to the new end (sfdisk --relocate is util-linux built-in,
+    # no extra packages needed), then re-read the partition table before parted
+    # runs — without this, parted sees stale GPT geometry and fails with
+    # "Unable to satisfy all constraints on the partition."
+    sudo sgdisk --move-second-header "${NBD_DEV}"
+    sudo partprobe "${NBD_DEV}" 2>/dev/null || true
+    sleep 1
+    sudo parted --script "${NBD_DEV}" resizepart 1 100%
+fi
+
 # Alpine cloud images have ext4 directly on the block device (no partition table).
-# Resize the filesystem to fill the resized image.
 echo "Checking filesystem..."
-sudo e2fsck -f -y "${NBD_DEV}" || {
+sudo e2fsck -f -y "${ROOT_PART}" || {
    ec=$?
    if [[ $ec -gt 1 ]]; then
-        echo "ERROR: e2fsck failed with exit code $ec on ${NBD_DEV}"
+        echo "ERROR: e2fsck failed with exit code $ec on ${ROOT_PART}"
        exit 1
    fi
 }

 echo "Resizing filesystem..."
-if ! sudo resize2fs "${NBD_DEV}"; then
-    echo "ERROR: resize2fs failed on ${NBD_DEV}"
+if ! sudo resize2fs "${ROOT_PART}"; then
+    echo "ERROR: resize2fs failed on ${ROOT_PART}"
    exit 1
 fi

 # Step 4: Mount and customize
 echo "Mounting root filesystem..."
-sudo mount "${NBD_DEV}" "$MOUNT_DIR"
+sudo mount "${ROOT_PART}" "$MOUNT_DIR"

-# Set up resolv.conf for DNS inside chroot
+# Set up resolv.conf for DNS inside chroot.
+# Ubuntu cloud images symlink /etc/resolv.conf → /run/systemd/resolve/stub-resolv.conf
+# which doesn't exist yet. Remove the symlink and write a real file.
+sudo rm -f "${MOUNT_DIR}/etc/resolv.conf"
 sudo cp /etc/resolv.conf "${MOUNT_DIR}/etc/resolv.conf"

+# Ubuntu chroot requires /proc /sys /dev bind mounts for systemd and DKMS
+if [[ "$DISTRO" == "ubuntu" ]]; then
+    sudo mount --bind /proc       "${MOUNT_DIR}/proc"
+    sudo mount --bind /sys        "${MOUNT_DIR}/sys"
+    sudo mount --bind /dev        "${MOUNT_DIR}/dev"
+    sudo mount --bind /dev/pts    "${MOUNT_DIR}/dev/pts"
+fi
+
 # Install packages
-if [[ -n "${APK_PACKAGES:-}" ]]; then
+if [[ "$DISTRO" == "alpine" ]] && [[ -n "${APK_PACKAGES:-}" ]]; then
    echo "Installing packages: ${APK_PACKAGES}..."
    sudo chroot "$MOUNT_DIR" /bin/sh -c "
 set -e
@@ -244,16 +315,30 @@ fi
 apk update
 apk add ${APK_PACKAGES}
 "
+elif [[ "$DISTRO" == "ubuntu" ]] && [[ -n "${APT_PACKAGES:-}" ]]; then
+    echo "Installing packages: ${APT_PACKAGES}..."
+    sudo chroot "$MOUNT_DIR" /bin/bash -c "
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get install -y --no-install-recommends ${APT_PACKAGES}
+"
 fi

-# Enable OpenRC services
+# Enable services
 if [[ -n "${ENABLE_SERVICES:-}" ]]; then
    echo "Enabling services: ${ENABLE_SERVICES}..."
    IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES"
    for svc in "${SERVICES[@]}"; do
-        if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then
-            echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?"
-            exit 1
+        if [[ "$DISTRO" == "alpine" ]]; then
+            if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then
+                echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?"
+                exit 1
+            fi
+        else
+            if ! sudo chroot "$MOUNT_DIR" /bin/bash -c "systemctl enable ${svc}"; then
+                echo "ERROR: Failed to enable service '${svc}'"
+                exit 1
+            fi
        fi
    done
 fi
@@ -282,21 +367,40 @@ if [[ -n "${SETUP_SCRIPT:-}" ]]; then
    echo "Running setup script: ${SETUP_SCRIPT}..."
    sudo cp "$setup_path" "${MOUNT_DIR}/tmp/setup.sh"
    sudo chmod 755 "${MOUNT_DIR}/tmp/setup.sh"
-    sudo chroot "$MOUNT_DIR" /tmp/setup.sh
+    if [[ "$DISTRO" == "ubuntu" ]]; then
+        sudo chroot "$MOUNT_DIR" /bin/bash /tmp/setup.sh
+    else
+        sudo chroot "$MOUNT_DIR" /tmp/setup.sh
+    fi
    sudo rm -f "${MOUNT_DIR}/tmp/setup.sh"
 fi

 # Step 6: Clean up and unmount
 echo "Cleaning up image..."
-sudo chroot "$MOUNT_DIR" /bin/sh -c '
+if [[ "$DISTRO" == "alpine" ]]; then
+    sudo chroot "$MOUNT_DIR" /bin/sh -c '
 apk cache clean 2>/dev/null || true
 rm -rf /var/cache/apk/* /tmp/*
 '
+else
+    sudo chroot "$MOUNT_DIR" /bin/bash -c '
+export DEBIAN_FRONTEND=noninteractive
+apt-get clean
+rm -rf /var/lib/apt/lists/* /tmp/*
+'
+fi

-# Restore original resolv.conf (cloud-init will set it on boot)
+# Restore the systemd-resolved symlink (Ubuntu default); cloud-init sets DNS on boot.
 sudo rm -f "${MOUNT_DIR}/etc/resolv.conf"
+sudo ln -sf /run/systemd/resolve/stub-resolv.conf "${MOUNT_DIR}/etc/resolv.conf"

 echo "Unmounting..."
+if [[ "$DISTRO" == "ubuntu" ]]; then
+    sudo umount "${MOUNT_DIR}/dev/pts"
+    sudo umount "${MOUNT_DIR}/dev"
+    sudo umount "${MOUNT_DIR}/sys"
+    sudo umount "${MOUNT_DIR}/proc"
+fi
 sudo umount "$MOUNT_DIR"
 sudo qemu-nbd --disconnect "${NBD_DEV}"

@@ -318,19 +422,20 @@ echo ""

 if [[ "$DO_IMPORT" == true ]]; then
    echo "Importing as AMI..."
-    IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro alpine --version "${ALPINE_VERSION}" --arch x86_64)
+    rm -f "$OUTPUT_IMAGE"
+    IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64)
    if [[ -n "${SYSTEM_TAG:-}" ]]; then
        IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
    fi
-    (cd "$PROJECT_DIR" && sudo -u spinifex-storage ./bin/spx admin images import "${IMPORT_ARGS[@]}")
+    spx admin images import "${IMPORT_ARGS[@]}"
 else
    echo "To import as AMI, run:"
-    echo "  cd $PROJECT_DIR && ./bin/spx admin images import \\"
+    echo "  spx admin images import \\"
    echo "    --file $OUTPUT_RAW \\"
    if [[ -n "${SYSTEM_TAG:-}" ]]; then
-        echo "    --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \\"
+        echo "    --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 \\"
        echo "    --tag ${SYSTEM_TAG}"
    else
-        echo "    --distro alpine --version ${ALPINE_VERSION} --arch x86_64"
+        echo "    --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64"
    fi
 fi
@@ -5,7 +5,7 @@ set -euo pipefail
 # Usage: scripts/check-coverage.sh <coverprofile> [quiet]

 PROFILE="${1:?Usage: check-coverage.sh <coverprofile> [quiet]}"
-MIN=65.0
+MIN=70.0
 QUIET="${2:-}"

 if [[ ! -s "$PROFILE" ]]; then
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Dashboard server for the Spinifex GPU demo.
+
+Proxies streams from the GPU instances and serves the web UI.
+
+Environment variables:
+  YOLO_HOST     Base URL of the YOLO instance,   e.g. http://10.0.0.3:8080
+  OLLAMA_HOST   Base URL of the Ollama instance, e.g. http://10.0.0.2:11434
+  OLLAMA_MODEL  Model name to use for chat       (default: llama3.1:70b)
+  PORT          Server port                       (default: 8000)
+"""
+
+import asyncio
+import json
+import os
+
+import httpx
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.responses import HTMLResponse, StreamingResponse
+from fastapi.staticfiles import StaticFiles
+
+YOLO_HOST = os.environ.get("YOLO_HOST", "http://localhost:8080")
+OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
+OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "llama3.1:70b")
+PORT = int(os.environ.get("PORT", "8000"))
+
+app = FastAPI()
+app.mount("/static", StaticFiles(directory="static"), name="static")
+
+
+@app.get("/", response_class=HTMLResponse)
+async def index():
+    with open("static/index.html") as f:
+        return f.read()
+
+
+@app.get("/proxy/video")
+async def proxy_video():
+    async def stream():
+        while True:
+            try:
+                async with httpx.AsyncClient(timeout=None) as client:
+                    async with client.stream("GET", f"{YOLO_HOST}/video") as resp:
+                        async for chunk in resp.aiter_bytes(8192):
+                            yield chunk
+            except Exception:
+                await asyncio.sleep(5)
+
+    return StreamingResponse(stream(), media_type="multipart/x-mixed-replace; boundary=frame")
+
+
+@app.get("/proxy/descriptions")
+async def proxy_descriptions():
+    async def stream():
+        while True:
+            try:
+                async with httpx.AsyncClient(timeout=httpx.Timeout(connect=10.0, read=None, write=None, pool=None)) as client:
+                    async with client.stream("GET", f"{YOLO_HOST}/descriptions") as resp:
+                        async for line in resp.aiter_lines():
+                            yield line + "\n"
+            except Exception:
+                yield 'data: {"type":"error","text":"YOLO stream offline, retrying..."}\n\n'
+                await asyncio.sleep(5)
+
+    return StreamingResponse(stream(), media_type="text/event-stream")
+
+
+@app.post("/chat")
+async def chat(request: Request):
+    body = await request.json()
+    prompt = body.get("prompt", "").strip()
+    if not prompt:
+        return {"error": "empty prompt"}
+
+    async def stream():
+        async with httpx.AsyncClient(timeout=120) as client:
+            async with client.stream(
+                "POST",
+                f"{OLLAMA_HOST}/v1/chat/completions",
+                json={
+                    "model": OLLAMA_MODEL,
+                    "messages": [{"role": "user", "content": prompt}],
+                    "stream": True,
+                },
+            ) as resp:
+                async for line in resp.aiter_lines():
+                    if not line.startswith("data: "):
+                        continue
+                    data_str = line[6:]
+                    if data_str.strip() == "[DONE]":
+                        yield f"data: {json.dumps({'token': '', 'done': True})}\n\n"
+                        break
+                    try:
+                        data = json.loads(data_str)
+                        token = data["choices"][0]["delta"].get("content", "")
+                        yield f"data: {json.dumps({'token': token, 'done': False})}\n\n"
+                    except (json.JSONDecodeError, KeyError, IndexError):
+                        pass
+
+    return StreamingResponse(stream(), media_type="text/event-stream")
+
+
+@app.get("/status")
+async def status():
+    results: dict[str, str] = {}
+    async with httpx.AsyncClient(timeout=3) as client:
+        for name, url in [
+            ("yolo", f"{YOLO_HOST}/health"),
+            ("ollama", f"{OLLAMA_HOST}/v1/models"),
+        ]:
+            try:
+                r = await client.get(url)
+                results[name] = "ok" if r.status_code < 500 else "error"
+            except Exception:
+                results[name] = "offline"
+    return results
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=PORT)
@@ -0,0 +1,116 @@
+#!/bin/bash
+# deploy.sh — Deploy and start the Spinifex GPU demo across four instances.
+#
+# Usage:
+#   DASHBOARD_IP=<ip> YOLO_IP=<ip> OLLAMA_IP=<ip> QWEN_IP=<ip> \
+#     VIDEO_PATH=/path/to/video.mp4 \
+#     scripts/demo/deploy.sh
+#
+# Required env vars:
+#   DASHBOARD_IP   4x-1: serves the web dashboard
+#   YOLO_IP        4x-3: runs YOLO inference
+#   OLLAMA_IP      4x-2: runs Ollama + llama3.1:70b
+#   QWEN_IP        12x:  runs Ollama + qwen3-vl:235b
+#   VIDEO_PATH     Local path to the .mp4 file to run YOLO on
+#
+# Optional:
+#   SSH_KEY        (default: ~/.ssh/spinifex-key)
+#   SSH_USER       (default: ec2-user)
+
+set -euo pipefail
+
+DASHBOARD_IP="${DASHBOARD_IP:?set DASHBOARD_IP}"
+YOLO_IP="${YOLO_IP:?set YOLO_IP}"
+OLLAMA_IP="${OLLAMA_IP:?set OLLAMA_IP}"
+QWEN_IP="${QWEN_IP:?set QWEN_IP}"
+VIDEO_PATH="${VIDEO_PATH:?set VIDEO_PATH}"
+
+SSH_KEY="${SSH_KEY:-$HOME/.ssh/spinifex-key}"
+SSH_USER="${SSH_USER:-ec2-user}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+_ssh() {
+    local ip="$1"; shift
+    ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        -o ConnectTimeout=10 -o BatchMode=yes \
+        -i "$SSH_KEY" "${SSH_USER}@${ip}" "$@"
+}
+
+_scp() {
+    scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        -i "$SSH_KEY" -r "$@"
+}
+
+# ── YOLO instance (4x-3) ────────────────────────────────────────────────────
+echo "==> [YOLO] Deploying to $YOLO_IP..."
+_scp "$SCRIPT_DIR/yolo_stream.py" "$SCRIPT_DIR/requirements-yolo.txt" \
+    "${SSH_USER}@${YOLO_IP}:~/"
+_scp "$VIDEO_PATH" "${SSH_USER}@${YOLO_IP}:~/demo_video.mp4"
+
+echo "   Installing Python deps..."
+_ssh "$YOLO_IP" '
+    . ~/yolo-rocm/bin/activate
+    pip install --quiet -r ~/requirements-yolo.txt
+'
+
+echo "   Starting yolo_stream.py..."
+_ssh "$YOLO_IP" "
+    pkill -f yolo_stream.py || true
+    sleep 1
+    . ~/yolo-rocm/bin/activate
+    nohup env \
+        VIDEO_PATH=~/demo_video.mp4 \
+        QWEN_HOST=http://${QWEN_IP}:11434 \
+        YOLO_DEVICE=0 \
+        PORT=8080 \
+        python ~/yolo_stream.py > ~/yolo_stream.log 2>&1 &
+    echo \"   PID \$!\"
+"
+echo "   YOLO stream: http://${YOLO_IP}:8080/video"
+
+# ── Dashboard instance (4x-1) ───────────────────────────────────────────────
+echo ""
+echo "==> [Dashboard] Deploying to $DASHBOARD_IP..."
+_ssh "$DASHBOARD_IP" 'mkdir -p ~/demo/static'
+_scp "$SCRIPT_DIR/dashboard_server.py" "$SCRIPT_DIR/requirements-dashboard.txt" \
+    "${SSH_USER}@${DASHBOARD_IP}:~/demo/"
+_scp "$SCRIPT_DIR/static/index.html" \
+    "${SSH_USER}@${DASHBOARD_IP}:~/demo/static/"
+
+echo "   Installing Python deps..."
+_ssh "$DASHBOARD_IP" '
+    python3 -m venv ~/demo-venv 2>/dev/null || true
+    . ~/demo-venv/bin/activate
+    pip install --quiet -r ~/demo/requirements-dashboard.txt
+'
+
+echo "   Starting dashboard_server.py..."
+_ssh "$DASHBOARD_IP" "
+    pkill -f dashboard_server.py || true
+    sleep 1
+    . ~/demo-venv/bin/activate
+    cd ~/demo
+    nohup env \
+        YOLO_HOST=http://${YOLO_IP}:8080 \
+        OLLAMA_HOST=http://${OLLAMA_IP}:11434 \
+        OLLAMA_MODEL=llama3.1:70b \
+        PORT=8000 \
+        python dashboard_server.py > ~/dashboard.log 2>&1 &
+    echo \"   PID \$!\"
+"
+
+echo ""
+echo "✅ Demo deployed"
+echo ""
+echo "   Dashboard:  http://${DASHBOARD_IP}:8000"
+echo "   YOLO feed:  http://${YOLO_IP}:8080/video"
+echo "   Ollama:     http://${OLLAMA_IP}:11434"
+echo "   Qwen3-VL:   http://${QWEN_IP}:11434"
+echo ""
+echo "   Logs:"
+echo "     ssh -i ${SSH_KEY} ${SSH_USER}@${YOLO_IP}      tail -f ~/yolo_stream.log"
+echo "     ssh -i ${SSH_KEY} ${SSH_USER}@${DASHBOARD_IP} tail -f ~/dashboard.log"
+echo ""
+echo "   NOTE: Ollama and qwen3-vl must already be running on their instances."
+echo "   If not: _ssh \$IP 'sudo systemctl start ollama && ollama run <model>'"
@@ -0,0 +1,4 @@
+fastapi
+uvicorn[standard]
+httpx
+python-multipart
@@ -0,0 +1,5 @@
+fastapi
+uvicorn[standard]
+httpx
+opencv-python-headless
+ultralytics
@@ -0,0 +1,498 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Spinifex — GPU Compute Demo</title>
+<style>
+* { margin: 0; padding: 0; box-sizing: border-box; }
+
+:root {
+  --bg:       #07070d;
+  --surface:  #0e0e18;
+  --border:   #1a1a2e;
+  --accent:   #6366f1;
+  --accent2:  #22d3ee;
+  --text:     #d4d4d8;
+  --muted:    #52525b;
+  --green:    #22c55e;
+  --red:      #ef4444;
+}
+
+body {
+  background: var(--bg);
+  color: var(--text);
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+  height: 100vh;
+  display: flex;
+  flex-direction: column;
+  overflow: hidden;
+}
+
+/* ── Header ── */
+header {
+  background: var(--surface);
+  border-bottom: 1px solid var(--border);
+  padding: 0 24px;
+  height: 52px;
+  display: flex;
+  align-items: center;
+  gap: 20px;
+  flex-shrink: 0;
+  z-index: 10;
+}
+
+.logo {
+  font-size: 17px;
+  font-weight: 700;
+  letter-spacing: -0.3px;
+  color: #fff;
+  white-space: nowrap;
+}
+.logo em { color: var(--accent); font-style: normal; }
+
+.divider { width: 1px; height: 24px; background: var(--border); }
+
+.subtitle {
+  font-size: 12px;
+  color: var(--muted);
+  letter-spacing: 0.3px;
+}
+
+.instances {
+  margin-left: auto;
+  display: flex;
+  gap: 20px;
+}
+
+.inst {
+  display: flex;
+  align-items: center;
+  gap: 6px;
+  font-size: 11px;
+  color: var(--muted);
+}
+
+.inst-dot {
+  width: 7px; height: 7px;
+  border-radius: 50%;
+  background: var(--green);
+}
+.inst-dot.offline { background: var(--red); }
+.inst-dot.live {
+  background: var(--green);
+  box-shadow: 0 0 0 0 var(--green);
+  animation: ring 2s infinite;
+}
+
+@keyframes ring {
+  0%   { box-shadow: 0 0 0 0 rgba(34,197,94,.6); }
+  70%  { box-shadow: 0 0 0 6px rgba(34,197,94,0); }
+  100% { box-shadow: 0 0 0 0 rgba(34,197,94,0); }
+}
+
+/* ── Layout ── */
+main {
+  flex: 1;
+  display: grid;
+  grid-template-columns: 1fr 400px;
+  grid-template-rows: 1fr 1fr;
+  gap: 10px;
+  padding: 10px;
+  overflow: hidden;
+}
+
+/* ── Panel ── */
+.panel {
+  background: var(--surface);
+  border: 1px solid var(--border);
+  border-radius: 10px;
+  display: flex;
+  flex-direction: column;
+  overflow: hidden;
+  min-height: 0;
+}
+
+.panel-head {
+  padding: 9px 14px;
+  background: rgba(0,0,0,.3);
+  border-bottom: 1px solid var(--border);
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  flex-shrink: 0;
+}
+
+.panel-head h2 {
+  font-size: 11px;
+  font-weight: 600;
+  text-transform: uppercase;
+  letter-spacing: 1px;
+  color: var(--muted);
+}
+
+.chip {
+  margin-left: auto;
+  font-size: 10px;
+  font-weight: 500;
+  padding: 2px 8px;
+  border-radius: 99px;
+  border: 1px solid var(--border);
+  color: var(--muted);
+  white-space: nowrap;
+}
+.chip.accent { border-color: rgba(99,102,241,.4); color: #818cf8; }
+.chip.cyan   { border-color: rgba(34,211,238,.3); color: var(--accent2); }
+
+/* ── Video panel ── */
+.video-panel { grid-row: 1 / 3; position: relative; }
+
+.video-wrap {
+  flex: 1;
+  background: #000;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  overflow: hidden;
+  min-height: 0;
+}
+
+.video-wrap img {
+  width: 100%;
+  height: 100%;
+  object-fit: contain;
+  display: block;
+}
+
+.video-footer {
+  padding: 8px 14px;
+  font-size: 10px;
+  color: var(--muted);
+  border-top: 1px solid var(--border);
+  flex-shrink: 0;
+  display: flex;
+  gap: 16px;
+}
+
+/* ── Description panel ── */
+.desc-content {
+  flex: 1;
+  overflow-y: auto;
+  padding: 14px;
+  display: flex;
+  flex-direction: column;
+  gap: 14px;
+  scroll-behavior: smooth;
+  min-height: 0;
+}
+
+.desc-entry {
+  padding-bottom: 14px;
+  border-bottom: 1px solid var(--border);
+}
+.desc-entry:last-child { border-bottom: none; padding-bottom: 0; }
+
+.desc-time {
+  font-size: 10px;
+  color: var(--muted);
+  margin-bottom: 5px;
+  font-variant-numeric: tabular-nums;
+}
+
+.desc-text {
+  font-size: 13px;
+  line-height: 1.65;
+  color: var(--text);
+}
+
+.cursor {
+  display: inline-block;
+  width: 2px; height: 13px;
+  background: var(--accent);
+  vertical-align: text-bottom;
+  margin-left: 1px;
+  animation: blink 1s step-end infinite;
+}
+@keyframes blink { 0%,100%{opacity:1} 50%{opacity:0} }
+
+.desc-placeholder {
+  font-size: 12px;
+  color: var(--muted);
+  font-style: italic;
+}
+
+/* ── Chat panel ── */
+.chat-messages {
+  flex: 1;
+  overflow-y: auto;
+  padding: 14px;
+  display: flex;
+  flex-direction: column;
+  gap: 10px;
+  min-height: 0;
+}
+
+.msg {
+  max-width: 88%;
+  padding: 9px 13px;
+  border-radius: 10px;
+  font-size: 13px;
+  line-height: 1.55;
+  word-break: break-word;
+}
+.msg.user {
+  background: #1e1e42;
+  align-self: flex-end;
+  color: #c7d2fe;
+}
+.msg.assistant {
+  background: #111120;
+  align-self: flex-start;
+  color: var(--text);
+  border: 1px solid var(--border);
+}
+
+.chat-foot {
+  border-top: 1px solid var(--border);
+  padding: 10px;
+  display: flex;
+  gap: 8px;
+  flex-shrink: 0;
+}
+
+.chat-foot input {
+  flex: 1;
+  background: var(--bg);
+  border: 1px solid var(--border);
+  border-radius: 7px;
+  padding: 8px 12px;
+  color: var(--text);
+  font-size: 13px;
+  outline: none;
+  transition: border-color .15s;
+}
+.chat-foot input:focus { border-color: var(--accent); }
+.chat-foot input::placeholder { color: var(--muted); }
+
+.send-btn {
+  background: var(--accent);
+  color: #fff;
+  border: none;
+  border-radius: 7px;
+  padding: 8px 16px;
+  font-size: 13px;
+  font-weight: 600;
+  cursor: pointer;
+  transition: background .15s, opacity .15s;
+  white-space: nowrap;
+}
+.send-btn:hover { background: #4f52d1; }
+.send-btn:disabled { opacity: .4; cursor: not-allowed; }
+
+/* scrollbar */
+::-webkit-scrollbar { width: 4px; }
+::-webkit-scrollbar-track { background: transparent; }
+::-webkit-scrollbar-thumb { background: var(--border); border-radius: 2px; }
+</style>
+</head>
+<body>
+
+<header>
+  <div class="logo">Spini<em>fex</em></div>
+  <div class="divider"></div>
+  <div class="subtitle">GPU Compute Platform &mdash; Live Demo</div>
+  <div class="instances">
+    <div class="inst">
+      <div class="inst-dot live" id="dot-yolo"></div>
+      <span>YOLO&nbsp;&middot;&nbsp;g7e.4xlarge</span>
+    </div>
+    <div class="inst">
+      <div class="inst-dot live" id="dot-qwen"></div>
+      <span>Qwen3-VL 235B&nbsp;&middot;&nbsp;g7e.12xlarge</span>
+    </div>
+    <div class="inst">
+      <div class="inst-dot live" id="dot-llama"></div>
+      <span>Llama&nbsp;3.1 70B&nbsp;&middot;&nbsp;g7e.4xlarge</span>
+    </div>
+  </div>
+</header>
+
+<main>
+
+  <!-- Left: YOLO video, full height -->
+  <div class="panel video-panel">
+    <div class="panel-head">
+      <h2>Live Object Detection</h2>
+      <span class="chip accent">YOLO11 &middot; AMD MI350X</span>
+    </div>
+    <div class="video-wrap">
+      <img id="video-feed" src="/proxy/video" alt="YOLO stream loading...">
+    </div>
+    <div class="video-footer">
+      <span>Real-time inference &mdash; AMD MI350X GPU</span>
+      <span id="fps-counter"></span>
+    </div>
+  </div>
+
+  <!-- Top right: qwen3-vl scene analysis -->
+  <div class="panel">
+    <div class="panel-head">
+      <h2>Scene Analysis</h2>
+      <span class="chip cyan">Qwen3-VL 235B &middot; MI350X</span>
+    </div>
+    <div class="desc-content" id="desc-content">
+      <p class="desc-placeholder">Waiting for first frame analysis&hellip;</p>
+    </div>
+  </div>
+
+  <!-- Bottom right: Llama3.1 chat -->
+  <div class="panel">
+    <div class="panel-head">
+      <h2>Chat</h2>
+      <span class="chip accent">Llama 3.1 70B &middot; MI350X</span>
+    </div>
+    <div class="chat-messages" id="chat-messages">
+      <div class="msg assistant">
+        Hello! I&rsquo;m Llama&nbsp;3.1&nbsp;70B running on an AMD MI350X GPU, orchestrated by Spinifex. Ask me anything.
+      </div>
+    </div>
+    <div class="chat-foot">
+      <input type="text" id="chat-input" placeholder="Ask Llama 3.1…" autocomplete="off">
+      <button class="send-btn" id="send-btn" onclick="sendChat()">Send</button>
+    </div>
+  </div>
+
+</main>
+
+<script>
+// ── Scene description stream ─────────────────────────────────────────────────
+const descContent = document.getElementById('desc-content');
+let activeEntry = null;
+let activeText  = '';
+
+function initDescriptions() {
+  const es = new EventSource('/proxy/descriptions');
+
+  es.onmessage = (e) => {
+    let data;
+    try { data = JSON.parse(e.data); } catch { return; }
+
+    if (data.type === 'token' && data.text) {
+      if (!activeEntry) {
+        // Remove placeholder if present
+        descContent.querySelector('.desc-placeholder')?.remove();
+
+        activeEntry = document.createElement('div');
+        activeEntry.className = 'desc-entry';
+        activeEntry.innerHTML = `<div class="desc-time">${new Date().toLocaleTimeString()}</div><div class="desc-text"></div>`;
+        descContent.appendChild(activeEntry);
+        activeText = '';
+      }
+      activeText += data.text;
+      const textEl = activeEntry.querySelector('.desc-text');
+      textEl.innerHTML = escHtml(activeText) + '<span class="cursor"></span>';
+      descContent.scrollTop = descContent.scrollHeight;
+    }
+
+    if (data.type === 'end' && activeEntry) {
+      activeEntry.querySelector('.desc-text').textContent = activeText;
+      activeEntry = null;
+      activeText  = '';
+      // Keep only the last 4 entries
+      const entries = descContent.querySelectorAll('.desc-entry');
+      entries.forEach((el, i) => { if (i < entries.length - 4) el.remove(); });
+    }
+  };
+
+  es.onerror = () => { setTimeout(initDescriptions, 3000); };
+}
+
+// ── Chat ─────────────────────────────────────────────────────────────────────
+const chatMessages = document.getElementById('chat-messages');
+const chatInput    = document.getElementById('chat-input');
+const sendBtn      = document.getElementById('send-btn');
+
+async function sendChat() {
+  const prompt = chatInput.value.trim();
+  if (!prompt || sendBtn.disabled) return;
+
+  chatInput.value = '';
+  sendBtn.disabled = true;
+
+  appendMsg('user', prompt);
+
+  const assistantEl = appendMsg('assistant', '');
+  let text = '';
+
+  try {
+    const resp = await fetch('/chat', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ prompt }),
+    });
+
+    const reader  = resp.body.getReader();
+    const decoder = new TextDecoder();
+    let buf = '';
+
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buf += decoder.decode(value, { stream: true });
+      const lines = buf.split('\n');
+      buf = lines.pop();
+      for (const line of lines) {
+        if (!line.startsWith('data: ')) continue;
+        let d;
+        try { d = JSON.parse(line.slice(6)); } catch { continue; }
+        text += d.token;
+        assistantEl.innerHTML = escHtml(text) + (d.done ? '' : '<span class="cursor"></span>');
+        chatMessages.scrollTop = chatMessages.scrollHeight;
+      }
+    }
+  } catch (err) {
+    assistantEl.textContent = '(connection error)';
+  }
+
+  sendBtn.disabled = false;
+  chatInput.focus();
+}
+
+function appendMsg(role, text) {
+  const el = document.createElement('div');
+  el.className = `msg ${role}`;
+  el.textContent = text;
+  chatMessages.appendChild(el);
+  chatMessages.scrollTop = chatMessages.scrollHeight;
+  return el;
+}
+
+chatInput.addEventListener('keydown', (e) => { if (e.key === 'Enter') sendChat(); });
+
+// ── Status polling ────────────────────────────────────────────────────────────
+async function pollStatus() {
+  try {
+    const data = await fetch('/status').then(r => r.json());
+    setDot('dot-yolo',  data.yolo   === 'ok');
+    setDot('dot-llama', data.ollama === 'ok');
+  } catch {}
+  setTimeout(pollStatus, 5000);
+}
+
+function setDot(id, ok) {
+  const el = document.getElementById(id);
+  if (!el) return;
+  el.className = 'inst-dot' + (ok ? ' live' : ' offline');
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+function escHtml(str) {
+  return str.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
+}
+
+// ── Boot ──────────────────────────────────────────────────────────────────────
+initDescriptions();
+pollStatus();
+</script>
+</body>
+</html>
@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+"""YOLO inference server for the Spinifex demo.
+
+Reads a video file, runs YOLO detection, and serves:
+  GET /video        — MJPEG stream of annotated frames
+  GET /descriptions — SSE stream of qwen3-vl scene descriptions
+  GET /health       — {"ok": true}
+
+Environment variables:
+  VIDEO_PATH        Path to input .mp4 (required)
+  QWEN_HOST         Base URL of the qwen3-vl Ollama instance, e.g. http://10.0.0.5:11434
+  YOLO_MODEL        Model name (default: yolo11n.pt)
+  YOLO_DEVICE       GPU device index (default: 0)
+  DESCRIBE_INTERVAL Seconds between qwen3-vl calls (default: 5.0)
+  PORT              Server port (default: 8080)
+"""
+
+import asyncio
+import base64
+import json
+import os
+import queue
+import threading
+import time
+
+import cv2
+import httpx
+import uvicorn
+from fastapi import FastAPI
+from fastapi.responses import StreamingResponse
+from ultralytics import YOLO
+
+VIDEO_PATH = os.environ["VIDEO_PATH"]
+QWEN_HOST = os.environ.get("QWEN_HOST", "")
+YOLO_MODEL = os.environ.get("YOLO_MODEL", "yolo11x.pt")
+YOLO_DEVICE = os.environ.get("YOLO_DEVICE", "0")
+DESCRIBE_INTERVAL = float(os.environ.get("DESCRIBE_INTERVAL", "5.0"))
+PORT = int(os.environ.get("PORT", "8080"))
+
+app = FastAPI()
+
+_frame_lock = threading.Lock()
+_latest_frame: bytes | None = None
+_desc_queue: queue.Queue = queue.Queue(maxsize=100)
+
+
+def _encode(frame, quality: int = 80) -> bytes:
+    _, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, quality])
+    return buf.tobytes()
+
+
+def _describe(img_b64: str) -> None:
+    if not QWEN_HOST:
+        return
+    try:
+        with httpx.Client(timeout=180) as client:
+            with client.stream(
+                "POST",
+                f"{QWEN_HOST}/api/generate",
+                json={
+                    "model": "qwen3-vl:235b",
+                    "prompt": "Briefly describe what is happening in this scene. Focus on the objects detected and any notable activity.",
+                    "images": [img_b64],
+                    "stream": True,
+                },
+            ) as resp:
+                for line in resp.iter_lines():
+                    if not line:
+                        continue
+                    try:
+                        data = json.loads(line)
+                        token = data.get("response", "")
+                        if token and not _desc_queue.full():
+                            _desc_queue.put(("token", token))
+                        if data.get("done"):
+                            _desc_queue.put(("end", ""))
+                    except json.JSONDecodeError:
+                        pass
+    except Exception as exc:
+        _desc_queue.put(("error", str(exc)))
+
+
+def inference_loop() -> None:
+    model = YOLO(YOLO_MODEL)
+    cap = cv2.VideoCapture(VIDEO_PATH)
+    if not cap.isOpened():
+        raise RuntimeError(f"Cannot open video: {VIDEO_PATH}")
+
+    fps = cap.get(cv2.CAP_PROP_FPS) or 25.0
+    frame_delay = 1.0 / fps
+    last_describe = 0.0
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
+            continue
+
+        results = model(frame, device=YOLO_DEVICE, verbose=False)
+        annotated = results[0].plot()
+
+        with _frame_lock:
+            global _latest_frame
+            _latest_frame = _encode(annotated)
+
+        now = time.monotonic()
+        if QWEN_HOST and now - last_describe >= DESCRIBE_INTERVAL:
+            last_describe = now
+            img_b64 = base64.b64encode(_encode(annotated, quality=60)).decode()
+            threading.Thread(target=_describe, args=(img_b64,), daemon=True).start()
+
+        time.sleep(frame_delay)
+
+
+@app.on_event("startup")
+async def startup() -> None:
+    threading.Thread(target=inference_loop, daemon=True).start()
+
+
+@app.get("/health")
+async def health():
+    return {"ok": True}
+
+
+@app.get("/video")
+async def video():
+    async def generate():
+        while True:
+            with _frame_lock:
+                frame = _latest_frame
+            if frame:
+                yield b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" + frame + b"\r\n"
+            await asyncio.sleep(0.04)
+
+    return StreamingResponse(
+        generate(),
+        media_type="multipart/x-mixed-replace; boundary=frame",
+    )
+
+
+@app.get("/descriptions")
+async def descriptions():
+    async def generate():
+        yield 'data: {"type":"ready"}\n\n'
+        loop = asyncio.get_event_loop()
+        while True:
+            try:
+                kind, text = await loop.run_in_executor(
+                    None, lambda: _desc_queue.get(timeout=30)
+                )
+                yield f"data: {json.dumps({'type': kind, 'text': text})}\n\n"
+            except queue.Empty:
+                yield 'data: {"type":"ping"}\n\n'
+
+    return StreamingResponse(generate(), media_type="text/event-stream")
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=PORT)
@@ -33,6 +33,8 @@ mkdir -p "$STAGING/systemd"
 cp "$PROJECT_ROOT/build/systemd/"* "$STAGING/systemd/"
 mkdir -p "$STAGING/scripts"
 cp "$PROJECT_ROOT/build/scripts/"* "$STAGING/scripts/"
+mkdir -p "$STAGING/udev"
+cp "$PROJECT_ROOT/build/udev/"* "$STAGING/udev/"
 cp "$PROJECT_ROOT/build/logrotate/spinifex" "$STAGING/logrotate-spinifex"
 tar czf /tmp/spinifex-local.tar.gz -C "$STAGING" .

@@ -89,9 +91,8 @@ if ! id -Gn 2>/dev/null | grep -qw spinifex; then
    newgrp spinifex
 fi

-echo "=== Building and importing LB image ==="
-cd "$PROJECT_ROOT" && make build-lb-agent
-"$PROJECT_ROOT/scripts/build-system-image.sh" "$PROJECT_ROOT/scripts/images/lb.conf" --import --quiet
+echo "=== Building and installing microVM artifacts ==="
+cd "$PROJECT_ROOT" && make build-lb-agent install-microvm

 echo "=== Done ==="
 echo "Services: sudo systemctl status spinifex.target"
@@ -13,7 +13,7 @@ set -euo pipefail

 PROFILE=""
 BASE_REF=""
-THRESHOLD=65
+THRESHOLD=60
 QUIET="${QUIET:-}"

 while [[ $# -gt 0 ]]; do
@@ -31,7 +31,7 @@ This directory contains OpenTofu/Terraform configuration files for deploying inf
   ```

 4. **AMI available**
-   Before running, ensure you have a Debian 12 AMI imported:
+   Before running, ensure you have a Debian 13 AMI imported:
   ```bash
   # Import an AMI using the spinifex CLI or aws CLI
   aws --endpoint-url http://localhost:8080 ec2 describe-images
@@ -105,7 +105,7 @@ The `main.tf` creates:
 4. **Route Table** - Routes `0.0.0.0/0` through IGW
 5. **Security Group** - Allows SSH (22), HTTP (80), HTTPS (443) inbound
 6. **Placement Group** - Spread strategy for HA
-7. **3 EC2 Instances** - t3.small Debian 12 VMs
+7. **3 EC2 Instances** - t3.small Debian 13 VMs
 8. **SSH Key Pair** - ED25519 key saved to `spinifex-test.pem`

 ## SSH Access
@@ -133,7 +133,7 @@ export AWS_SECRET_ACCESS_KEY=your_secret

 ### "Error: No AMI found"

-Import a Debian 12 AMI first, or modify `main.tf` to use a different AMI filter.
+Import a Debian 13 AMI first, or modify `main.tf` to use a different AMI filter.

 ### Connection refused

@@ -72,13 +72,13 @@ data "aws_availability_zones" "available" {
  state = "available"
 }

-data "aws_ami" "debian12" {
+data "aws_ami" "debian13" {
  most_recent = true
  owners      = ["000000000000"]

  filter {
    name   = "name"
-    values = ["*debian-12*"]
+    values = ["*debian-13*"]
  }

  filter {
@@ -200,7 +200,7 @@ resource "aws_placement_group" "test" {
 # ---------------------------------------------------------------------------

 resource "aws_instance" "test" {
-  ami           = data.aws_ami.debian12.id
+  ami           = data.aws_ami.debian13.id
  instance_type = "t3.small"

  subnet_id              = aws_subnet.test.id
@@ -57,15 +57,15 @@ data "aws_availability_zones" "available" {
  state = "available"
 }

-# Debian 12 AMI
+# Debian 13 AMI
 # For MulgaOS, need to import a local AMI first
-data "aws_ami" "debian12" {
+data "aws_ami" "debian13" {
  most_recent = true
  owners      = ["000000000000"] # Spinifex system images (on AWS use "136693071363" for Debian official)

  filter {
    name   = "name"
-    values = ["*debian-12*"]
+    values = ["*debian-13*"]
  }

  filter {
@@ -191,7 +191,7 @@ resource "aws_security_group" "web_ssh" {

 resource "aws_instance" "vm" {
  count         = 3
-  ami           = data.aws_ami.debian12.id
+  ami           = data.aws_ami.debian13.id
  instance_type = "t3.small"

  subnet_id              = aws_subnet.public.id
@@ -7,11 +7,11 @@ Builds a Proxmox VM template with all Spinifex dependencies pre-installed. Cloni
 | Component | Details |
 |---|---|
 | System packages | QEMU, nbdkit, libvirt, gcc, jq, curl, etc. (`make install-system`) |
-| Go | 1.26.2 (`make install-go`) |
+| Go | 1.26.3 (`make install-go`) |
 | AWS CLI | v2 (`make install-aws`) |
 | Repositories | spinifex, viperblock, predastore (with `go.work` configured) |
 | Module cache | `go mod download` completed for all repos |
-| Cloud images | Ubuntu 24.04 for nested VMs (`~/images/ubuntu-24.04.img`) |
+| Cloud images | Ubuntu 26.04 for nested VMs (`~/images/ubuntu-26.04.img`) |
 | Tuning | sysctl (rmem_max/wmem_max), kvm group, ufw disabled |
 | Generalized | Clean machine-id, SSH host keys, cloud-init — each clone boots fresh |

@@ -92,15 +92,15 @@ mkdir -p "$HOME/images"
 ARCH=$(dpkg --print-architecture)
 IMAGE_URL=""
 if [ "$ARCH" = "amd64" ]; then
-    IMAGE_URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+    IMAGE_URL="https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img"
 elif [ "$ARCH" = "arm64" ]; then
-    IMAGE_URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-arm64.img"
+    IMAGE_URL="https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-arm64.img"
 fi

 if [ -n "$IMAGE_URL" ]; then
-    echo "  Downloading Ubuntu 24.04 cloud image ($ARCH)..."
-    curl -L -o "$HOME/images/ubuntu-24.04.img" "$IMAGE_URL"
-    echo "  Saved to ~/images/ubuntu-24.04.img"
+    echo "  Downloading Ubuntu 26.04 cloud image ($ARCH)..."
+    curl -L -o "$HOME/images/ubuntu-26.04.img" "$IMAGE_URL"
+    echo "  Saved to ~/images/ubuntu-26.04.img"
 else
    echo "  Unknown arch ($ARCH), skipping cloud image download"
 fi
@@ -0,0 +1,62 @@
+#!/bin/bash
+set -euo pipefail
+
+# ubuntu-gpu-amd-setup.sh — Chroot setup for an AMD GPU-capable guest image.
+#
+# Installs linux-firmware (AMD firmware blobs required by the amdgpu kernel
+# module), ROCm CLI tools, Python toolchain, and common utilities. Rebuilds
+# initramfs so AMD firmware is available at early boot inside the guest.
+#
+# Host-side VFIO setup (driver blacklisting, IOMMU, initramfs) is handled
+# by `spx admin gpu setup` — this script is for the guest VM image only.
+#
+# Must run inside an Ubuntu 26.04 chroot with /proc /sys /dev bind-mounted.
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update -qq
+apt-get install -y --no-install-recommends curl gnupg ca-certificates
+
+# AMD ROCm apt repository — rocminfo and rocm-smi-lib are not in the Ubuntu
+# default repos, so we add AMD's signed repo before the main package install.
+UBUNTU_CODENAME=$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME}}")
+mkdir -p /etc/apt/keyrings
+curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key \
+    | gpg --dearmor -o /etc/apt/keyrings/rocm.gpg
+echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.3 ${UBUNTU_CODENAME} main" \
+    > /etc/apt/sources.list.d/rocm.list
+apt-get update -qq
+
+# linux-firmware carries the AMD GPU firmware blobs loaded by the amdgpu
+# kernel module at boot. Without them the GPU is invisible to guest userland.
+apt-get install -y -o Acquire::Retries=3 --no-install-recommends \
+    linux-image-generic \
+    linux-headers-generic \
+    linux-firmware \
+    initramfs-tools \
+    pciutils \
+    rocminfo rocm-smi-lib \
+    python3 python3-venv python3-pip \
+    git curl wget htop tmux \
+    ffmpeg libgl1 libglib2.0-0
+
+KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||')
+if [ -z "${KVER}" ]; then
+    echo "ERROR: No kernel found under /boot"
+    exit 1
+fi
+echo "Rebuilding initramfs for kernel: ${KVER}"
+update-initramfs -u -k "${KVER}"
+
+# Prevent unattended-upgrades from replacing linux-firmware mid-lifecycle;
+# a firmware change without a matching initramfs rebuild can make the GPU invisible.
+mkdir -p /etc/apt/apt.conf.d
+cat > /etc/apt/apt.conf.d/99-gpu-ami <<'EOF'
+Unattended-Upgrade::Package-Blacklist {
+    "linux-firmware";
+    "linux-image-";
+    "linux-headers-";
+};
+EOF
+
+echo "AMD GPU image setup complete: kernel=${KVER}, linux-firmware and ROCm userland installed"
@@ -0,0 +1,12 @@
+IMAGE_NAME="ubuntu-amd-gpu"
+IMAGE_DESCRIPTION="Ubuntu 26.04 AMD GPU guest image — linux-firmware, ROCm CLI, Python toolchain"
+DISTRO="ubuntu"
+UBUNTU_VERSION="26.04"
+UBUNTU_CODENAME="resolute"
+
+# 12G provides headroom for kernel headers, linux-firmware, and ROCm packages
+IMAGE_SIZE="12G"
+
+SETUP_SCRIPT="scripts/images/ubuntu-gpu-amd-setup.sh"
+
+SYSTEM_TAG="gpu-vendor=amd"
@@ -0,0 +1,107 @@
+#!/bin/bash
+set -euo pipefail
+
+# ubuntu-gpu-nvidia-setup.sh — Chroot setup for an NVIDIA GPU-capable guest image.
+#
+# Pre-builds the NVIDIA headless server driver via DKMS against a pinned kernel
+# so guests can use the GPU immediately after launch. Also installs Python
+# toolchain and common utilities.
+#
+# nouveau is blacklisted inside the guest image so the NVIDIA kernel module can
+# bind — this is distinct from the host-side blacklist that `spx admin gpu setup`
+# configures for VFIO passthrough.
+#
+# Host-side VFIO setup (IOMMU, vfio-pci binding, host initramfs) is handled
+# by `spx admin gpu setup` — this script is for the guest VM image only.
+#
+# Must run inside an Ubuntu 26.04 chroot with /proc /sys /dev bind-mounted
+# (required by DKMS during the nvidia package post-install hook).
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update -qq
+
+# Detect the cloud image's existing kernel BEFORE installing anything.
+# Installing linux-image-generic would upgrade to a newer kernel version;
+# DKMS would then build for that version but the VM boots the original
+# cloud image kernel, causing a version mismatch at modprobe time.
+KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||')
+if [[ -z "${KVER}" ]]; then
+    echo "ERROR: No kernel found under /boot"
+    exit 1
+fi
+echo "Target kernel: ${KVER}"
+
+# Install headers for the exact kernel already present, plus initramfs-tools.
+apt-get install -y --no-install-recommends \
+    "linux-headers-${KVER}" \
+    initramfs-tools
+
+# Install the NVIDIA server DKMS driver. Use the distro default version (no
+# hardcoded suffix) so Ubuntu 26.04's 580 driver is used instead of 550.
+# nvidia-dkms-server is a meta package that resolves to the current recommended
+# version; nvidia-utils-server brings nvidia-smi and related tools.
+apt-get install -y --no-install-recommends \
+    nvidia-dkms-server \
+    nvidia-utils-server
+
+# Detect the installed NVIDIA DKMS module name + version for explicit build.
+NVIDIA_DKMS_NAME=$(dkms status 2>/dev/null | awk -F'[,/ ]+' '/nvidia/{print $1; exit}')
+NVIDIA_DKMS_VER=$(dkms status 2>/dev/null  | awk -F'[,/ ]+' '/nvidia/{print $2; exit}')
+
+if [[ -n "${NVIDIA_DKMS_NAME}" && -n "${NVIDIA_DKMS_VER}" ]]; then
+    echo "Building NVIDIA DKMS module ${NVIDIA_DKMS_NAME}/${NVIDIA_DKMS_VER} for kernel ${KVER}..."
+    dkms install -m "${NVIDIA_DKMS_NAME}" -v "${NVIDIA_DKMS_VER}" --kernelver "${KVER}" --force 2>&1
+else
+    echo "WARNING: nvidia DKMS module not found in dkms status — driver install may have failed"
+fi
+
+# Verify the module was actually built.
+NVIDIA_KO=$(find "/lib/modules/${KVER}/updates" -name "nvidia.ko*" 2>/dev/null | head -1)
+if [[ -z "${NVIDIA_KO}" ]]; then
+    echo "ERROR: nvidia.ko not found under /lib/modules/${KVER}/updates after DKMS build"
+    exit 1
+fi
+echo "NVIDIA kernel module: ${NVIDIA_KO}"
+
+# Pin the exact kernel + headers so unattended-upgrades cannot replace the
+# kernel the DKMS module was built for.
+apt-mark hold \
+    "linux-image-${KVER}" \
+    "linux-headers-${KVER}"
+
+NVIDIA_PKGS=$(dpkg --get-selections 'nvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
+LIBNVIDIA_PKGS=$(dpkg --get-selections 'libnvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
+# shellcheck disable=SC2086
+[[ -n "${NVIDIA_PKGS}" ]]    && apt-mark hold ${NVIDIA_PKGS}
+# shellcheck disable=SC2086
+[[ -n "${LIBNVIDIA_PKGS}" ]] && apt-mark hold ${LIBNVIDIA_PKGS}
+
+# Blacklist nouveau inside the guest — nvidia.ko and nouveau conflict, and
+# without this blacklist the NVIDIA driver will not bind after boot.
+cat > /etc/modprobe.d/blacklist-nouveau.conf <<'EOF'
+blacklist nouveau
+options nouveau modeset=0
+EOF
+
+# Common tooling matching the AMD image.
+apt-get install -y -o Acquire::Retries=3 \
+    pciutils \
+    python3 python3-venv python3-pip \
+    git curl wget htop tmux \
+    ffmpeg libgl1 libglib2.0-0
+
+mkdir -p /etc/apt/apt.conf.d
+cat > /etc/apt/apt.conf.d/99-gpu-ami <<'EOF'
+Unattended-Upgrade::Package-Blacklist {
+    "linux-";
+    "nvidia-";
+    "libnvidia-";
+};
+EOF
+
+# Rebuild initramfs with nouveau blacklisted and NVIDIA module included.
+update-initramfs -u -k "${KVER}"
+
+
+echo "NVIDIA GPU image setup complete: kernel=${KVER}, NVIDIA driver + DKMS pre-built"
@@ -0,0 +1,12 @@
+IMAGE_NAME="ubuntu-nvidia-gpu"
+IMAGE_DESCRIPTION="Ubuntu 26.04 NVIDIA GPU guest image — NVIDIA server driver (DKMS pre-built), Python toolchain"
+DISTRO="ubuntu"
+UBUNTU_VERSION="26.04"
+UBUNTU_CODENAME="resolute"
+
+# 12G provides headroom for kernel headers, NVIDIA driver, and DKMS build artefacts
+IMAGE_SIZE="12G"
+
+SETUP_SCRIPT="scripts/images/ubuntu-gpu-nvidia-setup.sh"
+
+SYSTEM_TAG="gpu-vendor=nvidia"
@@ -0,0 +1,248 @@
+#!/bin/bash
+set -euo pipefail
+
+export AWS_PROFILE=spinifex
+
+# Resolve the actual user's home — $HOME is /root when run under sudo.
+_USER_HOME=$(getent passwd "${SUDO_USER:-$(id -un)}" | cut -d: -f6)
+SSH_KEY="${SSH_KEY:-${_USER_HOME}/.ssh/spinifex-key}"
+SSH_USER="${SSH_USER:-ec2-user}"
+SSH_TIMEOUT="${SSH_TIMEOUT:-300}"
+
+usage() {
+    cat <<'EOF'
+launch-gpu.sh — Launch a GPU instance from the Spinifex AMD or NVIDIA GPU AMI.
+
+Detects the GPU brand from the specified instance type and selects the matching
+pre-built AMI automatically. Use --amd or --nvidia to override.
+
+Usage:
+  scripts/launch-gpu.sh --type <instance-type> [--disk GB] [--amd|--nvidia]
+
+Options:
+  --type TYPE    EC2 instance type to launch (required)
+  --disk GB      Root disk size in GB (default: 50)
+  --ami  NAME    Override the AMI name to use (default: ubuntu-amd-gpu or ubuntu-nvidia-gpu)
+  --amd          Force AMD GPU AMI  (ubuntu-amd-gpu)
+  --nvidia       Force NVIDIA GPU AMI (ubuntu-nvidia-gpu)
+
+Env overrides:
+  SSH_KEY      Path to SSH private key (default: ~/.ssh/spinifex-key)
+  SSH_USER     SSH user inside guest  (default: ec2-user)
+  SSH_TIMEOUT  Seconds to wait for initial SSH (default: 300)
+EOF
+}
+
+INSTANCE_TYPE=""
+DISK_GB=50
+GPU_BRAND=""
+AMI_OVERRIDE=""
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --type)   INSTANCE_TYPE="$2"; shift 2 ;;
+        --disk)   DISK_GB="$2";        shift 2 ;;
+        --ami)    AMI_OVERRIDE="$2";   shift 2 ;;
+        --amd)    GPU_BRAND="amd";     shift ;;
+        --nvidia) GPU_BRAND="nvidia";  shift ;;
+        -h|--help) usage; exit 0 ;;
+        *) echo "Unknown option: $1"; usage; exit 1 ;;
+    esac
+done
+
+list_gpu_types() {
+    aws ec2 describe-instance-types \
+        --query 'InstanceTypes[?GpuInfo].[InstanceType,GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Manufacturer,GpuInfo.Gpus[0].Name]' \
+        --output table 2>/dev/null || true
+}
+
+if [ -z "$INSTANCE_TYPE" ]; then
+    echo "Error: --type is required."
+    echo ""
+    echo "Available GPU instance types on this node:"
+    list_gpu_types
+    exit 1
+fi
+
+# --- Validate instance type and fetch GPU info in one call ---
+echo "==> Checking instance type: ${INSTANCE_TYPE}"
+INSTANCE_INFO=$(aws ec2 describe-instance-types \
+    --query "InstanceTypes[?InstanceType=='${INSTANCE_TYPE}' && GpuInfo].[GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Manufacturer,GpuInfo.Gpus[0].Name]" \
+    --output text 2>/dev/null || true)
+
+if [ -z "$INSTANCE_INFO" ] || [ "$INSTANCE_INFO" = "None" ]; then
+    echo "'${INSTANCE_TYPE}' is not available or is not a GPU type on this node."
+    echo ""
+    echo "Available GPU instance types:"
+    list_gpu_types
+    exit 1
+fi
+
+IFS=$'\t' read -r GPU_COUNT GPU_MANUFACTURER GPU_NAME <<< "$INSTANCE_INFO"
+echo "   ${GPU_COUNT}× ${GPU_NAME} (${GPU_MANUFACTURER})"
+
+# --- Resolve GPU brand ---
+if [ -z "$GPU_BRAND" ]; then
+    case "$GPU_MANUFACTURER" in
+        AMD|amd)       GPU_BRAND="amd" ;;
+        NVIDIA|nvidia) GPU_BRAND="nvidia" ;;
+        *)
+            echo "Cannot auto-detect GPU brand from manufacturer '${GPU_MANUFACTURER}'."
+            echo "Use --amd or --nvidia to specify explicitly."
+            exit 1
+            ;;
+    esac
+fi
+
+case "$GPU_BRAND" in
+    amd)    AMI_NAME="ubuntu-amd-gpu" ;;
+    nvidia) AMI_NAME="ubuntu-nvidia-gpu" ;;
+esac
+
+# --- Resolve AMI ---
+AMI_LOOKUP="${AMI_OVERRIDE:-${AMI_NAME}}"
+echo "==> Resolving AMI: ${AMI_LOOKUP}"
+AMI_ID=$(aws ec2 describe-images \
+    --query "Images[?Name=='${AMI_LOOKUP}'].ImageId | [0]" --output text 2>/dev/null || true)
+if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
+    echo "No AMI found with name '${AMI_LOOKUP}'."
+    echo "Build and import it first, or specify the AMI name with --ami <name>"
+    echo "  scripts/build-system-image.sh scripts/images/ubuntu-gpu-${GPU_BRAND}.conf --import"
+    exit 1
+fi
+echo "   $AMI_ID"
+
+# --- Resolve subnet ---
+echo "==> Resolving subnet"
+SUBNET_ID=$(aws ec2 describe-subnets \
+    --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId | [0]' --output text 2>/dev/null || true)
+if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
+    echo "No public subnet found"
+    exit 1
+fi
+echo "   $SUBNET_ID"
+
+VPC_ID=$(aws ec2 describe-subnets --subnet-ids "$SUBNET_ID" \
+    --query 'Subnets[0].VpcId' --output text 2>/dev/null || true)
+if [ -z "$VPC_ID" ] || [ "$VPC_ID" = "None" ]; then
+    echo "Could not resolve VPC for subnet $SUBNET_ID"
+    exit 1
+fi
+
+# --- Ensure SSH security group ---
+# The default VPC SG only allows intra-SG traffic; we need a group that
+# permits port 22 from anywhere so the SSH wait below can connect.
+SSH_SG_NAME="spinifex-gpu-ssh"
+echo "==> Ensuring SSH security group (${SSH_SG_NAME})"
+SG_ID=$(aws ec2 describe-security-groups \
+    --filters "Name=group-name,Values=${SSH_SG_NAME}" "Name=vpc-id,Values=${VPC_ID}" \
+    --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || true)
+if [ -z "$SG_ID" ] || [ "$SG_ID" = "None" ]; then
+    SG_ID=$(aws ec2 create-security-group \
+        --group-name "$SSH_SG_NAME" \
+        --description "Allow SSH access to GPU instances" \
+        --vpc-id "$VPC_ID" \
+        --query 'GroupId' --output text)
+    aws ec2 authorize-security-group-ingress \
+        --group-id "$SG_ID" \
+        --protocol tcp \
+        --port 22 \
+        --cidr 0.0.0.0/0 >/dev/null
+    echo "   Created $SG_ID"
+else
+    echo "   Reusing $SG_ID"
+fi
+
+# --- Ensure SSH key ---
+if [ ! -f "$SSH_KEY" ]; then
+    echo "==> Generating SSH key"
+    sudo -u "${SUDO_USER:-$(id -un)}" mkdir -p "$(dirname "$SSH_KEY")"
+    sudo -u "${SUDO_USER:-$(id -un)}" ssh-keygen -t ed25519 -f "$SSH_KEY" -N ""
+fi
+# Delete before import — stale key pair material causes import to fail
+# silently, leaving the instance unreachable via SSH.
+echo "==> Importing SSH key"
+aws ec2 delete-key-pair --key-name spinifex-key >/dev/null 2>&1 || true
+aws ec2 import-key-pair --key-name spinifex-key \
+    --public-key-material "fileb://${SSH_KEY}.pub" >/dev/null 2>&1 || true
+
+# --- Launch ---
+echo "==> Launching ${INSTANCE_TYPE} (${DISK_GB} GB)..."
+ID=$(aws ec2 run-instances \
+    --image-id "$AMI_ID" \
+    --instance-type "$INSTANCE_TYPE" \
+    --key-name spinifex-key \
+    --subnet-id "$SUBNET_ID" \
+    --security-group-ids "$SG_ID" \
+    --count 1 \
+    --block-device-mappings "[{\"DeviceName\":\"/dev/sda1\",\"Ebs\":{\"VolumeSize\":${DISK_GB},\"VolumeType\":\"gp3\",\"DeleteOnTermination\":true}}]" \
+    --query 'Instances[0].InstanceId' --output text)
+if [ -z "$ID" ] || [ "$ID" = "None" ] || [ "$ID" = "null" ]; then
+    echo "run-instances returned no InstanceId"
+    exit 1
+fi
+echo "   $ID"
+
+# --- Wait for running ---
+echo "==> Waiting for running state..."
+COUNT=0; STATE="unknown"
+while [ $COUNT -lt 120 ]; do
+    STATE=$(aws ec2 describe-instances --instance-ids "$ID" \
+        --query "Reservations[0].Instances[0].State.Name" --output text 2>/dev/null || echo "unknown")
+    [ "$STATE" = "running" ] && break
+    [ "$STATE" = "terminated" ] && { echo "Instance terminated unexpectedly"; exit 1; }
+    sleep 2; COUNT=$((COUNT + 2))
+done
+[ "$STATE" = "running" ] || { echo "Instance did not reach running state (last: $STATE)"; exit 1; }
+
+# --- Wait for public IP ---
+echo "==> Waiting for public IP..."
+IP=""
+for _i in $(seq 1 120); do
+    IP=$(aws ec2 describe-instances --instance-ids "$ID" \
+        --query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null || true)
+    [ -n "$IP" ] && [ "$IP" != "None" ] && [ "$IP" != "null" ] && break
+    sleep 1
+done
+[ -n "$IP" ] && [ "$IP" != "None" ] || { echo "No public IP assigned after 120s"; exit 1; }
+echo "   $IP"
+
+# Clear stale known_hosts entry — instances reuse IPs across runs.
+KNOWN_HOSTS="$HOME/.ssh/known_hosts"
+[ -f "$KNOWN_HOSTS" ] && ssh-keygen -f "$KNOWN_HOSTS" -R "$IP" >/dev/null 2>&1 || true
+
+# --- Wait for SSH ---
+echo "==> Waiting for SSH..."
+_ssh() {
+    ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        -o ConnectTimeout=5 -o BatchMode=yes \
+        -i "$SSH_KEY" "${SSH_USER}@${IP}" "$@"
+}
+
+elapsed=0
+while [ "$elapsed" -lt "$SSH_TIMEOUT" ]; do
+    _ssh 'echo ready' >/dev/null 2>&1 && break
+    sleep 5; elapsed=$((elapsed + 5))
+done
+[ "$elapsed" -lt "$SSH_TIMEOUT" ] || { echo "SSH timeout after ${SSH_TIMEOUT}s"; exit 1; }
+
+# --- GPU visibility check ---
+echo "==> Checking GPU visibility..."
+lspci_out=$(_ssh 'lspci -nn 2>/dev/null' || true)
+if printf '%s\n' "$lspci_out" | grep -qiE "1002:|10de:|Instinct|Aqua|Processing accelerator|NVIDIA"; then
+    printf '%s\n' "$lspci_out" \
+        | grep -iE "1002:|10de:|Instinct|Aqua|Processing accelerator|NVIDIA" \
+        | sed 's/^/   ✓ /'
+else
+    echo "   WARNING: GPU not visible in lspci — check host-side passthrough assignment"
+    printf '%s\n' "$lspci_out" | head -20 | sed 's/^/   /'
+fi
+
+# --- Done ---
+echo ""
+echo "✅ ${INSTANCE_TYPE} ready — ${ID} (${IP})"
+echo ""
+echo "   ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no ${SSH_USER}@${IP}"
+echo ""
+echo "   Terminate when done:"
+echo "   AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids ${ID}"
@@ -268,10 +268,9 @@ sudo systemctl start spinifex.target
 # Wait for services to start
 sleep 5

-# --- Build + import LB image (needs services running) ---
-echo "==> Building and importing LB image"
-cd "$PROJECT_ROOT" && make build-lb-agent
-"$PROJECT_ROOT/scripts/build-system-image.sh" "$PROJECT_ROOT/scripts/images/lb.conf" --import --quiet
+# --- Build + install microVM artifacts (kernel + initramfs for direct-boot LBs) ---
+echo "==> Building and installing microVM artifacts"
+cd "$PROJECT_ROOT" && make build-lb-agent install-microvm

 # --- Smoke test ---
 sudo -u "$INVOKING_USER" "$SCRIPT_DIR/smoke-test.sh"
@@ -12,11 +12,11 @@ if [ -z "$SPINIFEX_AMI" ]; then
    echo "SPINIFEX_AMI not set, attempting to detect..."
    ARCH=$(uname -m)
    if [ "$ARCH" = "x86_64" ]; then
-        IMAGE_NAME="ami-ubuntu-24.04-x86_64"
+        IMAGE_NAME="ami-ubuntu-26.04-x86_64"
    elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
-        IMAGE_NAME="ami-ubuntu-24.04-arm64"
+        IMAGE_NAME="ami-ubuntu-26.04-arm64"
    else
-        IMAGE_NAME="ami-ubuntu-24.04-x86_64"
+        IMAGE_NAME="ami-ubuntu-26.04-x86_64"
    fi
    SPINIFEX_AMI=$(aws ec2 describe-images --query "Images[?Name=='$IMAGE_NAME'].ImageId" --output text)
    if [ -z "$SPINIFEX_AMI" ]; then
@@ -6,12 +6,12 @@ export AWS_PROFILE=spinifex
 # Detect architecture
 ARCH=$(uname -m)
 if [ "$ARCH" = "x86_64" ]; then
-    IMAGE_NAME="ami-ubuntu-24.04-x86_64"
+    IMAGE_NAME="ami-ubuntu-26.04-x86_64"
 elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
-    IMAGE_NAME="ami-ubuntu-24.04-arm64"
+    IMAGE_NAME="ami-ubuntu-26.04-arm64"
 else
    echo "Warning: Unknown architecture $ARCH, defaulting to x86_64"
-    IMAGE_NAME="ami-ubuntu-24.04-x86_64"
+    IMAGE_NAME="ami-ubuntu-26.04-x86_64"
 fi

 # Query available instance types from Spinifex and pick the first micro type
@@ -326,7 +326,7 @@ if [ -n "$WAN_BRIDGE" ]; then
                   /etc/systemd/network/15-spinifex-veth-wan.network \
                   /etc/systemd/network/16-spinifex-veth-wan-ovs.network
        sudo networkctl reload 2>/dev/null || true
-        sudo ovs-vsctl --if-exists del-port "$WAN_BRIDGE" veth-wan-ovs
+        sudo ovs-vsctl --if-exists del-port veth-wan-ovs 2>/dev/null || true
        sudo ip link del veth-wan-br 2>/dev/null || true
    fi

@@ -15,7 +15,7 @@
 #   VERBOSE                    Set to 1 to echo "[setup] <stage>" before each top-level step.
 #   SETUP_STAGES               Comma-separated subset of stages to run:
 #                                deps, aws, users, sudoers, files, directories,
-#                                env, systemd, logrotate, fixown, migrations
+#                                env, systemd, logrotate, udev, fixown, migrations
 #                              Unset = run every stage appropriate for the current mode.

 set -e
@@ -192,6 +192,7 @@ spinifex-vpcd ALL=(root) NOPASSWD: /usr/sbin/dhcpcd
 spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
 spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
 spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/systemctl is-active --quiet ovn-controller
+spinifex-vpcd ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
 SUDOERS
    $SUDO chmod 0440 /etc/sudoers.d/spinifex-network
    $SUDO visudo -cf /etc/sudoers.d/spinifex-network || fatal "Invalid sudoers syntax in spinifex-network"
@@ -213,8 +214,9 @@ install_apt_deps() {

        DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq \
            nbdkit \
-            $QEMU_PACKAGES qemu-utils qemu-kvm less \
+            $QEMU_PACKAGES qemu-utils qemu-kvm ovmf less \
            libvirt-daemon-system libvirt-clients \
+            pciutils \
            jq curl iproute2 netcat-openbsd wget unzip xz-utils file \
            ovn-central ovn-host openvswitch-switch dhcpcd-base \
            > /dev/null
@@ -344,6 +346,13 @@ install_files() {
        $SUDO install -m 0755 "$EXTRACT_DIR/setup.sh" /usr/local/share/spinifex/setup.sh
        info "  /usr/local/share/spinifex/setup.sh"
    fi
+
+    # microVM kernel + initramfs
+    $SUDO install -d /usr/share/spinifex/microvm
+    if [ -d "$EXTRACT_DIR/microvm" ]; then
+        $SUDO cp "$EXTRACT_DIR/microvm/"* /usr/share/spinifex/microvm/
+        info "  /usr/share/spinifex/microvm/*"
+    fi
 }

 # --- Create directories ---
@@ -378,11 +387,14 @@ create_directories() {
    # /run is tmpfs — direct mkdir doesn't survive reboot, and units have
    # ReadWritePaths= on these paths which fails namespace setup with ENOENT
    # if the dirs are absent at service start.
-    $SUDO install -m 0644 /dev/stdin /etc/tmpfiles.d/spinifex.conf <<'TMPEOF'
+    _tmpf=$(mktemp)
+    cat > "$_tmpf" <<'TMPEOF'
 # Type  Path               Mode  User                 Group                Age
 d       /run/spinifex      0770  root                 spinifex             -
 d       /run/spinifex/nbd  0770  spinifex-viperblock  spinifex-viperblock  -
 TMPEOF
+    $SUDO install -m 0644 "$_tmpf" /etc/tmpfiles.d/spinifex.conf
+    rm -f "$_tmpf"
    if [ "${ISO_BUILD:-0}" != "1" ]; then
        # Live systems: materialise the runtime dirs immediately so a re-run of
        # setup.sh on an existing host has correct /run state without a reboot.
@@ -608,6 +620,24 @@ install_logrotate() {
    info "Logrotate config installed"
 }

+# --- Install udev rules ---
+install_udev() {
+    stage "installing udev rules"
+    if [ ! -d "$EXTRACT_DIR/udev" ]; then
+        return
+    fi
+    $SUDO install -d /etc/udev/rules.d
+    for rule in "$EXTRACT_DIR"/udev/*; do
+        $SUDO install -m 0644 "$rule" "/etc/udev/rules.d/$(basename "$rule")"
+        info "  /etc/udev/rules.d/$(basename "$rule")"
+    done
+    if [ "${ISO_BUILD:-0}" != "1" ]; then
+        $SUDO udevadm control --reload-rules
+        $SUDO udevadm trigger --subsystem-match=vfio 2>/dev/null || true
+    fi
+    info "udev rules installed"
+}
+
 # --- Upgrade handling ---
 handle_upgrade() {
    if $SUDO systemctl is-active --quiet spinifex.target 2>/dev/null; then
@@ -688,10 +718,11 @@ main() {
        handle_upgrade
    fi

-    # Stages that need EXTRACT_DIR: files, directories, systemd, logrotate.
+    # Stages that need EXTRACT_DIR: files, directories, systemd, logrotate, udev.
    # Only download/extract when at least one such stage is enabled.
    if stage_enabled files || stage_enabled directories \
-        || stage_enabled systemd || stage_enabled logrotate; then
+        || stage_enabled systemd || stage_enabled logrotate \
+        || stage_enabled udev; then
        download_spinifex
    fi

@@ -705,6 +736,7 @@ main() {
    stage_enabled fixown     && fix_file_ownership
    stage_enabled systemd    && install_systemd
    stage_enabled logrotate  && install_logrotate
+    stage_enabled udev       && install_udev

    # Migrations are only safe on a live system (need a running NATS and a
    # persisted config file). Skip under ISO_BUILD and under any explicit
@@ -6,10 +6,31 @@
 #
 # Assumes services are running and AWS_PROFILE=spinifex is configured
 # by a prior 'spx admin init'.
+#
+# GPU passthrough test (opt-in):
+#   TEST_GPU=1 scripts/smoke-test.sh
+#   TEST_GPU=1 GPU_VENDOR_ID=10de GPU_DEVICE_ID=2236 scripts/smoke-test.sh
+#
+# GPU env vars:
+#   TEST_GPU=1            enable GPU passthrough checks
+#   GPU_VENDOR_ID         PCI vendor ID (default: auto-detect first NVIDIA VGA/3D device)
+#   GPU_DEVICE_ID         PCI device ID (default: auto-detect)
+#   GPU_FAMILY            instance family to test (default: g5)
 set -euo pipefail

+TEST_GPU="${TEST_GPU:-0}"
+GPU_VENDOR_ID="${GPU_VENDOR_ID:-}"
+GPU_DEVICE_ID="${GPU_DEVICE_ID:-}"
+GPU_FAMILY="${GPU_FAMILY:-g5}"
+
 export AWS_PROFILE=spinifex

+# AWS commands run as the invoking user so their ~/.aws/config (spinifex profile
+# + CA bundle) is used when the script is called via sudo.
+INVOKING_USER="${SUDO_USER:-$(id -un)}"
+INVOKING_HOME=$(getent passwd "$INVOKING_USER" | cut -d: -f6)
+aws_as_user() { sudo -u "$INVOKING_USER" env HOME="$INVOKING_HOME" AWS_PROFILE=spinifex aws "$@"; }
+
 # --- Wait for EC2 daemon to subscribe to NATS ---
 # Port 3000 (UI) becomes ready before the daemon finishes initialising.
 # Poll describe-key-pairs until it doesn't return InternalError.
@@ -17,7 +38,7 @@ echo "==> Waiting for EC2 daemon to be ready"
 DAEMON_TIMEOUT=60
 DAEMON_ELAPSED=0
 while [ $DAEMON_ELAPSED -lt $DAEMON_TIMEOUT ]; do
-    if aws ec2 describe-key-pairs --output text >/dev/null 2>&1; then
+    if aws_as_user ec2 describe-key-pairs --output text >/dev/null 2>&1; then
        break
    fi
    sleep 2
@@ -38,29 +59,64 @@ if [ ! -f "$SSH_KEY.pub" ]; then
 fi

 echo "==> Importing SSH key"
-! aws ec2 import-key-pair --key-name spinifex-key \
-    --public-key-material "fileb://$SSH_KEY.pub"
-aws ec2 describe-key-pairs
+aws_as_user ec2 import-key-pair --key-name spinifex-key \
+    --public-key-material "fileb://$SSH_KEY.pub" || true
+aws_as_user ec2 describe-key-pairs
+
+# --- GPU pre-flight (TEST_GPU=1 only) ---
+if [[ "$TEST_GPU" == "1" ]]; then
+    echo "==> [GPU] Checking IOMMU"
+    if ! dmesg | grep -qi "DMAR.*IOMMU enabled\|iommu.*Translated\|Adding to iommu group\|AMD-Vi.*enabled"; then
+        echo "❌ IOMMU not detected — add intel_iommu=on (or amd_iommu=on) and iommu=pt to kernel cmdline, then reboot"
+        exit 1
+    fi
+    echo "   IOMMU active"
+
+    echo "==> [GPU] Detecting NVIDIA GPU"
+    if [[ -z "$GPU_VENDOR_ID" || -z "$GPU_DEVICE_ID" ]]; then
+        GPU_LINE=$(lspci -nn | grep -i nvidia | grep -iE "VGA|3D" | head -1)
+        if [[ -z "$GPU_LINE" ]]; then
+            echo "❌ No NVIDIA GPU found via lspci — is the card seated and detectable?"
+            lspci | grep -i nvidia >&2 || true
+            exit 1
+        fi
+        echo "   Detected: $GPU_LINE"
+        PCI_IDS=$(echo "$GPU_LINE" | grep -oP '\[\K[0-9a-f]{4}:[0-9a-f]{4}(?=\])')
+        GPU_VENDOR_ID=$(echo "$PCI_IDS" | cut -d: -f1)
+        GPU_DEVICE_ID=$(echo "$PCI_IDS" | cut -d: -f2)
+    fi
+    echo "   GPU PCI ID: ${GPU_VENDOR_ID}:${GPU_DEVICE_ID}"
+
+    echo "==> [GPU] Verifying ${GPU_FAMILY} instance types advertised"
+    G5_TYPES=$(aws_as_user ec2 describe-instance-types \
+        --query "InstanceTypes[?starts_with(InstanceType, \`${GPU_FAMILY}\`)].InstanceType" \
+        --output text 2>/dev/null || true)
+    if [[ -z "$G5_TYPES" ]]; then
+        echo "❌ No ${GPU_FAMILY}.* instance types returned by describe-instance-types"
+        exit 1
+    fi
+    echo "   ${GPU_FAMILY} types available: $G5_TYPES"
+fi

 # --- Import AMI ---
 # --- Import AMI ---
 echo "==> Importing AMI"

-LOCAL_IMAGE="$HOME/images/ubuntu-24.04.img"
+LOCAL_IMAGE="$HOME/images/ubuntu-26.04.img"
 ARCH=$(uname -m)

 case "$ARCH" in
-    x86_64)        IMG_ARCH="x86_64"; IMAGE_NAME="ubuntu-24.04-x86_64" ;;
-    aarch64|arm64) IMG_ARCH="arm64";  IMAGE_NAME="ubuntu-24.04-arm64"  ;;
+    x86_64)        IMG_ARCH="x86_64"; IMAGE_NAME="ubuntu-26.04-x86_64" ;;
+    aarch64|arm64) IMG_ARCH="arm64";  IMAGE_NAME="ubuntu-26.04-arm64"  ;;
    *)
        echo "  Warning: unknown arch $ARCH, defaulting to x86_64"
-        IMG_ARCH="x86_64"; IMAGE_NAME="ubuntu-24.04-x86_64"
+        IMG_ARCH="x86_64"; IMAGE_NAME="ubuntu-26.04-x86_64"
        ;;
 esac

 AMI_NAME="ami-${IMAGE_NAME}"

-EXISTING_AMI_ID=$(aws ec2 describe-images \
+EXISTING_AMI_ID=$(aws_as_user ec2 describe-images \
    --query "Images[?Name=='${AMI_NAME}'] | [0].ImageId" \
    --output text)

@@ -70,7 +126,7 @@ else
    if [ -f "$LOCAL_IMAGE" ]; then
        echo "  Using local image: $LOCAL_IMAGE"
        sudo /usr/local/bin/spx admin images import \
-            --file "$LOCAL_IMAGE" --distro ubuntu --version 24.04 --arch "$IMG_ARCH"
+            --file "$LOCAL_IMAGE" --distro ubuntu --version 26.04 --arch "$IMG_ARCH"
    else
        echo "  Downloading image: $IMAGE_NAME"
        sudo /usr/local/bin/spx admin images import --name "$IMAGE_NAME"
@@ -79,13 +135,15 @@ fi

 # --- Launch smoke-test instance ---
 echo "==> Launching smoke-test instance"
-if grep -q 'AuthenticAMD' /proc/cpuinfo; then
+if [[ "$TEST_GPU" == "1" ]]; then
+    INSTANCE_TYPE="${GPU_FAMILY}.xlarge"
+elif grep -q 'AuthenticAMD' /proc/cpuinfo; then
    INSTANCE_TYPE="t3a.small"
 else
    INSTANCE_TYPE="t3.small"
 fi

-AMI_ID=$(aws ec2 describe-images \
+AMI_ID=$(aws_as_user ec2 describe-images \
    --query "Images[?Name=='${AMI_NAME}'] | [0].ImageId" \
    --output text)

@@ -94,7 +152,7 @@ if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
    exit 1
 fi

-SUBNET_ID=$(aws ec2 describe-subnets --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId | [0]' --output text)
+SUBNET_ID=$(aws_as_user ec2 describe-subnets --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId | [0]' --output text)
 if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
    echo "❌ No subnet found"
    exit 1
@@ -102,7 +160,7 @@ fi

 echo "  AMI: $AMI_ID  type: $INSTANCE_TYPE  subnet: $SUBNET_ID"

-INSTANCE_ID=$(aws ec2 run-instances \
+INSTANCE_ID=$(aws_as_user ec2 run-instances \
    --image-id "$AMI_ID" \
    --instance-type "$INSTANCE_TYPE" \
    --key-name spinifex-key \
@@ -121,7 +179,7 @@ echo "==> Waiting for instance to reach running state"
 COUNT=0
 STATE="unknown"
 while [ $COUNT -lt 60 ]; do
-    DESCRIBE=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" 2>/dev/null) || {
+    DESCRIBE=$(aws_as_user ec2 describe-instances --instance-ids "$INSTANCE_ID" 2>/dev/null) || {
        sleep 2; COUNT=$((COUNT + 1)); continue
    }
    STATE=$(echo "$DESCRIBE" | jq -r '.Reservations[0].Instances[0].State.Name // "not-found"')
@@ -144,7 +202,7 @@ echo "==> Waiting for public IP assignment (up to 300s)"
 SSH_INST_PORT=22
 SSH_INST_HOST=""
 for _i in $(seq 1 300); do
-    _ip=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" \
+    _ip=$(aws_as_user ec2 describe-instances --instance-ids "$INSTANCE_ID" \
        --query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null || true)
    if [ -n "$_ip" ] && [ "$_ip" != "None" ] && [ "$_ip" != "null" ]; then
        SSH_INST_HOST="$_ip"
@@ -195,4 +253,27 @@ if ! echo "$SSH_OUT" | grep -q "ec2-user"; then
    exit 1
 fi

-echo "✅ Smoke test passed — instance $INSTANCE_ID launched, running, and SSH-verified"
+# --- GPU guest verification (TEST_GPU=1 only) ---
+if [[ "$TEST_GPU" == "1" ]]; then
+    echo "==> [GPU] Verifying GPU visible inside guest"
+    GPU_VISIBLE=0
+    for _i in $(seq 1 12); do
+        if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+               -o ConnectTimeout=5 -o BatchMode=yes \
+               -p "$SSH_INST_PORT" -i "$SSH_KEY" \
+               ec2-user@"$SSH_INST_HOST" \
+               'lspci 2>/dev/null | grep -iE "nvidia|amdgpu|instinct"' 2>/dev/null; then
+            GPU_VISIBLE=1
+            break
+        fi
+        sleep 5
+    done
+    if [[ $GPU_VISIBLE -eq 0 ]]; then
+        echo "❌ GPU not visible in guest via lspci"
+        exit 1
+    fi
+    echo "   GPU visible in guest"
+    echo "✅ Smoke test passed (GPU passthrough) — instance $INSTANCE_ID launched with GPU and verified"
+else
+    echo "✅ Smoke test passed — instance $INSTANCE_ID launched, running, and SSH-verified"
+fi
@@ -314,6 +314,17 @@ if has_service "predastore"; then
    export SPINIFEX_PREDASTORE_TLS_CERT=$CONFIG_DIR/server.pem
    export SPINIFEX_PREDASTORE_TLS_KEY=$CONFIG_DIR/server.key

+    # Per-node predastore encryption key. Self-heal for devs who skipped
+    # `spx admin init`: generate with umask 0177 so the file is 0600 from
+    # creation (predastore rejects group/other-readable keys outright).
+    ENCRYPTION_KEY="$CONFIG_DIR/predastore/encryption.key"
+    if [ ! -f "$ENCRYPTION_KEY" ]; then
+        mkdir -p "$CONFIG_DIR/predastore"
+        ( umask 0177 && openssl rand -out "$ENCRYPTION_KEY" 32 )
+        echo "   Generated predastore encryption key: $ENCRYPTION_KEY"
+    fi
+    export SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE="$ENCRYPTION_KEY"
+
    # Auto-detect Predastore host:port from spinifex.toml [nodes.<name>.predastore] section
    PREDASTORE_BIND="0.0.0.0:8443"
    if [ -f "$CONFIG_DIR/spinifex.toml" ]; then
@@ -7,9 +7,9 @@ export AWS_PROFILE=spinifex

 ARCH=$(uname -m)
 case "$ARCH" in
-    x86_64)            IMAGE_NAME="ami-ubuntu-24.04-x86_64" ;;
-    aarch64|arm64)     IMAGE_NAME="ami-ubuntu-24.04-arm64" ;;
-    *)                 echo "Unknown arch $ARCH, default x86_64"; IMAGE_NAME="ami-ubuntu-24.04-x86_64" ;;
+    x86_64)            IMAGE_NAME="ami-ubuntu-26.04-x86_64" ;;
+    aarch64|arm64)     IMAGE_NAME="ami-ubuntu-26.04-arm64" ;;
+    *)                 echo "Unknown arch $ARCH, default x86_64"; IMAGE_NAME="ami-ubuntu-26.04-x86_64" ;;
 esac

 INSTANCE_TYPE=$(aws ec2 describe-instance-types \
@@ -0,0 +1,121 @@
+#!/bin/bash
+# test-ddil-phase1.sh — systemd-level smoke test for daemon-local-autonomy
+# Tier 1 (1a / 1b / 1c / 1d / 1d-strict). Run after `make deploy` on a single
+# node where spinifex-daemon and spinifex-nats are managed by systemd.
+#
+# Usage: scripts/test-ddil-phase1.sh
+# Requires: sudo, curl, jq
+#
+# Destructive on the local node — stops/starts spinifex-nats and restarts
+# spinifex-daemon. Do NOT run against a host carrying workloads. The cleanup
+# trap restores both services on exit.
+
+set -euo pipefail
+
+DAEMON_URL="${DAEMON_URL:-https://127.0.0.1:4432}"
+DAEMON_SVC=spinifex-daemon
+NATS_SVC=spinifex-nats
+OVERRIDE=/etc/systemd/system/${DAEMON_SVC}.service.d/99-test-ddil.conf
+NATS_BLOCK=/run/spinifex-nats-blocked
+NATS_BLOCK_DROPIN=/run/systemd/system/${NATS_SVC}.service.d/99-test-block.conf
+CURL=(curl -fsS -k --max-time 5)
+
+ok()   { printf '\033[32mPASS\033[0m %s\n' "$1"; }
+fail() { printf '\033[31mFAIL\033[0m %s\n' "$1"; exit 1; }
+info() { printf '\033[36m==>\033[0m %s\n' "$1"; }
+
+cleanup() {
+    info "cleanup: removing nats block, override, restoring services"
+    sudo rm -f "$NATS_BLOCK" "$NATS_BLOCK_DROPIN" "$OVERRIDE"
+    sudo rmdir "$(dirname "$NATS_BLOCK_DROPIN")" 2>/dev/null || true
+    sudo systemctl daemon-reload
+    sudo systemctl start "$NATS_SVC"     2>/dev/null || true
+    sudo systemctl restart "$DAEMON_SVC" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+field() { "${CURL[@]}" "${DAEMON_URL}/local/status" | jq -r ".$1 // empty"; }
+
+wait_field() {
+    local f=$1 want=$2 timeout=${3:-30} elapsed=0
+    while [ "$elapsed" -lt "$timeout" ]; do
+        [ "$(field "$f" 2>/dev/null || true)" = "$want" ] && return 0
+        sleep 1; elapsed=$((elapsed + 1))
+    done
+    return 1
+}
+
+# 0. baseline
+info "baseline: daemon active + cluster mode"
+sudo systemctl is-active --quiet "$DAEMON_SVC" || fail "daemon not running — run \`make deploy\` first"
+sudo systemctl start "$NATS_SVC"
+wait_field mode cluster 30 || fail "never reached mode=cluster (got: $(field mode))"
+[ "$(field nats)" = connected ] || fail "nats not connected (got: $(field nats))"
+ok "mode=cluster nats=connected"
+
+# 1. kill NATS → standalone, daemon survives
+info "1a/1c: kill NATS"
+retry_before=$(field nats_retry_count)
+sudo systemctl stop "$NATS_SVC"
+wait_field mode standalone 30   || fail "never went standalone (got: $(field mode))"
+wait_field nats disconnected 5  || fail "nats not disconnected (got: $(field nats))"
+sudo systemctl is-active --quiet "$DAEMON_SVC" || fail "daemon exited — should stay up"
+ok "standalone, nats=disconnected, daemon alive"
+
+# 2. /local/instances reachable during partition
+info "1b: /local/instances during partition"
+"${CURL[@]}" "${DAEMON_URL}/local/instances" | jq -e . >/dev/null \
+    || fail "/local/instances did not return JSON"
+ok "/local/instances served while NATS down"
+
+# 3. restart NATS → cluster mode + retry count advances + KV push observed
+# nats_retry_count counts disconnect→reconnect *cycles* (bumped from
+# onNATSReconnect, daemon.go:321). It stays flat during disconnect and
+# advances by 1 here when the reconnect callback fires.
+info "1c reconnect: restart NATS"
+before_sync=$(field last_kv_sync_at)
+sudo systemctl start "$NATS_SVC"
+wait_field mode cluster 60   || fail "did not return to cluster mode"
+wait_field nats connected 10 || fail "nats not reconnected"
+sleep 5
+retry_after=$(field nats_retry_count)
+[ "$retry_after" -gt "$retry_before" ] \
+    || fail "nats_retry_count did not advance on reconnect ($retry_before → $retry_after)"
+after_sync=$(field last_kv_sync_at)
+[ -n "$after_sync" ] && [ "$after_sync" != "$before_sync" ] \
+    || fail "last_kv_sync_at did not advance ($before_sync → $after_sync)"
+ok "cluster restored, retry $retry_before → $retry_after, KV resync at $after_sync"
+
+# 4. SPINIFEX_REQUIRE_NATS=1 → daemon exits within ~30s when NATS down.
+# Block spinifex-nats via a runtime drop-in with ConditionPathExists=!. This
+# defeats the daemon unit's Wants=spinifex-nats.service auto-start, which
+# would otherwise pull NATS back up on `systemctl restart spinifex-daemon`
+# and let the daemon connect immediately. mask refuses (real unit lives in
+# /etc) and --runtime mask is shadowed by /etc — the condition drop-in is
+# the only mechanism that works against an /etc-installed unit.
+info "1d-strict: SPINIFEX_REQUIRE_NATS=1 exits when NATS down"
+sudo mkdir -p "$(dirname "$NATS_BLOCK_DROPIN")"
+sudo tee "$NATS_BLOCK_DROPIN" >/dev/null <<EOF
+[Unit]
+ConditionPathExists=!$NATS_BLOCK
+EOF
+sudo touch "$NATS_BLOCK"
+sudo systemctl daemon-reload
+sudo systemctl stop "$NATS_SVC"
+sudo mkdir -p "$(dirname "$OVERRIDE")"
+sudo tee "$OVERRIDE" >/dev/null <<EOF
+[Service]
+Environment=SPINIFEX_REQUIRE_NATS=1
+Restart=no
+EOF
+sudo systemctl daemon-reload
+sudo systemctl restart "$DAEMON_SVC" || true
+start=$(date +%s)
+while sudo systemctl is-active --quiet "$DAEMON_SVC"; do
+    [ $(( $(date +%s) - start )) -gt 45 ] && fail "did not exit within 45s"
+    sleep 1
+done
+ok "exited after $(( $(date +%s) - start ))s"
+
+echo
+echo "DDIL Phase 1 systemd integration: ALL PASS"
@@ -5,8 +5,8 @@
 sudo apt update
 sudo apt install -y make gcc git

-wget https://go.dev/dl/go1.26.2.linux-amd64.tar.gz
-sudo tar -C /usr/local -xzf go1.26.2.linux-amd64.tar.gz
+wget https://go.dev/dl/go1.26.3.linux-amd64.tar.gz
+sudo tar -C /usr/local -xzf go1.26.3.linux-amd64.tar.gz

 echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.bashrc
 source $HOME/.bashrc
@@ -15,6 +15,7 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"text/template"
@@ -99,6 +100,10 @@ type ConfigSettings struct {
 	BootstrapIgwId      string
 	BootstrapCidr       string
 	BootstrapSubnetCidr string
+
+	// GPUPassthrough enables VFIO GPU passthrough in the daemon config.
+	// Sets gpu_passthrough = true under [nodes.<node>.daemon].
+	GPUPassthrough bool
 }

 // PredastoreNodeConfig describes a single Predastore node for multi-node config generation.
@@ -881,8 +886,8 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP s
 // multi-node Predastore cluster. Each node gets its own DB entry (port 6660)
 // and shard entry (port 9991) on a distinct IP. Node ID 1 is the bootstrap leader.
 func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region, natsToken, configDir, bindIP string) (string, error) {
-	if len(nodes) < 3 {
-		return "", fmt.Errorf("multi-node predastore requires at least 3 nodes, got %d", len(nodes))
+	if len(nodes) < 2 {
+		return "", fmt.Errorf("multi-node predastore requires at least 2 nodes, got %d", len(nodes))
 	}

 	data := struct {
@@ -931,3 +936,58 @@ func ParsePredastoreNodeIDFromConfig(tomlContent string, ip string) int {
 	}
 	return FindNodeIDByIP(cfg.DB, ip)
 }
+
+// SetGPUPassthrough idempotently writes gpu_passthrough = <enabled> for the
+// given node into spinifex.toml, preserving all other content and comments.
+// Returns nil without touching the file if the setting is already correct.
+func SetGPUPassthrough(tomlPath, node string, enabled bool) error {
+	raw, err := os.ReadFile(tomlPath)
+	if err != nil {
+		return fmt.Errorf("read %s: %w", tomlPath, err)
+	}
+	text := string(raw)
+	value := "false"
+	if enabled {
+		value = "true"
+	}
+
+	sectionHeader := "[nodes." + node + ".daemon]"
+
+	sectionStart := strings.Index(text, sectionHeader)
+	if sectionStart < 0 {
+		// No daemon section — append one.
+		text = strings.TrimRight(text, "\n") +
+			"\n\n[nodes." + node + ".daemon]\ngpu_passthrough = " + value + "\n"
+		return os.WriteFile(tomlPath, []byte(text), 0640) //nolint:gosec // spinifex.toml is root:spinifex 0640 so the daemon can read it
+	}
+
+	// Extract body of just this section (stops at the next section header).
+	bodyStart := sectionStart + len(sectionHeader)
+	rest := text[bodyStart:]
+	nextSection := strings.Index(rest, "\n[")
+	var body, suffix string
+	if nextSection < 0 {
+		body = rest
+	} else {
+		body = rest[:nextSection]
+		suffix = rest[nextSection:]
+	}
+
+	// Already correct — no-op.
+	if regexp.MustCompile(`gpu_passthrough\s*=\s*` + value).MatchString(body) {
+		return nil
+	}
+
+	// Key exists with wrong value — flip within this section only.
+	flipRe := regexp.MustCompile(`gpu_passthrough\s*=\s*(?:true|false)`)
+	var newBody string
+	if flipRe.MatchString(body) {
+		newBody = flipRe.ReplaceAllString(body, "gpu_passthrough = "+value)
+	} else {
+		// Key absent — insert right after section header.
+		newBody = "\ngpu_passthrough = " + value + body
+	}
+
+	text = text[:bodyStart] + newBody + suffix
+	return os.WriteFile(tomlPath, []byte(text), 0640) //nolint:gosec // spinifex.toml is root:spinifex 0640 so the daemon can read it
+}
@@ -738,10 +738,9 @@ func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) {

 	_, err := GenerateMultiNodePredastoreConfig(tmpl, []PredastoreNodeConfig{
 		{ID: 1, Host: "10.0.0.1"},
-		{ID: 2, Host: "10.0.0.2"},
 	}, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1")
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "at least 3 nodes")
+	assert.Contains(t, err.Error(), "at least 2 nodes")
 }

 func TestGenerateMultiNodePredastoreConfig_InvalidTemplate(t *testing.T) {
@@ -862,3 +861,72 @@ func TestChownRecursive_NonExistentPath(t *testing.T) {
 	}
 	ChownRecursive("/tmp/nonexistent-path-12345", currentUser)
 }
+
+// --- SetGPUPassthrough ---
+
+func writeToml(t *testing.T, content string) string {
+	t.Helper()
+	f, err := os.CreateTemp(t.TempDir(), "spinifex*.toml")
+	require.NoError(t, err)
+	_, err = f.WriteString(content)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+	return f.Name()
+}
+
+func readToml(t *testing.T, path string) string {
+	t.Helper()
+	b, err := os.ReadFile(path)
+	require.NoError(t, err)
+	return string(b)
+}
+
+func TestSetGPUPassthrough_NoOp(t *testing.T) {
+	toml := "[nodes.node1.daemon]\ngpu_passthrough = true\n"
+	path := writeToml(t, toml)
+	require.NoError(t, SetGPUPassthrough(path, "node1", true))
+	assert.Equal(t, toml, readToml(t, path))
+}
+
+func TestSetGPUPassthrough_Flip(t *testing.T) {
+	path := writeToml(t, "[nodes.node1.daemon]\ngpu_passthrough = false\n")
+	require.NoError(t, SetGPUPassthrough(path, "node1", true))
+	assert.Contains(t, readToml(t, path), "gpu_passthrough = true")
+}
+
+func TestSetGPUPassthrough_AddKeyToSection(t *testing.T) {
+	path := writeToml(t, "[nodes.node1.daemon]\nsome_other = true\n")
+	require.NoError(t, SetGPUPassthrough(path, "node1", true))
+	got := readToml(t, path)
+	assert.Contains(t, got, "gpu_passthrough = true")
+	assert.Contains(t, got, "some_other = true")
+}
+
+func TestSetGPUPassthrough_AppendSection(t *testing.T) {
+	path := writeToml(t, "[nodes.node1.network]\ncidr = \"10.0.0.0/24\"\n")
+	require.NoError(t, SetGPUPassthrough(path, "node1", true))
+	got := readToml(t, path)
+	assert.Contains(t, got, "[nodes.node1.daemon]")
+	assert.Contains(t, got, "gpu_passthrough = true")
+}
+
+func TestSetGPUPassthrough_DisableNoOp(t *testing.T) {
+	toml := "[nodes.node1.daemon]\ngpu_passthrough = false\n"
+	path := writeToml(t, toml)
+	require.NoError(t, SetGPUPassthrough(path, "node1", false))
+	assert.Equal(t, toml, readToml(t, path))
+}
+
+func TestSetGPUPassthrough_ReadError(t *testing.T) {
+	err := SetGPUPassthrough("/nonexistent/path/spinifex.toml", "node1", true)
+	require.Error(t, err)
+}
+
+func TestSetGPUPassthrough_SectionBoundary(t *testing.T) {
+	// gpu_passthrough = true exists but in a DIFFERENT node's section; should still write.
+	path := writeToml(t, "[nodes.node2.daemon]\ngpu_passthrough = true\n[nodes.node1.daemon]\nsome_key = 1\n")
+	require.NoError(t, SetGPUPassthrough(path, "node1", true))
+	got := readToml(t, path)
+	// node1 section should now have the key
+	assert.Contains(t, got, "[nodes.node1.daemon]\ngpu_passthrough = true")
+}
@@ -2,473 +2,47 @@ package awserrors

 import "testing"

-func TestErrorLookup(t *testing.T) {
-	expected := []struct {
-		code    string
-		http    int
-		message string
-	}{
-		{code: "AccountDisabled", http: 400, message: "The functionality you have requested has been administratively disabled for this account."},
-		{code: "ActiveVpcPeeringConnectionPerVpcLimitExceeded", http: 400, message: "You've reached the limit on the number of active VPC peering connections you can have for the specified VPC."},
-		{code: "AddressLimitExceeded", http: 400, message: "You've reached the limit on the number of Elastic IP addresses that you can allocate. For more information, see Elastic IP address limit."},
-		{code: "AsnConflict", http: 400, message: "The Autonomous System Numbers (ASNs) of the specified customer gateway and the specified virtual private gateway are the same."},
-		{code: "AttachmentLimitExceeded", http: 400, message: "You've reached the limit on the number of Amazon EBS volumes or network interfaces that can be attached to a single instance. The number of Amazon EBS volumes that you can attach to an instance depends on the instance type. For more information, see Amazon EBS volume limits for Amazon EC2 instances in the Amazon EC2 User Guide."},
-		{code: "AuthFailure", http: 403, message: "The provided credentials could not be validated. You might not be authorized to carry out the request; for example, trying to associate an Elastic IP address that is not yours, or trying to use an AMI for which you do not have permissions. Ensure that your account is authorized to use Amazon EC2, that your credit card details are correct, and that you are using the correct credentials."},
-		{code: "BandwidthLimitExceeded", http: 500, message: "You've reached the limit on the network bandwidth that is available to an Amazon EC2 instance. For more information, see Amazon EC2 instance network bandwidth."},
-		{code: "Blocked", http: 403, message: "Your account is currently blocked. Contact Support if you have questions."},
-		{code: "BootForVolumeTypeUnsupported", http: 400, message: "The specified volume type cannot be used as a boot volume. For more information, see Amazon EBS volume types."},
-		{code: "BundlingInProgress", http: 400, message: "The specified instance already has a bundling task in progress."},
-		{code: "CannotDelete", http: 400, message: "You cannot delete the 'default' security group in your VPC, but you can change its rules. For more information, see Amazon EC2 security groups."},
-		{code: "CapacityBlockDescribeLimitExceeded", http: 400, message: "You've reached the limit for this account. The returned message provides details."},
-		{code: "ClientInvalidParameterValue", http: 400, message: "A parameter specified in a request is not valid, is unsupported, or cannot be used. The returned message provides an explanation of the error value. For example, if you are launching an instance, you can't specify a security group and subnet that are in different VPCs."},
-		{code: "ClientVpnAuthorizationRuleLimitExceeded", http: 400, message: "You've reached the limit on the number of authorization rules that can be added to a single Client VPN endpoint."},
-		{code: "ClientVpnCertificateRevocationListLimitExceeded", http: 400, message: "You've reached the limit on the number of client certificate revocation lists that can be added to a single Client VPN endpoint."},
-		{code: "ClientVpnEndpointAssociationExists", http: 409, message: "The specified target network is already associated with the Client VPN endpoint."},
-		{code: "ClientVpnEndpointLimitExceeded", http: 400, message: "You've reached the limit on the number of Client VPN endpoints that you can create."},
-		{code: "ClientVpnRouteLimitExceeded", http: 400, message: "You've reached the limit on the number of routes that can be added to a single Client VPN endpoint."},
-		{code: "ClientVpnTerminateConnectionsLimitExceeded", http: 400, message: "The number of client connections you're attempting to terminate exceeds the limit."},
-		{code: "ConcurrentCreateImageNoRebootLimitExceeded", http: 400, message: "The maximum number of concurrent CreateImage requests for the instance has been reached. Wait for the current CreateImage requests to complete, and then retry your request."},
-		{code: "ConcurrentSnapshotLimitExceeded", http: 409, message: "You've reached the limit on the number of concurrent snapshots you can create on the specified volume. Wait until the 'pending' requests have completed, and check that you do not have snapshots that are in an incomplete state, such as 'error', which count against your concurrent snapshot limit."},
-		{code: "ConcurrentTagAccess", http: 400, message: "You can't run simultaneous commands to modify a tag for a specific resource. Allow sufficient wait time for the previous request to complete, then retry your request."},
-		{code: "CreditSpecificationUpdateInProgress", http: 400, message: "The default credit specification for the instance family is currently being updated. It takes about five minutes to complete. For more information, see Set the default credit specification for the account."},
-		{code: "CustomerGatewayLimitExceeded", http: 400, message: "You've reached the limit on the number of customer gateways you can create for the AWS Region. For more information, see Amazon VPC quotas."},
-		{code: "CustomerKeyHasBeenRevoked", http: 400, message: "The KMS key cannot be accessed. For more information, see Amazon EBS encryption."},
-		{code: "DeclarativePoliciesAccessDeniedException", http: 404, message: "You do not have sufficient access to perform this action, or the specified TargetId does not exist, or the specified TargetId is not in your organization. To generate an account status report for declarative policies, the caller must be the management account or a delegated administrator for the organization and the specified TargetId must belong to your organization."},
-		{code: "DeclarativePoliciesNotEnabledException", http: 400, message: "Trusted access is not enabled. Trusted access must be enabled for the service for which the declarative policy will enforce a baseline configuration. The API uses the following service principal to identify the EC2 service: ec2.amazonaws.com. For more information on how to enable trusted access with the AWS CLI and AWS SDKs, see Using Organizations with other AWS services."},
-		{code: "DefaultSubnetAlreadyExistsInAvailabilityZone", http: 409, message: "A default subnet already exists in the specified Availability Zone. You can have only one default subnet per Availability Zone."},
-		{code: "DefaultVpcAlreadyExists", http: 409, message: "A default VPC already exists in the AWS Region. You can only have one default VPC per Region."},
-		{code: "DefaultVpcDoesNotExist", http: 400, message: "There is no default VPC in which to carry out the request. If you've deleted your default VPC, you can create a new one. For more information, see Create a default VPC."},
-		{code: "DeleteConversionTaskError", http: 400, message: "The conversion task cannot be canceled."},
-		{code: "DependencyViolation", http: 400, message: "The specified object has dependent resources. A number of resources in a VPC may have dependent resources, which prevent you from deleting or detaching them. Remove the dependencies first, then retry your request. For example, this error occurs if you try to delete a security group in a VPC that is in use by another security group."},
-		{code: "DiskImageSizeTooLarge", http: 400, message: "The disk image exceeds the allowed limit (for instance or volume import)."},
-		{code: "DryRunOperation", http: 412, message: "The user has the required permissions, so the request would have succeeded, but the DryRun parameter was used."},
-		{code: "DuplicateSubnetsInSameZone", http: 400, message: "For an interface VPC endpoint, you can specify only one subnet per Availability Zone."},
-		{code: "EncryptedVolumesNotSupported", http: 400, message: "Encrypted Amazon EBS volumes may only be attached to instances that support Amazon EBS encryption. For more information, see Amazon EBS encryption."},
-		{code: "ExistingVpcEndpointConnections", http: 400, message: "You cannot delete a VPC endpoint service configuration or change the load balancers for the endpoint service if there are endpoints attached to the service."},
-		{code: "FilterLimitExceeded", http: 400, message: "The request uses too many filters or too many filter values."},
-		{code: "FleetNotInModifiableState", http: 400, message: "The Spot Fleet request must be in the active state in order to modify it. For more information, see Spot Fleet request types."},
-		{code: "FlowLogAlreadyExists", http: 409, message: "A flow log with the specified configuration already exists."},
-		{code: "FlowLogsLimitExceeded", http: 400, message: "You've reached the limit on the number of flow logs you can create. For more information, see Amazon VPC quotas."},
-		{code: "Gateway.NotAttached", http: 400, message: "An internet gateway is not attached to a VPC. If you are trying to detach an internet gateway, ensure that you specify the correct VPC. If you are trying to associate an Elastic IP address with a network interface or an instance, ensure that an internet gateway is attached to the relevant VPC."},
-		{code: "HostAlreadyCoveredByReservation", http: 400, message: "The specified Dedicated Host is already covered by a reservation."},
-		{code: "HostLimitExceeded", http: 400, message: "You've reached the limit on the number of Dedicated Hosts that you can allocate. For more information, see Dedicated Hosts."},
-		{code: "IdempotentInstanceTerminated", http: 400, message: "The request to launch an instance uses the same client token as a previous request for which the instance has been terminated."},
-		{code: "IdempotentParameterMismatch", http: 400, message: "The request uses the same client token as a previous, but non-identical request. Do not reuse a client token with different requests, unless the requests are identical."},
-		{code: "InaccessibleStorageLocation", http: 400, message: "The specified Amazon S3 URL cannot be accessed. Check the access permissions for the URL."},
-		{code: "InaccessibleStorageLocationException", http: 400, message: "The specified Amazon S3 bucket can't be accessed. An S3 bucket must be available before generating an account status report for declarative policies (you can create a new one or use an existing one), you must own the bucket, it must be in the same Region in which the request was made, and it must have an appropriate bucket policy. For a sample S3 policy, see Sample Amazon S3 policy under Examples."},
-		{code: "IncompatibleHostRequirements", http: 400, message: "There are no available or compatible Dedicated Hosts available on which to launch or start the instance."},
-		{code: "IncompleteSignature", http: 400, message: "The request signature does not conform to AWS standards."},
-		{code: "IncorrectInstanceState", http: 409, message: "The instance is in an incorrect state for the requested action. For example, some instance attributes, such as user data, can only be modified if the instance is in a 'stopped' state. If you are associating an Elastic IP address with a network interface, ensure that the instance that the interface is attached to is not in the 'pending' state."},
-		{code: "IncorrectModificationState", http: 400, message: "A new modification action on an EBS Elastic Volume cannot occur because the volume is currently being modified."},
-		{code: "IncorrectSpotRequestState", http: 400, message: "The Spot Instance request is in an incorrect state for the request. Spot request status information can help you track your Amazon EC2 Spot Instance requests. For more information, see Spot request status."},
-		{code: "IncorrectState", http: 400, message: "The resource is in an incorrect state for the request. This error can occur if you are trying to attach a volume that is still being created or detach a volume that is not in the 'available' state. Verify that the volume is in the 'available' state. If you are creating a snapshot, ensure that the previous request to create a snapshot on the same volume has completed. If you are deleting a virtual private gateway, ensure that it's detached from the VPC."},
-		{code: "IncorrectStateException", http: 400, message: "The resource is in an incorrect state for the request. This error can occur if you're trying to cancel the generation of an account status report for declarative policies that already has the complete, cancelled, or error status. Cancelation is only possible for reports with a running status. This error can also occur when you're trying to get a summary for a report that has not yet reached the completed status. This error can also occur if you start report generation while another report is being generated. Only one report per organization can be generated at a time. For more information, see Generating the account status report for declarative policies."},
-		{code: "InstanceCreditSpecification.NotSupported", http: 400, message: "The specified instance does not use CPU credits for CPU usage; only T2 instances use CPU credits for CPU usage."},
-		{code: "InstanceEventStartTimeCannotChange", http: 400, message: "The specified scheduled event start time does not meet the requirements for rescheduling a scheduled event. For more information, see Limitations."},
-		{code: "InstanceLimitExceeded", http: 400, message: "You've reached the limit on the number of instances you can run concurrently. This error can occur if you are launching an instance or if you are creating a Capacity Reservation. Capacity Reservations count towards your On-Demand Instance limits. If your request fails due to limit constraints, increase your On-Demand Instance limit for the required instance type and try again. For more information, see EC2 On-Demand instance limits."},
-		{code: "InstanceTpmEkPubNotFound", http: 400, message: "The public Trusted Platform Module (TPM) Endorsement Key (EK) cannot be found."},
-		{code: "InsufficientAddressCapacity", http: 503, message: "Not enough available addresses to satisfy your minimum request. Reduce the number of addresses you are requesting or wait for additional capacity to become available."},
-		{code: "InsufficientCapacity", http: 503, message: "There is not enough capacity to fulfill your import instance request. You can wait for additional capacity to become available."},
-		{code: "InsufficientCapacityOnHost", http: 400, message: "There is not enough capacity on the Dedicated Host to launch or start the instance."},
-		{code: "InsufficientFreeAddressesInSubnet", http: 503, message: "The specified subnet does not contain enough free private IP addresses to fulfill your request. Use the DescribeSubnets request to view how many IP addresses are available (unused) in your subnet. IP addresses associated with stopped instances are considered unavailable."},
-		{code: "InsufficientHostCapacity", http: 503, message: "There is not enough capacity to fulfill your Dedicated Host request. Reduce the number of Dedicated Hosts in your request, or wait for additional capacity to become available."},
-		{code: "InsufficientInstanceCapacity", http: 400, message: "There is not enough capacity to fulfill your request. This error can occur if you launch a new instance, restart a stopped instance, create a new Capacity Reservation, or modify an existing Capacity Reservation. Reduce the number of instances in your request, or wait for additional capacity to become available. You can also try launching an instance by selecting different instance types (which you can resize at a later stage). The returned message might also give specific guidance about how to solve the problem."},
-		{code: "InsufficientReservedInstanceCapacity", http: 503, message: "Not enough available Reserved Instances to satisfy your minimum request. Reduce the number of Reserved Instances in your request or wait for additional capacity to become available."},
-		{code: "InsufficientReservedInstancesCapacity", http: 400, message: "There is insufficient capacity for the requested Reserved Instances."},
-		{code: "InsufficientVolumeCapacity", http: 503, message: "There is not enough capacity to fulfill your EBS volume provision request. You can try to provision a different volume type, EBS volume in a different availability zone, or you can wait for additional capacity to become available."},
-		{code: "InterfaceInUseByTrafficMirrorSession", http: 409, message: "The Traffic Mirror source that you are trying to create uses an interface that is already associated with a session. An interface can only be associated with a session, or with a target, but not both."},
-		{code: "InterfaceInUseByTrafficMirrorTarget", http: 409, message: "The Traffic Mirror source that you are trying to create uses an interface that is already associated with a target. An interface can only be associated with a session, or with a target, but not both. If the interface is associated with a target, it cannot be associated with another target."},
-		{code: "InternalError", http: 500, message: "An internal error has occurred. Retry your request, but if the problem persists, contact us with details by posting a message on AWS re:Post."},
-		{code: "InternalFailure", http: 404, message: "The request processing has failed because of an unknown error, exception, or failure."},
-		{code: "InternetGatewayLimitExceeded", http: 400, message: "You've reached the limit on the number of internet gateways that you can create. For more information, see Amazon VPC quotas."},
-		{code: "InvalidAMIAttributeItemValue", http: 400, message: "The value of an item added to, or removed from, an image attribute is not valid. If you are specifying a userId, check that it is in the form of an AWS account ID, without hyphens."},
-		{code: "InvalidAMIID.Malformed", http: 400, message: "The specified AMI ID is malformed. Ensure that you provide the full AMI ID, in the form ami-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidAMIID.NotFound", http: 400, message: "The specified AMI doesn't exist. Check the AMI ID, and ensure that you specify the AWS Region in which the AMI is located, if it's not in the default Region. This error might also occur if you specified an incorrect kernel ID when launching an instance. This error might also occur if the AMI doesn't meet your Allowed AMIs criteria. To use this AMI, you must change the Allowed AMIs criteria. For more information, see Control the discovery and use of AMIs in Amazon EC2 with Allowed AMIs."},
-		{code: "InvalidAMIID.Unavailable", http: 400, message: "The specified AMI has been deregistered and is no longer available, or is not in a state from which you can launch an instance or modify attributes."},
-		{code: "InvalidAMIName.Duplicate", http: 409, message: "The specified AMI name is already in use by another AMI. If you have recently deregistered an AMI with the same name, allow enough time for the change to propagate through the system, and retry your request."},
-		{code: "InvalidAMIName.Malformed", http: 400, message: "AMI names must be between 3 and 128 characters long, and may only contain letters, numbers, and the following special characters: '-', '_', '.', '/', '(', and ')'."},
-		{code: "InvalidAction", http: 400, message: "The action or operation requested is not valid. Verify that the action is typed correctly."},
-		{code: "InvalidAddress.Locked", http: 400, message: "The specified Elastic IP address cannot be released from your account. A reverse DNS record may be associated with the Elastic IP address. To unlock the address, contact Support."},
-		{code: "InvalidAddress.Malformed", http: 400, message: "The specified IP address is not valid. Ensure that you provide the address in the form xx.xx.xx.xx; for example, 55.123.45.67"},
-		{code: "InvalidAddress.NotFound", http: 400, message: "The specified Elastic IP address that you are describing cannot be found. Ensure that you specify the AWS Region in which the IP address is located, if it's not in the default Region."},
-		{code: "InvalidAddressID.NotFound", http: 400, message: "The specified allocation ID for the Elastic IP address you are trying to release cannot be found. Ensure that you specify the AWS Region in which the IP address is located, if it's not in the default Region."},
-		{code: "InvalidAffinity", http: 400, message: "The specified affinity value is not valid."},
-		{code: "InvalidAllocationID.NotFound", http: 404, message: "The specified allocation ID you are trying to describe or associate does not exist. Ensure that you specify the AWS Region in which the IP address is located, if it's not in the default Region."},
-		{code: "InvalidAssociationID.NotFound", http: 404, message: "The specified association ID (for an Elastic IP address, a route table, or network ACL) does not exist. Ensure that you specify the AWS Region in which the association ID is located, if it's not in the default Region."},
-		{code: "InvalidAttachment.NotFound", http: 400, message: "Indicates an attempt to detach a volume from an instance to which it is not attached."},
-		{code: "InvalidAttachmentID.NotFound", http: 404, message: "The specified network interface attachment does not exist."},
-		{code: "InvalidAutoPlacement", http: 400, message: "The specified value for auto-placement is not valid."},
-		{code: "InvalidAvailabilityZone", http: 400, message: "The specified Availability Zone is not valid."},
-		{code: "InvalidBlockDeviceMapping", http: 400, message: "A block device mapping parameter is not valid. The returned message indicates the incorrect value."},
-		{code: "InvalidBundleID.NotFound", http: 400, message: "The specified bundle task ID cannot be found. Ensure that you specify the AWS Region in which the bundle task is located, if it's not in the default Region."},
-		{code: "InvalidCapacityBlockOfferingIdExpired", http: 400, message: "The Capacity Block offering ID is no longer available."},
-		{code: "InvalidCapacityBlockOfferingIdMalformed", http: 400, message: "The Capacity Block offering ID is malformed."},
-		{code: "InvalidCapacityBlockOfferingIdNotFound", http: 400, message: "The Capacity Block offering ID cannot be found for this account."},
-		{code: "InvalidCapacityReservationId.Malformed", http: 400, message: "The ID for the Capacity Reservation is malformed. Ensure that you specify the Capacity Reservation ID in the form cr-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidCapacityReservationId.NotFound", http: 404, message: "The specified Capacity Reservation ID does not exist."},
-		{code: "InvalidCapacityReservationState.PendingActivation", http: 400, message: "Your Capacity Block is not active yet."},
-		{code: "InvalidCarrierGatewayID.NotFound", http: 400, message: "The specified carrier gateway ID cannot be found. Ensure that you specify the AWS Region in which the carrier gateway is located, if it's not in the default Region."},
-		{code: "InvalidCharacter", http: 400, message: "A specified character is invalid."},
-		{code: "InvalidCidr.InUse", http: 409, message: "The specified inside tunnel CIDR is already in use by another VPN tunnel for the virtual private gateway."},
-		{code: "InvalidClientToken", http: 400, message: "The specified client token is not valid. For more information, see Ensuring idempotency."},
-		{code: "InvalidClientTokenId", http: 403, message: "The X.509 certificate or credentials provided do not exist in our records."},
-		{code: "InvalidClientVpnActiveAssociationNotFound", http: 409, message: "You cannot perform this action on the Client VPN endpoint while it is in the pending-association state."},
-		{code: "InvalidClientVpnAssociationIdNotFound", http: 400, message: "The specified target network association cannot be found."},
-		{code: "InvalidClientVpnConnection.IdNotFound", http: 400, message: "The specified Client VPN endpoint cannot be found."},
-		{code: "InvalidClientVpnConnection.UserNotFound", http: 400, message: "The specified user does not have an active connection to the specified Client VPN endpoint."},
-		{code: "InvalidClientVpnDuplicateAssociationException", http: 400, message: "The specified target network has already been associated with the Client VPN endpoint."},
-		{code: "InvalidClientVpnDuplicateAuthorizationRule", http: 400, message: "The specified authorization has already been added to the Client VPN endpoint."},
-		{code: "InvalidClientVpnDuplicateRoute", http: 400, message: "The specified route has already been added to the Client VPN endpoint."},
-		{code: "InvalidClientVpnEndpointAuthorizationRuleNotFound", http: 400, message: "The specified authorization rule cannot be found."},
-		{code: "InvalidClientVpnEndpointId.NotFound", http: 400, message: "The specified Client VPN Endpoint cannot be found."},
-		{code: "InvalidClientVpnRouteNotFound", http: 400, message: "The specified route cannot be found."},
-		{code: "InvalidClientVpnSubnetId.DifferentAccount", http: 400, message: "The specified subnet belongs to a different account."},
-		{code: "InvalidClientVpnSubnetId.DuplicateAz", http: 409, message: "You have already associated a subnet from this Availability Zone with the Client VPN endpoint."},
-		{code: "InvalidClientVpnSubnetId.NotFound", http: 400, message: "The specified subnet cannot be found in the VPN with which the Client VPN endpoint is associated."},
-		{code: "InvalidClientVpnSubnetId.OverlappingCidr", http: 400, message: "The specified target network's CIDR range overlaps with the Client VPN endpoint's client CIDR range."},
-		{code: "InvalidConversionTaskId", http: 400, message: "The specified conversion task ID (for instance or volume import) is not valid."},
-		{code: "InvalidConversionTaskId.Malformed", http: 400, message: "The specified conversion task ID (for instance or volume import) is malformed. Ensure that you've specified the ID in the form import-i-xxxxxxxx."},
-		{code: "InvalidCpuCredits", http: 400, message: "The specified CpuCredit value is invalid. Valid values are standard and unlimited."},
-		{code: "InvalidCpuCredits.Malformed", http: 400, message: "The specified CpuCredit value is invalid. Valid values are standard and unlimited."},
-		{code: "InvalidCustomerGateway.DuplicateIpAddress", http: 409, message: "There is a conflict among the specified gateway IP addresses. Each VPN connection in an AWS Region must be created with a unique customer gateway IP address (across all AWS accounts). For more information, see Your customer gateway device in the AWS Site-to-Site VPN User Guide."},
-		{code: "InvalidCustomerGatewayID.NotFound", http: 400, message: "The specified customer gateway ID cannot be found. Ensure that you specify the AWS Region in which the customer gateway is located, if it's not in the default Region."},
-		{code: "InvalidCustomerGatewayId.Malformed", http: 400, message: "The specified customer gateway ID is malformed, or cannot be found. Specify the ID in the form cgw-xxxxxxxx, and ensure that you specify the AWS Region in which the customer gateway is located, if it's not in the default Region."},
-		{code: "InvalidCustomerGatewayState", http: 400, message: "The customer gateway is not in the available state, and therefore cannot be used."},
-		{code: "InvalidDeclarativePoliciesReportId.Malformed", http: 400, message: "The specified account status report ID for declarative policies is malformed. Ensure that you specify the ID in the form p-xxxxxxxxxx. For more information, see Generating the account status report for declarative policies."},
-		{code: "InvalidDhcpOptionID.NotFound", http: 404, message: "The specified DHCP options set does not exist. Ensure that you specify the AWS Region in which the DHCP options set is located, if it's not in the default Region."},
-		{code: "InvalidDhcpOptionsID.NotFound", http: 404, message: "The specified DHCP options set does not exist. Ensure that you specify the AWS Region in which the DHCP options set is located, if it's not in the default Region."},
-		{code: "InvalidDhcpOptionsId.Malformed", http: 400, message: "The specified DHCP options set ID is malformed. Ensure that you provide the full DHCP options set ID in the request, in the form dopt-xxxxxxxx."},
-		{code: "InvalidEgressOnlyInternetGatewayId.Malformed", http: 400, message: "The specified egress-only internet gateway ID is malformed. Ensure that you specify the ID in the form eigw-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidEgressOnlyInternetGatewayId.NotFound", http: 404, message: "The specified egress-only internet gateway does not exist."},
-		{code: "InvalidElasticGpuID.Malformed", http: 400, message: "The specified Elastic GPU ID is malformed. Ensure that you specify the ID in the form egpu-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidElasticGpuID.NotFound", http: 404, message: "The specified Elastic GPU does not exist."},
-		{code: "InvalidExportTaskID.Malformed", http: 400, message: "The specified export task ID cannot be found."},
-		{code: "InvalidExportTaskID.NotFound", http: 400, message: "The specified export task ID is malformed. Ensure that you specify the ID in the form export-ami-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidFilter", http: 400, message: "The specified filter is not valid."},
-		{code: "InvalidFlowLogId.NotFound", http: 404, message: "The specified flow log does not exist."},
-		{code: "InvalidFormat", http: 400, message: "The specified disk format (for the instance or volume import) is not valid."},
-		{code: "InvalidFpgaImageID.Malformed", http: 400, message: "The specified Amazon FPGA image (AFI) ID is malformed. Ensure that you provide the full AFI ID in the request, in the form afi-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidFpgaImageID.NotFound", http: 404, message: "The specified Amazon FPGA image (AFI) ID does not exist. Ensure that you specify the AWS Region in which the AFI is located, if it's not in the default Region."},
-		{code: "InvalidGatewayID.NotFound", http: 404, message: "The specified gateway does not exist."},
-		{code: "InvalidGroup.Duplicate", http: 400, message: "You cannot create a security group with the same name as an existing security group in the same VPC."},
-		{code: "InvalidGroup.InUse", http: 409, message: "The specified security group can't be deleted because it's in use by another security group. You can remove dependencies by modifying or deleting rules in the affected security groups."},
-		{code: "InvalidGroup.NotFound", http: 404, message: "The specified security group does not exist. This error can occur because the ID of a recently created security group has not propagated through the system. For more information, see Ensuring idempotency. You can't specify a security group that is in a different AWS Region or VPC than the request."},
-		{code: "InvalidGroup.Reserved", http: 400, message: "The name 'default' is reserved, and cannot be used to create a new security group."},
-		{code: "InvalidGroupId.Malformed", http: 400, message: "The specified security group ID is malformed. Ensure that you provide the full security group ID in the request, in the form sg-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidHostConfiguration", http: 400, message: "The specified Dedicated Host configuration is not supported."},
-		{code: "InvalidHostID.Malformed", http: 400, message: "The specified Dedicated Host ID is not formed correctly. Ensure that you provide the full ID in the form h-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidHostID.NotFound", http: 404, message: "The specified Dedicated Host ID does not exist. Ensure that you specify the AWS Region in which the Dedicated Host is located, if it's not in the default Region."},
-		{code: "InvalidHostId", http: 400, message: "The specified Dedicated Host ID is not valid."},
-		{code: "InvalidHostId.Malformed", http: 400, message: "The specified Dedicated Host ID is not formed correctly. Ensure that you provide the full ID in the form h-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidHostId.NotFound", http: 404, message: "The specified Dedicated Host ID does not exist. Ensure that you specify the region in which the Dedicated Host is located, if it's not in the default region."},
-		{code: "InvalidHostReservationId.Malformed", http: 400, message: "The specified Dedicated Host Reservation ID is not formed correctly. Ensure that you provide the full ID in the form hr-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidHostReservationOfferingId.Malformed", http: 400, message: "The specified Dedicated Host Reservation offering is not formed correctly. Ensure that you provide the full ID in the form hro-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidHostState", http: 400, message: "The Dedicated Host must be in the available state to complete the operation."},
-		{code: "InvalidID", http: 400, message: "The specified ID for the resource you are trying to tag is not valid. Ensure that you provide the full resource ID; for example, ami-2bb65342 for an AMI. If you're using the command line tools on a Windows system, you might need to use quotation marks for the key-value pair; for example, \"Name=TestTag\"."},
-		{code: "InvalidIPAddress.InUse", http: 409, message: "The specified IP address is already in use. If you are trying to release an address, you must first disassociate it from the instance."},
-		{code: "InvalidIamInstanceProfileArn.Malformed", http: 400, message: "The specified IAM instance profile ARN is not valid. For more information about valid ARN formats, see Amazon Resource Names (ARNs)."},
-		{code: "InvalidInput", http: 400, message: "An input parameter in the request is not valid. For example, you may have specified an incorrect Reserved Instance listing ID in the request or the Reserved Instance you tried to list cannot be sold in the Reserved Instances Marketplace (for example, if it has a scope of Region, or is a Convertible Reserved Instance)."},
-		{code: "InvalidInstanceAttributeValue", http: 400, message: "The specified instance attribute value is not valid. This error is most commonly encountered when trying to set the InstanceType/--instance-type attribute to an unrecognized value."},
-		{code: "InvalidInstanceConnectEndpointId.Malformed", http: 400, message: "The specified EC2 Instance Connect Endpoint ID is malformed. Ensure that you specify the ID in the form eice-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidInstanceConnectEndpointId.NotFound", http: 404, message: "The specified EC2 Instance Connect Endpoint does not exist."},
-		{code: "InvalidInstanceCreditSpecification", http: 409, message: "If you are modifying the credit option for CPU usage for T2 instances, the request may not contain duplicate instance IDs."},
-		{code: "InvalidInstanceCreditSpecification.DuplicateInstanceId", http: 409, message: "If you are modifying the credit option for CPU usage for T2 instances, the request may not contain duplicate instance IDs."},
-		{code: "InvalidInstanceEventIDNotFound", http: 400, message: "The specified ID of the event whose date and time you are modifying cannot be found. Verify the ID of the event and try your request again."},
-		{code: "InvalidInstanceEventStartTime", http: 400, message: "The specified scheduled event start time does not meet the requirements for rescheduling a scheduled event. For more information, see Limitations."},
-		{code: "InvalidInstanceFamily", http: 400, message: "The instance family is not supported for this request. For example, the instance family for the Dedicated Host Reservation offering is different from the instance family of the Dedicated Hosts. Or, you can only modify the default credit specification for burstable performance instance families (T2, T3, and T3a). For more information, see Set the default credit specification for the account."},
-		{code: "InvalidInstanceID", http: 400, message: "This error can occur when trying to perform an operation on an instance that has multiple network interfaces. A network interface can have individual attributes; therefore, you may need to specify the network interface ID as part of the request, or use a different request. For example, each network interface in an instance can have a source/destination check flag. To modify this attribute, modify the network interface attribute, and not the instance attribute. To create a route in a route table, provide a specific network interface ID as part of the request."},
-		{code: "InvalidInstanceID.Malformed", http: 400, message: "The specified instance ID is malformed. Ensure that you provide the full instance ID in the request, in the form i-xxxxxxxx or i-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidInstanceID.NotFound", http: 404, message: "The specified instance does not exist. This error might occur because the ID of a recently created instance has not propagated through the system. For more information, see Ensuring idempotency."},
-		{code: "InvalidInstanceID.NotLinkable", http: 400, message: "The specified instance cannot be linked to the specified VPC. This error may also occur if the instance was recently launched, and its ID has not yet propagated through the system. Wait a few minutes, or wait until the instance is in the running state, and then try again."},
-		{code: "InvalidInstanceState", http: 400, message: "The instance is not in an appropriate state to complete the request. If you're modifying the instance placement, the instance must be in the stopped state."},
-		{code: "InvalidInstanceType", http: 400, message: "The instance type is not supported for this request. For example, you can only bundle instance store-backed Windows instances."},
-		{code: "InvalidInterface.IpAddressLimitExceeded", http: 400, message: "The number of private IP addresses for a specified network interface exceeds the limit for the type of instance you are trying to launch. For more information, see IP addresses per network interface per instance type."},
-		{code: "InvalidInternetGatewayID.NotFound", http: 404, message: "The specified internet gateway does not exist. Ensure that you specify the AWS Region in which the internet gateway is located, if it's not in the default Region."},
-		{code: "InvalidInternetGatewayId.Malformed", http: 400, message: "The specified internet gateway ID is malformed. Ensure that you provide the full ID in the request, in the form igw-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidKernelId.Malformed", http: 400, message: "The specified kernel ID is not valid. Ensure that you specify the kernel ID in the form aki-xxxxxxxx."},
-		{code: "InvalidKey.Format", http: 400, message: "The key pair is not specified in a valid OpenSSH public key format."},
-		{code: "InvalidKeyPair.Duplicate", http: 409, message: "The key pair name already exists in that AWS Region. If you are creating or importing a key pair, ensure that you use a unique name."},
-		{code: "InvalidKeyPair.Format", http: 400, message: "The format of the public key you are attempting to import is not valid."},
-		{code: "InvalidKeyPair.NotFound", http: 404, message: "The specified key pair name does not exist. Ensure that you specify the AWS Region in which the key pair is located, if it's not in the default Region."},
-		{code: "InvalidLaunchTargets", http: 400, message: "One or more specified targets are invalid. Verify the capacity for the Capacity Reservation selected or verify the ID."},
-		{code: "InvalidLaunchTemplateId.Malformed", http: 400, message: "The ID for the launch template is malformed. Ensure that you specify the launch template ID in the form lt-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidLaunchTemplateId.NotFound", http: 404, message: "The specified launch template ID does not exist. Ensure that you specify the AWS Region in which the launch template is located."},
-		{code: "InvalidLaunchTemplateId.VersionNotFound", http: 404, message: "The specified launch template version does not exist."},
-		{code: "InvalidLaunchTemplateName.AlreadyExistsException", http: 409, message: "The specified launch template name is already in use."},
-		{code: "InvalidLaunchTemplateName.MalformedException", http: 400, message: "The specified launch template name is invalid. A launch template name must be between 3 and 128 characters, and may contain letters, numbers, and the following characters: '-', '_', '.', '/', '(', and ')'."},
-		{code: "InvalidLaunchTemplateName.NotFoundException", http: 404, message: "The specified launch template name does not exist. Check the spelling of the name and ensure that you specify the AWS Region in which the launch template is located. Launch template names are case-sensitive."},
-		{code: "InvalidManifest", http: 400, message: "The specified AMI has an unparsable manifest, or you may not have access to the location of the manifest file in Amazon S3."},
-		{code: "InvalidMaxResults", http: 400, message: "The specified value for MaxResults is not valid."},
-		{code: "InvalidNatGatewayID.NotFound", http: 404, message: "The specified NAT gateway ID does not exist. Ensure that you specify the AWS Region in which the NAT gateway is located, if it's not in the default Region."},
-		{code: "InvalidNetworkAclEntry.NotFound", http: 404, message: "The specified network ACL entry does not exist."},
-		{code: "InvalidNetworkAclID.NotFound", http: 404, message: "The specified network ACL does not exist. Ensure that you specify the AWS Region in which the network ACL is located, if it's not in the default Region."},
-		{code: "InvalidNetworkAclId.Malformed", http: 400, message: "The specified network ACL ID is malformed. Ensure that you provide the ID in the form acl-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidNetworkInterface.InUse", http: 409, message: "The specified interface is currently in use and cannot be deleted or attached to another instance. Ensure that you have detached the network interface first. If a network interface is in use, you may also receive the InvalidParameterValue error."},
-		{code: "InvalidNetworkInterface.NotFound", http: 404, message: "The specified network interface does not exist."},
-		{code: "InvalidNetworkInterfaceAttachmentId.Malformed", http: 400, message: "The ID for the network interface attachment is malformed. Ensure that you use the attachment ID rather than the network interface ID, in the form eni-attach-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidNetworkInterfaceID.NotFound", http: 404, message: "The specified network interface does not exist. Ensure that you specify the AWS Region in which the network interface is located, if it's not in the default Region."},
-		{code: "InvalidNetworkInterfaceId.Malformed", http: 400, message: "The specified network interface ID is malformed. Ensure that you specify the network interface ID in the form eni-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidNetworkLoadBalancerArn.Malformed", http: 400, message: "The specified Network Load Balancer ARN is malformed. Ensure that you specify the ARN in the form arn:aws:elasticloadbalancing:region:account-id:loadbalancer/net/load-balancer-name/load-balancer-id."},
-		{code: "InvalidNetworkLoadBalancerArn.NotFound", http: 404, message: "The specified Network Load Balancer ARN does not exist."},
-		{code: "InvalidNextToken", http: 400, message: "The specified NextToken is not valid."},
-		{code: "InvalidOption.Conflict", http: 409, message: "A VPN connection between the virtual private gateway and the customer gateway already exists."},
-		{code: "InvalidPaginationToken", http: 403, message: "The specified pagination token is not valid or is expired."},
-		{code: "InvalidParameter", http: 400, message: "A parameter specified in a request is not valid, is unsupported, or cannot be used. The returned message provides an explanation of the error value. For example, if you are launching an instance, you can't specify a security group and subnet that are in different VPCs."},
-		{code: "InvalidParameterCombination", http: 400, message: "Indicates an incorrect combination of parameters, or a missing parameter. For example, trying to terminate an instance without specifying the instance ID."},
-		{code: "InvalidParameterDependency", http: 400, message: "Indicates an incorrect combination of parameters, or a missing parameter. For example, trying to terminate an instance without specifying the instance ID."},
-		{code: "InvalidParameterValue", http: 400, message: "A value specified in a parameter is not valid, is unsupported, or cannot be used. Ensure that you specify a resource by using its full ID. The returned message provides an explanation of the error value."},
-		{code: "InvalidPermission.Duplicate", http: 409, message: "The specified inbound or outbound rule already exists for that security group."},
-		{code: "InvalidPermission.Malformed", http: 400, message: "The specified security group rule is malformed. If you are specifying an IP address range, ensure that you use CIDR notation; for example, 203.0.113.0/24."},
-		{code: "InvalidPermission.NotFound", http: 404, message: "The specified rule does not exist in this security group."},
-		{code: "InvalidPlacementGroup.Duplicate", http: 409, message: "The specified placement group already exists in that AWS Region."},
-		{code: "InvalidPlacementGroup.InUse", http: 409, message: "The specified placement group is in use. If you are trying to delete a placement group, ensure that its instances have been terminated."},
-		{code: "InvalidPlacementGroup.Unknown", http: 400, message: "The specified placement group cannot be found. Ensure that you specify the AWS Region in which the placement group is located, if it's not in the default Region."},
-		{code: "InvalidPlacementGroupId.Malformed", http: 400, message: "The specified placement group ID is malformed. Ensure that you specify the ID in the form pg-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidPolicyDocument", http: 400, message: "The specified policy document is not a valid JSON policy document."},
-		{code: "InvalidPrefixListID.NotFound", http: 404, message: "The specified prefix list ID does not exist."},
-		{code: "InvalidPrefixListId.Malformed", http: 400, message: "The specified prefix list ID is malformed. Ensure that you provide the ID in the form pl-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidProductInfo", http: 400, message: "(AWS Marketplace) The product code is not valid."},
-		{code: "InvalidPurchaseToken.Expired", http: 403, message: "The specified purchase token has expired."},
-		{code: "InvalidPurchaseToken.Malformed", http: 400, message: "The specified purchase token is not valid."},
-		{code: "InvalidQuantity", http: 400, message: "The specified quantity of Dedicated Hosts is not valid."},
-		{code: "InvalidQueryParameter", http: 400, message: "The AWS query string is malformed or does not adhere to AWS standards."},
-		{code: "InvalidRamDiskId.Malformed", http: 400, message: "The specified RAM disk ID is not valid. Ensure that you specify the RAM disk ID in the form ari-xxxxxxxx."},
-		{code: "InvalidRegion", http: 400, message: "The specified AWS Region is not valid. For copying a snapshot or image, specify the source Region using its Region code, for example, us-west-2."},
-		{code: "InvalidRequest", http: 400, message: "The request is not valid. The returned message provides details about the nature of the error."},
-		{code: "InvalidReservationID.Malformed", http: 400, message: "The specified reservation ID is not valid."},
-		{code: "InvalidReservationID.NotFound", http: 404, message: "The specified reservation does not exist."},
-		{code: "InvalidReservedInstancesId", http: 404, message: "The specified Reserved Instance does not exist."},
-		{code: "InvalidReservedInstancesOfferingId", http: 404, message: "The specified Reserved Instances offering does not exist."},
-		{code: "InvalidResourceConfigurationArn.Malformed", http: 400, message: "The specified resource configuration ARN is malformed."},
-		{code: "InvalidResourceConfigurationArn.NotFound", http: 404, message: "The specified resource configuration ARN does not exist."},
-		{code: "InvalidResourceType.Unknown", http: 400, message: "The specified resource type is not supported or is not valid. To view resource types that support longer IDs, use DescribeIdFormat."},
-		{code: "InvalidRoute.InvalidState", http: 400, message: "The specified route is not valid."},
-		{code: "InvalidRoute.Malformed", http: 400, message: "The specified route is not valid. If you are deleting a route in a VPN connection, ensure that you've entered the value for the CIDR block correctly."},
-		{code: "InvalidRoute.NotFound", http: 404, message: "The specified route does not exist in the specified route table. Ensure that you indicate the exact CIDR range for the route in the request. This error can also occur if you've specified a route table ID in the request that does not exist."},
-		{code: "InvalidRouteTableID.NotFound", http: 404, message: "The specified route table does not exist. Ensure that you specify the AWS Region in which the route table is located, if it's not in the default Region."},
-		{code: "InvalidRouteTableId.Malformed", http: 400, message: "The specified route table ID is malformed. Ensure that you specify the route table ID in the form rtb-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidScheduledInstance", http: 404, message: "The specified Scheduled Instance does not exist."},
-		{code: "InvalidSecurity.RequestHasExpired", http: 400, message: "The difference between the request timestamp and the AWS server time is greater than 5 minutes. Ensure that your system clock is accurate and configured to use the correct time zone."},
-		{code: "InvalidSecurityGroupID.NotFound", http: 404, message: "The specified security group does not exist."},
-		{code: "InvalidSecurityGroupId.Malformed", http: 400, message: "The specified security group ID is not valid. Ensure that you specify the security group ID in the form sg-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidSecurityGroupRuleId.Malformed", http: 400, message: "The specified security group rule ID is not valid. Ensure that you specify the security group rule ID in the form sgr-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidSecurityGroupRuleId.NotFound", http: 404, message: "The specified security group rule does not exist."},
-		{code: "InvalidServiceName", http: 400, message: "The name of the service is not valid. To get a list of available service names, use DescribeVpcEndpointServices."},
-		{code: "InvalidSnapshot.InUse", http: 409, message: "The snapshot that you are trying to delete is in use by one or more AMIs."},
-		{code: "InvalidSnapshot.NotFound", http: 404, message: "The specified snapshot does not exist. Ensure that you specify the AWS Region in which the snapshot is located, if it's not in the default Region."},
-		{code: "InvalidSnapshotID.Malformed", http: 400, message: "The snapshot ID is not valid."},
-		{code: "InvalidSpotDatafeed.NotFound", http: 400, message: "You have no data feed for Spot Instances."},
-		{code: "InvalidSpotFleetRequestConfig", http: 400, message: "The Spot Fleet request configuration is not valid. Ensure that you provide valid values for all of the configuration parameters; for example, a valid AMI ID. Limits apply on the target capacity and the number of launch specifications per Spot Fleet request. For more information, see Fleet quotas."},
-		{code: "InvalidSpotFleetRequestId.Malformed", http: 400, message: "The specified Spot Fleet request ID is malformed. Ensure that you specify the Spot Fleet request ID in the form sfr- followed by 36 characters, including hyphens; for example, sfr-123f8fc2-11aa-22bb-33cc-example12710."},
-		{code: "InvalidSpotFleetRequestId.NotFound", http: 404, message: "The specified Spot Fleet request ID does not exist. Ensure that you specify the AWS Region in which the Spot Fleet request is located, if it's not in the default Region."},
-		{code: "InvalidSpotInstanceRequestID.Malformed", http: 400, message: "The specified Spot Instance request ID is not valid. Ensure that you specify the Spot Instance request ID in the form sir-xxxxxxxx."},
-		{code: "InvalidSpotInstanceRequestID.NotFound", http: 404, message: "The specified Spot Instance request ID does not exist. Ensure that you specify the AWS Region in which the Spot Instance request is located, if it's not in the default Region."},
-		{code: "InvalidState", http: 400, message: "The specified resource is not in the correct state for the request; for example, if you are trying to enable monitoring on a recently terminated instance, or if you are trying to create a snapshot when a previous identical request has not yet completed."},
-		{code: "InvalidStateTransition", http: 400, message: "The specified VPC peering connection is not in the correct state for the request. For example, you may be trying to accept a VPC peering request that has failed, or that was rejected."},
-		{code: "InvalidSubnet", http: 404, message: "The specified subnet ID is not valid or does not exist."},
-		{code: "InvalidSubnet.Conflict", http: 409, message: "The specified CIDR block conflicts with that of another subnet in your VPC."},
-		{code: "InvalidSubnet.Range", http: 400, message: "The CIDR block you've specified for the subnet is not valid. The allowed block size is between a /28 netmask and /16 netmask."},
-		{code: "InvalidSubnetID.Malformed", http: 400, message: "The specified subnet ID is malformed. Ensure that you specify the ID in the form subnet-xxxxxxxxxxxxxxxxx"},
-		{code: "InvalidSubnetID.NotFound", http: 404, message: "or InvalidSubnetId.NotFound \tThe specified subnet does not exist."},
-		{code: "InvalidTagKey.Malformed", http: 400, message: "The specified tag key is not valid. Tag keys cannot be empty or null, and cannot start with aws:."},
-		{code: "InvalidTargetArn.Unknown", http: 404, message: "The specified ARN for the specified user or role is not valid or does not exist."},
-		{code: "InvalidTargetException", http: 404, message: "The specified TargetId is not valid, does not exist, or is in another organization. You can only generate an account status report for declarative policies in your own organization. Ensure that you specify the TargetId in one of the following forms: r-xxxx, ou-xxxx-xxxxxxxx, or a 12-digit account ID in the form xxxxxxxxxxxx."},
-		{code: "InvalidTenancy", http: 400, message: "The tenancy of the instance or VPC is not supported for the requested action. For example, you cannot modify the tenancy of an instance or VPC that has a tenancy attribute of default."},
-		{code: "InvalidTime", http: 400, message: "The specified timestamp is not valid."},
-		{code: "InvalidTrafficMirrorFilterNotFound", http: 404, message: "The specified Traffic Mirror filter does not exist."},
-		{code: "InvalidTrafficMirrorFilterRuleNotFound", http: 404, message: "The specified Traffic Mirror filter rule does not exist."},
-		{code: "InvalidTrafficMirrorSessionNotFound", http: 404, message: "The specified Traffic Mirror session does not exist."},
-		{code: "InvalidTrafficMirrorTargetNoFound", http: 404, message: "The specified Traffic Mirror target does not exist."},
-		{code: "InvalidUserID.Malformed", http: 400, message: "The specified user or owner is not valid. If you are performing a DescribeImages request, you must specify a valid value for the owner or executableBy parameters, such as an AWS account ID. If you are performing a DescribeSnapshots request, you must specify a valid value for the owner or restorableBy parameters."},
-		{code: "InvalidVolume.NotFound", http: 404, message: "The specified volume does not exist."},
-		{code: "InvalidVolume.ZoneMismatch", http: 400, message: "The specified volume is not in the same Availability Zone as the specified instance. You can only attach an Amazon EBS volume to an instance if they are in the same Availability Zone."},
-		{code: "InvalidVolumeID.Duplicate", http: 409, message: "The Amazon EBS volume already exists."},
-		{code: "InvalidVolumeID.Malformed", http: 400, message: "The specified volume ID is not valid. Check the letter-number combination carefully."},
-		{code: "InvalidVolumeID.ZoneMismatch", http: 400, message: "The specified volume and instance are in different Availability Zones."},
-		{code: "InvalidVpcEndpoint.NotFound", http: 404, message: "The specified VPC endpoint does not exist. If you are performing a bulk request that is partially successful or unsuccessful, the response includes a list of the unsuccessful items. If the request succeeds, the list is empty."},
-		{code: "InvalidVpcEndpointId.Malformed", http: 400, message: "The specified VPC endpoint ID is malformed. Use the full VPC endpoint ID in the request, in the form vpce-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidVpcEndpointId.NotFound", http: 404, message: "The specified VPC endpoint does not exist. If you are performing a bulk request that is partially successful or unsuccessful, the response includes a list of the unsuccessful items. If the request succeeds, the list is empty."},
-		{code: "InvalidVpcEndpointService.NotFound", http: 404, message: "The specified VPC endpoint service does not exist. If you are performing a bulk request that is partially successful or unsuccessful, the response includes a list of the unsuccessful items. If the request succeeds, the list is empty."},
-		{code: "InvalidVpcEndpointServiceId.NotFound", http: 404, message: "The specified VPC endpoint service does not exist. If you are performing a bulk request that is partially successful or unsuccessful, the response includes a list of the unsuccessful items. If the request succeeds, the list is empty."},
-		{code: "InvalidVpcEndpointServiceIdId.Malformed", http: 400, message: "The specified VPC endpoint service ID is malformed. Ensure that you specify the ID in the form vpc-svc-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidVpcEndpointType", http: 400, message: "The specified VPC endpoint type is not valid."},
-		{code: "InvalidVpcID.Malformed", http: 400, message: "The specified VPC ID is malformed. Ensure that you've specified the ID in the form vpc-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidVpcID.NotFound", http: 404, message: "The specified VPC does not exist."},
-		{code: "InvalidVpcPeeringConnectionID.NotFound", http: 404, message: "The specified VPC peering connection ID does not exist."},
-		{code: "InvalidVpcPeeringConnectionId.Malformed", http: 400, message: "The specified VPC peering connection ID is malformed. Ensure that you provide the ID in the form pcx-xxxxxxxxxxxxxxxxx."},
-		{code: "InvalidVpcPeeringConnectionState.DnsHostnamesDisabled", http: 400, message: "To enable DNS hostname resolution for the VPC peering connection, DNS hostname support must be enabled for the VPCs."},
-		{code: "InvalidVpcRange", http: 400, message: "The specified CIDR block range is not valid. The block range must be between a /28 netmask and /16 netmask. For more information, see VPC CIDR blocks."},
-		{code: "InvalidVpcState", http: 400, message: "The specified VPC already has a virtual private gateway attached to it."},
-		{code: "InvalidVpnConnection.InvalidState", http: 400, message: "The VPN connection must be in the available state to complete the request."},
-		{code: "InvalidVpnConnection.InvalidType", http: 400, message: "The specified VPN connection does not support static routes."},
-		{code: "InvalidVpnConnectionID", http: 400, message: "The specified VPN connection ID cannot be found."},
-		{code: "InvalidVpnConnectionID.NotFound", http: 404, message: "The specified VPN connection ID does not exist."},
-		{code: "InvalidVpnGatewayAttachment.NotFound", http: 404, message: "An attachment between the specified virtual private gateway and specified VPC does not exist. This error can also occur if you've specified an incorrect VPC ID in the request."},
-		{code: "InvalidVpnGatewayID.NotFound", http: 404, message: "The specified virtual private gateway does not exist."},
-		{code: "InvalidVpnGatewayState", http: 400, message: "The virtual private gateway is not in an available state."},
-		{code: "InvalidZone.NotFound", http: 404, message: "The specified Availability Zone does not exist, or is not available for you to use. Use the DescribeAvailabilityZones request to list the Availability Zones that are currently available to you. Specify the full name of the Availability Zone: for example, us-east-1a."},
-		{code: "KeyPairLimitExceeded", http: 400, message: "You've reached the limit on the number of key pairs that you can have in this AWS Region. For more information, see Amazon EC2 key pairs."},
-		{code: "LegacySecurityGroup", http: 400, message: "Any VPC created using an API version older than 2011-01-01 may have the 2009-07-15-default security group. You must delete this security group before you can attach an internet gateway to the VPC."},
-		{code: "LimitPriceExceeded", http: 400, message: "The cost of the total order is greater than the specified limit price (instance count * price)."},
-		{code: "LogDestinationNotFound", http: 404, message: "The specified Amazon S3 bucket does not exist. Ensure that you have specified the ARN for an existing Amazon S3 bucket, and that the ARN is in the correct format."},
-		{code: "LogDestinationPermissionIssue", http: 400, message: "You do not have sufficient permissions to publish flow logs to the specific Amazon S3 bucket."},
-		{code: "MalformedQueryString", http: 400, message: "The query string contains a syntax error."},
-		{code: "MaxConfigLimitExceededException", http: 400, message: "You\u2019ve exceeded your maximum allowed Spot placement configurations. You can retry configurations that you used within the last 24 hours, or wait for 24 hours before specifying a new configuration. For more information, see Spot placement score."},
-		{code: "MaxIOPSLimitExceeded", http: 400, message: "You've reached the limit on your IOPS usage for the AWS Region. For more information, see Quotas for Amazon EBS."},
-		{code: "MaxScheduledInstanceCapacityExceeded", http: 400, message: "You've attempted to launch more instances than you purchased."},
-		{code: "MaxSpotFleetRequestCountExceeded", http: 400, message: "You've reached one or both of these limits: the total number of Spot Fleet requests that you can make, or the total number of instances in all Spot Fleets for the AWS Region (the target capacity). For more information, see Fleet quotas."},
-		{code: "MaxSpotInstanceCountExceeded", http: 400, message: "You've reached the limit on the number of Spot Instances that you can launch. The limit depends on the instance type. For more information, see Spot Instance limits."},
-		{code: "MaxTemplateLimitExceeded", http: 400, message: "You've reached the limit on the number of launch templates you can create. For more information, see Launch template restrictions."},
-		{code: "MaxTemplateVersionLimitExceeded", http: 400, message: "You've reached the limit on the number of launch template versions that you can create. For more information, see Launch template restrictions."},
-		{code: "MissingAction", http: 400, message: "The request is missing an action or a required parameter."},
-		{code: "MissingAuthenticationToken", http: 403, message: "The request must contain valid credentials."},
-		{code: "MissingInput", http: 400, message: "An input parameter is missing."},
-		{code: "MissingParameter", http: 400, message: "The request is missing a required parameter. Ensure that you have supplied all the required parameters for the request; for example, the resource ID."},
-		{code: "NatGatewayLimitExceeded", http: 400, message: "You've reached the limit on the number of NAT gateways that you can create. For more information, see Amazon VPC quotas."},
-		{code: "NatGatewayMalformed", http: 400, message: "The specified NAT gateway ID is not formed correctly. Ensure that you specify the NAT gateway ID in the form nat-xxxxxxxxxxxxxxxxx."},
-		{code: "NatGatewayNotFound", http: 404, message: "The specified NAT gateway does not exist."},
-		{code: "NetworkAclEntryAlreadyExists", http: 409, message: "The specified rule number already exists in this network ACL."},
-		{code: "NetworkAclEntryLimitExceeded", http: 400, message: "You've reached the limit on the number of rules that you can add to the network ACL. For more information, see Amazon VPC quotas."},
-		{code: "NetworkAclLimitExceeded", http: 400, message: "You've reached the limit on the number of network ACLs that you can create for the specified VPC. For more information, see Amazon VPC quotas."},
-		{code: "NetworkInterfaceLimitExceeded", http: 400, message: "You've reached the limit on the number of network interfaces that you can create. For more information, see Amazon VPC quotas."},
-		{code: "NetworkInterfaceNotSupported", http: 400, message: "The network interface is not supported for Traffic Mirroring."},
-		{code: "NetworkLoadBalancerNotFoundException", http: 404, message: "The specified Network Load Balancer does not exist."},
-		{code: "NlbInUseByTrafficMirrorTargetException", http: 400, message: "The Network Load Balancer is already configured as a Traffic Mirror target."},
-		{code: "NoSuchVersion", http: 404, message: "The specified API version does not exist."},
-		{code: "NonEBSInstance", http: 400, message: "The specified instance does not support Amazon EBS. Restart the instance and try again, to ensure that the code is run on an instance with updated code."},
-		{code: "NotExportable", http: 400, message: "The specified instance cannot be exported. You can only export certain instances. For more information, see Considerations for instance export."},
-		{code: "OperationNotPermitted", http: 400, message: "The specified operation is not allowed. This error can occur for a number of reasons; for example, you might be trying to terminate an instance that has termination protection enabled, or trying to detach the primary network interface (eth0) from an instance."},
-		{code: "OptInRequired", http: 403, message: "You are not authorized to use the requested service. Ensure that you have subscribed to the service you are trying to use. If you are new to AWS, your account might take some time to be activated while your credit card details are being verified."},
-		{code: "OutstandingVpcPeeringConnectionLimitExceeded", http: 400, message: "You've reached the limit on the number of VPC peering connection requests that you can create for the specified VPC."},
-		{code: "PendingSnapshotLimitExceeded", http: 409, message: "You've reached the limit on the number of Amazon EBS snapshots that you can have in the pending state."},
-		{code: "PendingVerification", http: 409, message: "Your account is pending verification. Until the verification process is complete, you may not be able to carry out requests with this account. If you have questions, contact Support."},
-		{code: "PendingVpcPeeringConnectionLimitExceeded", http: 409, message: "You've reached the limit on the number of pending VPC peering connections that you can have."},
-		{code: "PlacementGroupLimitExceeded", http: 400, message: "You've reached the limit on the number of placement groups that you can have."},
-		{code: "PrivateIpAddressLimitExceeded", http: 400, message: "You've reached the limit on the number of private IP addresses that you can assign to the specified network interface for that type of instance. For more information, see IP addresses per network interface."},
-		{code: "RequestEntityTooLarge", http: 413, message: "Request body exceeds the maximum allowed size."},
-		{code: "RequestExpired", http: 403, message: "The request reached the service more than 15 minutes after the date stamp on the request or more than 15 minutes after the request expiration date (such as for presigned URLs), or the date stamp on the request is more than 15 minutes in the future. If you're using temporary security credentials, this error can also occur if the credentials have expired. For more information, see Temporary security credentials in the IAM User Guide."},
-		{code: "RequestLimitExceeded", http: 503, message: "The maximum request rate permitted by the Amazon EC2 APIs has been exceeded for your account. For best results, use an increasing or variable sleep interval between requests. For more information, see Query API request rate."},
-		{code: "RequestResourceCountExceeded", http: 409, message: "Details in your Spot request exceed the numbers allowed by the Spot service in one of the following ways, depending on the action that generated the error: \u2014If you get this error when you submitted a request for Spot Instances, check the number of Spot Instances specified in your request. The number shouldn't exceed the 3,000 maximum allowed per request. Resend your Spot Instance request and specify a number less than 3,000. If your account's regional Spot request limit is greater than 3,000 instances, you can access these instances by submitting multiple smaller requests. \u2014If you get this error when you sent Describe Spot Instance requests, check the number of requests for Spot Instance data, the amount of data you requested, and how often you sent the request. The frequency with which you requested the data combined with the amount of data exceeds the levels allowed by the Spot service. Try again and submit fewer large Describe requests over longer intervals."},
-		{code: "ReservationCapacityExceeded", http: 400, message: "The targeted Capacity Reservation does not enough available instance capacity to fulfill your request. Either increase the instance capacity for the targeted Capacity Reservation, or target a different Capacity Reservation."},
-		{code: "ReservedInstancesCountExceeded", http: 400, message: "You've reached the limit for the number of Reserved Instances."},
-		{code: "ReservedInstancesLimitExceeded", http: 400, message: "Your current quota does not allow you to purchase the required number of Reserved Instances."},
-		{code: "ReservedInstancesUnavailable", http: 400, message: "The requested Reserved Instances are not available."},
-		{code: "Resource.AlreadyAssigned", http: 400, message: "The specified private IP address is already assigned to a resource. Unassign the private IP first, or use a different private IP address."},
-		{code: "Resource.AlreadyAssociated", http: 409, message: "The specified resource is already in use. For example, in EC2-VPC, you cannot associate an Elastic IP address with an instance if it's already associated with another instance. You also cannot attach an internet gateway to more than one VPC at a time."},
-		{code: "ResourceCountExceeded", http: 400, message: "You have exceeded the number of resources allowed for this request; for example, if you try to launch more instances than AWS allows in a single request. This limit is separate from your individual resource limit. If you get this error, break up your request into smaller requests; for example, if you are launching 15 instances, try launching 5 instances in 3 separate requests."},
-		{code: "ResourceCountLimitExceeded", http: 400, message: "You have exceeded a resource limit for creating routes."},
-		{code: "ResourceLimitExceeded", http: 400, message: "You have exceeded an Amazon EC2 resource limit. For example, you might have too many snapshot copies in progress."},
-		{code: "RetryableError", http: 400, message: "A request submitted by an AWS service on your behalf could not be completed. The requesting service might automatically retry the request."},
-		{code: "RouteAlreadyExists", http: 409, message: "A route for the specified CIDR block already exists in this route table."},
-		{code: "RouteLimitExceeded", http: 400, message: "You've reached the limit on the number of routes that you can add to a route table."},
-		{code: "RouteTableLimitExceeded", http: 400, message: "You've reached the limit on the number of route tables that you can create for the specified VPC. For more information about route table limits, see Amazon VPC quotas."},
-		{code: "RulesPerSecurityGroupLimitExceeded", http: 400, message: "You've reached the limit on the number of rules that you can add to a security group. For more information, see Amazon VPC quotas."},
-		{code: "ScheduledInstanceLimitExceeded", http: 400, message: "You've reached the limit on the number of Scheduled Instances that you can purchase."},
-		{code: "ScheduledInstanceParameterMismatch", http: 400, message: "The launch specification does not match the details for the Scheduled Instance."},
-		{code: "ScheduledInstanceSlotNotOpen", http: 400, message: "You can launch a Scheduled Instance only during its scheduled time periods."},
-		{code: "ScheduledInstanceSlotUnavailable", http: 400, message: "The requested Scheduled Instance is no longer available during this scheduled time period."},
-		{code: "SecurityGroupLimitExceeded", http: 400, message: "You've reached the limit on the number of security groups that you can create, or that you can assign to an instance."},
-		{code: "SecurityGroupsPerInstanceLimitExceeded", http: 400, message: "You've reached the limit on the number of security groups that you can assign to an instance. For more information, see Amazon EC2 security groups."},
-		{code: "SecurityGroupsPerInterfaceLimitExceeded", http: 400, message: "You've reached the limit on the number of security groups you can associate with the specified network interface. For more information, see Amazon VPC quotas."},
-		{code: "SerialConsoleSessionUnavailable", http: 403, message: "The serial console access is not enabled for this account. Use EnableSerialConsoleAccess to enable access."},
-		{code: "ServerInternal", http: 500, message: "An internal error has occurred. Retry your request, but if the problem persists, contact us with details by posting a message on AWS re:Post."},
-		{code: "ServiceUnavailable", http: 503, message: "The request has failed due to a temporary failure of the server."},
-		{code: "SignatureDoesNotMatch", http: 403, message: "The request signature that Amazon has does not match the signature that you provided. Check your AWS credentials and signing method."},
-		{code: "SnapshotCopyUnsupported.InterRegion", http: 400, message: "Inter-region snapshot copy is not supported for this AWS Region."},
-		{code: "SnapshotCreationPerVolumeRateExceeded", http: 400, message: "The rate limit for creating concurrent snapshots of an EBS volume has been exceeded. Wait at least 15 seconds between concurrent volume snapshots."},
-		{code: "SnapshotLimitExceeded", http: 400, message: "You've reached the limit on the number of Amazon EBS snapshots that you can create."},
-		{code: "SpotMaxPriceTooLow", http: 400, message: "The request can't be fulfilled yet because your maximum price is below the Spot price. In this case, no instance is launched and your request remains open."},
-		{code: "SubnetLimitExceeded", http: 400, message: "You've reached the limit on the number of subnets that you can create for the specified VPC. For more information, see Amazon VPC quotas."},
-		{code: "TagLimitExceeded", http: 400, message: "You've reached the limit on the number of tags that you can assign to the specified resource. For more information, see Tag restrictions."},
-		{code: "TagPolicyViolation", http: 400, message: "You attempted to create or update a resource with tags that are not compliant with the tag policy requirements for this account. For more information, see Grant permission to tag resources during creation."},
-		{code: "TargetCapacityLimitExceededException", http: 400, message: "The value for targetCapacity exceeds your limit on the amount of Spot placement target capacity you can explore. Reduce the targetCapacity value, and try again. For more information, see Spot placement score."},
-		{code: "Throttling", http: 503, message: "Rate exceeded."},
-		{code: "TrafficMirrorFilterInUse", http: 400, message: "The Traffic Mirror filter cannot be deleted because a Traffic Mirror session is currently using it."},
-		{code: "TrafficMirrorFilterLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror filters has been exceeded."},
-		{code: "TrafficMirrorFilterRuleAlreadyExists", http: 409, message: "The Traffic Mirror filter rule already exists."},
-		{code: "TrafficMirrorFilterRuleLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror filter rules has been exceeded."},
-		{code: "TrafficMirrorSessionLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror sessions has been exceeded."},
-		{code: "TrafficMirrorSessionsPerInterfaceLimitExceeded", http: 400, message: "The allowed number of Traffic Mirror sessions for the specified network interface has been exceeded."},
-		{code: "TrafficMirrorSessionsPerTargetLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror sessions for the specified Traffic Mirror target has been exceeded."},
-		{code: "TrafficMirrorSourcesPerTargetLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror sources for the specified Traffic Mirror target has been exceeded."},
-		{code: "TrafficMirrorTargetInUseException", http: 400, message: "The Traffic Mirror target cannot be deleted because a Traffic Mirror session is currently using it."},
-		{code: "TrafficMirrorTargetLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror targets has been exceeded."},
-		{code: "UnauthorizedOperation", http: 403, message: "You are not authorized to perform this operation. Check your IAM policies, and ensure that you are using the correct credentials. For more information, see Identity and access management for Amazon EC2. If the returned message is encoded, you can decode it using the DecodeAuthorizationMessage action. For more information, see DecodeAuthorizationMessage in the AWS Security Token Service API Reference."},
-		{code: "Unavailable", http: 503, message: "The server is overloaded and can't handle the request."},
-		{code: "UnavailableHostRequirements", http: 400, message: "There are no valid Dedicated Hosts available on which you can launch an instance."},
-		{code: "UnfulfillableCapacity", http: 400, message: "At this time there isn't enough spare capacity to fulfill your request for Spot Instances. You can wait a few minutes to see whether capacity becomes available for your request. Alternatively, create a more flexible request. For example, include additional instance types, include additional Availability Zones, or use the capacity-optimized allocation strategy."},
-		{code: "UnknownParameter", http: 404, message: "An unknown or unrecognized parameter was supplied. Requests that could cause this error include supplying a misspelled parameter or a parameter that is not supported for the specified API version."},
-		{code: "UnknownPrincipalType.Unsupported", http: 400, message: "The principal type is not supported."},
-		{code: "UnknownVolumeType", http: 400, message: "The specified volume type is unsupported. The supported volume types are gp2, io1, st1, sc1, and standard."},
-		{code: "Unsupported", http: 400, message: "The specified request is unsupported. For example, you might be trying to launch an instance in an Availability Zone that currently has constraints on that instance type. The returned message provides details of the unsupported request."},
-		{code: "UnsupportedException", http: 400, message: "Capacity Reservations are not supported for this Region."},
-		{code: "UnsupportedHibernationConfiguration", http: 400, message: "The instance could not be launched because one or more parameter values do not meet the prerequisites for enabling hibernation. For more information, see Hibernation Prerequisites. Alternatively, the instance could not be hibernated because it is not enabled for hibernation."},
-		{code: "UnsupportedHostConfiguration", http: 400, message: "The specified Dedicated Host configuration is unsupported. For more information about supported configurations, see Dedicated Hosts."},
-		{code: "UnsupportedInstanceAttribute", http: 400, message: "The specified attribute cannot be modified."},
-		{code: "UnsupportedInstanceTypeOnHost", http: 400, message: "The instance type is not supported on the Dedicated Host. For more information about supported instance types, see Amazon EC2 Dedicated Hosts Pricing."},
-		{code: "UnsupportedOperation", http: 400, message: "The specified request includes an unsupported operation. For example, you can't stop an instance that's instance store-backed. Or you might be trying to launch an instance type that is not supported by the specified AMI. The returned message provides details of the unsupported operation."},
-		{code: "UnsupportedProtocol", http: 400, message: "SOAP has been deprecated and is no longer supported."},
-		{code: "UnsupportedTenancy", http: 400, message: "The specified tenancy is unsupported. You can change the tenancy of a VPC to default only."},
-		{code: "UpdateLimitExceeded", http: 400, message: "The default credit specification for an instance family can be modified only once in a rolling 5-minute period, and up to four times in a rolling 24-hour period. For more information, see Set the default credit specification for the account."},
-		{code: "VPCIdNotSpecified", http: 400, message: "You have no default VPC in which to carry out the request. Specify a VPC or subnet ID or, in the case of security groups, specify the ID and not the security group name. If you deleted your default VPC, you can create a new one. For more information, see Create a default VPC."},
-		{code: "VPCResourceNotSpecified", http: 400, message: "The specified resource can be used only in a VPC; for example, T2 instances. Ensure that you have a VPC in your account, and then specify a subnet ID or network interface ID in the request."},
-		{code: "ValidationError", http: 400, message: "The input fails to satisfy the constraints specified by an AWS service."},
-		{code: "VcpuLimitExceeded", http: 400, message: "You've reached the limit on the number of vCPUs (virtual processing units) assigned to the running instances in your account. You are limited to running one or more On-Demand instances in an AWS account, and Amazon EC2 measures usage towards each limit based on the total number of vCPUs that are assigned to the running On-Demand instances in your AWS account. If your request fails due to limit constraints, increase your On-Demand instance limits and try again. For more information, see EC2 On-Demand instance limits."},
-		{code: "VolumeIOPSLimit", http: 400, message: "The maximum IOPS limit for the volume has been reached. For more information, see Amazon EBS volume types."},
-		{code: "VolumeInUse", http: 400, message: "The specified Amazon EBS volume is attached to an instance. Ensure that the specified volume is in an \u2018available\u2019 state."},
-		{code: "VolumeLimitExceeded", http: 400, message: "You've reached the limit on your Amazon EBS volume storage. For more information, see Quotas for Amazon EBS."},
-		{code: "VolumeModificationSizeLimitExceeded", http: 400, message: "You've reached the limit on your Amazon EBS volume modification storage in this Region. For more information, see Quotas for Amazon EBS."},
-		{code: "VolumeTypeNotAvailableInZone", http: 400, message: "The specified Availability Zone does not support Provisioned IOPS SSD volumes. Try launching your instance in a different Availability Zone, or don't specify a zone in the request. If you're creating a volume, try specifying a different Availability Zone in the request."},
-		{code: "VpcEndpointLimitExceeded", http: 400, message: "You've reached the limit on the number of VPC endpoints that you can create in the AWS Region. For more information, see Amazon VPC quotas."},
-		{code: "VpcLimitExceeded", http: 400, message: "You've reached the limit on the number of VPCs that you can create in the AWS Region. For more information about VPC limits, see Amazon VPC quotas."},
-		{code: "VpcPeeringConnectionAlreadyExists", http: 409, message: "A VPC peering connection between the VPCs already exists."},
-		{code: "VpcPeeringConnectionsPerVpcLimitExceeded", http: 400, message: "You've reached the limit on the number of VPC peering connections that you can have per VPC. For more information, see Amazon VPC quotas."},
-		{code: "VpnConnectionLimitExceeded", http: 400, message: "You've reached the limit on the number of VPN connections that you can create. For more information, see Amazon VPC quotas."},
-		{code: "VpnGatewayAttachmentLimitExceeded", http: 400, message: "You've reached the limit on the number of VPCs that can be attached to the specified virtual private gateway."},
-		{code: "VpnGatewayLimitExceeded", http: 400, message: "You've reached the limit on the number of virtual private gateways that you can create. For more information about limits, see Amazon VPC quotas."},
-		{code: "ZonesMismatched", http: 400, message: "The Availability Zone for the instance does not match that of the Dedicated Host"},
-
-		// IAM error codes
-		{code: "NoSuchEntity", http: 404, message: "The request was rejected because it referenced a resource entity that does not exist."},
-		{code: "EntityAlreadyExists", http: 409, message: "The request was rejected because it attempted to create a resource that already exists."},
-		{code: "DeleteConflict", http: 409, message: "The request was rejected because it attempted to delete a resource that has attached subordinate entities."},
-		{code: "LimitExceeded", http: 409, message: "The request was rejected because it attempted to create resources beyond the current AWS account limits."},
-		{code: "MalformedPolicyDocument", http: 400, message: "The policy document is malformed."},
-		{code: "AccessDenied", http: 403, message: "User is not authorized to perform this action."},
-		// ELBv2 error codes
-		{code: "LoadBalancerNotFound", http: 400, message: "One or more load balancers not found."},
-		{code: "TargetGroupNotFound", http: 400, message: "One or more target groups not found."},
-		{code: "ListenerNotFound", http: 400, message: "One or more listeners not found."},
-		{code: "DuplicateLoadBalancerName", http: 400, message: "A load balancer with the same name already exists."},
-		{code: "DuplicateTargetGroupName", http: 400, message: "A target group with the same name already exists."},
-		{code: "DuplicateListener", http: 400, message: "A listener with the specified port already exists on this load balancer."},
-		{code: "TooManyLoadBalancers", http: 400, message: "You've reached the limit on the number of load balancers for your account."},
-		{code: "TooManyTargetGroups", http: 400, message: "You've reached the limit on the number of target groups for your account."},
-		{code: "TooManyListeners", http: 400, message: "You've reached the limit on the number of listeners per load balancer."},
-		{code: "TooManyRegistrationsForTargetId", http: 400, message: "You've reached the limit on the number of times a target can be registered with a target group."},
-		{code: "InvalidTarget", http: 400, message: "The specified target does not exist, is not in the same VPC as the target group, or has an unsupported instance type."},
-		{code: "ResourceInUse", http: 400, message: "The target group is currently in use by a listener or rule."},
-		{code: "InvalidSecurityGroup", http: 400, message: "The specified security group does not exist."},
-		{code: "InvalidScheme", http: 400, message: "The specified scheme is not valid. Specify 'internet-facing' or 'internal'."},
-		{code: "SubnetNotFound", http: 400, message: "The specified subnet does not exist."},
-		{code: "AvailabilityZoneNotSupported", http: 400, message: "The specified Availability Zone is not supported."},
-		{code: "InvalidConfigurationRequest", http: 400, message: "Security groups are not supported for load balancers with type 'network'."},
+// TestErrorLookup_Structure asserts invariants of the ErrorLookup map without
+// duplicating every entry. Catches wholesale-deletion accidents, invalid HTTP
+// codes, empty messages, and regressions of SDK-surfaced codes — the failure
+// modes that actually matter — without locking in a 1:1 mirror that has to be
+// updated in lockstep with every new error code.
+func TestErrorLookup_Structure(t *testing.T) {
+	if len(ErrorLookup) < 400 {
+		t.Fatalf("ErrorLookup unexpectedly small: %d entries", len(ErrorLookup))
 	}

-	if len(ErrorLookup) != len(expected) {
-		t.Fatalf("expected %d errors, got %d", len(expected), len(ErrorLookup))
+	validHTTP := map[int]bool{400: true, 403: true, 404: true, 409: true, 412: true, 413: true, 500: true, 503: true}
+	for code, msg := range ErrorLookup {
+		if !validHTTP[msg.HTTPCode] {
+			t.Errorf("%s has invalid HTTPCode %d", code, msg.HTTPCode)
+		}
+		if msg.Message == "" {
+			t.Errorf("%s has empty Message", code)
+		}
 	}

-	for _, tt := range expected {
-		msg, ok := ErrorLookup[tt.code]
+	// Spot-check business-critical codes that the AWS SDK surfaces by name.
+	critical := map[string]int{
+		ErrorAuthFailure:                  403,
+		ErrorInvalidInstanceIDNotFound:    404,
+		ErrorInvalidAMIIDNotFound:         400,
+		ErrorInvalidKeyPairNotFound:       404,
+		ErrorInvalidVpcIDNotFound:         404,
+		ErrorInvalidGroupNotFound:         404,
+		ErrorInvalidSubnetIDNotFound:      404,
+		ErrorServerInternal:               500,
+		ErrorInsufficientInstanceCapacity: 400,
+		ErrorUnauthorizedOperation:        403,
+	}
+	for code, wantHTTP := range critical {
+		msg, ok := ErrorLookup[code]
 		if !ok {
-			t.Fatalf("missing error code %s", tt.code)
+			t.Errorf("critical code %q missing from ErrorLookup", code)
+			continue
 		}
-		if msg.HTTPCode != tt.http {
-			t.Fatalf("%s HTTPCode = %d, want %d", tt.code, msg.HTTPCode, tt.http)
-		}
-		if msg.Message != tt.message {
-			t.Fatalf("%s message = %q, want %q", tt.code, msg.Message, tt.message)
+		if msg.HTTPCode != wantHTTP {
+			t.Errorf("%s HTTPCode = %d, want %d", code, msg.HTTPCode, wantHTTP)
 		}
 	}
 }
--- a/Show More
+++ b/Show More