release: update go.mod for v1.3.0

Sibling changes in Spinifex to support changes made to Predastore in this PR.

.github/dependabot.yml

@@ -8,7 +8,7 @@ updates:
       day: monday
       timezone: Australia/Sydney
     cooldown:
-      default-days: 1
+      default-days: 3
     groups:
       go-dependencies:
         patterns:
@@ -22,7 +22,7 @@ updates:
       day: monday
       timezone: Australia/Sydney
     cooldown:
-      default-days: 1
+      default-days: 3
     ignore:
       - dependency-name: "*"
         update-types:
@@ -41,7 +41,7 @@ updates:
       day: monday
       timezone: Australia/Sydney
     cooldown:
-      default-days: 1
+      default-days: 3
     ignore:
       - dependency-name: "@types/node"
         update-types:

.github/workflows/e2e-baremetal.yml

@@ -0,0 +1,80 @@
+name: E2E Bare-Metal
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  statuses: write
+
+env:
+  GIT_BRANCH: ${{ github.ref_name }}
+
+jobs:
+  e2e_baremetal:
+    name: Bare-Metal Single-Node E2E
+    runs-on: [ self-hosted, ci-baremetal ]
+    timeout-minutes: 60
+    concurrency:
+      group: e2e-baremetal-cluster
+      cancel-in-progress: false
+    env:
+      BM_ISO_CACHE: /var/cache/spinifex-iso-builder
+    steps:
+      - name: Clean Workspace
+        run: rm -rf "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/checkout@v6
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v6
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: |
+            scripts/iso-builder
+            scripts/baremetal
+          path: mulga
+
+      - name: Install SSH Key
+        run: |
+          install -m 0700 -d ~/.ssh
+          echo "${{ secrets.BM_SSH_PRIVATE_KEY }}" > ~/.ssh/bm-ci
+          chmod 0600 ~/.ssh/bm-ci
+
+      - name: Build Artifacts
+        run: |
+          mulga/scripts/baremetal/bm-build-artifacts.sh \
+            "$GITHUB_WORKSPACE" \
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            "${RUNNER_TEMP}/bm-bins"
+
+      - name: Build PXE Artifacts
+        run: |
+          sudo make -C mulga/scripts/iso-builder \
+            build-rootfs build-squashfs build-initramfs \
+            CACHE="${BM_ISO_CACHE}" \
+            SPINIFEX_BIN="${RUNNER_TEMP}/bm-bins/spx" \
+            INSTALLER_BIN="${RUNNER_TEMP}/bm-bins/spinifex-installer" \
+            SPINIFEX_SRC="${GITHUB_WORKSPACE}" \
+            VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
+            VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always 2>/dev/null || echo dev)"
+
+      - name: Run Bare-Metal E2E
+        env:
+          BM_CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
+          BM_SSH_KEY: /home/tf-user/.ssh/bm-ci
+          BM_SKIP_EPDU: "1"
+          SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
+        run: |
+          sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
+            --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
+            --initrd    "${BM_ISO_CACHE}/initrd.img" \
+            --squashfs  "${BM_ISO_CACHE}/spinifex.squashfs" \
+            --tests-tar "${RUNNER_TEMP}/spinifex-tests.tar"

.github/workflows/e2e-ddil.yml

@@ -11,6 +11,9 @@ on:
         required: false
         default: ""
 
+permissions:
+  contents: read
+
 env:
   GIT_BRANCH: ${{ github.ref_name }}
 
@@ -53,6 +56,26 @@ jobs:
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
+      - name: Install gotestsum
+        env:
+          GOTESTSUM_VERSION: "1.12.0"
+        run: |
+          # Prebuilt tarball, not `go install`: gotestsum v1.12.0 pulls
+          # x/tools v0.19.0 whose tokeninternal static-assert pegs a
+          # narrow Go window (~1.22). The self-hosted runner's Go falls
+          # outside that window, so source-install fails with
+          # `invalid array length -delta * delta`. A prebuilt binary
+          # sidesteps toolchain coupling entirely.
+          BIN_DIR="${RUNNER_TEMP}/gotestsum-bin"
+          mkdir -p "${BIN_DIR}"
+          if [ ! -x "${BIN_DIR}/gotestsum" ]; then
+            curl -fsSL \
+              "https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/gotestsum_${GOTESTSUM_VERSION}_linux_amd64.tar.gz" \
+              | tar -xz -C "${BIN_DIR}" gotestsum
+          fi
+          echo "${BIN_DIR}" >> "${GITHUB_PATH}"
+          "${BIN_DIR}/gotestsum" --version
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
@@ -95,18 +118,73 @@ jobs:
           # tofu-cluster, wan_ip == cluster IP and the runner's outbound
           # IP is distinct — see e2e-ddil-test-harness.md §3 Scenario C.
           NODE_IPS=$(echo "$INVENTORY" | jq -r '[.nodes[].wan_ip] | join(",")')
+          PRIMARY_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
           echo "DDIL_NODES=${NODE_IPS}"
 
+          # Witness VMs talk to the cluster's AWS gateway via aws-sdk-go,
+          # which resolves credentials from the standard chain. spx admin
+          # init writes [spinifex] to ${SSH_USER}'s ~/.aws/credentials on
+          # node1 — pull them over SSH and export as the SDK's env-var
+          # entries so the harness can launch instances without depending
+          # on a cluster-bootstrapped profile being present on the runner.
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${PRIMARY_IP}" 'cat ~/.aws/credentials')
+          AWS_ACCESS_KEY_ID=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
+            echo "::error::Failed to extract spinifex AWS credentials from ${PRIMARY_IP}"
+            exit 1
+          fi
+
+          # Guest SSH key: bootstrap-install.sh imports spinifex-key into EC2;
+          # the matching private material was generated on the primary node at
+          # ~/.ssh/spinifex-key. Pull it to the runner so the harness can SSH
+          # the witness via its MapPublicIpOnLaunch-assigned EIP.
+          GUEST_KEY="${RUNNER_TEMP}/spinifex-key"
+          scp -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${PRIMARY_IP}:~/.ssh/spinifex-key" "${GUEST_KEY}"
+          chmod 600 "${GUEST_KEY}"
+
           mkdir -p "${RUNNER_TEMP}/ddil-results"
           cd "${GITHUB_WORKSPACE}/spinifex"
+          # gotestsum: human-readable PASS/FAIL/SKIP lines in Actions stdout
+          # while still emitting the machine-readable jsonfile artifact.
           DDIL_NODES="${NODE_IPS}" \
           DDIL_SSH_USER="${SSH_USER}" \
           DDIL_SSH_KEY="${SSH_KEY}" \
+          DDIL_GUEST_SSH_KEY="${GUEST_KEY}" \
           DDIL_QUARANTINED="${{ inputs.quarantined }}" \
           AWS_REGION="${REGION}" \
-            go test -tags=e2e -timeout 45m -json \
-              ./tests/e2e/ddil/scenarios/... \
-              | tee "${RUNNER_TEMP}/ddil-results/results.json"
+          AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
+          AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
+            gotestsum --jsonfile "${RUNNER_TEMP}/ddil-results/results.json" \
+                      --format testname \
+                      -- -tags=e2e -timeout 45m ./tests/e2e/ddil/scenarios/...
+
+      - name: Collect Cluster Logs
+        if: failure()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          # Pull spinifex-* journals + QEMU process listings from every
+          # node into the artifact bundle so failures can be triaged
+          # without re-provisioning the cluster. Best-effort — a missing
+          # node should not mask the original test failure.
+          set +e
+          INVENTORY=$(tofu output -json inventory)
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
+          LOG_DIR="${RUNNER_TEMP}/ddil-results/cluster-logs"
+          mkdir -p "${LOG_DIR}"
+
+          mapfile -t NODE_IPS < <(echo "$INVENTORY" | jq -r '.nodes[].wan_ip')
+          for ip in "${NODE_IPS[@]}"; do
+            echo "=== collecting ${ip} ==="
+            SSH="ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SSH_USER}@${ip}"
+            ${SSH} 'sudo journalctl -u "spinifex-*" --no-pager -n 5000' > "${LOG_DIR}/${ip}-spinifex.log" 2>&1
+            ${SSH} 'ps auxw | grep -E "qemu-system|spinifex" | grep -v grep' > "${LOG_DIR}/${ip}-procs.txt" 2>&1
+            ${SSH} 'ls -la /var/lib/spinifex 2>/dev/null; ls -la /var/log/spinifex 2>/dev/null' > "${LOG_DIR}/${ip}-state.txt" 2>&1
+          done
 
       - name: Upload DDIL Results
         if: always()

.github/workflows/e2e-nightly.yml

@@ -12,8 +12,11 @@ on:
 permissions:
   contents: read
 
+# Scheduled runs target dev (active development trunk). GitHub loads scheduled
+# workflow files from the repo's default branch, so this file must be kept in
+# sync on main for the schedule to use it.
 env:
-  GIT_BRANCH: ${{ github.ref_name }}
+  GIT_BRANCH: dev
   UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
 
 jobs:

.github/workflows/e2e.yml

@@ -19,6 +19,7 @@ on:
         required: false
 
 permissions:
+  contents: read
   statuses: write
 
 env:
@@ -28,7 +29,7 @@ jobs:
   e2e_single:
     name: Single-Node E2E
     runs-on: [ self-hosted, ci-single ]
-    timeout-minutes: 30
+    timeout-minutes: 35
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
     steps:
@@ -97,7 +98,7 @@ jobs:
   e2e_pseudo_multi:
     name: Pseudo Multi-Node E2E
     runs-on: [ self-hosted, ci-pseudo ]
-    timeout-minutes: 30
+    timeout-minutes: 25
     env:
       TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log
     steps:
@@ -161,7 +162,7 @@ jobs:
   e2e_real_multi:
     name: Real Multi-Node E2E
     runs-on: [ self-hosted, ci-multi ]
-    timeout-minutes: 35
+    timeout-minutes: 30
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
     steps:

.github/workflows/spinifex.yml

@@ -17,6 +17,9 @@ on:
       - dev
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   build_and_test:
     name: Build and Test

.github/workflows/ui.yml

@@ -17,6 +17,9 @@ on:
       - .github/workflows/ui.yml
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   ui_build_and_test:
     name: UI Build and Test
@@ -46,8 +49,8 @@ jobs:
       - name: Install Dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: Run Check
-        run: pnpm check
+      - name: Run Lint
+        run: pnpm lint
 
       - name: Run Tests with Coverage
         run: pnpm test:coverage

Makefile

@@ -149,9 +149,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.2 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.3 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.2.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi

README.md

@@ -93,9 +93,24 @@ AWS-Compatible Interfaces – Provision infrastructure with awscli, Terraform, o
 
 Installation requires an Ubuntu / Debian system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex.
 
+### Bare Metal ISO
+
+The recommended installation is a [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) for bare-metal hardware.
+
+```bash
+curl -fLO https://iso.mulgadc.com/spinifex.iso
+```
+
+Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware. The install guide walks through the full process.
+
+
 ### Single Node Install
 
-The installation is straightforward to set up and running on a single node for testing purposes.
+The installation is straightforward to set up and running on a single node for testing purposes. Debian 13 is currently supported, additional Linux distributions are on the immediate roadmap.
+
+>*Prerequisite:* Linux bridge for networking.
+
+Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) prerequisites for setup details.
 
 ```bash
 curl -fsSL https://install.mulgadc.com | bash

build/scripts/predastore-start.sh

@@ -5,17 +5,20 @@
 
 CONF=/etc/spinifex/spinifex.toml
 BIND="0.0.0.0:8443"
-NODE_ID="0"
+# -1 = co-located mode (all [[db]] peers in one process). Single-node spinifex
+# templates list 3 DB peers on 127.0.0.1; only -1 can serve that topology.
+# Multi-node spinifex.toml emits node_id = N (N >= 1) which overrides below.
+# predastore rejects NODE_ID=0 since d3a02f0 (strconv.Atoi-of-garbage guard).
+NODE_ID="-1"
 
 if [ -f "$CONF" ]; then
     H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
     if [ -n "$H" ]; then
         BIND="$H"
-        BACKEND="distributed"
     fi
 
     N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
-    if [ -n "$N" ] && [ "$N" != "0" ]; then
+    if [ -n "$N" ]; then
         NODE_ID="$N"
     fi
 fi

build/systemd/regenerate-ssh-host-keys.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Regenerate SSH host keys
+ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key
+Before=ssh.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/ssh-keygen -A
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

build/systemd/spinifex-daemon.service

@@ -2,27 +2,67 @@
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service
-Wants=spinifex-nats.service
+# No Wants=spinifex-nats.service: Wants= reactivates a stopped unit during
+# `systemctl restart spinifex-daemon`, which would mask the standalone-startup
+# path the Tier 1 DDIL Scenario B exercises. spinifex.target Wants= covers
+# cold boot.
 
 [Service]
 Type=simple
 User=spinifex-daemon
 Group=spinifex
-ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+# DDIL Tier 1 (siv-17): no ExecStartPre wait-for-nats. The daemon starts in
+# standalone mode and reconnects to NATS in the background — process-exiting
+# on a missing NATS would kill the local VM management plane.
+#
+# Opt-in strict-startup (§1d-strict, siv-44): set SPINIFEX_REQUIRE_NATS=1 to
+# restore pre-DDIL fail-fast UX. First NATS connect uses a ~30s bounded retry
+# and the daemon aborts on timeout. Intended for dev/test/single-node deploys
+# where silent NATS misconfig is more confusing than helpful. Do NOT set this
+# in the shipped unit — it re-introduces the SPOF that Tier 1 removed.
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5
+# DDIL Tier 1 (mulga-siv-58): KillMode=process so systemd only signals the
+# daemon's main PID, not the cgroup. QEMU children stay alive across
+# `systemctl restart spinifex-daemon` and the new daemon reattaches to them
+# via the persisted local state file + pidfile + QMP socket. With the
+# default control-group mode, restart would terminate every guest VM —
+# defeating Scenario B's "daemon restart preserves running instances".
+KillMode=process
+# 15s upper bound on the SIGTERM grace window. The new SIGTERM handler
+# skips VM teardown (it now just persists state and exits), so 15s is
+# plenty of slack for the cluster-manager + NATS unsubscribe path.
+TimeoutStopSec=15
 OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
+# DDIL Tier 1 (mulga-siv-58): QEMU pidfiles, QMP sockets, and the
+# shared qemu.log all resolve via utils.pidPath() — which falls back
+# to os.TempDir() (=/tmp) when $HOME/spinifex doesn't exist. With
+# PrivateTmp=yes, each daemon start gets a fresh /tmp namespace, so
+# pidfiles written by QEMU (running in the OLD daemon's namespace)
+# are invisible to the new daemon. The /local/instances handler then
+# reports PID=0 and Scenario B fails its "live QEMU PID" assertion.
+# Pin XDG_RUNTIME_DIR to the host-wide tmpfs at /run/spinifex (created
+# by tmpfiles.d, already in ReadWritePaths) so pidPath() returns a
+# stable, namespace-independent path that both daemon incarnations and
+# QEMU children resolve identically.
+Environment=XDG_RUNTIME_DIR=/run/spinifex
 
-# Filesystem access — NoNewPrivileges omitted (daemon uses sudo for ip, ovs-vsctl)
+# Filesystem access — NoNewPrivileges omitted (daemon uses sudo for ip, ovs-vsctl, dhcpcd)
+# CAP_SYS_ADMIN: required for GPU passthrough sysfs operations (vfio group management).
+# CAP_DAC_OVERRIDE: sysfs driver bind/unbind paths (/sys/bus/pci/drivers/vfio-pci/bind,
+# unbind) are root:root 0200; non-root processes need CAP_DAC_OVERRIDE to open them.
+AmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE
 ProtectSystem=full
 PrivateDevices=no
 DevicePolicy=closed
 DeviceAllow=/dev/kvm rw
 DeviceAllow=/dev/net/tun rw
+DeviceAllow=char-vfio rw
+DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
 ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd
 
@@ -50,6 +90,9 @@ RemoveIPC=yes
 LimitCORE=0
 LimitNOFILE=65536
 LimitNPROC=4096
+# VFIO GPU passthrough requires locking all guest RAM into physical memory.
+# Without this, QEMU fails with RLIMIT_MEMLOCK exceeded when pinning guest pages.
+LimitMEMLOCK=infinity
 
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-predastore.service

@@ -17,6 +17,7 @@ Environment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.
 Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
 Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key
+Environment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key
 
 # Filesystem access
 NoNewPrivileges=yes

build/udev/99-spinifex-vfio.rules

@@ -0,0 +1,5 @@
+# Allow spinifex group to access VFIO devices for GPU passthrough.
+# QEMU (launched by spinifex-daemon, group spinifex) must open /dev/vfio/N
+# to use vfio-pci passthrough devices. The kernel creates these nodes as
+# root:root 0600 by default; this rule grants group-level read-write access.
+SUBSYSTEM=="vfio", GROUP="spinifex", MODE="0660"

cmd/dhcptest/main.go

@@ -0,0 +1,172 @@
+// dhcptest is a diagnostic tool that runs a DHCP DORA handshake using the
+// same nclient4 library and options that spinifex-vpcd uses for gateway pool
+// leases. Run it on a node to verify DHCP works at the library level before
+// debugging the full spinifex stack.
+//
+// Usage (must run as root — needs AF_PACKET):
+//
+//	sudo dhcptest --iface=br-wan
+//	sudo dhcptest --iface=br-wan --mac=02:00:00:7c:b7:b9 --promisc
+//	sudo dhcptest --iface=br-wan --spinifex-id=gateway-wan1
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/insomniacslk/dhcp/dhcpv4"
+	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
+)
+
+func main() {
+	iface := flag.String("iface", "br-wan", "Interface/bridge to bind the DHCP client to")
+	macStr := flag.String("mac", "", "Hardware address for DORA (chaddr + AF_PACKET filter). If empty, uses the real interface MAC.")
+	spinifexID := flag.String("spinifex-id", "", "Generate a spinifex virtual MAC from this resource ID (e.g. 'gateway-wan1'). Overrides --mac.")
+	clientID := flag.String("client-id", "", "DHCP option 61 client identifier. Defaults to --spinifex-id or --mac value.")
+	hostname := flag.String("hostname", "", "DHCP option 12 hostname")
+	vendorClass := flag.String("vendor-class", "mulga-spinifex-gw", "DHCP option 60 vendor class identifier")
+	promisc := flag.Bool("promisc", false, "Enable promiscuous mode on the interface before the test (mirrors vpcd's setBridgePromisc)")
+	timeout := flag.Duration("timeout", 10*time.Second, "Per-attempt DHCP timeout")
+	retries := flag.Int("retries", 3, "Number of DORA attempts before giving up")
+	verbose := flag.Bool("verbose", false, "Print full DHCP packet contents")
+	flag.Parse()
+
+	// Resolve the hardware address to use.
+	var hwAddr net.HardwareAddr
+	var err error
+
+	switch {
+	case *spinifexID != "":
+		hwAddr, err = net.ParseMAC(generateMAC(*spinifexID))
+		if err != nil {
+			log.Fatalf("generateMAC(%q): %v", *spinifexID, err)
+		}
+		fmt.Printf("spinifex virtual MAC for %q: %s\n", *spinifexID, hwAddr)
+	case *macStr != "":
+		hwAddr, err = net.ParseMAC(*macStr)
+		if err != nil {
+			log.Fatalf("parse --mac %q: %v", *macStr, err)
+		}
+	default:
+		hwAddr, err = ifaceMAC(*iface)
+		if err != nil {
+			log.Fatalf("get MAC for %s: %v", *iface, err)
+		}
+		fmt.Printf("using real interface MAC: %s\n", hwAddr)
+	}
+
+	// Resolve client-id: default to spinifex-id, then MAC string.
+	cid := *clientID
+	if cid == "" {
+		if *spinifexID != "" {
+			cid = *spinifexID
+		} else {
+			cid = hwAddr.String()
+		}
+	}
+
+	fmt.Printf("iface:        %s\n", *iface)
+	fmt.Printf("hw-addr:      %s\n", hwAddr)
+	fmt.Printf("client-id:    %s\n", cid)
+	fmt.Printf("hostname:     %s\n", *hostname)
+	fmt.Printf("vendor-class: %s\n", *vendorClass)
+	fmt.Printf("promisc:      %v\n", *promisc)
+	fmt.Printf("timeout:      %s  retries: %d\n", *timeout, *retries)
+	fmt.Println()
+
+	if *promisc {
+		if err := setPromisc(*iface, true); err != nil {
+			log.Printf("WARNING: failed to set promisc on %s: %v", *iface, err)
+		} else {
+			fmt.Printf("set %s promisc ON\n\n", *iface)
+		}
+	}
+
+	// Build nclient4 options — identical to spinifex/services/vpcd/dhcp/nclient4.go.
+	opts := []nclient4.ClientOpt{
+		nclient4.WithHWAddr(hwAddr),
+		nclient4.WithTimeout(*timeout),
+		nclient4.WithRetry(*retries),
+	}
+	if *verbose {
+		opts = append(opts, nclient4.WithLogger(nclient4.ShortSummaryLogger{Printfer: log.New(os.Stderr, "[nclient4] ", 0)}))
+	}
+
+	client, err := nclient4.New(*iface, opts...)
+	if err != nil {
+		log.Fatalf("nclient4.New(%s): %v", *iface, err)
+	}
+	defer client.Close()
+
+	// Identity modifiers — identical to identityModifiers() in vpcd dhcp package.
+	var mods []dhcpv4.Modifier
+	if cid != "" {
+		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(cid))))
+	}
+	if *hostname != "" {
+		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptHostName(*hostname)))
+	}
+	if *vendorClass != "" {
+		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClassIdentifier(*vendorClass)))
+	}
+
+	fmt.Println("Sending DHCP Discover...")
+	start := time.Now()
+	ctx, cancel := context.WithTimeout(context.Background(), *timeout*time.Duration(*retries+1))
+	defer cancel()
+
+	lease, err := client.Request(ctx, mods...)
+	if err != nil {
+		fmt.Printf("\nFAILED after %s: %v\n", time.Since(start).Round(time.Millisecond), err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("SUCCESS in %s\n\n", time.Since(start).Round(time.Millisecond))
+	ack := lease.ACK
+	fmt.Printf("  Your IP:     %s\n", ack.YourIPAddr)
+	fmt.Printf("  Subnet:      %s\n", ack.SubnetMask())
+	fmt.Printf("  Gateway:     %v\n", ack.Router())
+	fmt.Printf("  DNS:         %v\n", ack.DNS())
+	fmt.Printf("  Server ID:   %s\n", ack.ServerIdentifier())
+	fmt.Printf("  Lease time:  %s\n", ack.IPAddressLeaseTime(24*time.Hour))
+	if *verbose {
+		fmt.Printf("\n--- Full ACK ---\n%s\n", ack.Summary())
+	}
+}
+
+// generateMAC is the same deterministic hash used by vpcd's topology.go.
+// It produces a locally-administered MAC (02:00:00:xx:xx:xx) from a resource ID.
+func generateMAC(resourceID string) string {
+	h := uint32(0)
+	for _, c := range resourceID {
+		h = h*31 + uint32(c)
+	}
+	return fmt.Sprintf("02:00:00:%02x:%02x:%02x", (h>>16)&0xff, (h>>8)&0xff, h&0xff)
+}
+
+func ifaceMAC(name string) (net.HardwareAddr, error) {
+	ifc, err := net.InterfaceByName(name)
+	if err != nil {
+		return nil, err
+	}
+	return ifc.HardwareAddr, nil
+}
+
+func setPromisc(iface string, on bool) error {
+	state := "on"
+	if !on {
+		state = "off"
+	}
+	out, err := exec.Command("ip", "link", "set", iface, "promisc", state).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(out)))
+	}
+	return nil
+}

cmd/installer/autoinstall/autoinstall.go

@@ -134,6 +134,10 @@ func buildConfig() (*install.Config, error) {
 		}
 	}
 
+	if os.Getenv("SPINIFEX_GPU_PASSTHROUGH") == "1" {
+		cfg.GPUPassthrough = true
+	}
+
 	role := strings.ToLower(os.Getenv("SPINIFEX_ROLE"))
 	if role == "" {
 		role = "init"

cmd/installer/autoinstall/autoinstall_test.go

@@ -0,0 +1,17 @@
+package autoinstall
+
+import (
+	"os"
+	"testing"
+)
+
+func TestLoadReturnsNilWhenNotAuto(t *testing.T) {
+	os.Unsetenv("SPINIFEX_AUTO")
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if cfg != nil {
+		t.Fatal("expected nil config when SPINIFEX_AUTO is not set")
+	}
+}

cmd/installer/firstboot/firstboot.go

@@ -43,6 +43,15 @@ type Config struct {
 	// the headless path. Passed to `spx admin init --email=<value>` when set;
 	// omitted entirely when empty.
 	Email string
+	// GPUPassthrough enables VFIO GPU passthrough by passing --gpu-passthrough
+	// to `spx admin init`, which writes gpu_passthrough = true in the daemon config.
+	GPUPassthrough bool
+	// InstallCallback, when non-empty, is curled once at the end of firstboot
+	// after the success marker is written. Generic phone-home hook for
+	// provisioning controllers (PXE/MAAS-style flows). Best-effort: non-2xx
+	// and network failures are swallowed so the install is not gated on
+	// controller reachability.
+	InstallCallback string
 }
 
 // Write drops the firstboot script and systemd unit into root, which should be
@@ -78,6 +87,14 @@ func writeScript(root string, cfg Config) error {
 		setupOVN += fmt.Sprintf(" --encap-ip=%s", cfg.EncapIP)
 	}
 
+	callbackBlock := ""
+	if cfg.InstallCallback != "" {
+		callbackBlock = fmt.Sprintf(
+			"\ncurl -fsS --max-time 10 --retry 2 --retry-delay 2 -o /dev/null %s || true\n",
+			shellEscapeSingle(cfg.InstallCallback),
+		)
+	}
+
 	script := fmt.Sprintf(`#!/bin/bash
 # Spinifex firstboot — runs once after ISO installation. On success, writes
 # /var/lib/spinifex/.firstboot-done and the systemd unit's
@@ -146,6 +163,18 @@ SETUP_STAGES=fixown /usr/local/share/spinifex/setup.sh
 # setup.sh's create_directories — it resolves to /etc/spinifex, so
 # {BaseDir}/config/master.key automatically points at /etc/spinifex/master.key.
 
+# Bootstrap the spinifex user's SSH directory with a generated key pair so
+# ~/.ssh exists and the operator can immediately use ssh-keygen / ssh-copy-id
+# without hitting "No such file or directory" errors documented in the setup guide.
+if [ ! -f /home/spinifex/.ssh/id_ed25519 ]; then
+    mkdir -p /home/spinifex/.ssh
+    chmod 700 /home/spinifex/.ssh
+    ssh-keygen -t ed25519 -f /home/spinifex/.ssh/id_ed25519 -N "" -C "spinifex@$(hostname)"
+    chown -R spinifex:spinifex /home/spinifex/.ssh
+    chmod 600 /home/spinifex/.ssh/id_ed25519
+    chmod 644 /home/spinifex/.ssh/id_ed25519.pub
+fi
+
 # Copy AWS credentials to the spinifex user's home directory.
 # spx admin init runs with HOME=/root (set by the systemd unit), so credentials
 # land in /root/.aws/. Copy them to the spinifex user's home so the operator
@@ -176,7 +205,7 @@ systemctl enable spinifex.target spinifex-banner.service
 # firstboot from the top.
 mkdir -p "$(dirname "$DONE_MARKER")"
 touch "$DONE_MARKER"
-
+%s
 systemctl start spinifex.target
 
 # Wait for the daemon to bring up external networking. When external_mode=pool
@@ -196,7 +225,7 @@ if grep -q 'external_mode.*pool' /etc/spinifex/spinifex.toml 2>/dev/null; then
         echo "[firstboot] warning: br-ext not up after 30s — external networking may be delayed"
     fi
 fi
-`, cfg.Hostname, setupOVN, clusterCmd)
+`, cfg.Hostname, setupOVN, clusterCmd, callbackBlock)
 
 	path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
 	return os.WriteFile(path, []byte(script), 0o755)
@@ -210,11 +239,15 @@ func buildClusterCmd(cfg Config) string {
 		// validator already rejects whitespace and @-chains.
 		emailFlag = " --email=" + shellEscapeSingle(cfg.Email)
 	}
+	gpuFlag := ""
+	if cfg.GPUPassthrough {
+		gpuFlag = " --gpu-passthrough"
+	}
 	switch cfg.ClusterRole {
 	case "join":
 		return fmt.Sprintf("spx admin join --node %s --host %s%s", cfg.Hostname, cfg.JoinAddr, emailFlag)
 	default:
-		return fmt.Sprintf("spx admin init --node %s --nodes 1%s", cfg.Hostname, emailFlag)
+		return fmt.Sprintf("spx admin init --node %s --nodes 1%s%s", cfg.Hostname, emailFlag, gpuFlag)
 	}
 }
 

cmd/installer/firstboot/firstboot_test.go

@@ -0,0 +1,100 @@
+package firstboot
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func makeRootDirs(t *testing.T, root string) {
+	t.Helper()
+	for _, d := range []string{
+		"usr/local/bin",
+		"etc/systemd/system",
+		"etc/systemd/system/multi-user.target.wants",
+	} {
+		if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
+			t.Fatalf("mkdir %s: %v", d, err)
+		}
+	}
+}
+
+func TestWriteScriptNoCallbackWhenEmpty(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+
+	script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
+	if err != nil {
+		t.Fatalf("read script: %v", err)
+	}
+	if strings.Contains(string(script), "curl") {
+		t.Error("script should not contain curl when InstallCallback is empty")
+	}
+}
+
+func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
+	cfg := Config{
+		Hostname:        "test-node",
+		ClusterRole:     "init",
+		InstallCallback: callbackURL,
+	}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+
+	script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
+	if err != nil {
+		t.Fatalf("read script: %v", err)
+	}
+	content := string(script)
+
+	if !strings.Contains(content, "curl") {
+		t.Error("script missing curl command")
+	}
+	if !strings.Contains(content, callbackURL) {
+		t.Errorf("script missing callback URL %q", callbackURL)
+	}
+}
+
+func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
+	cfg := Config{
+		Hostname:        "node1",
+		ClusterRole:     "init",
+		InstallCallback: callbackURL,
+	}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+
+	script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
+	if err != nil {
+		t.Fatalf("read script: %v", err)
+	}
+	content := string(script)
+
+	doneIdx := strings.Index(content, "touch \"$DONE_MARKER\"")
+	curlIdx := strings.Index(content, "curl")
+	if doneIdx < 0 {
+		t.Fatal("done marker not found in script")
+	}
+	if curlIdx < 0 {
+		t.Fatal("curl not found in script")
+	}
+	if curlIdx < doneIdx {
+		t.Error("curl must appear after done marker write")
+	}
+}

cmd/installer/install/config.go

@@ -67,4 +67,9 @@ type Config struct {
 	// Required on interactive installs; may be empty on headless/CI installs
 	// when SPINIFEX_EMAIL is not supplied on the kernel cmdline.
 	Email string
+
+	// GPUPassthrough enables VFIO GPU passthrough on this node.
+	// Set via SPINIFEX_GPU_PASSTHROUGH=1 on headless installs.
+	// Passes --gpu-passthrough to `spx admin init` during firstboot.
+	GPUPassthrough bool
 }

cmd/installer/install/install.go

@@ -662,11 +662,12 @@ func (c *Config) toFirstbootConfig() firstboot.Config {
 		encapIP = c.LANAddress
 	}
 	return firstboot.Config{
-		Hostname:    c.Hostname,
-		EncapIP:     encapIP,
-		ClusterRole: c.ClusterRole,
-		JoinAddr:    c.JoinAddr,
-		Email:       c.Email,
+		Hostname:        c.Hostname,
+		EncapIP:         encapIP,
+		ClusterRole:     c.ClusterRole,
+		JoinAddr:        c.JoinAddr,
+		Email:           c.Email,
+		InstallCallback: strings.TrimSpace(os.Getenv("SPINIFEX_INSTALL_CALLBACK")),
 	}
 }
 

cmd/installer/install/install_test.go

@@ -0,0 +1,27 @@
+package install
+
+import (
+	"os"
+	"testing"
+)
+
+func TestToFirstbootConfigIncludesInstallCallback(t *testing.T) {
+	const url = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
+	t.Setenv("SPINIFEX_INSTALL_CALLBACK", url)
+
+	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	fb := cfg.toFirstbootConfig()
+	if fb.InstallCallback != url {
+		t.Errorf("InstallCallback = %q, want %q", fb.InstallCallback, url)
+	}
+}
+
+func TestToFirstbootConfigEmptyCallbackWhenEnvUnset(t *testing.T) {
+	os.Unsetenv("SPINIFEX_INSTALL_CALLBACK")
+
+	cfg := &Config{Hostname: "node1", ClusterRole: "init"}
+	fb := cfg.toFirstbootConfig()
+	if fb.InstallCallback != "" {
+		t.Errorf("InstallCallback = %q, want empty", fb.InstallCallback)
+	}
+}

cmd/spinifex/cmd/admin.go

@@ -45,6 +45,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/formation"
+	"github.com/mulgadc/spinifex/spinifex/gpu"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/utils"
@@ -199,13 +200,13 @@ spx admin images list
 - fetches from remote endpoint for common/trusted images to bootstrap environment, or baked in from compile.
 
 // If --name specified, download
-spx admin images import --name debian-12-x86_64
+spx admin images import --name debian-13-x86_64
 
 // List available images
 spx admin images list
 
 // Manually import a path
-spx admin images import --file /path/to/image --distro debian --version 12 --arch x86_64
+spx admin images import --file /path/to/image --distro debian --version 13 --arch x86_64
 
 -> x <-
 */
@@ -274,6 +275,7 @@ func init() {
 	adminInitCmd.Flags().String("gateway-ip", "", "OVN gateway router's external IP for SNAT (default: pool range_start for pool mode, required for nat mode without DHCP)")
 	adminInitCmd.Flags().Int("external-prefix-len", 24, "External pool subnet prefix length (auto-detected)")
 	adminInitCmd.Flags().Bool("no-external", false, "Disable external networking (overlay-only, no internet for VMs)")
+	adminInitCmd.Flags().Bool("gpu-passthrough", false, "Enable VFIO GPU passthrough (sets gpu_passthrough = true in daemon config)")
 
 	// Flags for admin join
 	adminJoinCmd.Flags().String("region", "ap-southeast-2", "Region for this node")
@@ -660,6 +662,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	externalPrefixLen, _ := cmd.Flags().GetInt("external-prefix-len")
 	gatewayIP, _ := cmd.Flags().GetString("gateway-ip")
 	noExternal, _ := cmd.Flags().GetBool("no-external")
+	gpuPassthrough, _ := cmd.Flags().GetBool("gpu-passthrough")
 
 	// Fire telemetry in background (completes during init work, waited at end)
 	noTelemetry, _ := cmd.Flags().GetBool("no-telemetry")
@@ -901,6 +904,16 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 	fmt.Printf("   System creds: %s\n", filepath.Join(configDir, "system-credentials.json"))
 
+	// Predastore encryption key is per-node and never transmitted; generate
+	// it locally now so the service has it on first start.
+	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error generating predastore encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
+	fmt.Printf("   Key: %s\n", predastoreKeyPath)
+
 	fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
 	fmt.Printf("   Access Key:  %s\n", bootstrapResult.AdminAccessKey)
 	fmt.Printf("   Secret Key:  %s\n", bootstrapResult.AdminSecretKey)
@@ -1072,6 +1085,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		BootstrapIgwId:      bootstrapIgwId,
 		BootstrapCidr:       handlers_ec2_vpc.DefaultVPCCidr,
 		BootstrapSubnetCidr: handlers_ec2_vpc.DefaultSubnetCidr,
+
+		GPUPassthrough: gpuPassthrough,
 	}
 
 	// Print external networking summary
@@ -1153,6 +1168,17 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	fmt.Println("\n🔐 Generated IAM master key")
 	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 
+	// Predastore encryption key is per-node and never distributed via the
+	// formation server. Generate only the leader's own key here; each
+	// joiner generates its own during `spx admin join`.
+	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error generating predastore encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
+	fmt.Printf("   Key: %s\n", predastoreKeyPath)
+
 	// Read CA cert/key for distribution to joining nodes
 	caCertData, err := os.ReadFile(filepath.Join(configDir, "ca.pem"))
 	if err != nil {
@@ -1623,6 +1649,16 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	fmt.Println("✅ IAM master key received from leader")
 	fmt.Printf("✅ Bootstrap file written: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 
+	// Predastore encryption key is per-node: generate locally rather than
+	// receiving from the leader. Each node only opens fragments it sealed
+	// itself, so there is no cluster-wide predastore key to share.
+	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error generating predastore encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("✅ Predastore encryption key generated: %s\n", predastoreKeyPath)
+
 	// Generate server cert signed by CA with this node's bind IP
 	if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
@@ -2094,6 +2130,27 @@ func writeBootstrapFiles(configDir, bootstrapDir string, masterKey []byte, acces
 	}, nil
 }
 
+// writePredastoreEncryptionKey generates a fresh 32-byte AES-256 master key
+// for this node's predastore data dir and writes it to
+// <configDir>/predastore/encryption.key with mode 0600. The key is per-node:
+// every node generates its own at init/join time and never transmits it,
+// because predastore only opens fragments on the node that sealed them.
+func writePredastoreEncryptionKey(configDir string) (string, error) {
+	predastoreDir := filepath.Join(configDir, "predastore")
+	if err := os.MkdirAll(predastoreDir, 0750); err != nil {
+		return "", fmt.Errorf("create predastore config dir: %w", err)
+	}
+	key, err := handlers_iam.GenerateMasterKey()
+	if err != nil {
+		return "", fmt.Errorf("generate predastore encryption key: %w", err)
+	}
+	keyPath := filepath.Join(predastoreDir, "encryption.key")
+	if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil {
+		return "", fmt.Errorf("save predastore encryption key: %w", err)
+	}
+	return keyPath, nil
+}
+
 // writeSystemCredentials writes the system access key to a plaintext JSON file.
 // The daemon reads this at startup to inject credentials into ALB VM cloud-init
 // for SigV4-authenticated communication with the AWS gateway.
@@ -2331,17 +2388,17 @@ func runAdminBanner(cmd *cobra.Command, _ []string) {
 	}
 
 	banner := fmt.Sprintf(`
-  +----------------------------------------------------+
-  |         Spinifex  —  Mulga Defense Corporation     |
-  +----------------------------------------------------+
-  |  Node:      %-39s|
-  |  Login:     %-39s|
-  |  Dashboard: %-39s|
-  |  API:       %-39s|
-  |  SSH:       %-39s|
-  +----------------------------------------------------+
-  |  AWS credentials:  cat ~/.aws/credentials          |
-  +----------------------------------------------------+
+  +--------------------------------------------------------------+
+  |          Spinifex  —  Mulga Defense Corporation              |
+  +--------------------------------------------------------------+
+  |  Node:      %-49s|
+  |  Login:     %-49s|
+  |  Dashboard: %-49s|
+  |  API:       %-49s|
+  |  SSH:       %-49s|
+  +--------------------------------------------------------------+
+  |  AWS credentials:  cat ~/.aws/credentials                    |
+  +--------------------------------------------------------------+
 
 `,
 		hostname,
@@ -2351,6 +2408,8 @@ func runAdminBanner(cmd *cobra.Command, _ []string) {
 		"spinifex@"+currentIP,
 	)
 
+	banner += gpuBannerSection()
+
 	// Write to /etc/issue — displayed on the console before the login prompt.
 	// Overwrite entirely; this is a purpose-built appliance so we own this file.
 	if err := os.WriteFile("/etc/issue", []byte(banner), 0o644); err != nil {
@@ -2390,6 +2449,81 @@ func appendBannerToMotd(banner string) error {
 	return os.WriteFile(motdPath, []byte(base+sentinel+banner), 0o644)
 }
 
+// gpuBannerSection returns an optional banner box section describing GPU state.
+// Returns "" when no GPU hardware is detected. Safe to call at boot before the
+// daemon starts — all checks are sysfs/file reads, no NATS required.
+func gpuBannerSection() string {
+	devices, err := gpu.Discover()
+	if err != nil || len(devices) == 0 {
+		return ""
+	}
+
+	iommuEntries, _ := os.ReadDir("/sys/kernel/iommu_groups/")
+	iommuActive := len(iommuEntries) > 0
+
+	_, vfioErr := os.Stat("/sys/module/vfio_pci")
+	vfioPresent := vfioErr == nil
+
+	passthroughEnabled := false
+	cfgPath := DefaultConfigFile()
+	if cfg, err := config.LoadConfig(cfgPath); err == nil {
+		if nodeCfg, ok := cfg.Nodes[cfg.Node]; ok {
+			passthroughEnabled = nodeCfg.Daemon.GPUPassthrough
+		}
+	}
+
+	models := gpuModelSummary(devices)
+
+	var statusLine, hintLine string
+	switch {
+	case passthroughEnabled:
+		statusLine = "Passthrough enabled"
+	case iommuActive && vfioPresent:
+		statusLine = "Ready to enable"
+		hintLine = "sudo spx admin gpu enable"
+	default:
+		statusLine = "Setup required"
+		hintLine = "sudo spx admin gpu setup"
+	}
+
+	const (
+		sep    = "  +--------------------------------------------------------------+\n"
+		maxVal = 55
+	)
+	if len([]rune(models)) > maxVal {
+		models = string([]rune(models)[:maxVal-3]) + "..."
+	}
+
+	section := sep +
+		fmt.Sprintf("  |  GPU: %-55s|\n", models) +
+		fmt.Sprintf("  |       %-55s|\n", statusLine)
+	if hintLine != "" {
+		section += fmt.Sprintf("  |       %-55s|\n", hintLine)
+	}
+	section += sep + "\n"
+	return section
+}
+
+func gpuModelSummary(devices []gpu.GPUDevice) string {
+	counts := make(map[string]int)
+	var order []string
+	for _, d := range devices {
+		if counts[d.Model] == 0 {
+			order = append(order, d.Model)
+		}
+		counts[d.Model]++
+	}
+	var parts []string
+	for _, m := range order {
+		if n := counts[m]; n > 1 {
+			parts = append(parts, fmt.Sprintf("%dx %s", n, m))
+		} else {
+			parts = append(parts, m)
+		}
+	}
+	return strings.Join(parts, ", ")
+}
+
 // parseNodeConf reads a KEY=VALUE shell-format file and returns a map.
 // Lines starting with # and blank lines are ignored.
 func parseNodeConf(path string) map[string]string {

cmd/spinifex/cmd/admin_gpu.go

@@ -0,0 +1,447 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/gpu"
+	"github.com/mulgadc/spinifex/spinifex/types"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var adminGpuCmd = &cobra.Command{
+	Use:   "gpu",
+	Short: "Manage GPU passthrough for a node",
+}
+
+var adminGpuStatusCmd = &cobra.Command{
+	Use:   "status",
+	Short: "Show GPU hardware and passthrough state",
+	Run:   runAdminGpuStatus,
+}
+
+var adminGpuEnableCmd = &cobra.Command{
+	Use:   "enable",
+	Short: "Enable GPU passthrough on a node",
+	Run:   runAdminGpuEnable,
+}
+
+var adminGpuDisableCmd = &cobra.Command{
+	Use:   "disable",
+	Short: "Disable GPU passthrough on a node (blocked while GPU instances are running)",
+	Run:   runAdminGpuDisable,
+}
+
+var adminGpuSetupCmd = &cobra.Command{
+	Use:   "setup",
+	Short: "Configure this host for GPU passthrough (IOMMU, vfio-pci, nouveau blacklist)",
+	Long: `Idempotent host setup for GPU passthrough. Run once before a reboot to configure
+GRUB, vfio-pci early binding, and the nouveau blacklist; then run again after the
+reboot to verify bindings and enable GPU passthrough in the daemon.
+
+Must be run as root.`,
+	Run: runAdminGpuSetup,
+}
+
+func init() {
+	adminCmd.AddCommand(adminGpuCmd)
+	adminGpuCmd.AddCommand(adminGpuStatusCmd)
+	adminGpuCmd.AddCommand(adminGpuEnableCmd)
+	adminGpuCmd.AddCommand(adminGpuDisableCmd)
+	adminGpuCmd.AddCommand(adminGpuSetupCmd)
+
+	adminGpuStatusCmd.Flags().String("node", "", "Target node name (default: local node)")
+	// enable, disable, and setup write to the local spinifex.toml and signal the
+	// local daemon — they must be run directly on the target host.
+}
+
+// gpuNodeStatus queries NATS and returns the NodeStatusResponse for the target node.
+func gpuNodeStatus(targetNode string) (*types.NodeStatusResponse, error) {
+	cfg, nc, err := loadConfigAndConnect()
+	if err != nil {
+		return nil, err
+	}
+	defer nc.Close()
+
+	if targetNode == "" {
+		targetNode = cfg.Node
+	}
+
+	responses, err := collectResponses(nc, "spinifex.node.status", 3*time.Second)
+	if err != nil {
+		return nil, fmt.Errorf("collect node status: %w", err)
+	}
+	for _, raw := range responses {
+		var resp types.NodeStatusResponse
+		if err := json.Unmarshal(raw, &resp); err != nil {
+			continue
+		}
+		if resp.Node == targetNode {
+			return &resp, nil
+		}
+	}
+	return nil, fmt.Errorf("node %q not found or not responding", targetNode)
+}
+
+func runAdminGpuStatus(cmd *cobra.Command, _ []string) {
+	targetNode, _ := cmd.Flags().GetString("node")
+	resp, err := gpuNodeStatus(targetNode)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	iommu := "unknown"
+	vfio := "unknown"
+	if resp.GPUCapable || len(resp.GPUModels) > 0 {
+		iommu = "active"
+		vfio = "loaded"
+	}
+
+	fmt.Printf("Node:            %s\n", resp.Node)
+	if len(resp.GPUModels) > 0 {
+		fmt.Printf("GPU hardware:    %s\n", strings.Join(resp.GPUModels, ", "))
+	} else {
+		fmt.Printf("GPU hardware:    none detected\n")
+	}
+	fmt.Printf("IOMMU:           %s\n", iommu)
+	fmt.Printf("vfio-pci:        %s\n", vfio)
+
+	if resp.GPUPassthrough {
+		fmt.Printf("Passthrough:     enabled\n")
+		fmt.Printf("GPU pool:        %d/%d allocated\n", resp.AllocGPUs, resp.TotalGPUs)
+
+		var g5Types []string
+		for _, cap := range resp.InstanceTypes {
+			if strings.HasPrefix(cap.Name, "g5.") {
+				g5Types = append(g5Types, cap.Name)
+			}
+		}
+		if len(g5Types) > 0 {
+			fmt.Printf("Instance types:  %s\n", strings.Join(g5Types, " "))
+		}
+	} else if resp.GPUCapable {
+		fmt.Printf("Passthrough:     disabled\n")
+		fmt.Printf("Instance types:  (none — run 'spx admin gpu enable' to activate g5.*)\n")
+	} else {
+		fmt.Printf("Passthrough:     disabled\n")
+		fmt.Printf("Instance types:  (prerequisites not met)\n")
+	}
+}
+
+// gpuToggle writes the TOML setting, sends SIGHUP, and polls until the daemon confirms the new state.
+// enable/disable always operate on the local node; run the command directly on the target host.
+func gpuToggle(_ *cobra.Command, enable bool) {
+	cfgPath := viper.GetString("config")
+	if cfgPath == "" {
+		cfgPath = DefaultConfigFile()
+	}
+	cfg, err := config.LoadConfig(cfgPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error loading config: %v\n", err)
+		os.Exit(1)
+	}
+	localNode := cfg.Node
+
+	// Check current state first.
+	resp, err := gpuNodeStatus(localNode)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	if enable {
+		if resp.GPUPassthrough {
+			fmt.Println("GPU passthrough is already enabled.")
+			return
+		}
+		if !resp.GPUCapable {
+			fmt.Fprintln(os.Stderr, "Error: prerequisites not met on this node.")
+			fmt.Fprintln(os.Stderr, "  Run 'sudo spx admin gpu setup' to configure the host.")
+			os.Exit(1)
+		}
+	} else {
+		if !resp.GPUPassthrough {
+			fmt.Println("GPU passthrough is already disabled.")
+			return
+		}
+		if resp.AllocGPUs > 0 {
+			fmt.Fprintf(os.Stderr, "Error: %d GPU instance(s) are running. Terminate them first.\n", resp.AllocGPUs)
+			os.Exit(1)
+		}
+	}
+
+	// Write the TOML setting.
+	if err := admin.SetGPUPassthrough(cfgPath, localNode, enable); err != nil {
+		fmt.Fprintf(os.Stderr, "Error writing config: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Signal the daemon to reload.
+	out, err := exec.Command("systemctl", "kill", "-s", "HUP", "spinifex-daemon").CombinedOutput()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error sending SIGHUP: %v\n%s\n", err, out)
+		os.Exit(1)
+	}
+
+	// Poll until the daemon confirms the new state.
+	action := "enable"
+	if !enable {
+		action = "disable"
+	}
+	fmt.Printf("Waiting for daemon to %s GPU passthrough", action)
+	deadline := time.Now().Add(30 * time.Second)
+	for time.Now().Before(deadline) {
+		time.Sleep(1 * time.Second)
+		fmt.Print(".")
+		r, err := gpuNodeStatus(localNode)
+		if err != nil {
+			continue
+		}
+		if r.GPUPassthrough == enable {
+			fmt.Println(" done.")
+			runAdminGpuStatus(adminGpuStatusCmd, nil)
+			return
+		}
+	}
+	fmt.Println()
+	fmt.Fprintln(os.Stderr, "Timed out waiting for daemon. Check: journalctl -u spinifex-daemon -n 50")
+	os.Exit(1)
+}
+
+func runAdminGpuEnable(cmd *cobra.Command, _ []string) {
+	gpuToggle(cmd, true)
+}
+
+func runAdminGpuDisable(cmd *cobra.Command, _ []string) {
+	gpuToggle(cmd, false)
+}
+
+func runAdminGpuSetup(_ *cobra.Command, _ []string) {
+	if os.Getuid() != 0 {
+		fmt.Fprintln(os.Stderr, "Error: spx admin gpu setup must be run as root.")
+		os.Exit(1)
+	}
+
+	fmt.Println("==> Detecting GPU")
+	devices, err := gpu.Discover()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "GPU discovery failed: %v\n", err)
+		os.Exit(1)
+	}
+	if len(devices) == 0 {
+		fmt.Fprintln(os.Stderr, "No NVIDIA or AMD GPU found — is the card seated?")
+		os.Exit(1)
+	}
+	for _, d := range devices {
+		fmt.Printf("    %s - %s\n", d.PCIAddress, d.Model)
+	}
+
+	fmt.Println("==> Collecting PCI IDs for vfio-pci")
+	ids := gpuSetupCollectVFIOIDs(devices)
+	idsCSV := strings.Join(ids, ",")
+	fmt.Printf("    IDs: %s\n", idsCSV)
+
+	rebootNeeded := false
+
+	fmt.Println("==> Checking IOMMU")
+	iommuEntries, _ := os.ReadDir("/sys/kernel/iommu_groups/")
+	if len(iommuEntries) > 0 {
+		fmt.Println("    Active")
+	} else {
+		params := gpuSetupIOMMUParams()
+		changed, err := gpuSetupAddGRUBParams(params)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to update GRUB: %v\n", err)
+			os.Exit(1)
+		}
+		if changed {
+			fmt.Printf("    GRUB updated with: %s\n", params)
+		} else {
+			fmt.Println("    GRUB already has IOMMU params — refreshing")
+		}
+		ug := exec.Command("update-grub")
+		ug.Stdout, ug.Stderr = os.Stdout, os.Stderr
+		if err := ug.Run(); err != nil {
+			fmt.Fprintf(os.Stderr, "update-grub failed: %v\n", err)
+			os.Exit(1)
+		}
+		rebootNeeded = true
+	}
+
+	fmt.Println("==> Configuring vfio udev rule")
+	const vfioUdevRule = "/etc/udev/rules.d/99-spinifex-vfio.rules"
+	if _, err := os.Stat(vfioUdevRule); os.IsNotExist(err) {
+		rule := "SUBSYSTEM==\"vfio\", GROUP=\"spinifex\", MODE=\"0660\"\n"
+		if err := os.WriteFile(vfioUdevRule, []byte(rule), 0o644); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to write vfio udev rule: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("    Rule installed")
+		_ = exec.Command("udevadm", "control", "--reload-rules").Run()
+		_ = exec.Command("udevadm", "trigger", "--subsystem-match=vfio").Run()
+	} else {
+		fmt.Println("    Rule already present")
+	}
+
+	fmt.Println("==> Configuring nouveau blacklist")
+	const nouveauConf = "/etc/modprobe.d/blacklist-nouveau.conf"
+	if _, err := os.Stat(nouveauConf); os.IsNotExist(err) {
+		if err := os.WriteFile(nouveauConf, []byte("blacklist nouveau\noptions nouveau modeset=0\n"), 0o644); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to write nouveau blacklist: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("    nouveau blacklisted")
+		rebootNeeded = true
+	} else {
+		fmt.Println("    Already blacklisted")
+	}
+
+	fmt.Println("==> Configuring vfio-pci early binding")
+	const vfioPCIConf = "/etc/modprobe.d/vfio-pci.conf"
+	existing, _ := os.ReadFile(vfioPCIConf)
+	if !strings.Contains(string(existing), "ids="+idsCSV) {
+		if err := os.WriteFile(vfioPCIConf, []byte("options vfio-pci ids="+idsCSV+"\n"), 0o644); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to write vfio-pci.conf: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("    vfio-pci.conf written (ids=%s)\n", idsCSV)
+		rebootNeeded = true
+	} else {
+		fmt.Printf("    Already configured for %s\n", idsCSV)
+	}
+
+	const initramfsModules = "/etc/initramfs-tools/modules"
+	modData, _ := os.ReadFile(initramfsModules)
+	if !strings.Contains(string(modData), "vfio_pci") {
+		f, err := os.OpenFile(initramfsModules, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to open initramfs modules: %v\n", err)
+			os.Exit(1)
+		}
+		_, werr := fmt.Fprint(f, "\n# vfio early binding for GPU passthrough\nvfio\nvfio_iommu_type1\nvfio_pci\n")
+		f.Close()
+		if werr != nil {
+			fmt.Fprintf(os.Stderr, "Failed to write initramfs modules: %v\n", werr)
+			os.Exit(1)
+		}
+		fmt.Println("    vfio modules added to initramfs")
+		rebootNeeded = true
+	} else {
+		fmt.Println("    vfio modules already in initramfs")
+	}
+
+	if rebootNeeded {
+		fmt.Println("==> Updating initramfs")
+		ui := exec.Command("update-initramfs", "-u")
+		ui.Stdout, ui.Stderr = os.Stdout, os.Stderr
+		if err := ui.Run(); err != nil {
+			fmt.Fprintf(os.Stderr, "update-initramfs failed: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("\nSetup complete — reboot required.")
+		fmt.Println("  sudo reboot")
+		fmt.Println("  Then re-run: sudo spx admin gpu setup")
+		return
+	}
+
+	fmt.Println("==> Verifying vfio-pci module")
+	if _, err := os.Stat("/sys/module/vfio_pci"); os.IsNotExist(err) {
+		fmt.Fprintln(os.Stderr, "vfio_pci not loaded — check: journalctl -b | grep vfio")
+		os.Exit(1)
+	}
+	fmt.Println("    Loaded")
+
+	fmt.Println("==> Verifying GPU driver binding")
+	for _, d := range devices {
+		driverLink := "/sys/bus/pci/devices/" + d.PCIAddress + "/driver"
+		target, err := os.Readlink(driverLink)
+		driver := filepath.Base(target)
+		if err != nil || driver != "vfio-pci" {
+			fmt.Fprintf(os.Stderr, "%s is bound to %q, not vfio-pci\n", d.PCIAddress, driver)
+			fmt.Fprintf(os.Stderr, "  Check: ls -la %s\n", driverLink)
+			os.Exit(1)
+		}
+		fmt.Printf("    %s - vfio-pci\n", d.PCIAddress)
+	}
+
+	fmt.Println("==> Enabling GPU passthrough")
+	gpuToggle(nil, true)
+}
+
+// gpuSetupCollectVFIOIDs collects vendor:device PCI ID pairs for each GPU and
+// all sibling PCI functions on the same device slot (e.g. GPU + HDMI audio).
+// All siblings must bind to vfio-pci together for IOMMU group isolation.
+func gpuSetupCollectVFIOIDs(devices []gpu.GPUDevice) []string {
+	seen := make(map[string]bool)
+	var ids []string
+	for _, d := range devices {
+		dot := strings.LastIndex(d.PCIAddress, ".")
+		if dot < 0 {
+			continue
+		}
+		prefix := d.PCIAddress[:dot]
+		matches, _ := filepath.Glob("/sys/bus/pci/devices/" + prefix + ".*")
+		for _, m := range matches {
+			vb, err1 := os.ReadFile(m + "/vendor")
+			db, err2 := os.ReadFile(m + "/device")
+			if err1 != nil || err2 != nil {
+				continue
+			}
+			v := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(string(vb))), "0x")
+			dv := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(string(db))), "0x")
+			id := v + ":" + dv
+			if !seen[id] {
+				seen[id] = true
+				ids = append(ids, id)
+			}
+		}
+	}
+	return ids
+}
+
+func gpuSetupIOMMUParams() string {
+	data, _ := os.ReadFile("/proc/cpuinfo")
+	if strings.Contains(string(data), "GenuineIntel") {
+		return "intel_iommu=on iommu=pt"
+	}
+	return "amd_iommu=on iommu=pt"
+}
+
+// gpuSetupAddGRUBParams appends iommu kernel params to GRUB_CMDLINE_LINUX_DEFAULT.
+// Returns (true, nil) if the file was modified, (false, nil) if params were already present.
+func gpuSetupAddGRUBParams(params string) (bool, error) {
+	const grubFile = "/etc/default/grub"
+	data, err := os.ReadFile(grubFile)
+	if err != nil {
+		return false, fmt.Errorf("read %s: %w", grubFile, err)
+	}
+	content := string(data)
+	if strings.Contains(content, "intel_iommu=on") || strings.Contains(content, "amd_iommu=on") {
+		return false, nil
+	}
+	re := regexp.MustCompile(`(GRUB_CMDLINE_LINUX_DEFAULT="[^"]*)"`)
+	updated := re.ReplaceAllStringFunc(content, func(match string) string {
+		groups := re.FindStringSubmatch(match)
+		if len(groups) < 2 {
+			return match
+		}
+		return groups[1] + " " + params + `"`
+	})
+	if updated == content {
+		return false, fmt.Errorf("GRUB_CMDLINE_LINUX_DEFAULT not found in %s", grubFile)
+	}
+	if err := os.WriteFile(grubFile, []byte(updated), 0o644); err != nil {
+		return false, fmt.Errorf("write %s: %w", grubFile, err)
+	}
+	return true, nil
+}

cmd/spinifex/cmd/admin_test.go

@@ -22,7 +22,7 @@ import (
 func TestPrintChecksumError(t *testing.T) {
 	image := utils.Images{Checksum: "https://example.com/SUMS", ChecksumType: "sha512"}
 	const imageFile = "/var/lib/img.tar.xz"
-	const imageName = "debian-12-x86_64"
+	const imageName = "debian-13-x86_64"
 
 	errs := []error{
 		fmt.Errorf("%w: expected abc got def", utils.ErrChecksumMismatch),
@@ -262,3 +262,51 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {
 	require.NoError(t, err)
 	assert.Contains(t, content2, `nats_url = "nats://10.11.12.1:4222"`)
 }
+
+// writePredastoreEncryptionKey is called from both `spx admin init` and `spx
+// admin join`; both depend on it producing a 32-byte file at 0600 in the
+// expected layout. Predastore's keyfile loader rejects anything that isn't
+// exactly that, so a regression here would break service startup on every
+// node.
+func TestWritePredastoreEncryptionKey(t *testing.T) {
+	configDir := t.TempDir()
+
+	keyPath, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join(configDir, "predastore", "encryption.key"), keyPath)
+
+	info, err := os.Stat(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, int64(32), info.Size(), "predastore master key must be exactly 32 bytes")
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "predastore master key must be mode 0600")
+
+	// Per-node invariant: two successive calls (on what would be two
+	// different nodes) must produce different key material. If they ever
+	// returned the same bytes we'd have silently lost the per-node
+	// blast-radius property.
+	configDir2 := t.TempDir()
+	keyPath2, err := writePredastoreEncryptionKey(configDir2)
+	require.NoError(t, err)
+
+	key1, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+	key2, err := os.ReadFile(keyPath2)
+	require.NoError(t, err)
+	assert.NotEqual(t, key1, key2, "per-node keys must differ")
+}
+
+// Joiners run `spx admin join` against a configDir that doesn't yet contain
+// a predastore/ subdir. The helper must create it (predastore subdir is
+// owned by spinifex-storage in prod, but the helper just needs the dir to
+// exist).
+func TestWritePredastoreEncryptionKey_CreatesMissingDir(t *testing.T) {
+	configDir := t.TempDir()
+	// Don't pre-create the predastore subdir — simulate a fresh joiner.
+
+	keyPath, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+
+	dirInfo, err := os.Stat(filepath.Dir(keyPath))
+	require.NoError(t, err)
+	assert.True(t, dirInfo.IsDir())
+}

cmd/spinifex/cmd/service.go

@@ -115,6 +115,13 @@ var predastoreStartCmd = &cobra.Command{
 			return
 		}
 
+		encryptionKeyFile := viper.GetString("encryption-key-file")
+
+		if encryptionKeyFile == "" {
+			fmt.Println("Encryption key file is not set")
+			return
+		}
+
 		nodeID := viper.GetInt("node-id")
 		pprofEnabled := viper.GetBool("pprof")
 		pprofOutput := viper.GetString("pprof-output")
@@ -128,6 +135,8 @@ var predastoreStartCmd = &cobra.Command{
 			TlsCert:    tlsCert,
 			TlsKey:     tlsKey,
 
+			EncryptionKeyFile: encryptionKeyFile,
+
 			NodeID: nodeID,
 
 			PprofEnabled:    pprofEnabled,
@@ -858,13 +867,20 @@ func init() {
 	viper.BindEnv("tls-key", "SPINIFEX_PREDASTORE_TLS_KEY")
 	viper.BindPFlag("tls-key", predastoreCmd.PersistentFlags().Lookup("tls-key"))
 
+	// Predastore at-rest encryption master key (per node; mode 0600)
+	predastoreCmd.PersistentFlags().String("encryption-key-file", "", "Path to this node's 32-byte AES-256 master key file (required)")
+	viper.BindEnv("encryption-key-file", "SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE")
+	viper.BindPFlag("encryption-key-file", predastoreCmd.PersistentFlags().Lookup("encryption-key-file"))
+
 	// Predastore Backend
 	predastoreCmd.PersistentFlags().String("backend", "distributed", "Predastore (S3) backend")
 	viper.BindEnv("backend", "SPINIFEX_PREDASTORE_BACKEND")
 	viper.BindPFlag("backend", predastoreCmd.PersistentFlags().Lookup("backend"))
 
-	// Predastore Node ID
-	predastoreCmd.PersistentFlags().Int("node-id", 0, "Predastore (S3) node ID")
+	// Predastore Node ID. Default -1 is dev mode (launch every configured
+	// QUIC node in-process). Production deployments set this to the node's
+	// real ID (>= 1) via SPINIFEX_PREDASTORE_NODE_ID or --node-id.
+	predastoreCmd.PersistentFlags().Int("node-id", -1, "Predastore (S3) node ID (-1 = dev mode, >= 1 = production)")
 	viper.BindEnv("node-id", "SPINIFEX_PREDASTORE_NODE_ID")
 	viper.BindPFlag("node-id", predastoreCmd.PersistentFlags().Lookup("node-id"))
 

cmd/spinifex/cmd/templates/predastore-multinode.toml

@@ -1,6 +1,6 @@
 # Predastore S3-compatible storage configuration file in TOML format.
 # Generated for multi-node cluster
-version = "3"
+version = "4"
 region = "{{.Region}}"
 
 host = "0.0.0.0"
@@ -39,10 +39,12 @@ region = "{{.Region}}"
 type = "distributed"
 public = false
 encryption = ""
+account_id = "000000000000"
 
 [[auth]]
 access_key_id = "{{.AccessKey}}"
 secret_access_key = "{{.SecretKey}}"
+account_id = "000000000000"
 policy = [
   { bucket = "predastore", actions = ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
 ]

cmd/spinifex/cmd/templates/predastore.toml

@@ -1,6 +1,6 @@
 # Predastore S3-compatible storage configuration file in TOML format.
 # MULTI NODE used for testing
-version = "3"
+version = "4"
 region = "{{.Region}}"
 
 # Optionally configure port and host, or specify via daemon CLI options
@@ -73,10 +73,12 @@ region = "ap-southeast-2"
 type = "distributed"
 public = false
 encryption = ""
+account_id = "000000000000"
 
 [[auth]]
 access_key_id = "{{.AccessKey}}"
 secret_access_key = "{{.SecretKey}}"
+account_id = "000000000000"
 policy = [
   { bucket = "predastore", actions = ["s3:ListBucket",  "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
 ]

cmd/spinifex/cmd/templates/spinifex.toml

@@ -52,6 +52,7 @@ advertise = "{{.AdvertiseIP}}"
 {{- end}}
 region = "{{.Region}}"
 az = "{{.Az}}"
+data_dir = "{{.DataDir}}"
 {{- if .Services}}
 services = [{{range $i, $s := .Services}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
 {{- end}}
@@ -67,6 +68,9 @@ dev_networking = false
 {{- else}}
 dev_networking = true
 {{- end}}
+{{- if .GPUPassthrough}}
+gpu_passthrough = true
+{{- end}}
 
 [nodes.{{.Node}}.awsgw]
 host = "{{.BindIP}}:9999"

cmd/spinifex/cmd/top.go

@@ -88,7 +88,7 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 	sort.Strings(nodeNames)
 
 	nodeTable := pterm.TableData{
-		{"NAME", "CPU (used/total)", "MEM (used/total)", "VMs"},
+		{"NAME", "CPU (used/total)", "MEM (used/total)", "GPU (used/total)", "VMs"},
 	}
 
 	// Aggregate instance type capacity across all nodes
@@ -96,10 +96,17 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 
 	for _, name := range nodeNames {
 		if resp, ok := respondedNodes[name]; ok {
+			gpuCol := "-"
+			if resp.GPUPassthrough {
+				gpuCol = fmt.Sprintf("%d/%d", resp.AllocGPUs, resp.TotalGPUs)
+			} else if resp.GPUCapable {
+				gpuCol = fmt.Sprintf("0/%d*", len(resp.GPUModels))
+			}
 			nodeTable = append(nodeTable, []string{
 				resp.Node,
 				fmt.Sprintf("%d/%d", resp.AllocVCPU, resp.TotalVCPU),
 				fmt.Sprintf("%s/%s", formatMemGB(resp.AllocMemGB), formatMemGB(resp.TotalMemGB)),
+				gpuCol,
 				strconv.Itoa(resp.VMCount),
 			})
 
@@ -120,6 +127,7 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 				"-",
 				"-",
 				"-",
+				"-",
 			})
 		}
 	}

docs/administration/spinifex-admin-cli/README.md

@@ -77,7 +77,7 @@ Prints per-node CPU/memory usage and cluster-wide instance type availability.
 
 ```bash
 spx admin images list
-spx admin images import --name debian-12-arm64
+spx admin images import --name debian-13-arm64
 ```
 
 Catalog imports verify the image against the catalog-declared SHA-256/SHA-512 digest before extraction. Use `--file` to import operator-supplied media (verification skipped — operator is responsible for integrity), or `--force` to re-download after a checksum mismatch.

docs/compute/launching-instances/README.md

@@ -157,7 +157,7 @@ If the AMI is for a different architecture (e.g. arm64 on an x86_64 host), impor
 
 ```bash
 spx admin images list
-spx admin images import --name debian-12-x86_64
+spx admin images import --name debian-13-x86_64
 ```
 
 ### Cannot SSH Into Instance

docs/getting-started/install-airgapped/README.md

@@ -74,8 +74,8 @@ Mirror the cloud image you intend to run as guest VMs:
 
 ```bash
 mkdir -p images
-curl -fsSL "https://cdimage.debian.org/cdimage/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2" \
-  -o images/debian-12-amd64.qcow2
+curl -fsSL "https://cloud.debian.org/images/cloud/trixie/20260501-2465/debian-13-genericcloud-amd64-20260501-2465.qcow2" \
+  -o images/debian-13-amd64.qcow2
 ```
 
 ## Step 3. Assemble Transfer Media
@@ -140,8 +140,8 @@ sudo systemctl start spinifex.target
 Register the pre-staged image with Spinifex:
 
 ```bash
-sudo spx admin images import --file /mnt/usb/images/debian-12-amd64.qcow2 \
-  --distro debian --version 12 --arch x86_64
+sudo spx admin images import --file /mnt/usb/images/debian-13-amd64.qcow2 \
+  --distro debian --version 13 --arch x86_64
 ```
 
 ## Step 9. Verify

docs/getting-started/install-multi-node/README.md

@@ -22,6 +22,7 @@ resources:
 ## Table of Contents
 
 - [Overview](#overview)
+- [Prerequisites](#prerequisites)
 - [Instructions](#instructions)
 - [Troubleshooting](#troubleshooting)
 
@@ -37,6 +38,22 @@ A Spinifex cluster distributes services across multiple servers for high availab
 - UDP port 6081 open between hosts (Geneve tunnels)
 - TCP ports 4222, 4248, 6641, 6642 open between hosts (NATS, OVN)
 
+## Prerequisites
+
+> [!IMPORTANT]
+> **Prerequisite — WAN bridge required on every node.**
+>
+> Before running the installer on any server, that server's WAN interface **must** already be enslaved to a Linux bridge named `br-wan`. The host IP, default route, and DHCP must all live on the bridge — not on the bare NIC.
+>
+> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in the next release.
+>
+> **Verify on every node before continuing:**
+>
+> - `ip -br link show br-wan` — bridge exists and is `UP`
+> - `ip route` — default route's `dev` is `br-wan`
+>
+> **Setup references:** [VPC Networking → Bridge Setup](/docs/vpc-networking#bridge-setup-physical-network-wiring) for the topology.
+
 ## Instructions
 
 ## Step 1. Install Spinifex on Each Server

docs/getting-started/install-source/README.md

@@ -31,8 +31,8 @@ This guide builds Spinifex from source. For production deployments, the [binary
 
 **Requirements:**
 
-- Ubuntu 22.04+ / Debian 12+
-- Go 1.26.2+
+- Ubuntu 26.04 / Debian 13
+- Go 1.26.3+
 - GCC, make, pkg-config
 - QEMU/KVM, OVN/Open vSwitch, AWS CLI v2, dhcpcd-base
 

docs/getting-started/install/README.md

@@ -22,6 +22,7 @@ resources:
 ## Table of Contents
 
 - [Overview](#overview)
+- [Prerequisites](#prerequisites)
 - [Instructions](#instructions)
 - [Troubleshooting](#troubleshooting)
 
@@ -35,8 +36,8 @@ This guide installs Spinifex on a single server using the binary installer. For
 
 **Supported Operating Systems:**
 
-- Ubuntu 22.04 / 24.04 / 25.10
-- Debian 12 / 13
+- Ubuntu 26.04
+- Debian 13
 
 **What Gets Installed:**
 
@@ -47,6 +48,22 @@ This guide installs Spinifex on a single server using the binary installer. For
 - Viperblock (EBS-compatible block storage)
 - AWS CLI v2
 
+## Prerequisites
+
+> [!IMPORTANT]
+> **Prerequisite — WAN bridge required.**
+>
+> Before running the installer, the host's WAN interface **must** already be enslaved to a Linux bridge named `br-wan`. The host IP, default route, and DHCP must all live on the bridge — not on the bare NIC.
+>
+> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in the next release.
+>
+> **Verify before continuing:**
+>
+> - `ip -br link show br-wan` — bridge exists and is `UP`
+> - `ip route` — default route's `dev` is `br-wan`
+>
+> **Setup references:** [VPC Networking → Bridge Setup](/docs/vpc-networking#bridge-setup-physical-network-wiring) for the topology.
+
 ## Instructions
 
 ## Step 1. Install Spinifex

docs/getting-started/setting-up-your-cluster/README.md

@@ -58,23 +58,23 @@ spx admin images list
 
 ```
 NAME                    | DISTRO | VERSION | ARCH   | BOOT
-debian-12-arm64         | debian | 12      | arm64  | bios
-debian-12-x86_64        | debian | 12      | x86_64 | bios
+debian-13-arm64         | debian | 13      | arm64  | bios
+debian-13-x86_64        | debian | 13      | x86_64 | bios
 lb-alpine-3.21.6-x86_64 | alpine | 3.21.6  | x86_64 | bios
-ubuntu-24.04-arm64      | ubuntu | 24.04   | arm64  | bios
-ubuntu-24.04-x86_64     | ubuntu | 24.04   | x86_64 | bios
+ubuntu-26.04-arm64      | ubuntu | 26.04   | arm64  | bios
+ubuntu-26.04-x86_64     | ubuntu | 26.04   | x86_64 | bios
 ```
 
 Import an image:
 
 ```bash
-spx admin images import --name ubuntu-24.04-x86_64
+spx admin images import --name ubuntu-26.04-x86_64
 ```
 
 Or import a local image file:
 
 ```bash
-spx admin images import --file ~/images/ubuntu-24.04.img --distro ubuntu --version 24.04 --arch x86_64
+spx admin images import --file ~/images/ubuntu-26.04.img --distro ubuntu --version 26.04 --arch x86_64
 ```
 
 Verify the import and note the AMI ID:
@@ -173,7 +173,7 @@ Launch an instance in the public subnet, note this will select an instance type
 > **Note:** Replace `AMI_NAME` with the previously imported image above with the `ami-` prefix.
 
 ```bash
-AMI_NAME="ami-ubuntu-24.04-x86_64"
+AMI_NAME="ami-ubuntu-26.04-x86_64"
 
 AMI_ID=$(aws ec2 describe-images \
   --filters "Name=name,Values=$AMI_NAME" \

docs/gpu-passthrough/GPU-PASSTHROUGH.md

@@ -0,0 +1,47 @@
+# Configuring GPU Passthrough for Spinifex Nodes
+
+
+Spinifex is designed for bare-metal hardware, edge nodes and data-centre use. Follow this guide to configure your Spinifex node to enable GPU Passthrough to its VM instances if hardware is available.
+
+>**Note:** this tutorial is for x86 architecture.
+
+
+
+### Getting Started - GPU Setup
+By default, Spinifex will probe for the presence of a GPU on its machine. After booting, the presence and availability of GPU hardware is printed in the info banner:
+![Installer complete](../../.github/assets/images/gpu-banner1.png)
+
+To enable GPU passthrough, you may first need to run:
+```bash
+sudo spx admin gpu setup
+```
+Note that this command will blacklist the GPU's drivers from the host machine such that they can be used by VM instances Therefore the GPU will **NOT** be available to the host machine until after the VM instance  It will also require rebooting the host machine before the changes take effect.
+
+### Enable Passthrough
+After reboot, the banner message should reflect that GPU passthrough is **ready** but not **enabled**.
+
+![Installer complete](../../.github/assets/images/gpu-banner2.png)
+
+This will require you to run:
+```bash
+sudo spx admin gpu enable
+```
+After running this command, you should see:
+
+![Installer complete](../../.github/assets/images/gpu-enabled.png)
+
+The host machine is now configured to provide GPU passthrough to EC2 instances on the host machine.
+
+Lastly, import the Spinifex GPU enabled AMI: [link]
+
+As a final check, spin up an EC2 instance, SSH into it, and check that the host machine's GPU is visible from the EC2 instance.
+
+### Multiple GPUs
+Spinifex is able to detect multiple GPUs. After running the setup commands, the total number of allocatable GPUs is printed.
+
+### Mapping GPUs to EC2 instances
+By defualt, any consumer grade GPUs are mapped to the g5 EC2 instance class, which is the most basic "Accelerated Computing" Ec2 instance provided by AWS. If using a consumer GPU, running:
+```bash
+aws ec2 describe-instance-types
+```
+will return the various sizes of g5 class. All other AWS "Accelerated Computing" EC2 instance classes map directly to their associated GPU, so the Spinifex host machine will need to have the specified GPU if a specific EC2 instance class is desired.

docs/security/network-connections/README.md

@@ -105,8 +105,8 @@ Spinifex nodes initiate a small, fixed set of outbound connections.
 
 | Destination | Purpose | Protocol | Verification |
 |-------------|---------|----------|--------------|
-| `https://cdimage.debian.org/cdimage/cloud/bookworm/latest` | Debian 12 cloud image download | HTTPS | TLS + checksum verification |
-| `https://cloud-images.ubuntu.com/noble/current` | Ubuntu 24.04 LTS cloud image download | HTTPS | TLS + checksum verification |
+| `https://cloud.debian.org/images/cloud/trixie/20260501-2465` | Debian 13 cloud image download | HTTPS | TLS + checksum verification |
+| `https://cloud-images.ubuntu.com/resolute/current` | Ubuntu 26.04 LTS cloud image download | HTTPS | TLS + checksum verification |
 | `https://d2yp8ipz5jfqcw.cloudfront.net` | Alpine image for managed HAProxy load-balancer | HTTPS | TLS + checksum verification |
 | `https://install.mulgadc.com/install` | One-shot install telemetry POST on `spx admin init` / `join`. | HTTPS | TLS |
 

docs/terraform-workbooks/bastion-private-subnet/README.md

@@ -56,7 +56,7 @@ Deploy a VPC with public and private subnets where the private subnet has no int
 **Prerequisites:**
 
 - Spinifex installed and running (see [Installing Spinifex](/docs/install))
-- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- An Ubuntu 26.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
 - OpenTofu or Terraform installed
 
 ## Instructions
@@ -149,7 +149,7 @@ The private route table should have no `0.0.0.0/0` route. Also verify the privat
 
 ### AMI Not Found
 
-Ensure you have imported an Ubuntu 24.04 image:
+Ensure you have imported an Ubuntu 26.04 image:
 
 ```bash
 aws ec2 describe-images --owners 000000000000 --profile spinifex

docs/terraform-workbooks/nginx-alb/README.md

@@ -58,7 +58,7 @@ Deploy two Nginx web servers behind an internet-facing Application Load Balancer
 | Elastic IP | `nginx-alb-nat-eip` | Public address for the NAT Gateway |
 | NAT Gateway | `nginx-alb-nat` | Outbound internet for the private subnets (cloud-init apt bootstrap) |
 | Security Group | `nginx-alb-sg` | Allows SSH (22) and HTTP (80) inbound |
-| EC2 Instances | `nginx-alb-1`, `nginx-alb-2` | Ubuntu 24.04 with Nginx via cloud-init (private subnets) |
+| EC2 Instances | `nginx-alb-1`, `nginx-alb-2` | Ubuntu 26.04 with Nginx via cloud-init (private subnets) |
 | ALB | `nginx-alb` | Internet-facing Application Load Balancer on port 80 |
 | Target Group | `nginx-alb-tg` | HTTP health-checked group for both instances |
 | Listener | HTTP :80 | Forwards traffic to the target group |
@@ -66,7 +66,7 @@ Deploy two Nginx web servers behind an internet-facing Application Load Balancer
 **Prerequisites:**
 
 - Spinifex installed and running (see [Installing Spinifex](/docs/install))
-- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- An Ubuntu 26.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
 - OpenTofu or Terraform installed
 
 ## Instructions
@@ -96,10 +96,10 @@ spx admin images import --name lb-alpine-3.21.6-x86_64
 
 ### Step 2.1 Install Debian AMI
 
-Next, install the Debian 12 AMI which is used in the example to host the `nginx` webservers as an EC2 instance.
+Next, install the Debian 13 AMI which is used in the example to host the `nginx` webservers as an EC2 instance.
 
 ```bash
-spx admin images import --name debian-12-x86_64
+spx admin images import --name debian-13-x86_64
 ```
 
 ### Step 3. Deploy
@@ -184,7 +184,7 @@ tofu destroy
 
 ### AMI Not Found
 
-Ensure you have imported an Ubuntu 24.04 image. Check available AMIs:
+Ensure you have imported an Ubuntu 26.04 image. Check available AMIs:
 
 ```bash
 aws ec2 describe-images --owners 000000000000 --profile spinifex
@@ -193,7 +193,7 @@ aws ec2 describe-images --owners 000000000000 --profile spinifex
 If missing import:
 
 ```bash
-spx admin images import --name ubuntu-24.04-x86_64
+spx admin images import --name ubuntu-26.04-x86_64
 ```
 
 ### Provider Connection Refused

docs/terraform-workbooks/nginx-webserver/README.md

@@ -43,7 +43,7 @@ Deploy a complete Nginx web server on Spinifex using Terraform/OpenTofu. This wo
 **Prerequisites:**
 
 - Spinifex installed and running (see [Installing Spinifex](/docs/install))
-- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- An Ubuntu 26.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
 - OpenTofu or Terraform installed
 
 ## Instructions
@@ -96,7 +96,7 @@ tofu destroy
 
 ### AMI Not Found
 
-Ensure you have imported an Ubuntu 24.04 image. Check available AMIs:
+Ensure you have imported an Ubuntu 26.04 image. Check available AMIs:
 
 ```bash
 aws ec2 describe-images --owners 000000000000 --profile spinifex

docs/terraform-workbooks/nginx-webserver/main.tf

@@ -220,7 +220,7 @@ resource "aws_security_group" "web" {
 }
 
 # ---------------------------------------------------------------------------
-# EC2 Instance — Ubuntu 24.04 with Nginx installed via cloud-init
+# EC2 Instance — Ubuntu 26.04 with Nginx installed via cloud-init
 # ---------------------------------------------------------------------------
 
 resource "aws_instance" "nginx" {

docs/terraform-workbooks/s3-webapp/README.md

@@ -53,7 +53,7 @@ Deploy an EC2 instance running a Flask file-sharing web application backed by S3
 
 - Spinifex installed and running (see [Installing Spinifex](/docs/install))
 - Predastore running (S3 API on port 8443)
-- An Ubuntu 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- An Ubuntu 26.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
 - OpenTofu or Terraform installed
 - The EC2 instance must be able to reach Predastore — use the host's br-wan IP, not localhost
 
@@ -153,7 +153,7 @@ cat /opt/webapp/.env
 
 ### AMI Not Found
 
-Ensure you have imported an Ubuntu 24.04 image:
+Ensure you have imported an Ubuntu 26.04 image:
 
 ```bash
 aws ec2 describe-images --owners 000000000000 --profile spinifex

go.mod

@@ -1,6 +1,6 @@
 module github.com/mulgadc/spinifex
 
-go 1.26.2
+go 1.26.3
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
@@ -12,20 +12,19 @@ require (
 	github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mdlayher/packet v1.1.2
-	github.com/mulgadc/predastore v1.2.0
-	github.com/mulgadc/viperblock v1.2.1
-	github.com/nats-io/nats-server/v2 v2.12.7
-	github.com/nats-io/nats.go v1.51.0
+	github.com/mulgadc/predastore v1.3.0
+	github.com/mulgadc/viperblock v1.3.0
+	github.com/nats-io/nats-server/v2 v2.14.0
+	github.com/nats-io/nats.go v1.52.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
-	github.com/pelletier/go-toml/v2 v2.3.0
+	github.com/pelletier/go-toml/v2 v2.3.1
 	github.com/pterm/pterm v0.12.83
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
-	golang.org/x/crypto v0.50.0
-	golang.org/x/sys v0.43.0
-	gopkg.in/ini.v1 v1.67.1
+	go.uber.org/goleak v1.3.0
+	golang.org/x/crypto v0.51.0
+	gopkg.in/ini.v1 v1.67.2
 )
 
 require (
@@ -80,7 +79,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/klauspost/compress v1.18.5 // indirect
-	github.com/klauspost/reedsolomon v1.13.3 // indirect
+	github.com/klauspost/reedsolomon v1.14.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
@@ -88,6 +87,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.21 // indirect
+	github.com/mdlayher/packet v1.1.2 // indirect
 	github.com/mdlayher/socket v0.4.1 // indirect
 	github.com/minio/crc64nvme v1.1.1 // indirect
 	github.com/minio/highwayhash v1.0.4 // indirect
@@ -120,15 +120,15 @@ require (
 	go.opentelemetry.io/otel/trace v1.42.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
-	golang.org/x/mod v0.34.0 // indirect
+	golang.org/x/mod v0.35.0 // indirect
 	golang.org/x/net v0.53.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
-	golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c // indirect
-	golang.org/x/term v0.42.0 // indirect
-	golang.org/x/text v0.36.0 // indirect
+	golang.org/x/sys v0.44.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect
+	golang.org/x/term v0.43.0 // indirect
+	golang.org/x/text v0.37.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
-	golang.org/x/tools v0.43.0 // indirect
+	golang.org/x/tools v0.44.0 // indirect
 	golang.org/x/vuln v1.1.4 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -31,30 +31,30 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.6 h1:1AX0AthnBQzMx1vbmir3Y4WsnJgiydmnJjiLu+LvXOg=
-github.com/aws/aws-sdk-go-v2 v1.41.6/go.mod h1:dy0UzBIfwSeot4grGvY1AqFWN5zgziMmWGzysDnHFcQ=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 h1:adBsCIIpLbLmYnkQU+nAChU5yhVTvu5PerROm+/Kq2A=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9/go.mod h1:uOYhgfgThm/ZyAuJGNQ5YgNyOlYfqnGpTHXvk3cpykg=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.15 h1:fyvgWTszojq8hEnMi8PPBTvZdTtEVmAVyo+NFLHBhH4=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.15/go.mod h1:gJiYyMOjNg8OEdRWOf3CrFQxM2a98qmrtjx1zuiQfB8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22 h1:GmLa5Kw1ESqtFpXsx5MmC84QWa/ZrLZvlJGa2y+4kcQ=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22/go.mod h1:6sW9iWm9DK9YRpRGga/qzrzNLgKpT2cIxb7Vo2eNOp0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22 h1:dY4kWZiSaXIzxnKlj17nHnBcXXBfac6UlsAx2qL6XrU=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22/go.mod h1:KIpEUx0JuRZLO7U6cbV204cWAEco2iC3l061IxlwLtI=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23 h1:FPXsW9+gMuIeKmz7j6ENWcWtBGTe1kH8r9thNt5Uxx4=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23/go.mod h1:7J8iGMdRKk6lw2C+cMIphgAnT8uTwBwNOsGkyOCm80U=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8 h1:HtOTYcbVcGABLOVuPYaIihj6IlkqubBwFj10K5fxRek=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8/go.mod h1:VsK9abqQeGlzPgUr+isNWzPlK2vKe9INMLWnY65f5Xs=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.14 h1:xnvDEnw+pnj5mctWiYuFbigrEzSm35x7k4KS/ZkCANg=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.14/go.mod h1:yS5rNogD8e0Wu9+l3MUwr6eENBzEeGejvINpN5PAYfY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22 h1:PUmZeJU6Y1Lbvt9WFuJ0ugUK2xn6hIWUBBbKuOWF30s=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22/go.mod h1:nO6egFBoAaoXze24a2C0NjQCvdpk8OueRoYimvEB9jo=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22 h1:SE+aQ4DEqG53RRCAIHlCf//B2ycxGH7jFkpnAh/kKPM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22/go.mod h1:ES3ynECd7fYeJIL6+oax+uIEljmfps0S70BaQzbMd/o=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0 h1:7G26Sae6PMKn4kMcU5JzNfrm1YrKwyOhowXPYR2WiWY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0/go.mod h1:Fw9aqhJicIVee1VytBBjH+l+5ov6/PhbtIK/u3rt/ls=
-github.com/aws/smithy-go v1.25.0 h1:Sz/XJ64rwuiKtB6j98nDIPyYrV1nVNJ4YU74gttcl5U=
-github.com/aws/smithy-go v1.25.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
+github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
+github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
+github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -236,8 +236,8 @@ github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuOb
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
-github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
+github.com/klauspost/reedsolomon v1.14.0 h1:5YSZeclzSYg5nl349+GDG/agDtQ6MZiwUYXvVKN1Jx0=
+github.com/klauspost/reedsolomon v1.14.0/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -286,20 +286,20 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/predastore v1.2.0 h1:E4+ehAm2YEi2n5t4DmqguURMUXwTSDrK7792zVzw7JM=
-github.com/mulgadc/predastore v1.2.0/go.mod h1:+NWROaNCtpPE9f36fUxCHwskW8CuHHEIvjsDQrwOB00=
-github.com/mulgadc/viperblock v1.2.1 h1:G3cSgh/yspZd7Wdg3C9QGqkqzYbEfYpusd6uMddSDmg=
-github.com/mulgadc/viperblock v1.2.1/go.mod h1:CisHy13Uam0gUg5dxV0rKryJF2yGbgfDv09FYApRaNk=
+github.com/mulgadc/predastore v1.3.0 h1:Zf0rIuyQ1mi9Gpd4ihLqik1gqn8L/VKLf6P6Tk9Qs0Q=
+github.com/mulgadc/predastore v1.3.0/go.mod h1:nH5lBc8iXIw4fXXXNAnrJiRO47yZ+ZBCjaqjiCRoGOQ=
+github.com/mulgadc/viperblock v1.3.0 h1:vrDCNqNpLmolbTbgEHMSz8CBjoc8xkmQdtwKhZfrqew=
+github.com/mulgadc/viperblock v1.3.0/go.mod h1:WCP9h+u2DvN8/+ccai5oMOa48hOJPUzSbh7Lia0Q8nY=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
 github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
-github.com/nats-io/nats-server/v2 v2.12.7 h1:prQ9cPiWHcnwfT81Wi5lU9LL8TLY+7pxDru6fQYLCQQ=
-github.com/nats-io/nats-server/v2 v2.12.7/go.mod h1:dOnmkprKMluTmTF7/QHZioxlau3sKHUM/LBPy9AiBPw=
-github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI=
-github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
+github.com/nats-io/nats-server/v2 v2.14.0 h1:+8q0HrDFotwLLcGH/legOEOnowunhK+aZ4GYBIWpQlM=
+github.com/nats-io/nats-server/v2 v2.14.0/go.mod h1:ImVUUDvfClJbb6cuJQRc1VmgDCXKM5ds0OoiG9MVOKo=
+github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc=
+github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
 github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
 github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -308,8 +308,8 @@ github.com/ovn-kubernetes/libovsdb v0.8.1 h1:M2J8bcJt5mXCom0HqzfEtuHkT80CTSQRcYG
 github.com/ovn-kubernetes/libovsdb v0.8.1/go.mod h1:ZlnHLzagmLOSvyd9qfxBIZp6wOSOw0IsRsc+6lNUGbU=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM=
-github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc=
+github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -424,14 +424,14 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
-golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
+golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
+golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
-golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
+golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
+golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -484,33 +484,33 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc=
-golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
+golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
+golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4=
+golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
-golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
+golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
+golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
-golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
+golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
+golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
-golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
+golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
+golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
 golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
 golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
 golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM=
@@ -534,8 +534,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k=
-gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
+gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss=
+gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -548,5 +548,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
-pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
+pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc=
+pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk=

scripts/ansible/vars/defaults.yml

@@ -32,34 +32,34 @@ spinifex_nat_gateway_ip: ""
 
 # Preflight checks.
 spinifex_ports:
-  - { port: 4222, service: NATS }
-  - { port: 8443, service: Predastore }
-  - { port: 9999, service: "Spinifex Gateway" }
+    - { port: 4222, service: NATS }
+    - { port: 8443, service: Predastore }
+    - { port: 9999, service: "Spinifex Gateway" }
 
 spinifex_required_commands:
-  - go
-  - make
-  - nbdkit
+    - go
+    - make
+    - nbdkit
 spinifex_optional_commands:
-  - air
+    - air
 spinifex_ovn_commands:
-  - ovs-vsctl
-  - ovn-controller
-  - ovn-nbctl
+    - ovs-vsctl
+    - ovn-controller
+    - ovn-nbctl
 
 spinifex_components:
-  - viperblock
-  - predastore
+    - viperblock
+    - predastore
 
 # Process patterns for teardown.
 # Bracket trick ([b]in/spx etc.) prevents the pattern literal from matching
 # our own pgrep/shell cmdline. Exclude filter drops benign log followers and
 # ansible's own wrappers that also mention these service names.
-spinifex_process_pattern: '([b]in/spx|spinifex-[u]i|nats-[s]erver|[p]redastore|[v]iperblock|[v]pcd)'
-spinifex_process_exclude: '(journalctl|ansible|/bin/sh|pgrep|pkill|tail -[fF])'
+spinifex_process_pattern: "([b]in/spx|spinifex-[u]i|nats-[s]erver|[p]redastore|[v]iperblock|[v]pcd)"
+spinifex_process_exclude: "(journalctl|ansible|/bin/sh|pgrep|pkill|tail -[fF])"
 spinifex_qemu_processes:
-  - qemu-system-x86_64
-  - qemu-system-aarch64
+    - qemu-system-x86_64
+    - qemu-system-aarch64
 
 # Invoking user (when run under sudo).
 spinifex_invoking_user: "{{ lookup('env','SUDO_USER') | default(ansible_facts['user_id'], true) }}"
@@ -85,6 +85,6 @@ spinifex_lb_image_conf: "{{ spinifex_root }}/scripts/images/lb.conf"
 
 # Smoketest image — parameterised so tofu examples can share the same base.
 spinifex_smoketest_distro: debian
-spinifex_smoketest_version: "12"
+spinifex_smoketest_version: "13"
 # Local image file path (if present, imported directly; otherwise fall back to --name).
 spinifex_smoketest_local_image: "~/images/{{ spinifex_smoketest_distro }}-{{ spinifex_smoketest_version }}.img"

scripts/build-system-image.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 set -euo pipefail
 
-# build-system-image.sh — Build a minimal Alpine AMI from a manifest
+# build-system-image.sh — Build a minimal system AMI from a manifest
 #
-# Creates a pre-baked Alpine Linux image with custom packages, binaries,
-# and setup scripts installed, ready for import as a Spinifex AMI.
+# Supports Alpine Linux and Ubuntu 24.04 as base distros.
+# Creates a pre-baked image with custom packages, binaries, and setup scripts
+# installed, ready for import as a Spinifex AMI.
 #
 # Requirements: qemu-nbd, qemu-img, sudo (for mount/chroot), curl
 # Usage: ./scripts/build-system-image.sh <manifest.conf> [--import]
@@ -46,8 +47,11 @@ done
 # shellcheck source=/dev/null
 source "$MANIFEST"
 
+# Default distro is alpine for backward compatibility
+DISTRO="${DISTRO:-alpine}"
+
 # Validate required manifest fields
-for field in IMAGE_NAME ALPINE_VERSION IMAGE_SIZE; do
+for field in IMAGE_NAME IMAGE_SIZE; do
     if [[ -z "${!field:-}" ]]; then
         echo "ERROR: Manifest missing required field: $field"
         exit 1
@@ -56,16 +60,48 @@ done
 
 # Derived paths
 BUILD_DIR="/tmp/${IMAGE_NAME}-image-build"
-ALPINE_IMAGE="generic_alpine-${ALPINE_VERSION}-x86_64-bios-cloudinit-r0.qcow2"
-ALPINE_URL="https://dl-cdn.alpinelinux.org/alpine/v${ALPINE_VERSION%.*}/releases/cloud/${ALPINE_IMAGE}"
-OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-alpine.qcow2"
-OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-alpine.raw"
 NBD_DEV="/dev/nbd0"
 MOUNT_DIR="${BUILD_DIR}/mnt"
 
+case "$DISTRO" in
+    alpine)
+        if [[ -z "${ALPINE_VERSION:-}" ]]; then
+            echo "ERROR: Manifest missing required field: ALPINE_VERSION"
+            exit 1
+        fi
+        SOURCE_IMAGE="generic_alpine-${ALPINE_VERSION}-x86_64-bios-cloudinit-r0.qcow2"
+        SOURCE_URL="https://dl-cdn.alpinelinux.org/alpine/v${ALPINE_VERSION%.*}/releases/cloud/${SOURCE_IMAGE}"
+        OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-alpine.qcow2"
+        OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-alpine.raw"
+        DISTRO_VERSION="${ALPINE_VERSION}"
+        ROOT_PART="${NBD_DEV}"
+        ;;
+    ubuntu)
+        if [[ -z "${UBUNTU_VERSION:-}" ]]; then
+            echo "ERROR: Manifest missing required field: UBUNTU_VERSION"
+            exit 1
+        fi
+        SOURCE_IMAGE="ubuntu-${UBUNTU_VERSION}-minimal-cloudimg-amd64.img"
+        SOURCE_URL="https://cloud-images.ubuntu.com/minimal/releases/noble/release/${SOURCE_IMAGE}"
+        OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-ubuntu.qcow2"
+        OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-ubuntu.raw"
+        DISTRO_VERSION="${UBUNTU_VERSION}"
+        ROOT_PART="${NBD_DEV}p1"
+        ;;
+    *)
+        echo "ERROR: Unknown DISTRO: $DISTRO (supported: alpine, ubuntu)"
+        exit 1
+        ;;
+esac
+
 cleanup() {
     echo "Cleaning up..."
-    sudo umount "${MOUNT_DIR}" 2>/dev/null || true
+    # Ubuntu chroot has bind mounts that must be released first
+    sudo umount "${MOUNT_DIR}/dev/pts" 2>/dev/null || true
+    sudo umount "${MOUNT_DIR}/dev"     2>/dev/null || true
+    sudo umount "${MOUNT_DIR}/sys"     2>/dev/null || true
+    sudo umount "${MOUNT_DIR}/proc"    2>/dev/null || true
+    sudo umount "${MOUNT_DIR}"         2>/dev/null || true
     sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
     exec 9>&- 2>/dev/null || true  # release nbd lock if held
     echo "Done."
@@ -80,7 +116,7 @@ fi
 
 echo "=== System Image Builder ==="
 echo "Image:   ${IMAGE_NAME} — ${IMAGE_DESCRIPTION:-}"
-echo "Alpine:  ${ALPINE_VERSION}"
+echo "Distro:  ${DISTRO} ${DISTRO_VERSION}"
 echo "Size:    ${IMAGE_SIZE}"
 echo "Build:   ${BUILD_DIR}"
 echo ""
@@ -96,6 +132,11 @@ if ! command -v qemu-img &>/dev/null; then
     exit 1
 fi
 
+if [[ "$DISTRO" == "ubuntu" ]] && ! command -v parted &>/dev/null; then
+    echo "ERROR: parted not found. Install parted."
+    exit 1
+fi
+
 # Build binaries if BUILD_COMMANDS is set
 if [[ -n "${BUILD_COMMANDS:-}" ]]; then
     echo "Building binaries: ${BUILD_COMMANDS}"
@@ -105,7 +146,7 @@ if [[ -n "${BUILD_COMMANDS:-}" ]]; then
     fi
 fi
 
-# Verify binaries exist and are statically linked
+# Verify binaries exist; Alpine also requires static linking (uses musl)
 if [[ -n "${INSTALL_BINARIES:-}" ]]; then
     IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
     for pair in "${BINARY_PAIRS[@]}"; do
@@ -115,7 +156,7 @@ if [[ -n "${INSTALL_BINARIES:-}" ]]; then
             echo "ERROR: Binary not found: $src_path"
             exit 1
         fi
-        if ! file "$src_path" | grep -q "statically linked"; then
+        if [[ "$DISTRO" == "alpine" ]] && ! file "$src_path" | grep -q "statically linked"; then
             echo "ERROR: $src_path is not statically linked (Alpine uses musl — dynamic glibc binaries will fail)"
             echo "  Rebuild with: CGO_ENABLED=0 go build ..."
             exit 1
@@ -147,7 +188,7 @@ if [[ -f "$OUTPUT_RAW" ]] && [[ $(( $(date +%s) - $(stat -c %Y "$OUTPUT_RAW") ))
 
     if [[ "$DO_IMPORT" == true ]]; then
         echo "Importing as AMI..."
-        IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro alpine --version "${ALPINE_VERSION}" --arch x86_64)
+        IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64)
         if [[ -n "${SYSTEM_TAG:-}" ]]; then
             IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
         fi
@@ -158,20 +199,20 @@ fi
 
 mkdir -p "$BUILD_DIR" "$MOUNT_DIR"
 
-# Step 1: Download Alpine cloud image
-if [[ -f "${BUILD_DIR}/${ALPINE_IMAGE}" ]]; then
-    echo "Alpine image already downloaded."
+# Step 1: Download base cloud image
+if [[ -f "${BUILD_DIR}/${SOURCE_IMAGE}" ]]; then
+    echo "Base image already downloaded."
 else
-    echo "Downloading Alpine ${ALPINE_VERSION} cloud image..."
-    if ! curl --fail -L -o "${BUILD_DIR}/${ALPINE_IMAGE}" "$ALPINE_URL"; then
-        rm -f "${BUILD_DIR}/${ALPINE_IMAGE}"
-        echo "ERROR: Failed to download Alpine image from $ALPINE_URL"
+    echo "Downloading ${DISTRO} ${DISTRO_VERSION} cloud image..."
+    if ! curl --fail -L -o "${BUILD_DIR}/${SOURCE_IMAGE}" "$SOURCE_URL"; then
+        rm -f "${BUILD_DIR}/${SOURCE_IMAGE}"
+        echo "ERROR: Failed to download image from $SOURCE_URL"
         exit 1
     fi
-    # Verify the download is a valid qcow2 image
-    if ! qemu-img info "${BUILD_DIR}/${ALPINE_IMAGE}" &>/dev/null; then
-        rm -f "${BUILD_DIR}/${ALPINE_IMAGE}"
-        echo "ERROR: Downloaded file is not a valid qcow2 image"
+    # Verify the download is a valid qcow2/img
+    if ! qemu-img info "${BUILD_DIR}/${SOURCE_IMAGE}" &>/dev/null; then
+        rm -f "${BUILD_DIR}/${SOURCE_IMAGE}"
+        echo "ERROR: Downloaded file is not a valid disk image"
         exit 1
     fi
 fi
@@ -179,9 +220,9 @@ fi
 # Step 2: Copy image for customization
 rm -f "$OUTPUT_IMAGE"
 echo "Copying image for customization..."
-cp "${BUILD_DIR}/${ALPINE_IMAGE}" "$OUTPUT_IMAGE"
+cp "${BUILD_DIR}/${SOURCE_IMAGE}" "$OUTPUT_IMAGE"
 
-# Resize the image (Alpine cloud images are ~200MB, need room for packages)
+# Resize the image to provide room for packages
 qemu-img resize "$OUTPUT_IMAGE" "$IMAGE_SIZE"
 
 # Step 3: Connect via qemu-nbd
@@ -194,7 +235,7 @@ fi
 sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
 sudo qemu-nbd --connect="${NBD_DEV}" "$OUTPUT_IMAGE"
 
-# Wait for the nbd device to be ready (kernel needs time to scan the block device)
+# Wait for the nbd device to be ready
 for i in $(seq 1 10); do
     if sudo blockdev --getsize64 "${NBD_DEV}" &>/dev/null; then
         break
@@ -203,32 +244,57 @@ for i in $(seq 1 10); do
 done
 sleep 1
 
+# Ubuntu images have a partition table; wait for partition devices then resize
+# the partition to fill the expanded image before resizing the filesystem.
+if [[ "$DISTRO" == "ubuntu" ]]; then
+    for i in $(seq 1 10); do
+        if [[ -e "${ROOT_PART}" ]]; then break; fi
+        sleep 1
+    done
+    echo "Resizing partition..."
+    # After qemu-img resize the GPT backup header sits at the old end of disk.
+    # Move it to the new end before parted tries to resize, otherwise parted
+    # refuses with "Unable to satisfy all constraints on the partition."
+    sudo sgdisk --move-second-header "${NBD_DEV}" 2>/dev/null || true
+    sudo parted --script "${NBD_DEV}" resizepart 1 100%
+fi
+
 # Alpine cloud images have ext4 directly on the block device (no partition table).
-# Resize the filesystem to fill the resized image.
 echo "Checking filesystem..."
-sudo e2fsck -f -y "${NBD_DEV}" || {
+sudo e2fsck -f -y "${ROOT_PART}" || {
     ec=$?
     if [[ $ec -gt 1 ]]; then
-        echo "ERROR: e2fsck failed with exit code $ec on ${NBD_DEV}"
+        echo "ERROR: e2fsck failed with exit code $ec on ${ROOT_PART}"
         exit 1
     fi
 }
 
 echo "Resizing filesystem..."
-if ! sudo resize2fs "${NBD_DEV}"; then
-    echo "ERROR: resize2fs failed on ${NBD_DEV}"
+if ! sudo resize2fs "${ROOT_PART}"; then
+    echo "ERROR: resize2fs failed on ${ROOT_PART}"
     exit 1
 fi
 
 # Step 4: Mount and customize
 echo "Mounting root filesystem..."
-sudo mount "${NBD_DEV}" "$MOUNT_DIR"
+sudo mount "${ROOT_PART}" "$MOUNT_DIR"
 
-# Set up resolv.conf for DNS inside chroot
+# Set up resolv.conf for DNS inside chroot.
+# Ubuntu cloud images symlink /etc/resolv.conf → /run/systemd/resolve/stub-resolv.conf
+# which doesn't exist yet. Remove the symlink and write a real file.
+sudo rm -f "${MOUNT_DIR}/etc/resolv.conf"
 sudo cp /etc/resolv.conf "${MOUNT_DIR}/etc/resolv.conf"
 
+# Ubuntu chroot requires /proc /sys /dev bind mounts for systemd and DKMS
+if [[ "$DISTRO" == "ubuntu" ]]; then
+    sudo mount --bind /proc       "${MOUNT_DIR}/proc"
+    sudo mount --bind /sys        "${MOUNT_DIR}/sys"
+    sudo mount --bind /dev        "${MOUNT_DIR}/dev"
+    sudo mount --bind /dev/pts    "${MOUNT_DIR}/dev/pts"
+fi
+
 # Install packages
-if [[ -n "${APK_PACKAGES:-}" ]]; then
+if [[ "$DISTRO" == "alpine" ]] && [[ -n "${APK_PACKAGES:-}" ]]; then
     echo "Installing packages: ${APK_PACKAGES}..."
     sudo chroot "$MOUNT_DIR" /bin/sh -c "
 set -e
@@ -244,16 +310,30 @@ fi
 apk update
 apk add ${APK_PACKAGES}
 "
+elif [[ "$DISTRO" == "ubuntu" ]] && [[ -n "${APT_PACKAGES:-}" ]]; then
+    echo "Installing packages: ${APT_PACKAGES}..."
+    sudo chroot "$MOUNT_DIR" /bin/bash -c "
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get install -y --no-install-recommends ${APT_PACKAGES}
+"
 fi
 
-# Enable OpenRC services
+# Enable services
 if [[ -n "${ENABLE_SERVICES:-}" ]]; then
     echo "Enabling services: ${ENABLE_SERVICES}..."
     IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES"
     for svc in "${SERVICES[@]}"; do
-        if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then
-            echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?"
-            exit 1
+        if [[ "$DISTRO" == "alpine" ]]; then
+            if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then
+                echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?"
+                exit 1
+            fi
+        else
+            if ! sudo chroot "$MOUNT_DIR" /bin/bash -c "systemctl enable ${svc}"; then
+                echo "ERROR: Failed to enable service '${svc}'"
+                exit 1
+            fi
         fi
     done
 fi
@@ -282,21 +362,40 @@ if [[ -n "${SETUP_SCRIPT:-}" ]]; then
     echo "Running setup script: ${SETUP_SCRIPT}..."
     sudo cp "$setup_path" "${MOUNT_DIR}/tmp/setup.sh"
     sudo chmod 755 "${MOUNT_DIR}/tmp/setup.sh"
-    sudo chroot "$MOUNT_DIR" /tmp/setup.sh
+    if [[ "$DISTRO" == "ubuntu" ]]; then
+        sudo chroot "$MOUNT_DIR" /bin/bash /tmp/setup.sh
+    else
+        sudo chroot "$MOUNT_DIR" /tmp/setup.sh
+    fi
     sudo rm -f "${MOUNT_DIR}/tmp/setup.sh"
 fi
 
 # Step 6: Clean up and unmount
 echo "Cleaning up image..."
-sudo chroot "$MOUNT_DIR" /bin/sh -c '
+if [[ "$DISTRO" == "alpine" ]]; then
+    sudo chroot "$MOUNT_DIR" /bin/sh -c '
 apk cache clean 2>/dev/null || true
 rm -rf /var/cache/apk/* /tmp/*
 '
+else
+    sudo chroot "$MOUNT_DIR" /bin/bash -c '
+export DEBIAN_FRONTEND=noninteractive
+apt-get clean
+rm -rf /var/lib/apt/lists/* /tmp/*
+'
+fi
 
-# Restore original resolv.conf (cloud-init will set it on boot)
+# Restore the systemd-resolved symlink (Ubuntu default); cloud-init sets DNS on boot.
 sudo rm -f "${MOUNT_DIR}/etc/resolv.conf"
+sudo ln -sf /run/systemd/resolve/stub-resolv.conf "${MOUNT_DIR}/etc/resolv.conf"
 
 echo "Unmounting..."
+if [[ "$DISTRO" == "ubuntu" ]]; then
+    sudo umount "${MOUNT_DIR}/dev/pts"
+    sudo umount "${MOUNT_DIR}/dev"
+    sudo umount "${MOUNT_DIR}/sys"
+    sudo umount "${MOUNT_DIR}/proc"
+fi
 sudo umount "$MOUNT_DIR"
 sudo qemu-nbd --disconnect "${NBD_DEV}"
 
@@ -318,7 +417,7 @@ echo ""
 
 if [[ "$DO_IMPORT" == true ]]; then
     echo "Importing as AMI..."
-    IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro alpine --version "${ALPINE_VERSION}" --arch x86_64)
+    IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64)
     if [[ -n "${SYSTEM_TAG:-}" ]]; then
         IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
     fi
@@ -328,9 +427,9 @@ else
     echo "  cd $PROJECT_DIR && ./bin/spx admin images import \\"
     echo "    --file $OUTPUT_RAW \\"
     if [[ -n "${SYSTEM_TAG:-}" ]]; then
-        echo "    --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \\"
+        echo "    --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 \\"
         echo "    --tag ${SYSTEM_TAG}"
     else
-        echo "    --distro alpine --version ${ALPINE_VERSION} --arch x86_64"
+        echo "    --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64"
     fi
 fi

scripts/dev-install.sh

@@ -33,6 +33,8 @@ mkdir -p "$STAGING/systemd"
 cp "$PROJECT_ROOT/build/systemd/"* "$STAGING/systemd/"
 mkdir -p "$STAGING/scripts"
 cp "$PROJECT_ROOT/build/scripts/"* "$STAGING/scripts/"
+mkdir -p "$STAGING/udev"
+cp "$PROJECT_ROOT/build/udev/"* "$STAGING/udev/"
 cp "$PROJECT_ROOT/build/logrotate/spinifex" "$STAGING/logrotate-spinifex"
 tar czf /tmp/spinifex-local.tar.gz -C "$STAGING" .
 

scripts/iac/aws/README.md

@@ -31,7 +31,7 @@ This directory contains OpenTofu/Terraform configuration files for deploying inf
    ```
 
 4. **AMI available**
-   Before running, ensure you have a Debian 12 AMI imported:
+   Before running, ensure you have a Debian 13 AMI imported:
    ```bash
    # Import an AMI using the spinifex CLI or aws CLI
    aws --endpoint-url http://localhost:8080 ec2 describe-images
@@ -105,7 +105,7 @@ The `main.tf` creates:
 4. **Route Table** - Routes `0.0.0.0/0` through IGW
 5. **Security Group** - Allows SSH (22), HTTP (80), HTTPS (443) inbound
 6. **Placement Group** - Spread strategy for HA
-7. **3 EC2 Instances** - t3.small Debian 12 VMs
+7. **3 EC2 Instances** - t3.small Debian 13 VMs
 8. **SSH Key Pair** - ED25519 key saved to `spinifex-test.pem`
 
 ## SSH Access
@@ -133,7 +133,7 @@ export AWS_SECRET_ACCESS_KEY=your_secret
 
 ### "Error: No AMI found"
 
-Import a Debian 12 AMI first, or modify `main.tf` to use a different AMI filter.
+Import a Debian 13 AMI first, or modify `main.tf` to use a different AMI filter.
 
 ### Connection refused
 

scripts/iac/aws/basic/main.tf

@@ -72,13 +72,13 @@ data "aws_availability_zones" "available" {
   state = "available"
 }
 
-data "aws_ami" "debian12" {
+data "aws_ami" "debian13" {
   most_recent = true
   owners      = ["000000000000"]
 
   filter {
     name   = "name"
-    values = ["*debian-12*"]
+    values = ["*debian-13*"]
   }
 
   filter {
@@ -200,7 +200,7 @@ resource "aws_placement_group" "test" {
 # ---------------------------------------------------------------------------
 
 resource "aws_instance" "test" {
-  ami           = data.aws_ami.debian12.id
+  ami           = data.aws_ami.debian13.id
   instance_type = "t3.small"
 
   subnet_id              = aws_subnet.test.id

scripts/iac/aws/main.tf

@@ -57,15 +57,15 @@ data "aws_availability_zones" "available" {
   state = "available"
 }
 
-# Debian 12 AMI
+# Debian 13 AMI
 # For MulgaOS, need to import a local AMI first
-data "aws_ami" "debian12" {
+data "aws_ami" "debian13" {
   most_recent = true
   owners      = ["000000000000"] # Spinifex system images (on AWS use "136693071363" for Debian official)
 
   filter {
     name   = "name"
-    values = ["*debian-12*"]
+    values = ["*debian-13*"]
   }
 
   filter {
@@ -191,7 +191,7 @@ resource "aws_security_group" "web_ssh" {
 
 resource "aws_instance" "vm" {
   count         = 3
-  ami           = data.aws_ami.debian12.id
+  ami           = data.aws_ami.debian13.id
   instance_type = "t3.small"
 
   subnet_id              = aws_subnet.public.id

scripts/iac/proxmox-template/README.md

@@ -7,11 +7,11 @@ Builds a Proxmox VM template with all Spinifex dependencies pre-installed. Cloni
 | Component | Details |
 |---|---|
 | System packages | QEMU, nbdkit, libvirt, gcc, jq, curl, etc. (`make install-system`) |
-| Go | 1.26.2 (`make install-go`) |
+| Go | 1.26.3 (`make install-go`) |
 | AWS CLI | v2 (`make install-aws`) |
 | Repositories | spinifex, viperblock, predastore (with `go.work` configured) |
 | Module cache | `go mod download` completed for all repos |
-| Cloud images | Ubuntu 24.04 for nested VMs (`~/images/ubuntu-24.04.img`) |
+| Cloud images | Ubuntu 26.04 for nested VMs (`~/images/ubuntu-26.04.img`) |
 | Tuning | sysctl (rmem_max/wmem_max), kvm group, ufw disabled |
 | Generalized | Clean machine-id, SSH host keys, cloud-init — each clone boots fresh |
 

scripts/iac/proxmox-template/scripts/provision.sh

@@ -92,15 +92,15 @@ mkdir -p "$HOME/images"
 ARCH=$(dpkg --print-architecture)
 IMAGE_URL=""
 if [ "$ARCH" = "amd64" ]; then
-    IMAGE_URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+    IMAGE_URL="https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img"
 elif [ "$ARCH" = "arm64" ]; then
-    IMAGE_URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-arm64.img"
+    IMAGE_URL="https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-arm64.img"
 fi
 
 if [ -n "$IMAGE_URL" ]; then
-    echo "  Downloading Ubuntu 24.04 cloud image ($ARCH)..."
-    curl -L -o "$HOME/images/ubuntu-24.04.img" "$IMAGE_URL"
-    echo "  Saved to ~/images/ubuntu-24.04.img"
+    echo "  Downloading Ubuntu 26.04 cloud image ($ARCH)..."
+    curl -L -o "$HOME/images/ubuntu-26.04.img" "$IMAGE_URL"
+    echo "  Saved to ~/images/ubuntu-26.04.img"
 else
     echo "  Unknown arch ($ARCH), skipping cloud image download"
 fi

scripts/images/ubuntu-gpu-setup.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+set -euo pipefail
+
+# ubuntu-gpu-setup.sh — Chroot setup for Ubuntu 24.04 GPU AMI
+#
+# Installs NVIDIA headless server driver with DKMS module pre-built against
+# a pinned kernel. Blacklists nouveau. Prevents unattended-upgrades from
+# invalidating the pre-built module.
+#
+# Must run inside an Ubuntu 24.04 chroot with /proc /sys /dev bind-mounted
+# (required by DKMS during the nvidia package post-install hook).
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+
+# Install a pinned kernel and its headers so DKMS has a stable build target.
+# linux-image-generic and linux-headers-generic pull in the current HWE stack.
+apt-get install -y --no-install-recommends \
+    linux-image-generic \
+    linux-headers-generic
+
+# Capture the installed kernel version before adding the driver.
+KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||')
+if [[ -z "${KVER}" ]]; then
+    echo "ERROR: No kernel found under /boot after linux-image-generic install"
+    exit 1
+fi
+echo "Target kernel: ${KVER}"
+
+# Install the NVIDIA headless server driver.
+# The nvidia-dkms-550-server post-install hook runs `dkms autoinstall`, which
+# builds the module against every kernel found under /lib/modules/ — including
+# ${KVER} whose headers we just installed above.
+apt-get install -y --no-install-recommends \
+    nvidia-headless-550-server \
+    nvidia-utils-550-server
+
+# Verify the DKMS module was registered and built.
+if dkms status 2>/dev/null | grep -q "nvidia"; then
+    echo "NVIDIA DKMS module status: $(dkms status 2>/dev/null | grep nvidia | head -1)"
+else
+    echo "WARNING: nvidia DKMS module not found in dkms status — manual verification required"
+fi
+
+# Pin the kernel so unattended-upgrades cannot replace it and invalidate
+# the pre-built DKMS module.
+apt-mark hold \
+    linux-image-generic \
+    linux-headers-generic \
+    "linux-image-${KVER}" \
+    "linux-headers-${KVER}"
+
+# Pin all installed NVIDIA packages for the same reason.
+NVIDIA_PKGS=$(dpkg --get-selections 'nvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
+LIBNVIDIA_PKGS=$(dpkg --get-selections 'libnvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
+# shellcheck disable=SC2086
+[[ -n "${NVIDIA_PKGS}" ]]    && apt-mark hold ${NVIDIA_PKGS}
+# shellcheck disable=SC2086
+[[ -n "${LIBNVIDIA_PKGS}" ]] && apt-mark hold ${LIBNVIDIA_PKGS}
+
+# Blacklist nouveau so it does not conflict with the NVIDIA driver on boot.
+cat > /etc/modprobe.d/blacklist-nouveau.conf << 'EOF'
+blacklist nouveau
+options nouveau modeset=0
+EOF
+
+# Prevent unattended-upgrades from updating kernel or NVIDIA packages,
+# which would break the pre-built DKMS module.
+mkdir -p /etc/apt/apt.conf.d
+cat > /etc/apt/apt.conf.d/99-gpu-ami << 'EOF'
+Unattended-Upgrade::Package-Blacklist {
+    "linux-";
+    "nvidia-";
+    "libnvidia-";
+};
+EOF
+
+# Rebuild initramfs with nouveau blacklisted and NVIDIA module included.
+update-initramfs -u -k "${KVER}"
+
+echo "GPU AMI setup complete: kernel=${KVER}, NVIDIA 550 driver installed, DKMS module pre-built"

scripts/images/ubuntu-gpu.conf

@@ -0,0 +1,12 @@
+IMAGE_NAME="ubuntu-gpu"
+IMAGE_DESCRIPTION="Ubuntu 24.04 GPU-ready AMI (NVIDIA A10G / G5 family)"
+DISTRO="ubuntu"
+UBUNTU_VERSION="24.04"
+
+# 12G provides headroom for kernel headers, NVIDIA driver, and DKMS build artefacts
+IMAGE_SIZE="12G"
+
+# NVIDIA driver install and DKMS pre-build are handled in the setup script
+SETUP_SCRIPT="scripts/images/ubuntu-gpu-setup.sh"
+
+SYSTEM_TAG="gpu"

scripts/run-bench.sh

@@ -12,11 +12,11 @@ if [ -z "$SPINIFEX_AMI" ]; then
     echo "SPINIFEX_AMI not set, attempting to detect..."
     ARCH=$(uname -m)
     if [ "$ARCH" = "x86_64" ]; then
-        IMAGE_NAME="ami-ubuntu-24.04-x86_64"
+        IMAGE_NAME="ami-ubuntu-26.04-x86_64"
     elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
-        IMAGE_NAME="ami-ubuntu-24.04-arm64"
+        IMAGE_NAME="ami-ubuntu-26.04-arm64"
     else
-        IMAGE_NAME="ami-ubuntu-24.04-x86_64"
+        IMAGE_NAME="ami-ubuntu-26.04-x86_64"
     fi
     SPINIFEX_AMI=$(aws ec2 describe-images --query "Images[?Name=='$IMAGE_NAME'].ImageId" --output text)
     if [ -z "$SPINIFEX_AMI" ]; then

scripts/run-instance.sh

@@ -6,12 +6,12 @@ export AWS_PROFILE=spinifex
 # Detect architecture
 ARCH=$(uname -m)
 if [ "$ARCH" = "x86_64" ]; then
-    IMAGE_NAME="ami-ubuntu-24.04-x86_64"
+    IMAGE_NAME="ami-ubuntu-26.04-x86_64"
 elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
-    IMAGE_NAME="ami-ubuntu-24.04-arm64"
+    IMAGE_NAME="ami-ubuntu-26.04-arm64"
 else
     echo "Warning: Unknown architecture $ARCH, defaulting to x86_64"
-    IMAGE_NAME="ami-ubuntu-24.04-x86_64"
+    IMAGE_NAME="ami-ubuntu-26.04-x86_64"
 fi
 
 # Query available instance types from Spinifex and pick the first micro type

scripts/setup.sh

@@ -15,7 +15,7 @@
 #   VERBOSE                    Set to 1 to echo "[setup] <stage>" before each top-level step.
 #   SETUP_STAGES               Comma-separated subset of stages to run:
 #                                deps, aws, users, sudoers, files, directories,
-#                                env, systemd, logrotate, fixown, migrations
+#                                env, systemd, logrotate, udev, fixown, migrations
 #                              Unset = run every stage appropriate for the current mode.
 
 set -e
@@ -192,6 +192,7 @@ spinifex-vpcd ALL=(root) NOPASSWD: /usr/sbin/dhcpcd
 spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
 spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
 spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/systemctl is-active --quiet ovn-controller
+spinifex-vpcd ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
 SUDOERS
     $SUDO chmod 0440 /etc/sudoers.d/spinifex-network
     $SUDO visudo -cf /etc/sudoers.d/spinifex-network || fatal "Invalid sudoers syntax in spinifex-network"
@@ -213,8 +214,9 @@ install_apt_deps() {
 
         DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq \
             nbdkit \
-            $QEMU_PACKAGES qemu-utils qemu-kvm less \
+            $QEMU_PACKAGES qemu-utils qemu-kvm ovmf less \
             libvirt-daemon-system libvirt-clients \
+            pciutils \
             jq curl iproute2 netcat-openbsd wget unzip xz-utils file \
             ovn-central ovn-host openvswitch-switch dhcpcd-base \
             > /dev/null
@@ -608,6 +610,24 @@ install_logrotate() {
     info "Logrotate config installed"
 }
 
+# --- Install udev rules ---
+install_udev() {
+    stage "installing udev rules"
+    if [ ! -d "$EXTRACT_DIR/udev" ]; then
+        return
+    fi
+    $SUDO install -d /etc/udev/rules.d
+    for rule in "$EXTRACT_DIR"/udev/*; do
+        $SUDO install -m 0644 "$rule" "/etc/udev/rules.d/$(basename "$rule")"
+        info "  /etc/udev/rules.d/$(basename "$rule")"
+    done
+    if [ "${ISO_BUILD:-0}" != "1" ]; then
+        $SUDO udevadm control --reload-rules
+        $SUDO udevadm trigger --subsystem-match=vfio 2>/dev/null || true
+    fi
+    info "udev rules installed"
+}
+
 # --- Upgrade handling ---
 handle_upgrade() {
     if $SUDO systemctl is-active --quiet spinifex.target 2>/dev/null; then
@@ -688,10 +708,11 @@ main() {
         handle_upgrade
     fi
 
-    # Stages that need EXTRACT_DIR: files, directories, systemd, logrotate.
+    # Stages that need EXTRACT_DIR: files, directories, systemd, logrotate, udev.
     # Only download/extract when at least one such stage is enabled.
     if stage_enabled files || stage_enabled directories \
-        || stage_enabled systemd || stage_enabled logrotate; then
+        || stage_enabled systemd || stage_enabled logrotate \
+        || stage_enabled udev; then
         download_spinifex
     fi
 
@@ -705,6 +726,7 @@ main() {
     stage_enabled fixown     && fix_file_ownership
     stage_enabled systemd    && install_systemd
     stage_enabled logrotate  && install_logrotate
+    stage_enabled udev       && install_udev
 
     # Migrations are only safe on a live system (need a running NATS and a
     # persisted config file). Skip under ISO_BUILD and under any explicit

scripts/smoke-test.sh

@@ -6,8 +6,23 @@
 #
 # Assumes services are running and AWS_PROFILE=spinifex is configured
 # by a prior 'spx admin init'.
+#
+# GPU passthrough test (opt-in):
+#   TEST_GPU=1 scripts/smoke-test.sh
+#   TEST_GPU=1 GPU_VENDOR_ID=10de GPU_DEVICE_ID=2236 scripts/smoke-test.sh
+#
+# GPU env vars:
+#   TEST_GPU=1            enable GPU passthrough checks
+#   GPU_VENDOR_ID         PCI vendor ID (default: auto-detect first NVIDIA VGA/3D device)
+#   GPU_DEVICE_ID         PCI device ID (default: auto-detect)
+#   GPU_FAMILY            instance family to test (default: g5)
 set -euo pipefail
 
+TEST_GPU="${TEST_GPU:-0}"
+GPU_VENDOR_ID="${GPU_VENDOR_ID:-}"
+GPU_DEVICE_ID="${GPU_DEVICE_ID:-}"
+GPU_FAMILY="${GPU_FAMILY:-g5}"
+
 export AWS_PROFILE=spinifex
 
 # --- Wait for EC2 daemon to subscribe to NATS ---
@@ -42,19 +57,54 @@ echo "==> Importing SSH key"
     --public-key-material "fileb://$SSH_KEY.pub"
 aws ec2 describe-key-pairs
 
+# --- GPU pre-flight (TEST_GPU=1 only) ---
+if [[ "$TEST_GPU" == "1" ]]; then
+    echo "==> [GPU] Checking IOMMU"
+    if ! dmesg | grep -qi "DMAR.*IOMMU enabled\|iommu.*Translated\|Adding to iommu group\|AMD-Vi.*enabled"; then
+        echo "❌ IOMMU not detected — add intel_iommu=on (or amd_iommu=on) and iommu=pt to kernel cmdline, then reboot"
+        exit 1
+    fi
+    echo "   IOMMU active"
+
+    echo "==> [GPU] Detecting NVIDIA GPU"
+    if [[ -z "$GPU_VENDOR_ID" || -z "$GPU_DEVICE_ID" ]]; then
+        GPU_LINE=$(lspci -nn | grep -i nvidia | grep -iE "VGA|3D" | head -1)
+        if [[ -z "$GPU_LINE" ]]; then
+            echo "❌ No NVIDIA GPU found via lspci — is the card seated and detectable?"
+            lspci | grep -i nvidia >&2 || true
+            exit 1
+        fi
+        echo "   Detected: $GPU_LINE"
+        PCI_IDS=$(echo "$GPU_LINE" | grep -oP '\[\K[0-9a-f]{4}:[0-9a-f]{4}(?=\])')
+        GPU_VENDOR_ID=$(echo "$PCI_IDS" | cut -d: -f1)
+        GPU_DEVICE_ID=$(echo "$PCI_IDS" | cut -d: -f2)
+    fi
+    echo "   GPU PCI ID: ${GPU_VENDOR_ID}:${GPU_DEVICE_ID}"
+
+    echo "==> [GPU] Verifying ${GPU_FAMILY} instance types advertised"
+    G5_TYPES=$(aws ec2 describe-instance-types \
+        --query "InstanceTypes[?starts_with(InstanceType, \`${GPU_FAMILY}\`)].InstanceType" \
+        --output text 2>/dev/null || true)
+    if [[ -z "$G5_TYPES" ]]; then
+        echo "❌ No ${GPU_FAMILY}.* instance types returned by describe-instance-types"
+        exit 1
+    fi
+    echo "   ${GPU_FAMILY} types available: $G5_TYPES"
+fi
+
 # --- Import AMI ---
 # --- Import AMI ---
 echo "==> Importing AMI"
 
-LOCAL_IMAGE="$HOME/images/ubuntu-24.04.img"
+LOCAL_IMAGE="$HOME/images/ubuntu-26.04.img"
 ARCH=$(uname -m)
 
 case "$ARCH" in
-    x86_64)        IMG_ARCH="x86_64"; IMAGE_NAME="ubuntu-24.04-x86_64" ;;
-    aarch64|arm64) IMG_ARCH="arm64";  IMAGE_NAME="ubuntu-24.04-arm64"  ;;
+    x86_64)        IMG_ARCH="x86_64"; IMAGE_NAME="ubuntu-26.04-x86_64" ;;
+    aarch64|arm64) IMG_ARCH="arm64";  IMAGE_NAME="ubuntu-26.04-arm64"  ;;
     *)
         echo "  Warning: unknown arch $ARCH, defaulting to x86_64"
-        IMG_ARCH="x86_64"; IMAGE_NAME="ubuntu-24.04-x86_64"
+        IMG_ARCH="x86_64"; IMAGE_NAME="ubuntu-26.04-x86_64"
         ;;
 esac
 
@@ -70,7 +120,7 @@ else
     if [ -f "$LOCAL_IMAGE" ]; then
         echo "  Using local image: $LOCAL_IMAGE"
         sudo /usr/local/bin/spx admin images import \
-            --file "$LOCAL_IMAGE" --distro ubuntu --version 24.04 --arch "$IMG_ARCH"
+            --file "$LOCAL_IMAGE" --distro ubuntu --version 26.04 --arch "$IMG_ARCH"
     else
         echo "  Downloading image: $IMAGE_NAME"
         sudo /usr/local/bin/spx admin images import --name "$IMAGE_NAME"
@@ -79,7 +129,9 @@ fi
 
 # --- Launch smoke-test instance ---
 echo "==> Launching smoke-test instance"
-if grep -q 'AuthenticAMD' /proc/cpuinfo; then
+if [[ "$TEST_GPU" == "1" ]]; then
+    INSTANCE_TYPE="${GPU_FAMILY}.xlarge"
+elif grep -q 'AuthenticAMD' /proc/cpuinfo; then
     INSTANCE_TYPE="t3a.small"
 else
     INSTANCE_TYPE="t3.small"
@@ -195,4 +247,27 @@ if ! echo "$SSH_OUT" | grep -q "ec2-user"; then
     exit 1
 fi
 
-echo "✅ Smoke test passed — instance $INSTANCE_ID launched, running, and SSH-verified"
+# --- GPU guest verification (TEST_GPU=1 only) ---
+if [[ "$TEST_GPU" == "1" ]]; then
+    echo "==> [GPU] Verifying GPU visible inside guest"
+    GPU_VISIBLE=0
+    for _i in $(seq 1 12); do
+        if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+               -o ConnectTimeout=5 -o BatchMode=yes \
+               -p "$SSH_INST_PORT" -i "$SSH_KEY" \
+               ec2-user@"$SSH_INST_HOST" \
+               'lspci 2>/dev/null | grep -i nvidia' 2>/dev/null; then
+            GPU_VISIBLE=1
+            break
+        fi
+        sleep 5
+    done
+    if [[ $GPU_VISIBLE -eq 0 ]]; then
+        echo "❌ NVIDIA GPU not visible in guest via lspci"
+        exit 1
+    fi
+    echo "   GPU visible in guest"
+    echo "✅ Smoke test passed (GPU passthrough) — instance $INSTANCE_ID launched with GPU and verified"
+else
+    echo "✅ Smoke test passed — instance $INSTANCE_ID launched, running, and SSH-verified"
+fi

scripts/start-dev.sh

@@ -314,6 +314,17 @@ if has_service "predastore"; then
     export SPINIFEX_PREDASTORE_TLS_CERT=$CONFIG_DIR/server.pem
     export SPINIFEX_PREDASTORE_TLS_KEY=$CONFIG_DIR/server.key
 
+    # Per-node predastore encryption key. Self-heal for devs who skipped
+    # `spx admin init`: generate with umask 0177 so the file is 0600 from
+    # creation (predastore rejects group/other-readable keys outright).
+    ENCRYPTION_KEY="$CONFIG_DIR/predastore/encryption.key"
+    if [ ! -f "$ENCRYPTION_KEY" ]; then
+        mkdir -p "$CONFIG_DIR/predastore"
+        ( umask 0177 && openssl rand -out "$ENCRYPTION_KEY" 32 )
+        echo "   Generated predastore encryption key: $ENCRYPTION_KEY"
+    fi
+    export SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE="$ENCRYPTION_KEY"
+
     # Auto-detect Predastore host:port from spinifex.toml [nodes.<name>.predastore] section
     PREDASTORE_BIND="0.0.0.0:8443"
     if [ -f "$CONFIG_DIR/spinifex.toml" ]; then

scripts/start-then-terminate-pending.sh

@@ -7,9 +7,9 @@ export AWS_PROFILE=spinifex
 
 ARCH=$(uname -m)
 case "$ARCH" in
-    x86_64)            IMAGE_NAME="ami-ubuntu-24.04-x86_64" ;;
-    aarch64|arm64)     IMAGE_NAME="ami-ubuntu-24.04-arm64" ;;
-    *)                 echo "Unknown arch $ARCH, default x86_64"; IMAGE_NAME="ami-ubuntu-24.04-x86_64" ;;
+    x86_64)            IMAGE_NAME="ami-ubuntu-26.04-x86_64" ;;
+    aarch64|arm64)     IMAGE_NAME="ami-ubuntu-26.04-arm64" ;;
+    *)                 echo "Unknown arch $ARCH, default x86_64"; IMAGE_NAME="ami-ubuntu-26.04-x86_64" ;;
 esac
 
 INSTANCE_TYPE=$(aws ec2 describe-instance-types \

scripts/test-ddil-phase1.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+# test-ddil-phase1.sh — systemd-level smoke test for daemon-local-autonomy
+# Tier 1 (1a / 1b / 1c / 1d / 1d-strict). Run after `make deploy` on a single
+# node where spinifex-daemon and spinifex-nats are managed by systemd.
+#
+# Usage: scripts/test-ddil-phase1.sh
+# Requires: sudo, curl, jq
+#
+# Destructive on the local node — stops/starts spinifex-nats and restarts
+# spinifex-daemon. Do NOT run against a host carrying workloads. The cleanup
+# trap restores both services on exit.
+
+set -euo pipefail
+
+DAEMON_URL="${DAEMON_URL:-https://127.0.0.1:4432}"
+DAEMON_SVC=spinifex-daemon
+NATS_SVC=spinifex-nats
+OVERRIDE=/etc/systemd/system/${DAEMON_SVC}.service.d/99-test-ddil.conf
+NATS_BLOCK=/run/spinifex-nats-blocked
+NATS_BLOCK_DROPIN=/run/systemd/system/${NATS_SVC}.service.d/99-test-block.conf
+CURL=(curl -fsS -k --max-time 5)
+
+ok()   { printf '\033[32mPASS\033[0m %s\n' "$1"; }
+fail() { printf '\033[31mFAIL\033[0m %s\n' "$1"; exit 1; }
+info() { printf '\033[36m==>\033[0m %s\n' "$1"; }
+
+cleanup() {
+    info "cleanup: removing nats block, override, restoring services"
+    sudo rm -f "$NATS_BLOCK" "$NATS_BLOCK_DROPIN" "$OVERRIDE"
+    sudo rmdir "$(dirname "$NATS_BLOCK_DROPIN")" 2>/dev/null || true
+    sudo systemctl daemon-reload
+    sudo systemctl start "$NATS_SVC"     2>/dev/null || true
+    sudo systemctl restart "$DAEMON_SVC" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+field() { "${CURL[@]}" "${DAEMON_URL}/local/status" | jq -r ".$1 // empty"; }
+
+wait_field() {
+    local f=$1 want=$2 timeout=${3:-30} elapsed=0
+    while [ "$elapsed" -lt "$timeout" ]; do
+        [ "$(field "$f" 2>/dev/null || true)" = "$want" ] && return 0
+        sleep 1; elapsed=$((elapsed + 1))
+    done
+    return 1
+}
+
+# 0. baseline
+info "baseline: daemon active + cluster mode"
+sudo systemctl is-active --quiet "$DAEMON_SVC" || fail "daemon not running — run \`make deploy\` first"
+sudo systemctl start "$NATS_SVC"
+wait_field mode cluster 30 || fail "never reached mode=cluster (got: $(field mode))"
+[ "$(field nats)" = connected ] || fail "nats not connected (got: $(field nats))"
+ok "mode=cluster nats=connected"
+
+# 1. kill NATS → standalone, daemon survives
+info "1a/1c: kill NATS"
+retry_before=$(field nats_retry_count)
+sudo systemctl stop "$NATS_SVC"
+wait_field mode standalone 30   || fail "never went standalone (got: $(field mode))"
+wait_field nats disconnected 5  || fail "nats not disconnected (got: $(field nats))"
+sudo systemctl is-active --quiet "$DAEMON_SVC" || fail "daemon exited — should stay up"
+ok "standalone, nats=disconnected, daemon alive"
+
+# 2. /local/instances reachable during partition
+info "1b: /local/instances during partition"
+"${CURL[@]}" "${DAEMON_URL}/local/instances" | jq -e . >/dev/null \
+    || fail "/local/instances did not return JSON"
+ok "/local/instances served while NATS down"
+
+# 3. restart NATS → cluster mode + retry count advances + KV push observed
+# nats_retry_count counts disconnect→reconnect *cycles* (bumped from
+# onNATSReconnect, daemon.go:321). It stays flat during disconnect and
+# advances by 1 here when the reconnect callback fires.
+info "1c reconnect: restart NATS"
+before_sync=$(field last_kv_sync_at)
+sudo systemctl start "$NATS_SVC"
+wait_field mode cluster 60   || fail "did not return to cluster mode"
+wait_field nats connected 10 || fail "nats not reconnected"
+sleep 5
+retry_after=$(field nats_retry_count)
+[ "$retry_after" -gt "$retry_before" ] \
+    || fail "nats_retry_count did not advance on reconnect ($retry_before → $retry_after)"
+after_sync=$(field last_kv_sync_at)
+[ -n "$after_sync" ] && [ "$after_sync" != "$before_sync" ] \
+    || fail "last_kv_sync_at did not advance ($before_sync → $after_sync)"
+ok "cluster restored, retry $retry_before → $retry_after, KV resync at $after_sync"
+
+# 4. SPINIFEX_REQUIRE_NATS=1 → daemon exits within ~30s when NATS down.
+# Block spinifex-nats via a runtime drop-in with ConditionPathExists=!. This
+# defeats the daemon unit's Wants=spinifex-nats.service auto-start, which
+# would otherwise pull NATS back up on `systemctl restart spinifex-daemon`
+# and let the daemon connect immediately. mask refuses (real unit lives in
+# /etc) and --runtime mask is shadowed by /etc — the condition drop-in is
+# the only mechanism that works against an /etc-installed unit.
+info "1d-strict: SPINIFEX_REQUIRE_NATS=1 exits when NATS down"
+sudo mkdir -p "$(dirname "$NATS_BLOCK_DROPIN")"
+sudo tee "$NATS_BLOCK_DROPIN" >/dev/null <<EOF
+[Unit]
+ConditionPathExists=!$NATS_BLOCK
+EOF
+sudo touch "$NATS_BLOCK"
+sudo systemctl daemon-reload
+sudo systemctl stop "$NATS_SVC"
+sudo mkdir -p "$(dirname "$OVERRIDE")"
+sudo tee "$OVERRIDE" >/dev/null <<EOF
+[Service]
+Environment=SPINIFEX_REQUIRE_NATS=1
+Restart=no
+EOF
+sudo systemctl daemon-reload
+sudo systemctl restart "$DAEMON_SVC" || true
+start=$(date +%s)
+while sudo systemctl is-active --quiet "$DAEMON_SVC"; do
+    [ $(( $(date +%s) - start )) -gt 45 ] && fail "did not exit within 45s"
+    sleep 1
+done
+ok "exited after $(( $(date +%s) - start ))s"
+
+echo
+echo "DDIL Phase 1 systemd integration: ALL PASS"

scripts/workload-performance.sh

@@ -5,8 +5,8 @@
 sudo apt update
 sudo apt install -y make gcc git
 
-wget https://go.dev/dl/go1.26.2.linux-amd64.tar.gz
-sudo tar -C /usr/local -xzf go1.26.2.linux-amd64.tar.gz
+wget https://go.dev/dl/go1.26.3.linux-amd64.tar.gz
+sudo tar -C /usr/local -xzf go1.26.3.linux-amd64.tar.gz
 
 echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.bashrc
 source $HOME/.bashrc

spinifex/admin/admin.go

@@ -15,6 +15,7 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"text/template"
@@ -99,6 +100,10 @@ type ConfigSettings struct {
 	BootstrapIgwId      string
 	BootstrapCidr       string
 	BootstrapSubnetCidr string
+
+	// GPUPassthrough enables VFIO GPU passthrough in the daemon config.
+	// Sets gpu_passthrough = true under [nodes.<node>.daemon].
+	GPUPassthrough bool
 }
 
 // PredastoreNodeConfig describes a single Predastore node for multi-node config generation.
@@ -931,3 +936,58 @@ func ParsePredastoreNodeIDFromConfig(tomlContent string, ip string) int {
 	}
 	return FindNodeIDByIP(cfg.DB, ip)
 }
+
+// SetGPUPassthrough idempotently writes gpu_passthrough = <enabled> for the
+// given node into spinifex.toml, preserving all other content and comments.
+// Returns nil without touching the file if the setting is already correct.
+func SetGPUPassthrough(tomlPath, node string, enabled bool) error {
+	raw, err := os.ReadFile(tomlPath)
+	if err != nil {
+		return fmt.Errorf("read %s: %w", tomlPath, err)
+	}
+	text := string(raw)
+	value := "false"
+	if enabled {
+		value = "true"
+	}
+
+	sectionHeader := "[nodes." + node + ".daemon]"
+
+	sectionStart := strings.Index(text, sectionHeader)
+	if sectionStart < 0 {
+		// No daemon section — append one.
+		text = strings.TrimRight(text, "\n") +
+			"\n\n[nodes." + node + ".daemon]\ngpu_passthrough = " + value + "\n"
+		return os.WriteFile(tomlPath, []byte(text), 0640) //nolint:gosec // spinifex.toml is root:spinifex 0640 so the daemon can read it
+	}
+
+	// Extract body of just this section (stops at the next section header).
+	bodyStart := sectionStart + len(sectionHeader)
+	rest := text[bodyStart:]
+	nextSection := strings.Index(rest, "\n[")
+	var body, suffix string
+	if nextSection < 0 {
+		body = rest
+	} else {
+		body = rest[:nextSection]
+		suffix = rest[nextSection:]
+	}
+
+	// Already correct — no-op.
+	if regexp.MustCompile(`gpu_passthrough\s*=\s*` + value).MatchString(body) {
+		return nil
+	}
+
+	// Key exists with wrong value — flip within this section only.
+	flipRe := regexp.MustCompile(`gpu_passthrough\s*=\s*(?:true|false)`)
+	var newBody string
+	if flipRe.MatchString(body) {
+		newBody = flipRe.ReplaceAllString(body, "gpu_passthrough = "+value)
+	} else {
+		// Key absent — insert right after section header.
+		newBody = "\ngpu_passthrough = " + value + body
+	}
+
+	text = text[:bodyStart] + newBody + suffix
+	return os.WriteFile(tomlPath, []byte(text), 0640) //nolint:gosec // spinifex.toml is root:spinifex 0640 so the daemon can read it
+}

spinifex/admin/admin_test.go

@@ -862,3 +862,72 @@ func TestChownRecursive_NonExistentPath(t *testing.T) {
 	}
 	ChownRecursive("/tmp/nonexistent-path-12345", currentUser)
 }
+
+// --- SetGPUPassthrough ---
+
+func writeToml(t *testing.T, content string) string {
+	t.Helper()
+	f, err := os.CreateTemp(t.TempDir(), "spinifex*.toml")
+	require.NoError(t, err)
+	_, err = f.WriteString(content)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+	return f.Name()
+}
+
+func readToml(t *testing.T, path string) string {
+	t.Helper()
+	b, err := os.ReadFile(path)
+	require.NoError(t, err)
+	return string(b)
+}
+
+func TestSetGPUPassthrough_NoOp(t *testing.T) {
+	toml := "[nodes.node1.daemon]\ngpu_passthrough = true\n"
+	path := writeToml(t, toml)
+	require.NoError(t, SetGPUPassthrough(path, "node1", true))
+	assert.Equal(t, toml, readToml(t, path))
+}
+
+func TestSetGPUPassthrough_Flip(t *testing.T) {
+	path := writeToml(t, "[nodes.node1.daemon]\ngpu_passthrough = false\n")
+	require.NoError(t, SetGPUPassthrough(path, "node1", true))
+	assert.Contains(t, readToml(t, path), "gpu_passthrough = true")
+}
+
+func TestSetGPUPassthrough_AddKeyToSection(t *testing.T) {
+	path := writeToml(t, "[nodes.node1.daemon]\nsome_other = true\n")
+	require.NoError(t, SetGPUPassthrough(path, "node1", true))
+	got := readToml(t, path)
+	assert.Contains(t, got, "gpu_passthrough = true")
+	assert.Contains(t, got, "some_other = true")
+}
+
+func TestSetGPUPassthrough_AppendSection(t *testing.T) {
+	path := writeToml(t, "[nodes.node1.network]\ncidr = \"10.0.0.0/24\"\n")
+	require.NoError(t, SetGPUPassthrough(path, "node1", true))
+	got := readToml(t, path)
+	assert.Contains(t, got, "[nodes.node1.daemon]")
+	assert.Contains(t, got, "gpu_passthrough = true")
+}
+
+func TestSetGPUPassthrough_DisableNoOp(t *testing.T) {
+	toml := "[nodes.node1.daemon]\ngpu_passthrough = false\n"
+	path := writeToml(t, toml)
+	require.NoError(t, SetGPUPassthrough(path, "node1", false))
+	assert.Equal(t, toml, readToml(t, path))
+}
+
+func TestSetGPUPassthrough_ReadError(t *testing.T) {
+	err := SetGPUPassthrough("/nonexistent/path/spinifex.toml", "node1", true)
+	require.Error(t, err)
+}
+
+func TestSetGPUPassthrough_SectionBoundary(t *testing.T) {
+	// gpu_passthrough = true exists but in a DIFFERENT node's section; should still write.
+	path := writeToml(t, "[nodes.node2.daemon]\ngpu_passthrough = true\n[nodes.node1.daemon]\nsome_key = 1\n")
+	require.NoError(t, SetGPUPassthrough(path, "node1", true))
+	got := readToml(t, path)
+	// node1 section should now have the key
+	assert.Contains(t, got, "[nodes.node1.daemon]\ngpu_passthrough = true")
+}

spinifex/config/config.go

@@ -123,13 +123,30 @@ type PredastoreConfig struct {
 	NodeID    int    `json:"NodeID" mapstructure:"node_id"`
 }
 
+// GPUModelOverride maps a PCI vendor/device ID to a GPU instance family for
+// dev or test nodes that carry consumer GPUs not in the production model list.
+// Add entries under [[nodes.<node>.daemon.gpu_model_overrides]] in spinifex.toml.
+type GPUModelOverride struct {
+	VendorID     string `json:"VendorID" mapstructure:"vendor_id"`
+	DeviceID     string `json:"DeviceID" mapstructure:"device_id"`
+	Family       string `json:"Family" mapstructure:"family"`
+	Manufacturer string `json:"Manufacturer" mapstructure:"manufacturer"`
+	Name         string `json:"Name" mapstructure:"name"`
+	MemoryMiB    int64  `json:"MemoryMiB" mapstructure:"memory_mib"`
+	// XVGAOff forces x-vga=off for this GPU in QEMU passthrough, overriding the
+	// default (on for consumer GPUs, off for known datacenter cards).
+	XVGAOff bool `json:"XVGAOff" mapstructure:"xvga_off"`
+}
+
 // DaemonConfig holds the daemon configuration
 type DaemonConfig struct {
-	Host          string `json:"Host" mapstructure:"host"`
-	TLSKey        string `json:"TLSKey" mapstructure:"tlskey"`
-	TLSCert       string `json:"TLSCert" mapstructure:"tlscert"`
-	DevNetworking bool   `json:"DevNetworking" mapstructure:"dev_networking"` // VPC instances get both TAP + hostfwd for SSH dev access
-	MgmtBridge    string `json:"MgmtBridge" mapstructure:"mgmt_bridge"`       // Linux bridge for system instance control plane (default "br-mgmt")
+	Host              string             `json:"Host" mapstructure:"host"`
+	TLSKey            string             `json:"TLSKey" mapstructure:"tlskey"`
+	TLSCert           string             `json:"TLSCert" mapstructure:"tlscert"`
+	DevNetworking     bool               `json:"DevNetworking" mapstructure:"dev_networking"`          // VPC instances get both TAP + hostfwd for SSH dev access
+	MgmtBridge        string             `json:"MgmtBridge" mapstructure:"mgmt_bridge"`                // Linux bridge for system instance control plane (default "br-mgmt")
+	GPUPassthrough    bool               `json:"GPUPassthrough" mapstructure:"gpu_passthrough"`        // Enable VFIO GPU passthrough for g5.* instance types
+	GPUModelOverrides []GPUModelOverride `json:"GPUModelOverrides" mapstructure:"gpu_model_overrides"` // Dev/test GPU mappings not in the production model list
 }
 
 // NATSConfig holds the NATS configuration

spinifex/daemon/daemon_handlers.go

@@ -69,10 +69,7 @@ func (d *Daemon) handleEC2Events(msg *nats.Msg) {
 
 	slog.Debug("Received message", "subject", msg.Subject, "data", string(msg.Data))
 
-	d.Instances.Mu.Lock()
-	instance, ok := d.Instances.VMS[command.ID]
-	d.Instances.Mu.Unlock()
-
+	instance, ok := d.vmMgr.Get(command.ID)
 	if !ok {
 		slog.Warn("Instance is not running on this node", "id", command.ID)
 		respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound)
@@ -90,11 +87,29 @@ func (d *Daemon) handleEC2Events(msg *nats.Msg) {
 	case command.Attributes.DetachVolume:
 		d.handleDetachVolume(msg, command, instance)
 	case command.Attributes.StartInstance:
-		d.handleStartInstance(msg, command, instance)
+		if err := d.instanceService.StartInstance(instance, command); err != nil {
+			respondWithError(msg, awserrors.ValidErrorCode(err.Error()))
+			return
+		}
+		if err := msg.Respond(fmt.Appendf(nil, `{"status":"running","instanceId":"%s"}`, instance.ID)); err != nil {
+			slog.Error("Failed to respond to NATS request", "err", err)
+		}
 	case command.Attributes.RebootInstance:
-		d.handleRebootInstance(msg, command, instance)
+		if err := d.instanceService.RebootInstance(instance, command); err != nil {
+			respondWithError(msg, awserrors.ValidErrorCode(err.Error()))
+			return
+		}
+		if err := msg.Respond([]byte(`{}`)); err != nil {
+			slog.Error("Failed to respond to NATS request", "err", err)
+		}
 	case command.Attributes.StopInstance, command.Attributes.TerminateInstance:
-		d.handleStopOrTerminateInstance(msg, command, instance)
+		if err := d.instanceService.StopOrTerminateInstance(instance, command); err != nil {
+			respondWithError(msg, awserrors.ValidErrorCode(err.Error()))
+			return
+		}
+		if err := msg.Respond([]byte(`{}`)); err != nil {
+			slog.Error("Failed to respond to NATS request", "err", err)
+		}
 	default:
 		slog.Warn("Unhandled EC2 instance command", "id", command.ID, "attributes", command.Attributes)
 		respondWithError(msg, awserrors.ErrorServerInternal)
@@ -158,31 +173,45 @@ func (d *Daemon) daemonIP() string {
 func (d *Daemon) handleNodeStatus(msg *nats.Msg) {
 	totalVCPU, totalMemGB, reservedVCPU, reservedMemGB, allocVCPU, allocMemGB, caps := d.resourceMgr.GetResourceStats()
 
-	d.Instances.Mu.Lock()
 	vmCount := 0
-	for _, v := range d.Instances.VMS {
+	d.vmMgr.ForEach(func(v *vm.VM) {
 		if v.Status == vm.StateRunning {
 			vmCount++
 		}
+	})
+
+	totalGPUs, allocGPUs := 0, 0
+	if d.gpuManager != nil {
+		totalGPUs = d.gpuManager.TotalCount()
+		allocGPUs = d.gpuManager.AllocatedCount()
+	}
+
+	var gpuModelNames []string
+	for _, dev := range d.gpuProbe.Devices {
+		gpuModelNames = append(gpuModelNames, dev.Model)
 	}
-	d.Instances.Mu.Unlock()
 
 	resp := types.NodeStatusResponse{
-		Node:          d.node,
-		Status:        "Ready",
-		Host:          d.daemonIP(),
-		Region:        d.config.Region,
-		AZ:            d.config.AZ,
-		Uptime:        int64(time.Since(d.startTime).Seconds()),
-		Services:      d.config.GetServices(),
-		TotalVCPU:     totalVCPU,
-		TotalMemGB:    totalMemGB,
-		ReservedVCPU:  reservedVCPU,
-		ReservedMemGB: reservedMemGB,
-		AllocVCPU:     allocVCPU,
-		AllocMemGB:    allocMemGB,
-		VMCount:       vmCount,
-		InstanceTypes: caps,
+		Node:           d.node,
+		Status:         "Ready",
+		Host:           d.daemonIP(),
+		Region:         d.config.Region,
+		AZ:             d.config.AZ,
+		Uptime:         int64(time.Since(d.startTime).Seconds()),
+		Services:       d.config.GetServices(),
+		TotalVCPU:      totalVCPU,
+		TotalMemGB:     totalMemGB,
+		ReservedVCPU:   reservedVCPU,
+		ReservedMemGB:  reservedMemGB,
+		AllocVCPU:      allocVCPU,
+		AllocMemGB:     allocMemGB,
+		TotalGPUs:      totalGPUs,
+		AllocGPUs:      allocGPUs,
+		GPUCapable:     d.gpuProbe.Capable,
+		GPUPassthrough: d.gpuManager != nil,
+		GPUModels:      gpuModelNames,
+		VMCount:        vmCount,
+		InstanceTypes:  caps,
 	}
 
 	// Query service roles concurrently to halve worst-case latency (500ms vs 1s).
@@ -288,9 +317,8 @@ func fetchPredastoreRole(url string, client *http.Client) string {
 // handleNodeVMs responds with the list of VMs running on this node.
 // Used by the CLI: spx get vms.
 func (d *Daemon) handleNodeVMs(msg *nats.Msg) {
-	d.Instances.Mu.Lock()
-	vms := make([]types.VMInfo, 0, len(d.Instances.VMS))
-	for _, v := range d.Instances.VMS {
+	vms := make([]types.VMInfo, 0, d.vmMgr.Count())
+	d.vmMgr.ForEach(func(v *vm.VM) {
 		info := types.VMInfo{
 			InstanceID:   v.ID,
 			Status:       string(v.Status),
@@ -306,8 +334,7 @@ func (d *Daemon) handleNodeVMs(msg *nats.Msg) {
 			info.LaunchTime = v.Instance.LaunchTime.Unix()
 		}
 		vms = append(vms, info)
-	}
-	d.Instances.Mu.Unlock()
+	})
 
 	resp := types.NodeVMsResponse{
 		Node: d.node,

spinifex/daemon/daemon_handlers_console.go

@@ -33,10 +33,7 @@ func (d *Daemon) handleEC2GetConsoleOutput(msg *nats.Msg) {
 	instanceID := *input.InstanceId
 
 	// Find the instance on this node
-	d.Instances.Mu.Lock()
-	instance, exists := d.Instances.VMS[instanceID]
-	d.Instances.Mu.Unlock()
-
+	instance, exists := d.vmMgr.Get(instanceID)
 	if !exists {
 		respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound)
 		return

spinifex/daemon/daemon_handlers_image.go

@@ -62,25 +62,27 @@ func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) {
 	instanceID := *input.InstanceId
 
 	// Extract all instance context in a single critical section
-	d.Instances.Mu.Lock()
-	instance, ok := d.Instances.VMS[instanceID]
-	var status vm.InstanceState
-	var rootVolumeID, sourceImageID string
-	if ok {
-		status = instance.Status
-		if instance.Instance != nil {
-			for _, bdm := range instance.Instance.BlockDeviceMappings {
+	var (
+		instance      *vm.VM
+		status        vm.InstanceState
+		rootVolumeID  string
+		sourceImageID string
+	)
+	ok := d.vmMgr.UpdateState(instanceID, func(v *vm.VM) {
+		instance = v
+		status = v.Status
+		if v.Instance != nil {
+			for _, bdm := range v.Instance.BlockDeviceMappings {
 				if bdm.Ebs != nil && bdm.Ebs.VolumeId != nil {
 					rootVolumeID = *bdm.Ebs.VolumeId
 					break
 				}
 			}
-			if instance.Instance.ImageId != nil {
-				sourceImageID = *instance.Instance.ImageId
+			if v.Instance.ImageId != nil {
+				sourceImageID = *v.Instance.ImageId
 			}
 		}
-	}
-	d.Instances.Mu.Unlock()
+	})
 
 	if !ok {
 		slog.Warn("CreateImage: instance not found", "instanceId", instanceID)

spinifex/daemon/daemon_handlers_instance_test.go

@@ -0,0 +1,590 @@
+package daemon
+
+import (
+	"encoding/json"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
+	"github.com/mulgadc/spinifex/spinifex/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Tests in this file exercise the stopped/terminated daemon handlers in
+// daemon_handlers_instance.go against an in-memory vm.StateStore fake.
+// They cover error-injection paths (KV write/delete failures, retry, list
+// errors) that the JetStream-backed integration tests in daemon_handlers_test.go
+// cannot reach with a real backing bucket.
+
+// fakeStateStore is an in-memory vm.StateStore for daemon-handler unit tests.
+// Each method has an optional error knob so a test can drive specific failure
+// branches without standing up an embedded JetStream server.
+type fakeStateStore struct {
+	mu sync.Mutex
+
+	saveRunningErr error
+
+	stopped          map[string]*vm.VM
+	loadStoppedErr   error
+	writeStoppedErr  error
+	listStoppedErr   error
+	deleteStoppedErr error
+
+	// deleteStoppedFailFirst makes the first DeleteStoppedInstance call fail
+	// and the second succeed — exercises the handler's single retry.
+	deleteStoppedFailFirst bool
+	deleteStoppedAttempts  int
+
+	terminated         map[string]*vm.VM
+	writeTerminatedErr error
+	listTerminatedErr  error
+}
+
+func newFakeStateStore() *fakeStateStore {
+	return &fakeStateStore{
+		stopped:    map[string]*vm.VM{},
+		terminated: map[string]*vm.VM{},
+	}
+}
+
+func (f *fakeStateStore) SaveRunningState(_ string, _ map[string]*vm.VM) error {
+	return f.saveRunningErr
+}
+
+func (f *fakeStateStore) LoadRunningState(_ string) (map[string]*vm.VM, error) {
+	return map[string]*vm.VM{}, nil
+}
+
+func (f *fakeStateStore) WriteStoppedInstance(id string, v *vm.VM) error {
+	if f.writeStoppedErr != nil {
+		return f.writeStoppedErr
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.stopped[id] = v
+	return nil
+}
+
+func (f *fakeStateStore) LoadStoppedInstance(id string) (*vm.VM, error) {
+	if f.loadStoppedErr != nil {
+		return nil, f.loadStoppedErr
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if v, ok := f.stopped[id]; ok {
+		return v, nil
+	}
+	return nil, nil
+}
+
+func (f *fakeStateStore) DeleteStoppedInstance(id string) error {
+	f.mu.Lock()
+	f.deleteStoppedAttempts++
+	attempt := f.deleteStoppedAttempts
+	f.mu.Unlock()
+
+	if f.deleteStoppedErr != nil {
+		return f.deleteStoppedErr
+	}
+	if f.deleteStoppedFailFirst && attempt == 1 {
+		return errors.New("simulated transient delete failure")
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	delete(f.stopped, id)
+	return nil
+}
+
+func (f *fakeStateStore) ListStoppedInstances() ([]*vm.VM, error) {
+	if f.listStoppedErr != nil {
+		return nil, f.listStoppedErr
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]*vm.VM, 0, len(f.stopped))
+	for _, v := range f.stopped {
+		out = append(out, v)
+	}
+	return out, nil
+}
+
+func (f *fakeStateStore) WriteTerminatedInstance(id string, v *vm.VM) error {
+	if f.writeTerminatedErr != nil {
+		return f.writeTerminatedErr
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.terminated[id] = v
+	return nil
+}
+
+func (f *fakeStateStore) ListTerminatedInstances() ([]*vm.VM, error) {
+	if f.listTerminatedErr != nil {
+		return nil, f.listTerminatedErr
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]*vm.VM, 0, len(f.terminated))
+	for _, v := range f.terminated {
+		out = append(out, v)
+	}
+	return out, nil
+}
+
+var _ vm.StateStore = (*fakeStateStore)(nil)
+
+// daemonWithFakeStateStore returns a daemon wired with an in-memory NATS
+// connection (via createTestDaemon) and the supplied fake StateStore.
+// The daemon does not have JetStream initialized. Rewires d.instanceService
+// to point at the fake store so handlers that delegate to InstanceService
+// (e.g. ModifyInstanceAttribute) see the injected state.
+func daemonWithFakeStateStore(t *testing.T, store *fakeStateStore) *Daemon {
+	t.Helper()
+	d := createTestDaemon(t, sharedNATSURL)
+	d.stateStore = store
+	d.instanceService = handlers_ec2_instance.NewInstanceServiceImpl(
+		d.config, d.resourceMgr.instanceTypes, d.natsConn,
+		objectstore.NewMemoryObjectStore(), d.vmMgr, d.resourceMgr, store,
+	)
+	return d
+}
+
+// requestHandler subscribes fn to subject, sends a request with an
+// X-Account-ID header, and returns the reply. The subscription is cleaned up
+// when the test ends.
+func requestHandler(t *testing.T, nc *nats.Conn, subject string, fn nats.MsgHandler, accountID string, body []byte) *nats.Msg {
+	t.Helper()
+	sub, err := nc.Subscribe(subject, fn)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	msg := nats.NewMsg(subject)
+	msg.Data = body
+	msg.Header.Set(utils.AccountIDHeader, accountID)
+	reply, err := nc.RequestMsg(msg, 5*time.Second)
+	require.NoError(t, err)
+	return reply
+}
+
+func decodeError(t *testing.T, data []byte) map[string]any {
+	t.Helper()
+	var resp map[string]any
+	require.NoError(t, json.Unmarshal(data, &resp))
+	return resp
+}
+
+// stoppedVMFixture builds a minimally-valid stopped VM for handler tests.
+func stoppedVMFixture(id, accountID string) *vm.VM {
+	return &vm.VM{
+		ID:           id,
+		Status:       vm.StateStopped,
+		InstanceType: "t3.micro",
+		AccountID:    accountID,
+		Reservation: &ec2.Reservation{
+			ReservationId: aws.String("r-" + id),
+			OwnerId:       aws.String(accountID),
+		},
+		Instance: &ec2.Instance{
+			InstanceId:   aws.String(id),
+			InstanceType: aws.String("t3.micro"),
+		},
+	}
+}
+
+// --- handleEC2StartStoppedInstance ---
+
+func TestHandleEC2StartStoppedInstance_LoadError(t *testing.T) {
+	store := newFakeStateStore()
+	store.loadStoppedErr = errors.New("kv unavailable")
+	d := daemonWithFakeStateStore(t, store)
+
+	body, _ := json.Marshal(handlers_ec2_instance.StartStoppedInstanceInput{InstanceID: "i-load-fail"})
+	reply := requestHandler(t, d.natsConn, "ec2.start.test1", d.handleEC2StartStoppedInstance, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2StartStoppedInstance_StateStoreNil(t *testing.T) {
+	d := createTestDaemon(t, sharedNATSURL)
+	// d.stateStore intentionally left nil.
+
+	body, _ := json.Marshal(handlers_ec2_instance.StartStoppedInstanceInput{InstanceID: "i-no-store"})
+	reply := requestHandler(t, d.natsConn, "ec2.start.test2", d.handleEC2StartStoppedInstance, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2StartStoppedInstance_CrossTenantRejected(t *testing.T) {
+	store := newFakeStateStore()
+	store.stopped["i-foreign"] = stoppedVMFixture("i-foreign", "999988887777")
+	d := daemonWithFakeStateStore(t, store)
+
+	body, _ := json.Marshal(handlers_ec2_instance.StartStoppedInstanceInput{InstanceID: "i-foreign"})
+	reply := requestHandler(t, d.natsConn, "ec2.start.test3", d.handleEC2StartStoppedInstance, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, decodeError(t, reply.Data)["Code"])
+
+	// The instance must remain in shared KV — cross-tenant rejection cannot
+	// also remove it (would be a leak across accounts).
+	store.mu.Lock()
+	_, stillStopped := store.stopped["i-foreign"]
+	store.mu.Unlock()
+	assert.True(t, stillStopped, "cross-tenant rejection must not delete the stopped instance")
+}
+
+func TestHandleEC2StartStoppedInstance_InstanceTypeUnknown(t *testing.T) {
+	store := newFakeStateStore()
+	v := stoppedVMFixture("i-unknown-type", testAccountID)
+	v.InstanceType = "definitely.not.a.real.type"
+	store.stopped[v.ID] = v
+	d := daemonWithFakeStateStore(t, store)
+
+	body, _ := json.Marshal(handlers_ec2_instance.StartStoppedInstanceInput{InstanceID: v.ID})
+	reply := requestHandler(t, d.natsConn, "ec2.start.test4", d.handleEC2StartStoppedInstance, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, decodeError(t, reply.Data)["Code"])
+}
+
+// --- handleEC2TerminateStoppedInstance ---
+
+func TestHandleEC2TerminateStoppedInstance_LoadError(t *testing.T) {
+	store := newFakeStateStore()
+	store.loadStoppedErr = errors.New("kv unavailable")
+	d := daemonWithFakeStateStore(t, store)
+
+	body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: "i-load-fail"})
+	reply := requestHandler(t, d.natsConn, "ec2.terminate.test1", d.handleEC2TerminateStoppedInstance, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2TerminateStoppedInstance_StateStoreNil(t *testing.T) {
+	d := createTestDaemon(t, sharedNATSURL)
+
+	body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: "i-no-store"})
+	reply := requestHandler(t, d.natsConn, "ec2.terminate.test2", d.handleEC2TerminateStoppedInstance, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+// WriteTerminatedInstance failure must abort BEFORE the stopped-bucket
+// delete — otherwise an instance can vanish from both buckets.
+func TestHandleEC2TerminateStoppedInstance_WriteTerminatedFailureAborts(t *testing.T) {
+	store := newFakeStateStore()
+	store.writeTerminatedErr = errors.New("terminated bucket write failed")
+	v := stoppedVMFixture("i-write-term-fail", testAccountID)
+	store.stopped[v.ID] = v
+	d := daemonWithFakeStateStore(t, store)
+
+	body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: v.ID})
+	reply := requestHandler(t, d.natsConn, "ec2.terminate.test3", d.handleEC2TerminateStoppedInstance, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+
+	store.mu.Lock()
+	_, stillStopped := store.stopped[v.ID]
+	_, inTerminated := store.terminated[v.ID]
+	attempts := store.deleteStoppedAttempts
+	store.mu.Unlock()
+	assert.True(t, stillStopped, "stopped entry must remain when terminated write fails (caller can retry)")
+	assert.False(t, inTerminated, "no terminated entry should exist after write failure")
+	assert.Equal(t, 0, attempts, "DeleteStoppedInstance must not be called when terminated write fails")
+}
+
+// First stopped-bucket delete fails, second succeeds — instance must end up
+// only in the terminated bucket and the handler must still respond success.
+func TestHandleEC2TerminateStoppedInstance_DeleteRetrySucceeds(t *testing.T) {
+	store := newFakeStateStore()
+	store.deleteStoppedFailFirst = true
+	v := stoppedVMFixture("i-retry-success", testAccountID)
+	store.stopped[v.ID] = v
+	d := daemonWithFakeStateStore(t, store)
+
+	body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: v.ID})
+	reply := requestHandler(t, d.natsConn, "ec2.terminate.test4", d.handleEC2TerminateStoppedInstance, testAccountID, body)
+
+	var resp map[string]string
+	require.NoError(t, json.Unmarshal(reply.Data, &resp))
+	assert.Equal(t, "terminated", resp["status"])
+
+	store.mu.Lock()
+	_, stillStopped := store.stopped[v.ID]
+	_, inTerminated := store.terminated[v.ID]
+	attempts := store.deleteStoppedAttempts
+	store.mu.Unlock()
+	assert.False(t, stillStopped, "stopped entry must be removed after retry success")
+	assert.True(t, inTerminated, "terminated entry must be present")
+	assert.Equal(t, 2, attempts, "DeleteStoppedInstance must be retried exactly once")
+}
+
+// Both stopped-bucket deletes fail — the handler must still return success
+// (the terminated-bucket write is the source of truth) and must NOT roll back
+// the terminated write.
+func TestHandleEC2TerminateStoppedInstance_DeleteAlwaysFailsKeepsTerminated(t *testing.T) {
+	store := newFakeStateStore()
+	store.deleteStoppedErr = errors.New("delete persistently broken")
+	v := stoppedVMFixture("i-retry-fail", testAccountID)
+	store.stopped[v.ID] = v
+	d := daemonWithFakeStateStore(t, store)
+
+	body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: v.ID})
+	reply := requestHandler(t, d.natsConn, "ec2.terminate.test5", d.handleEC2TerminateStoppedInstance, testAccountID, body)
+
+	var resp map[string]string
+	require.NoError(t, json.Unmarshal(reply.Data, &resp))
+	assert.Equal(t, "terminated", resp["status"], "handler must report success — terminated write succeeded")
+
+	store.mu.Lock()
+	_, inTerminated := store.terminated[v.ID]
+	store.mu.Unlock()
+	assert.True(t, inTerminated, "terminated entry must NOT be rolled back when stopped delete fails")
+}
+
+func TestHandleEC2TerminateStoppedInstance_CrossTenantRejected(t *testing.T) {
+	store := newFakeStateStore()
+	store.stopped["i-foreign-term"] = stoppedVMFixture("i-foreign-term", "999988887777")
+	d := daemonWithFakeStateStore(t, store)
+
+	body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: "i-foreign-term"})
+	reply := requestHandler(t, d.natsConn, "ec2.terminate.test6", d.handleEC2TerminateStoppedInstance, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, decodeError(t, reply.Data)["Code"])
+
+	store.mu.Lock()
+	_, inTerminated := store.terminated["i-foreign-term"]
+	_, stillStopped := store.stopped["i-foreign-term"]
+	store.mu.Unlock()
+	assert.False(t, inTerminated, "foreign-tenant terminate must not write to terminated bucket")
+	assert.True(t, stillStopped, "foreign-tenant terminate must not delete the stopped entry")
+}
+
+// --- handleEC2ModifyInstanceAttribute ---
+
+func TestHandleEC2ModifyInstanceAttribute_WriteFailureReturnsServerInternal(t *testing.T) {
+	store := newFakeStateStore()
+	store.writeStoppedErr = errors.New("kv write failed")
+	v := stoppedVMFixture("i-mod-write-fail", testAccountID)
+	store.stopped[v.ID] = v
+	d := daemonWithFakeStateStore(t, store)
+
+	input := &ec2.ModifyInstanceAttributeInput{
+		InstanceId:   aws.String(v.ID),
+		InstanceType: &ec2.AttributeValue{Value: aws.String("t3.large")},
+	}
+	body, _ := json.Marshal(input)
+	reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test1", d.handleEC2ModifyInstanceAttribute, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2ModifyInstanceAttribute_LoadFailureReturnsServerInternal(t *testing.T) {
+	store := newFakeStateStore()
+	store.loadStoppedErr = errors.New("kv unavailable")
+	d := daemonWithFakeStateStore(t, store)
+
+	input := &ec2.ModifyInstanceAttributeInput{
+		InstanceId:   aws.String("i-mod-load-fail"),
+		InstanceType: &ec2.AttributeValue{Value: aws.String("t3.large")},
+	}
+	body, _ := json.Marshal(input)
+	reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test2", d.handleEC2ModifyInstanceAttribute, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2ModifyInstanceAttribute_NilInstanceFieldGuard(t *testing.T) {
+	// Stored VM with a valid status but a nil Instance pointer — the handler
+	// must reject this as a data-integrity violation rather than NPE.
+	store := newFakeStateStore()
+	v := &vm.VM{
+		ID:           "i-mod-nil-inst",
+		Status:       vm.StateStopped,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		Reservation:  &ec2.Reservation{ReservationId: aws.String("r-x"), OwnerId: aws.String(testAccountID)},
+		Instance:     nil,
+	}
+	store.stopped[v.ID] = v
+	d := daemonWithFakeStateStore(t, store)
+
+	input := &ec2.ModifyInstanceAttributeInput{
+		InstanceId:   aws.String(v.ID),
+		InstanceType: &ec2.AttributeValue{Value: aws.String("t3.large")},
+	}
+	body, _ := json.Marshal(input)
+	reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test3", d.handleEC2ModifyInstanceAttribute, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2ModifyInstanceAttribute_EmptyInstanceTypeRejected(t *testing.T) {
+	store := newFakeStateStore()
+	v := stoppedVMFixture("i-mod-empty-type", testAccountID)
+	store.stopped[v.ID] = v
+	d := daemonWithFakeStateStore(t, store)
+
+	input := &ec2.ModifyInstanceAttributeInput{
+		InstanceId:   aws.String(v.ID),
+		InstanceType: &ec2.AttributeValue{Value: aws.String("")},
+	}
+	body, _ := json.Marshal(input)
+	reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test4", d.handleEC2ModifyInstanceAttribute, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceAttributeValue, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2ModifyInstanceAttribute_CrossTenantRejected(t *testing.T) {
+	store := newFakeStateStore()
+	store.stopped["i-mod-foreign"] = stoppedVMFixture("i-mod-foreign", "999988887777")
+	d := daemonWithFakeStateStore(t, store)
+
+	input := &ec2.ModifyInstanceAttributeInput{
+		InstanceId:   aws.String("i-mod-foreign"),
+		InstanceType: &ec2.AttributeValue{Value: aws.String("t3.large")},
+	}
+	body, _ := json.Marshal(input)
+	reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test5", d.handleEC2ModifyInstanceAttribute, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, decodeError(t, reply.Data)["Code"])
+}
+
+// --- handleEC2DescribeInstanceAttribute ---
+
+func TestHandleEC2DescribeInstanceAttribute_StoppedFallback_LoadError(t *testing.T) {
+	store := newFakeStateStore()
+	store.loadStoppedErr = errors.New("kv unavailable")
+	d := daemonWithFakeStateStore(t, store)
+	// d.vmMgr has no running instance, so the handler falls through to the
+	// stopped KV branch — which now errors.
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String("i-describe-load-fail"),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	}
+	body, _ := json.Marshal(input)
+	reply := requestHandler(t, d.natsConn, "ec2.DescribeInstanceAttribute.test1", d.handleEC2DescribeInstanceAttribute, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2DescribeInstanceAttribute_StoppedFallback_HitsKV(t *testing.T) {
+	store := newFakeStateStore()
+	v := stoppedVMFixture("i-describe-stopped", testAccountID)
+	v.InstanceType = "t3.medium"
+	v.Instance.InstanceType = aws.String("t3.medium")
+	store.stopped[v.ID] = v
+	d := daemonWithFakeStateStore(t, store)
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(v.ID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	}
+	body, _ := json.Marshal(input)
+	reply := requestHandler(t, d.natsConn, "ec2.DescribeInstanceAttribute.test2", d.handleEC2DescribeInstanceAttribute, testAccountID, body)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &output))
+	require.NotNil(t, output.InstanceType)
+	require.NotNil(t, output.InstanceType.Value)
+	assert.Equal(t, "t3.medium", *output.InstanceType.Value)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_StateStoreNil(t *testing.T) {
+	d := createTestDaemon(t, sharedNATSURL)
+	// d.stateStore left nil; vmMgr also empty -> falls through to KV branch
+	// which short-circuits with ServerInternal.
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String("i-no-store-describe"),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	}
+	body, _ := json.Marshal(input)
+	reply := requestHandler(t, d.natsConn, "ec2.DescribeInstanceAttribute.test3", d.handleEC2DescribeInstanceAttribute, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+// --- handleEC2DescribeStoppedInstances / handleEC2DescribeTerminatedInstances ---
+
+func TestHandleEC2DescribeStoppedInstances_ListError(t *testing.T) {
+	store := newFakeStateStore()
+	store.listStoppedErr = errors.New("list failed")
+	d := daemonWithFakeStateStore(t, store)
+
+	reply := requestHandler(t, d.natsConn, "ec2.DescribeStoppedInstances.test1", d.handleEC2DescribeStoppedInstances, testAccountID, []byte("{}"))
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2DescribeTerminatedInstances_ListError(t *testing.T) {
+	store := newFakeStateStore()
+	store.listTerminatedErr = errors.New("list failed")
+	d := daemonWithFakeStateStore(t, store)
+
+	reply := requestHandler(t, d.natsConn, "ec2.DescribeTerminatedInstances.test1", d.handleEC2DescribeTerminatedInstances, testAccountID, []byte("{}"))
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2DescribeStoppedInstances_StateStoreNil(t *testing.T) {
+	d := createTestDaemon(t, sharedNATSURL)
+
+	reply := requestHandler(t, d.natsConn, "ec2.DescribeStoppedInstances.test2", d.handleEC2DescribeStoppedInstances, testAccountID, []byte("{}"))
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2DescribeTerminatedInstances_StateStoreNil(t *testing.T) {
+	d := createTestDaemon(t, sharedNATSURL)
+
+	reply := requestHandler(t, d.natsConn, "ec2.DescribeTerminatedInstances.test2", d.handleEC2DescribeTerminatedInstances, testAccountID, []byte("{}"))
+	assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
+}
+
+func TestHandleEC2DescribeStoppedInstances_CrossAccountIsolation(t *testing.T) {
+	store := newFakeStateStore()
+	store.stopped["i-mine"] = stoppedVMFixture("i-mine", testAccountID)
+	store.stopped["i-yours"] = stoppedVMFixture("i-yours", "999988887777")
+	d := daemonWithFakeStateStore(t, store)
+
+	reply := requestHandler(t, d.natsConn, "ec2.DescribeStoppedInstances.test3", d.handleEC2DescribeStoppedInstances, testAccountID, []byte("{}"))
+
+	var output ec2.DescribeInstancesOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &output))
+
+	var seen []string
+	for _, r := range output.Reservations {
+		for _, inst := range r.Instances {
+			if inst.InstanceId != nil {
+				seen = append(seen, *inst.InstanceId)
+			}
+		}
+	}
+	assert.ElementsMatch(t, []string{"i-mine"}, seen, "caller must only see their own instances")
+}
+
+func TestHandleEC2DescribeStoppedInstances_EmptyReturnsValidShape(t *testing.T) {
+	store := newFakeStateStore()
+	d := daemonWithFakeStateStore(t, store)
+
+	reply := requestHandler(t, d.natsConn, "ec2.DescribeStoppedInstances.test4", d.handleEC2DescribeStoppedInstances, testAccountID, []byte("{}"))
+
+	var output ec2.DescribeInstancesOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &output))
+	assert.Empty(t, output.Reservations, "empty store must produce empty reservation list")
+}
+
+// Two instances sharing a ReservationId must collapse into a single
+// reservation with both Instances attached.
+func TestHandleEC2DescribeStoppedInstances_ReservationGrouping(t *testing.T) {
+	store := newFakeStateStore()
+
+	a := stoppedVMFixture("i-grp-a", testAccountID)
+	b := stoppedVMFixture("i-grp-b", testAccountID)
+	a.Reservation.ReservationId = aws.String("r-shared")
+	b.Reservation.ReservationId = aws.String("r-shared")
+	store.stopped[a.ID] = a
+	store.stopped[b.ID] = b
+
+	d := daemonWithFakeStateStore(t, store)
+
+	reply := requestHandler(t, d.natsConn, "ec2.DescribeStoppedInstances.test5", d.handleEC2DescribeStoppedInstances, testAccountID, []byte("{}"))
+
+	var output ec2.DescribeInstancesOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &output))
+	require.Len(t, output.Reservations, 1, "shared ReservationId must collapse into one reservation")
+	require.NotNil(t, output.Reservations[0].ReservationId)
+	assert.Equal(t, "r-shared", *output.Reservations[0].ReservationId)
+	assert.Len(t, output.Reservations[0].Instances, 2)
+}

spinifex/daemon/daemon_handlers_volume.go

@@ -2,30 +2,26 @@ package daemon
 
 import (
 	"encoding/json"
-	"fmt"
+	"errors"
 	"log/slog"
-	"strconv"
-	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
-	"github.com/mulgadc/spinifex/spinifex/qmp"
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/spinifex/spinifex/vm"
 	"github.com/nats-io/nats.go"
 )
 
-// handleAttachVolume performs a three-phase hot-plug:
-//
-//	Phase 1: ebs.mount via NATS   (rolls back with ebs.unmount)
-//	Phase 2: QMP blockdev-add     (rolls back Phase 1)
-//	Phase 3: QMP device_add       (rolls back Phase 2 + Phase 1)
+// handleAttachVolume validates the volume request against the volume
+// service (existence, ownership, availability, AZ) and dispatches the
+// QMP/state-machine pipeline to vm.Manager.AttachVolume. The manager owns
+// every QMP and persistence side-effect; the daemon only emits the AWS API
+// response.
 func (d *Daemon) handleAttachVolume(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) {
 	slog.Info("Attaching volume to instance", "instanceId", command.ID)
 
-	// Validate AttachVolumeData
 	if command.AttachVolumeData == nil || command.AttachVolumeData.VolumeID == "" {
 		slog.Error("AttachVolume: missing attach volume data")
 		respondWithError(msg, awserrors.ErrorInvalidParameterValue)
@@ -33,20 +29,17 @@ func (d *Daemon) handleAttachVolume(msg *nats.Msg, command types.EC2InstanceComm
 	}
 
 	volumeID := command.AttachVolumeData.VolumeID
-	device := command.AttachVolumeData.Device
-
-	// Validate instance is running
-	d.Instances.Mu.Lock()
-	status := instance.Status
-	d.Instances.Mu.Unlock()
 
+	// Surface IncorrectInstanceState before any volume-side lookups so the
+	// caller sees the instance-state error first when both apply.
+	status := d.vmMgr.Status(instance)
 	if status != vm.StateRunning {
-		slog.Error("AttachVolume: instance not running", "instanceId", command.ID, "status", status)
+		slog.Error("AttachVolume: instance not running",
+			"instanceId", command.ID, "status", status)
 		respondWithError(msg, awserrors.ErrorIncorrectInstanceState)
 		return
 	}
 
-	// Validate volume exists and is available
 	volCfg, err := d.volumeService.GetVolumeConfig(volumeID)
 	if err != nil {
 		slog.Error("AttachVolume: failed to get volume config", "volumeId", volumeID, "err", err)
@@ -54,401 +47,86 @@ func (d *Daemon) handleAttachVolume(msg *nats.Msg, command types.EC2InstanceComm
 		return
 	}
 
-	// Account scoping: verify the caller owns this volume.
-	// Pre-Phase4 volumes (empty TenantID) are root-only; otherwise the caller
-	// must match the recorded tenant exactly.
 	callerAccountID := utils.AccountIDFromMsg(msg)
 	if !volumeVisibleTo(volCfg.VolumeMetadata.TenantID, callerAccountID) {
-		slog.Warn("AttachVolume: account does not own volume", "volumeId", volumeID, "callerAccount", callerAccountID, "ownerAccount", volCfg.VolumeMetadata.TenantID)
+		slog.Warn("AttachVolume: account does not own volume",
+			"volumeId", volumeID,
+			"callerAccount", callerAccountID,
+			"ownerAccount", volCfg.VolumeMetadata.TenantID)
 		respondWithError(msg, awserrors.ErrorInvalidVolumeNotFound)
 		return
 	}
 
 	if volCfg.VolumeMetadata.State != "available" {
-		slog.Error("AttachVolume: volume not available", "volumeId", volumeID, "state", volCfg.VolumeMetadata.State)
+		slog.Error("AttachVolume: volume not available",
+			"volumeId", volumeID, "state", volCfg.VolumeMetadata.State)
 		respondWithError(msg, awserrors.ErrorVolumeInUse)
 		return
 	}
 
-	// Check AZ compatibility — volume and instance must be in the same AZ
 	if volCfg.VolumeMetadata.AvailabilityZone != "" && d.config.AZ != "" &&
 		volCfg.VolumeMetadata.AvailabilityZone != d.config.AZ {
 		slog.Error("AttachVolume: volume and instance are in different AZs",
-			"volumeId", volumeID, "volumeAZ", volCfg.VolumeMetadata.AvailabilityZone,
+			"volumeId", volumeID,
+			"volumeAZ", volCfg.VolumeMetadata.AvailabilityZone,
 			"instanceAZ", d.config.AZ)
 		respondWithError(msg, awserrors.ErrorInvalidVolumeZoneMismatch)
 		return
 	}
 
-	// Determine device name
-	if device == "" {
-		d.Instances.Mu.Lock()
-		device = nextAvailableDevice(instance)
-		d.Instances.Mu.Unlock()
-		if device == "" {
-			slog.Error("AttachVolume: no available device names")
-			respondWithError(msg, awserrors.ErrorAttachmentLimitExceeded)
-			return
-		}
-	}
-
-	// Create EBS mount request
-	ebsRequest := types.EBSRequest{
-		Name:       volumeID,
-		DeviceName: device,
-	}
-
-	ebsMountData, err := json.Marshal(ebsRequest)
+	res, err := d.vmMgr.AttachVolume(instance.ID, volumeID, command.AttachVolumeData.Device)
 	if err != nil {
-		slog.Error("AttachVolume: failed to marshal ebs.mount request", "err", err)
-		respondWithError(msg, awserrors.ErrorServerInternal)
+		respondWithError(msg, attachDetachErrorCode(err))
 		return
 	}
 
-	mountReply, err := d.natsConn.Request(d.ebsTopic("mount"), ebsMountData, 30*time.Second)
-	if err != nil {
-		slog.Error("AttachVolume: ebs.mount failed", "volumeId", volumeID, "err", err)
-		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
-	}
-
-	var mountResp types.EBSMountResponse
-	if err := json.Unmarshal(mountReply.Data, &mountResp); err != nil {
-		slog.Error("AttachVolume: failed to unmarshal mount response", "err", err)
-		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
-	}
-
-	if mountResp.Error != "" {
-		slog.Error("AttachVolume: mount returned error", "volumeId", volumeID, "err", mountResp.Error)
-		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
-	}
-
-	nbdURI := mountResp.URI
-	if nbdURI == "" {
-		slog.Error("AttachVolume: mount response has empty URI", "volumeId", volumeID)
-		d.rollbackEBSMount(ebsRequest)
-		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
-	}
-	ebsRequest.NBDURI = nbdURI
-
-	// Parse NBDURI for QMP blockdev-add
-	serverType, socketPath, nbdHost, nbdPort, err := utils.ParseNBDURI(nbdURI)
-	if err != nil {
-		slog.Error("AttachVolume: failed to parse NBDURI", "uri", nbdURI, "err", err)
-		// Rollback: unmount
-		d.rollbackEBSMount(ebsRequest)
-		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
-	}
-
-	// Build QMP server argument
-	var serverArg map[string]any
-	if serverType == "unix" {
-		serverArg = map[string]any{"type": "unix", "path": socketPath}
-	} else {
-		serverArg = map[string]any{"type": "inet", "host": nbdHost, "port": strconv.Itoa(nbdPort)}
-	}
-
-	nodeName := fmt.Sprintf("nbd-%s", volumeID)
-	deviceID := fmt.Sprintf("vdisk-%s", volumeID)
-	iothreadID := fmt.Sprintf("ioth-%s", volumeID)
-
-	// QMP object-add: create iothread for this volume
-	iothreadCmd := qmp.QMPCommand{
-		Execute: "object-add",
-		Arguments: map[string]any{
-			"qom-type": "iothread",
-			"id":       iothreadID,
-		},
-	}
-	_, err = d.SendQMPCommand(instance.QMPClient, iothreadCmd, instance.ID)
-	if err != nil {
-		slog.Error("AttachVolume: QMP object-add iothread failed", "volumeId", volumeID, "err", err)
-		d.rollbackEBSMount(ebsRequest)
-		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
-	}
-
-	// QMP blockdev-add
-	blockdevCmd := qmp.QMPCommand{
-		Execute: "blockdev-add",
-		Arguments: map[string]any{
-			"node-name": nodeName,
-			"driver":    "nbd",
-			"server":    serverArg,
-			"export":    "",
-			"read-only": false,
-		},
-	}
-
-	_, err = d.SendQMPCommand(instance.QMPClient, blockdevCmd, instance.ID)
-	if err != nil {
-		slog.Error("AttachVolume: QMP blockdev-add failed", "volumeId", volumeID, "err", err)
-		d.rollbackEBSMount(ebsRequest)
-		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
-	}
-
-	// Determine which hotplug root port to use based on device letter.
-	// /dev/sdf -> hotplug1, /dev/sdg -> hotplug2, etc.
-	hotplugBus := ""
-	if len(device) > 0 {
-		letter := device[len(device)-1]
-		if letter >= 'f' && letter <= 'p' {
-			hotplugBus = fmt.Sprintf("hotplug%d", letter-'f'+1)
-		}
-	}
-
-	// QMP device_add
-	deviceAddArgs := map[string]any{
-		"driver":   "virtio-blk-pci",
-		"id":       deviceID,
-		"drive":    nodeName,
-		"iothread": iothreadID,
-	}
-	if hotplugBus != "" {
-		deviceAddArgs["bus"] = hotplugBus
-	}
-	deviceAddCmd := qmp.QMPCommand{
-		Execute:   "device_add",
-		Arguments: deviceAddArgs,
-	}
-
-	_, err = d.SendQMPCommand(instance.QMPClient, deviceAddCmd, instance.ID)
-	if err != nil {
-		slog.Error("AttachVolume: QMP device_add failed, rolling back blockdev", "volumeId", volumeID, "err", err)
-		// Rollback: blockdev-del then unmount
-		if _, delErr := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{
-			Execute:   "blockdev-del",
-			Arguments: map[string]any{"node-name": nodeName},
-		}, instance.ID); delErr != nil {
-			slog.Error("AttachVolume: rollback blockdev-del failed, skipping EBS unmount", "volumeId", volumeID, "err", delErr)
-		} else {
-			d.rollbackEBSMount(ebsRequest)
-		}
-		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
-	}
-
-	// Discover actual guest device name via QMP query-block.
-	// Use the retry variant because query-block may not include the device
-	// immediately after device_add.
-	guestDevice := device // fallback to AWS API name
-	deviceMap, qmpErr := queryGuestDeviceMapWait(d, instance.QMPClient, instance.ID, deviceID)
-	if qmpErr != nil {
-		slog.Warn("AttachVolume: failed to query guest device map, using API device name", "volumeId", volumeID, "err", qmpErr)
-	} else if gd, ok := deviceMap[deviceID]; ok {
-		guestDevice = gd
-		slog.Info("AttachVolume: discovered guest device", "volumeId", volumeID, "qemuDevice", deviceID, "guestDevice", guestDevice)
-	} else {
-		slog.Error("AttachVolume: device not found in QMP device map after retries, using API device name",
-			"volumeId", volumeID, "qemuDevice", deviceID, "deviceMap", deviceMap)
-	}
-
-	// Update instance state: replace existing entry for this volume (handles
-	// stop/start cycles that keep stale entries) or append a new one.
-	instance.EBSRequests.Mu.Lock()
-	replaced := false
-	for i, req := range instance.EBSRequests.Requests {
-		if req.Name == volumeID {
-			instance.EBSRequests.Requests[i] = ebsRequest
-			replaced = true
-			break
-		}
-	}
-	if !replaced {
-		instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, ebsRequest)
-	}
-	instance.EBSRequests.Mu.Unlock()
-
-	// Update BlockDeviceMappings on the ec2.Instance using actual guest device name
-	d.Instances.Mu.Lock()
-	if instance.Instance != nil {
-		now := time.Now()
-		mapping := &ec2.InstanceBlockDeviceMapping{}
-		mapping.SetDeviceName(guestDevice)
-		mapping.Ebs = &ec2.EbsInstanceBlockDevice{}
-		mapping.Ebs.SetVolumeId(volumeID)
-		mapping.Ebs.SetAttachTime(now)
-		mapping.Ebs.SetDeleteOnTermination(false)
-		mapping.Ebs.SetStatus("attached")
-		instance.Instance.BlockDeviceMappings = append(instance.Instance.BlockDeviceMappings, mapping)
-	}
-	d.Instances.Mu.Unlock()
-
-	// Update volume metadata in S3
-	if err := d.volumeService.UpdateVolumeState(volumeID, "in-use", command.ID, guestDevice); err != nil {
-		slog.Error("AttachVolume: failed to update volume metadata", "volumeId", volumeID, "err", err)
-	}
-
-	// Persist state
-	if err := d.WriteState(); err != nil {
-		slog.Error("AttachVolume: failed to write state", "err", err)
-	}
-
-	d.respondWithVolumeAttachment(msg, volumeID, command.ID, guestDevice, "attached")
-	slog.Info("Volume attached successfully", "volumeId", volumeID, "instanceId", command.ID, "apiDevice", device, "guestDevice", guestDevice)
+	d.respondWithVolumeAttachment(msg, volumeID, command.ID, res.GuestDevice, "attached")
 }
 
-// handleDetachVolume performs a three-phase hot-unplug (reverse of attach):
-//
-//	Phase 1: QMP device_del    (remove guest device)
-//	Phase 2: QMP blockdev-del  (remove block node)
-//	Phase 3: ebs.unmount NATS  (stop NBD server)
+// handleDetachVolume dispatches the QMP/state-machine pipeline to
+// vm.Manager.DetachVolume and emits the AWS API response.
 func (d *Daemon) handleDetachVolume(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) {
 	slog.Info("Detaching volume from instance", "instanceId", command.ID)
 
-	// Validate DetachVolumeData
 	if command.DetachVolumeData == nil || command.DetachVolumeData.VolumeID == "" {
 		slog.Error("DetachVolume: missing detach volume data")
 		respondWithError(msg, awserrors.ErrorInvalidParameterValue)
 		return
 	}
 
-	volumeID := command.DetachVolumeData.VolumeID
-	device := command.DetachVolumeData.Device
-	force := command.DetachVolumeData.Force
-
-	// Validate instance is running
-	d.Instances.Mu.Lock()
-	status := instance.Status
-	d.Instances.Mu.Unlock()
-
-	if status != vm.StateRunning {
-		slog.Error("DetachVolume: instance not running", "instanceId", command.ID, "status", status)
-		respondWithError(msg, awserrors.ErrorIncorrectInstanceState)
+	deviceName, err := d.vmMgr.DetachVolume(
+		instance.ID,
+		command.DetachVolumeData.VolumeID,
+		command.DetachVolumeData.Device,
+		command.DetachVolumeData.Force,
+	)
+	if err != nil {
+		respondWithError(msg, attachDetachErrorCode(err))
 		return
 	}
 
-	// Find the volume in EBSRequests
-	instance.EBSRequests.Mu.Lock()
-	var ebsReq types.EBSRequest
-	found := false
-	for _, req := range instance.EBSRequests.Requests {
-		if req.Name == volumeID {
-			ebsReq = req
-			found = true
-			break
-		}
-	}
-	instance.EBSRequests.Mu.Unlock()
+	d.respondWithVolumeAttachment(msg, command.DetachVolumeData.VolumeID, command.ID, deviceName, "detaching")
+}
 
-	if !found {
-		slog.Error("DetachVolume: volume not attached to instance", "volumeId", volumeID, "instanceId", command.ID)
-		respondWithError(msg, awserrors.ErrorIncorrectState)
-		return
-	}
-
-	// Reject detaching boot/EFI/CloudInit volumes
-	if ebsReq.Boot || ebsReq.EFI || ebsReq.CloudInit {
-		slog.Error("DetachVolume: cannot detach boot/EFI/CloudInit volume", "volumeId", volumeID)
-		respondWithError(msg, awserrors.ErrorOperationNotPermitted)
-		return
-	}
-
-	// Optional device cross-check
-	if device != "" && ebsReq.DeviceName != "" && device != ebsReq.DeviceName {
-		slog.Error("DetachVolume: device mismatch", "requested", device, "actual", ebsReq.DeviceName)
-		respondWithError(msg, awserrors.ErrorInvalidParameterValue)
-		return
-	}
-
-	deviceID := fmt.Sprintf("vdisk-%s", volumeID)
-	nodeName := fmt.Sprintf("nbd-%s", volumeID)
-	iothreadID := fmt.Sprintf("ioth-%s", volumeID)
-
-	// Phase 1: QMP device_del (remove guest device).
-	// Idempotent: a prior detach may have already removed the device but
-	// failed at blockdev-del, leaving the volume's external state intact.
-	// Treat DeviceNotFound as success so the retry can drive blockdev-del
-	// to completion without bailing on the now-absent guest device.
-	_, err := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{
-		Execute:   "device_del",
-		Arguments: map[string]any{"id": deviceID},
-	}, instance.ID)
+// attachDetachErrorCode maps a vm.Manager error returned by AttachVolume
+// or DetachVolume to the AWS API error code that the SDK expects.
+func attachDetachErrorCode(err error) string {
 	switch {
-	case err == nil:
-	case isQMPDeviceNotFound(err):
-		slog.Info("DetachVolume: guest device already removed (resuming detach)", "volumeId", volumeID, "err", err)
-	case force:
-		slog.Warn("DetachVolume: QMP device_del failed (force=true, continuing)", "volumeId", volumeID, "err", err)
+	case errors.Is(err, vm.ErrInstanceNotFound):
+		return awserrors.ErrorInvalidInstanceIDNotFound
+	case errors.Is(err, vm.ErrInvalidTransition):
+		return awserrors.ErrorIncorrectInstanceState
+	case errors.Is(err, vm.ErrAttachmentLimitExceeded):
+		return awserrors.ErrorAttachmentLimitExceeded
+	case errors.Is(err, vm.ErrVolumeNotAttached):
+		return awserrors.ErrorIncorrectState
+	case errors.Is(err, vm.ErrVolumeNotDetachable):
+		return awserrors.ErrorOperationNotPermitted
+	case errors.Is(err, vm.ErrVolumeDeviceMismatch):
+		return awserrors.ErrorInvalidParameterValue
 	default:
-		slog.Error("DetachVolume: QMP device_del failed", "volumeId", volumeID, "err", err)
-		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
+		return awserrors.ErrorServerInternal
 	}
-
-	// Brief pause for guest to acknowledge PCI removal
-	time.Sleep(d.detachDelay)
-
-	// Phase 2: QMP blockdev-del (remove block node).
-	// Poll-retry on "node is in use": after device_del, NBD client teardown
-	// and any in-flight I/O can hold the block node briefly. Bailing here
-	// without retry leaves the guest device gone but the block node intact,
-	// and the next AWS-CLI retry hits DeviceNotFound on device_del with no
-	// path forward (mulga-siv-41).
-	blockdevErr := d.tryBlockdevDel(instance, nodeName)
-	if blockdevErr != nil {
-		// Block node still referenced after retry budget exhausted; do not
-		// clean up state or unmount — tearing down the NBD server would
-		// crash the VM, and removing metadata would allow the volume to be
-		// double-attached.
-		slog.Error("DetachVolume: QMP blockdev-del failed, leaving volume state intact", "volumeId", volumeID, "err", blockdevErr)
-		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
-	}
-
-	// Phase 2b: QMP object-del (remove iothread, best-effort)
-	_, iothreadErr := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{
-		Execute:   "object-del",
-		Arguments: map[string]any{"id": iothreadID},
-	}, instance.ID)
-	if iothreadErr != nil {
-		slog.Warn("DetachVolume: QMP object-del iothread failed (non-fatal)", "volumeId", volumeID, "err", iothreadErr)
-	}
-
-	// Phase 3: ebs.unmount via NATS (best-effort)
-	d.rollbackEBSMount(ebsReq)
-
-	// State cleanup: remove volume from EBSRequests (search by name to avoid stale index)
-	instance.EBSRequests.Mu.Lock()
-	for i, req := range instance.EBSRequests.Requests {
-		if req.Name == volumeID {
-			instance.EBSRequests.Requests = append(instance.EBSRequests.Requests[:i], instance.EBSRequests.Requests[i+1:]...)
-			break
-		}
-	}
-	instance.EBSRequests.Mu.Unlock()
-
-	// Remove from BlockDeviceMappings
-	d.Instances.Mu.Lock()
-	if instance.Instance != nil {
-		filtered := make([]*ec2.InstanceBlockDeviceMapping, 0, len(instance.Instance.BlockDeviceMappings))
-		for _, bdm := range instance.Instance.BlockDeviceMappings {
-			if bdm.Ebs != nil && bdm.Ebs.VolumeId != nil && *bdm.Ebs.VolumeId == volumeID {
-				continue
-			}
-			filtered = append(filtered, bdm)
-		}
-		instance.Instance.BlockDeviceMappings = filtered
-	}
-	d.Instances.Mu.Unlock()
-
-	// Update volume metadata to "available"
-	if err := d.volumeService.UpdateVolumeState(volumeID, "available", "", ""); err != nil {
-		slog.Error("DetachVolume: failed to update volume metadata", "volumeId", volumeID, "err", err)
-	}
-
-	// Persist state
-	if err := d.WriteState(); err != nil {
-		slog.Error("DetachVolume: failed to write state", "err", err)
-	}
-
-	d.respondWithVolumeAttachment(msg, volumeID, command.ID, ebsReq.DeviceName, "detaching")
-	slog.Info("Volume detached successfully", "volumeId", volumeID, "instanceId", command.ID)
 }
 
 func (d *Daemon) handleEC2CreateVolume(msg *nats.Msg) {
@@ -517,62 +195,3 @@ func (d *Daemon) handleEC2ModifyVolume(msg *nats.Msg) {
 func (d *Daemon) handleEC2DeleteVolume(msg *nats.Msg) {
 	handleNATSRequest(msg, d.volumeService.DeleteVolume)
 }
-
-// tryBlockdevDel issues blockdev-del with bounded retry on "is in use"
-// errors. After device_del, NBD client teardown and any in-flight guest
-// I/O can briefly hold the block node; QEMU surfaces this as a
-// GenericError carrying "Node <name> is in use". Polling at d.detachDelay
-// gives the I/O drain enough time without a hardcoded long sleep.
-func (d *Daemon) tryBlockdevDel(instance *vm.VM, nodeName string) error {
-	const maxAttempts = 20 // 20 × detachDelay (default 1s) = 20s budget
-	var lastErr error
-	for attempt := 1; attempt <= maxAttempts; attempt++ {
-		_, err := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{
-			Execute:   "blockdev-del",
-			Arguments: map[string]any{"node-name": nodeName},
-		}, instance.ID)
-		if err == nil {
-			if attempt > 1 {
-				slog.Info("DetachVolume: blockdev-del succeeded after retry",
-					"nodeName", nodeName, "attempts", attempt)
-			}
-			return nil
-		}
-		lastErr = err
-		if !isQMPNodeInUse(err) {
-			return err
-		}
-		if attempt == maxAttempts {
-			break
-		}
-		slog.Debug("DetachVolume: blockdev-del busy, retrying",
-			"nodeName", nodeName, "attempt", attempt, "err", err)
-		time.Sleep(d.detachDelay)
-	}
-	return lastErr
-}
-
-// isQMPDeviceNotFound returns true when err is a QMP DeviceNotFound class
-// error from device_del. Used to make device_del idempotent across detach
-// retries — the second AWS-CLI call must not bail when QEMU reports the
-// device is already gone (mulga-siv-41).
-func isQMPDeviceNotFound(err error) bool {
-	if err == nil {
-		return false
-	}
-	s := err.Error()
-	return strings.Contains(s, "DeviceNotFound")
-}
-
-// isQMPNodeInUse returns true when err is a QMP GenericError reporting a
-// block node still in use. blockdev-del fails this way while NBD client
-// teardown and queued I/O drain after device_del; the caller polls until
-// the node is released or the budget expires. Match "in use" rather than
-// "is in use" — QEMU phrases this as both "X is in use" and "X is still
-// in use" depending on the path.
-func isQMPNodeInUse(err error) bool {
-	if err == nil {
-		return false
-	}
-	return strings.Contains(err.Error(), "in use")
-}

spinifex/daemon/daemon_handlers_volume_test.go

@@ -0,0 +1,88 @@
+package daemon
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/stretchr/testify/assert"
+)
+
+// TestAttachDetachErrorCode locks down the manager-error → AWS-API-code
+// mapping that handleAttachVolume and handleDetachVolume both call. Wrong
+// mapping silently breaks AWS-SDK retry semantics: clients expect 4xx
+// codes for caller-fixable problems and 5xx for server faults. A future
+// edit that drops a sentinel branch would otherwise pass the mechanical
+// "tests still compile" bar with no signal.
+func TestAttachDetachErrorCode(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want string
+	}{
+		{
+			name: "ErrInstanceNotFound maps to InvalidInstanceID.NotFound",
+			err:  vm.ErrInstanceNotFound,
+			want: awserrors.ErrorInvalidInstanceIDNotFound,
+		},
+		{
+			name: "wrapped ErrInstanceNotFound still matches via errors.Is",
+			err:  fmt.Errorf("manager: %w", vm.ErrInstanceNotFound),
+			want: awserrors.ErrorInvalidInstanceIDNotFound,
+		},
+		{
+			name: "ErrInvalidTransition maps to IncorrectInstanceState",
+			err:  vm.ErrInvalidTransition,
+			want: awserrors.ErrorIncorrectInstanceState,
+		},
+		{
+			name: "wrapped ErrInvalidTransition still matches",
+			err:  fmt.Errorf("cannot attach in state stopped: %w", vm.ErrInvalidTransition),
+			want: awserrors.ErrorIncorrectInstanceState,
+		},
+		{
+			name: "ErrAttachmentLimitExceeded maps to AttachmentLimitExceeded",
+			err:  vm.ErrAttachmentLimitExceeded,
+			want: awserrors.ErrorAttachmentLimitExceeded,
+		},
+		{
+			name: "ErrVolumeNotAttached maps to IncorrectState",
+			err:  vm.ErrVolumeNotAttached,
+			want: awserrors.ErrorIncorrectState,
+		},
+		{
+			name: "wrapped ErrVolumeNotAttached still matches",
+			err:  fmt.Errorf("%w: vol-1", vm.ErrVolumeNotAttached),
+			want: awserrors.ErrorIncorrectState,
+		},
+		{
+			name: "ErrVolumeNotDetachable maps to OperationNotPermitted",
+			err:  vm.ErrVolumeNotDetachable,
+			want: awserrors.ErrorOperationNotPermitted,
+		},
+		{
+			name: "ErrVolumeDeviceMismatch maps to InvalidParameterValue",
+			err:  vm.ErrVolumeDeviceMismatch,
+			want: awserrors.ErrorInvalidParameterValue,
+		},
+		{
+			name: "unknown error falls through to ServerInternal",
+			err:  errors.New("QMP blockdev-add: connection refused"),
+			want: awserrors.ErrorServerInternal,
+		},
+		{
+			name: "wrapped unknown error falls through to ServerInternal",
+			err:  fmt.Errorf("manager: %w", errors.New("nbdkit timeout")),
+			want: awserrors.ErrorServerInternal,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := attachDetachErrorCode(tt.err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

spinifex/daemon/daemon_handlers_vpc.go

@@ -78,19 +78,27 @@ func (d *Daemon) handleAccountCreated(msg *nats.Msg) {
 	if _, err := d.vpcService.EnsureDefaultVPC(evt.AccountID); err != nil {
 		slog.Error("Failed to create default VPC for new account",
 			"accountID", evt.AccountID, "error", err)
+		// Skip IGW setup — the VPC is missing or half-built. The next daemon
+		// startup or handleAccountCreated event will retry.
+		return
 	}
-	d.ensureDefaultVPCInfrastructure()
+	d.ensureDefaultVPCInfrastructure(nil)
 }
 
-// ensureDefaultVPCInfrastructure creates an IGW and default security group for
-// each default VPC that doesn't already have them. Matches AWS behavior where
-// the default VPC comes with an attached IGW and a default security group.
-func (d *Daemon) ensureDefaultVPCInfrastructure() {
+// ensureDefaultVPCInfrastructure attaches an IGW and a default 0.0.0.0/0 route
+// to each account's default VPC. The default SG is provisioned by
+// EnsureDefaultVPC / CreateVpc itself, so this routine is IGW-only. Accounts
+// in skipAccounts are not touched — used to avoid attaching infrastructure to
+// a half-built VPC when EnsureDefaultVPC failed earlier in startup.
+func (d *Daemon) ensureDefaultVPCInfrastructure(skipAccounts map[string]struct{}) {
 	if d.igwService == nil || d.vpcService == nil {
 		return
 	}
 
 	for _, accountID := range []string{utils.GlobalAccountID, admin.DefaultAccountID()} {
+		if _, skip := skipAccounts[accountID]; skip {
+			continue
+		}
 		// Find the default VPC for this account
 		descOut, err := d.vpcService.DescribeVpcs(&ec2.DescribeVpcsInput{}, accountID)
 		if err != nil {
@@ -150,59 +158,6 @@ func (d *Daemon) ensureDefaultVPCInfrastructure() {
 		if d.routeTableService != nil {
 			d.ensureDefaultIGWRoute(accountID, defaultVpcId)
 		}
-
-		// Check if default security group exists for this VPC
-		sgOut, err := d.vpcService.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{}, accountID)
-		if err != nil {
-			continue
-		}
-		hasDefaultSG := false
-		for _, sg := range sgOut.SecurityGroups {
-			if sg.VpcId != nil && *sg.VpcId == defaultVpcId && sg.GroupName != nil && *sg.GroupName == "default" {
-				hasDefaultSG = true
-				break
-			}
-		}
-		if !hasDefaultSG {
-			desc := "default VPC security group"
-			groupName := "default"
-			createSGOut, err := d.vpcService.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
-				GroupName:   &groupName,
-				Description: &desc,
-				VpcId:       &defaultVpcId,
-			}, accountID)
-			if err != nil {
-				slog.Error("Failed to create default security group", "vpcId", defaultVpcId, "err", err)
-				continue
-			}
-			sgId := *createSGOut.GroupId
-
-			// Default SG rules (AWS behavior):
-			// - Allow all inbound from same SG (self-referencing)
-			// - Allow all outbound to 0.0.0.0/0
-			allProto := "-1"
-			allCidr := "0.0.0.0/0"
-			_, _ = d.vpcService.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
-				GroupId: &sgId,
-				IpPermissions: []*ec2.IpPermission{
-					{
-						IpProtocol:       &allProto,
-						UserIdGroupPairs: []*ec2.UserIdGroupPair{{GroupId: &sgId}},
-					},
-				},
-			}, accountID)
-			_, _ = d.vpcService.AuthorizeSecurityGroupEgress(&ec2.AuthorizeSecurityGroupEgressInput{
-				GroupId: &sgId,
-				IpPermissions: []*ec2.IpPermission{
-					{
-						IpProtocol: &allProto,
-						IpRanges:   []*ec2.IpRange{{CidrIp: &allCidr}},
-					},
-				},
-			}, accountID)
-			slog.Info("Created default security group for default VPC",
-				"groupId", sgId, "vpcId", defaultVpcId, "accountID", accountID)
-		}
 	}
 }
 

spinifex/daemon/daemon_system_instance.go

@@ -225,7 +225,7 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 				vpcID = *result.NetworkInterfaces[0].VpcId
 			}
 			portName := "port-" + instance.ENIId
-			d.publishNATEvent("vpc.add-nat", vpcID, publicIP, privateIP, portName, instance.ENIMac)
+			utils.PublishNATEvent(d.natsConn, "vpc.add-nat", vpcID, publicIP, privateIP, portName, instance.ENIMac)
 			instance.PublicIP = publicIP
 			instance.PublicIPPool = poolName
 			slog.Info("LaunchSystemInstance: allocated public IP via direct IPAM",
@@ -237,15 +237,13 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 	}
 
 	// Add to daemon state so LaunchInstance can find it
-	d.Instances.Mu.Lock()
-	d.Instances.VMS[instance.ID] = instance
-	d.Instances.Mu.Unlock()
+	d.vmMgr.Insert(instance)
 
 	if err := d.WriteState(); err != nil {
 		slog.Warn("LaunchSystemInstance: failed to write state", "instanceId", instance.ID, "err", err)
 	}
 
-	// Subscribe to per-instance NATS topic for terminate commands
+	// Subscribe to per-instance NATS topic for terminate commands.
 	d.mu.Lock()
 	sub, subErr := d.natsConn.Subscribe(fmt.Sprintf("ec2.cmd.%s", instance.ID), d.handleEC2Events)
 	if subErr != nil {
@@ -257,7 +255,7 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 
 	// Pre-compute dev MAC for dual-NIC cloud-init
 	if d.config.Daemon.DevNetworking && instance.ENIId != "" {
-		instance.DevMAC = generateDevMAC(instance.ID)
+		instance.DevMAC = vm.GenerateDevMAC(instance.ID)
 	}
 
 	// Management NIC: allocate IP, generate MAC, create TAP on br-mgmt
@@ -278,7 +276,8 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 			instance.MgmtMAC = generateMgmtMAC(instance.ID)
 			instance.MgmtIP = mgmtIP
 
-			tapName, tapErr := SetupMgmtTapDevice(instance.ID, instance.MgmtMAC, mgmtBridge)
+			tapName := vm.MgmtTapName(instance.ID)
+			tapErr := d.networkPlumber.SetupTap(vm.TapSpec{Name: tapName, Bridge: mgmtBridge})
 			if tapErr != nil {
 				d.mgmtIPAllocator.Release(instance.ID)
 				instance.MgmtMAC = ""
@@ -289,7 +288,6 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 				}
 				slog.Error("LaunchSystemInstance: failed to setup mgmt tap", "instanceId", instance.ID, "err", tapErr)
 			} else {
-				instance.MgmtTap = tapName
 				slog.Info("LaunchSystemInstance: mgmt NIC configured",
 					"instanceId", instance.ID, "mgmtIP", mgmtIP, "mgmtMAC", instance.MgmtMAC, "mgmtTap", tapName)
 			}
@@ -324,7 +322,7 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 	}
 
 	// Launch QEMU VM
-	if err := d.LaunchInstance(instance); err != nil {
+	if err := d.vmMgr.Run(instance); err != nil {
 		d.cleanupFailedSystemInstance(instance, instanceType)
 		return nil, fmt.Errorf("launch instance: %w", err)
 	}
@@ -347,23 +345,15 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 
 // TerminateSystemInstance stops and cleans up a system-managed VM.
 func (d *Daemon) TerminateSystemInstance(instanceID string) error {
-	d.Instances.Mu.Lock()
-	instance, exists := d.Instances.VMS[instanceID]
-	d.Instances.Mu.Unlock()
-
+	instance, exists := d.vmMgr.Get(instanceID)
 	if !exists {
 		return fmt.Errorf("instance %s not found", instanceID)
 	}
 
-	// Transition to shutting-down
-	if err := d.TransitionState(instance, vm.StateShuttingDown); err != nil {
-		return fmt.Errorf("transition to shutting-down: %w", err)
-	}
-
 	// Release EIP through the EIP service for system VMs whose public IP was
 	// allocated via AllocateAddress (internet-facing ALBs). Clears the fields
-	// so the generic teardown path in stopInstance doesn't try to
-	// double-release the same IP through externalIPAM directly.
+	// so vm.Manager.Terminate's ReleasePublicIP doesn't double-release the
+	// same IP via externalIPAM.
 	if instance.PublicIPAllocID != "" && d.eipService != nil {
 		eniAccount := instance.AccountID
 		if instance.PublicIPAssocID != "" {
@@ -380,100 +370,38 @@ func (d *Daemon) TerminateSystemInstance(instanceID string) error {
 		} else {
 			slog.Info("TerminateSystemInstance: released EIP", "instanceId", instanceID, "ip", instance.PublicIP, "allocationId", instance.PublicIPAllocID)
 		}
-		instance.PublicIP = ""
-		instance.PublicIPPool = ""
-		instance.PublicIPAllocID = ""
-		instance.PublicIPAssocID = ""
+		d.vmMgr.UpdateState(instance.ID, func(v *vm.VM) {
+			v.PublicIP = ""
+			v.PublicIPPool = ""
+			v.PublicIPAllocID = ""
+			v.PublicIPAssocID = ""
+		})
 	}
 
-	// Stop the VM (QEMU shutdown, volume unmount, tap cleanup)
-	if err := d.stopInstance(map[string]*vm.VM{instanceID: instance}, true); err != nil {
-		slog.Error("TerminateSystemInstance: stopInstance failed", "instanceId", instanceID, "err", err)
-		return fmt.Errorf("stop instance: %w", err)
-	}
-
-	if err := d.TransitionState(instance, vm.StateTerminated); err != nil {
-		slog.Warn("TerminateSystemInstance: failed to transition to terminated", "instanceId", instanceID, "err", err)
-	}
-
-	// Write to terminated KV bucket (auto-expires)
-	if d.jsManager != nil {
-		if err := d.jsManager.WriteTerminatedInstance(instanceID, instance); err != nil {
-			slog.Warn("TerminateSystemInstance: failed to write terminated state", "instanceId", instanceID, "err", err)
-		}
-	}
-
-	// Clean up local state
-	d.Instances.Mu.Lock()
-	delete(d.Instances.VMS, instanceID)
-	d.Instances.Mu.Unlock()
-
-	// Unsubscribe from per-instance NATS topic
-	d.mu.Lock()
-	if sub, ok := d.natsSubscriptions[instanceID]; ok {
-		if err := sub.Unsubscribe(); err != nil {
-			slog.Warn("TerminateSystemInstance: failed to unsubscribe", "instanceId", instanceID, "err", err)
-		}
-		delete(d.natsSubscriptions, instanceID)
-	}
-	d.mu.Unlock()
-
-	// Deallocate resources
-	if it, ok := d.resourceMgr.instanceTypes[instance.InstanceType]; ok {
-		d.resourceMgr.deallocate(it)
+	if err := d.vmMgr.Terminate(instanceID); err != nil {
+		slog.Error("TerminateSystemInstance: vmMgr.Terminate failed", "instanceId", instanceID, "err", err)
+		return fmt.Errorf("terminate: %w", err)
 	}
 
 	slog.Info("TerminateSystemInstance completed", "instanceId", instanceID)
 	return nil
 }
 
-// cleanupFailedSystemInstance handles cleanup when a system instance launch fails
-// after partial setup (state added, volumes created, etc).
-func (d *Daemon) cleanupFailedSystemInstance(instance *vm.VM, instanceType *ec2.InstanceTypeInfo) {
-	d.markInstanceFailed(instance, "system_instance_launch_failed")
-	d.resourceMgr.deallocate(instanceType)
-
-	// Clean up management TAP and release IP
-	if instance.MgmtTap != "" {
-		if err := CleanupMgmtTapDevice(instance.MgmtTap); err != nil {
-			slog.Warn("Failed to cleanup mgmt tap on failed launch", "tap", instance.MgmtTap, "err", err)
-		}
-		if d.mgmtIPAllocator != nil {
-			d.mgmtIPAllocator.Release(instance.ID)
-		}
-	}
-
-	// Clean up ENI if we auto-created one (not pre-created)
-	// Pre-created ENIs are managed by the caller (e.g. ELBv2 service)
-	if instance.ENIId != "" && d.vpcService != nil {
-		// Only clean up ENIs we created (description starts with "System interface")
-		result, err := d.vpcService.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{
-			NetworkInterfaceIds: []*string{aws.String(instance.ENIId)},
-		}, utils.GlobalAccountID)
-		if err == nil && len(result.NetworkInterfaces) > 0 {
-			desc := aws.StringValue(result.NetworkInterfaces[0].Description)
-			if len(desc) >= 16 && desc[:16] == "System interface" {
-				if _, delErr := d.vpcService.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
-					NetworkInterfaceId: aws.String(instance.ENIId),
-				}, utils.GlobalAccountID); delErr != nil {
-					slog.Warn("Failed to cleanup system ENI", "eniId", instance.ENIId, "err", delErr)
-				}
-			}
-		}
-	}
+// cleanupFailedSystemInstance handles cleanup when a system instance launch
+// fails after partial setup (state added, volumes created, etc). Delegates
+// to vm.Manager.MarkFailed which runs the synchronous teardown chain
+// (volume unmount/delete, tap cleanup, ENI delete, IP release, resource
+// deallocation, state migration to terminated KV).
+func (d *Daemon) cleanupFailedSystemInstance(instance *vm.VM, _ *ec2.InstanceTypeInfo) {
+	d.vmMgr.MarkFailed(instance, "system_instance_launch_failed")
 }
 
 // WaitForSystemInstance polls until the instance reaches running state or times out.
 func (d *Daemon) WaitForSystemInstance(instanceID string, timeout time.Duration) error {
 	deadline := time.Now().Add(timeout)
 	for time.Now().Before(deadline) {
-		d.Instances.Mu.Lock()
-		inst, exists := d.Instances.VMS[instanceID]
 		var status vm.InstanceState
-		if exists {
-			status = inst.Status
-		}
-		d.Instances.Mu.Unlock()
+		exists := d.vmMgr.UpdateState(instanceID, func(v *vm.VM) { status = v.Status })
 
 		if !exists {
 			return fmt.Errorf("instance %s disappeared", instanceID)

spinifex/daemon/daemon_system_instance_test.go

@@ -7,14 +7,16 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/vm"
 )
 
-func TestWaitForSystemInstance_AlreadyRunning(t *testing.T) {
-	d := &Daemon{
-		Instances: vm.Instances{
-			VMS: map[string]*vm.VM{
-				"i-test1": {ID: "i-test1", Status: vm.StateRunning},
-			},
-		},
+func newDaemonWithVMs(vms ...*vm.VM) *Daemon {
+	d := &Daemon{vmMgr: vm.NewManager()}
+	for _, v := range vms {
+		d.vmMgr.Insert(v)
 	}
+	return d
+}
+
+func TestWaitForSystemInstance_AlreadyRunning(t *testing.T) {
+	d := newDaemonWithVMs(&vm.VM{ID: "i-test1", Status: vm.StateRunning})
 
 	err := d.WaitForSystemInstance("i-test1", 1*time.Second)
 	if err != nil {
@@ -23,11 +25,7 @@ func TestWaitForSystemInstance_AlreadyRunning(t *testing.T) {
 }
 
 func TestWaitForSystemInstance_NotFound(t *testing.T) {
-	d := &Daemon{
-		Instances: vm.Instances{
-			VMS: map[string]*vm.VM{},
-		},
-	}
+	d := newDaemonWithVMs()
 
 	err := d.WaitForSystemInstance("i-nonexistent", 500*time.Millisecond)
 	if err == nil {
@@ -36,13 +34,7 @@ func TestWaitForSystemInstance_NotFound(t *testing.T) {
 }
 
 func TestWaitForSystemInstance_ErrorState(t *testing.T) {
-	d := &Daemon{
-		Instances: vm.Instances{
-			VMS: map[string]*vm.VM{
-				"i-failed": {ID: "i-failed", Status: vm.StateError},
-			},
-		},
-	}
+	d := newDaemonWithVMs(&vm.VM{ID: "i-failed", Status: vm.StateError})
 
 	err := d.WaitForSystemInstance("i-failed", 1*time.Second)
 	if err == nil {
@@ -52,20 +44,12 @@ func TestWaitForSystemInstance_ErrorState(t *testing.T) {
 
 func TestWaitForSystemInstance_TransitionsToRunning(t *testing.T) {
 	inst := &vm.VM{ID: "i-pending", Status: vm.StateProvisioning}
-	d := &Daemon{
-		Instances: vm.Instances{
-			VMS: map[string]*vm.VM{
-				"i-pending": inst,
-			},
-		},
-	}
+	d := newDaemonWithVMs(inst)
 
 	// Transition to running after a short delay
 	go func() {
 		time.Sleep(200 * time.Millisecond)
-		d.Instances.Mu.Lock()
-		inst.Status = vm.StateRunning
-		d.Instances.Mu.Unlock()
+		d.vmMgr.UpdateState(inst.ID, func(v *vm.VM) { v.Status = vm.StateRunning })
 	}()
 
 	err := d.WaitForSystemInstance("i-pending", 2*time.Second)
@@ -75,13 +59,7 @@ func TestWaitForSystemInstance_TransitionsToRunning(t *testing.T) {
 }
 
 func TestWaitForSystemInstance_Timeout(t *testing.T) {
-	d := &Daemon{
-		Instances: vm.Instances{
-			VMS: map[string]*vm.VM{
-				"i-stuck": {ID: "i-stuck", Status: vm.StateProvisioning},
-			},
-		},
-	}
+	d := newDaemonWithVMs(&vm.VM{ID: "i-stuck", Status: vm.StateProvisioning})
 
 	err := d.WaitForSystemInstance("i-stuck", 600*time.Millisecond)
 	if err == nil {

spinifex/daemon/disconnect_test.go

@@ -0,0 +1,455 @@
+package daemon
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestDaemonModeDefaultStandalone — Mode() returns standalone before Start().
+func TestDaemonModeDefaultStandalone(t *testing.T) {
+	clusterCfg := &config.ClusterConfig{
+		Node:  "node-1",
+		Nodes: map[string]config.Config{"node-1": {}},
+	}
+	d, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
+
+	assert.Equal(t, DaemonModeStandalone, d.Mode())
+	assert.Equal(t, int64(0), d.NATSRetryCount())
+}
+
+// TestDaemonModeFlipsOnConnectAndDisconnect — connecting flips to cluster,
+// killing NATS flips back to standalone.
+func TestDaemonModeFlipsOnConnectAndDisconnect(t *testing.T) {
+	port := freePortForTest(t)
+	ns := startTestNATSOnPortForTest(t, port)
+
+	clusterCfg := &config.ClusterConfig{
+		Node:  "node-1",
+		Nodes: map[string]config.Config{"node-1": {NATS: config.NATSConfig{Host: ns.ClientURL()}}},
+	}
+	d, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
+
+	require.NoError(t, d.connectNATS())
+	defer d.natsConn.Close()
+	assert.Equal(t, DaemonModeCluster, d.Mode())
+
+	ns.Shutdown()
+	require.Eventually(t, func() bool { return d.Mode() == DaemonModeStandalone }, 3*time.Second, 50*time.Millisecond,
+		"mode should flip to standalone after NATS shutdown")
+}
+
+// TestDaemonReconnectBumpsRetryCount — reconnect callback fires, count++.
+func TestDaemonReconnectBumpsRetryCount(t *testing.T) {
+	port := freePortForTest(t)
+	ns := startTestNATSOnPortForTest(t, port)
+
+	clusterCfg := &config.ClusterConfig{
+		Node:  "node-1",
+		Nodes: map[string]config.Config{"node-1": {NATS: config.NATSConfig{Host: ns.ClientURL()}}},
+	}
+	d, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
+
+	require.NoError(t, d.connectNATS())
+	defer d.natsConn.Close()
+	require.Equal(t, int64(0), d.NATSRetryCount())
+
+	ns.Shutdown()
+	require.Eventually(t, func() bool { return !d.natsConn.IsConnected() }, 3*time.Second, 50*time.Millisecond)
+
+	startTestNATSOnPortForTest(t, port)
+	require.Eventually(t, func() bool { return d.Mode() == DaemonModeCluster }, 5*time.Second, 50*time.Millisecond,
+		"mode should flip back to cluster on reconnect")
+	assert.GreaterOrEqual(t, d.NATSRetryCount(), int64(1))
+}
+
+func freePortForTest(t *testing.T) int {
+	t.Helper()
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	addr, ok := l.Addr().(*net.TCPAddr)
+	require.True(t, ok)
+	port := addr.Port
+	require.NoError(t, l.Close())
+	return port
+}
+
+func startTestNATSOnPortForTest(t *testing.T, port int) *server.Server {
+	t.Helper()
+	opts := &server.Options{Host: "127.0.0.1", Port: port, NoLog: true, NoSigs: true}
+	ns, err := server.NewServer(opts)
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(func() { ns.Shutdown() })
+	return ns
+}
+
+// TestMode_DefaultStandalone — both signals false ⇒ standalone (zero-value
+// guard for callers reading Mode() before any initialisation).
+func TestMode_DefaultStandalone(t *testing.T) {
+	d := &Daemon{}
+	assert.Equal(t, DaemonModeStandalone, d.Mode())
+}
+
+// TestMode_RequiresBothSignals — Mode() reports cluster only when both NATS
+// is connected and peers are reachable.
+func TestMode_RequiresBothSignals(t *testing.T) {
+	d := &Daemon{}
+
+	d.natsConnected.Store(true)
+	d.peersReachable.Store(false)
+	assert.Equal(t, DaemonModeStandalone, d.Mode(), "nats up, peers unreachable ⇒ standalone")
+
+	d.natsConnected.Store(false)
+	d.peersReachable.Store(true)
+	assert.Equal(t, DaemonModeStandalone, d.Mode(), "nats down, peers reachable ⇒ standalone")
+
+	d.natsConnected.Store(true)
+	d.peersReachable.Store(true)
+	assert.Equal(t, DaemonModeCluster, d.Mode(), "both up ⇒ cluster")
+}
+
+// TestOnNATSDisconnect_FlipsMode — direct unit test of the disconnect callback
+// without needing an actual NATS disconnect roundtrip.
+func TestOnNATSDisconnect_FlipsMode(t *testing.T) {
+	d := &Daemon{}
+	d.natsConnected.Store(true)
+	d.peersReachable.Store(true)
+	require.Equal(t, DaemonModeCluster, d.Mode())
+
+	d.onNATSDisconnect(nil, nil)
+
+	assert.Equal(t, DaemonModeStandalone, d.Mode())
+}
+
+// TestOnNATSReconnect_NoJetStreamManager_DoesNotPanic — reconnect with
+// jsManager nil marks NATS connected + bumps counter but skips the
+// goroutine WriteState. peersReachable is forced true so the derived mode
+// flips back to cluster.
+func TestOnNATSReconnect_NoJetStreamManager_DoesNotPanic(t *testing.T) {
+	d := &Daemon{}
+	d.peersReachable.Store(true)
+
+	d.onNATSReconnect(nil)
+
+	assert.Equal(t, DaemonModeCluster, d.Mode())
+	assert.Equal(t, int64(1), d.NATSRetryCount())
+}
+
+// TestLocalStatePath_NilConfig_UsesDefault — defensive path used by callers
+// that build a Daemon without populating Config (older test fixtures, etc).
+func TestLocalStatePath_NilConfig_UsesDefault(t *testing.T) {
+	d := &Daemon{}
+	assert.Equal(t, "/var/lib/spinifex/state/instance-state.json", d.localStatePath())
+}
+
+// TestLocalStatePath_RootedAtDataDir — verifies 1a's DataDir-rooted layout.
+func TestLocalStatePath_RootedAtDataDir(t *testing.T) {
+	d := &Daemon{config: &config.Config{DataDir: "/var/lib/spinifex/n1"}}
+	assert.Equal(t, "/var/lib/spinifex/n1/state/instance-state.json", d.localStatePath())
+}
+
+// TestDaemonWriteState_LocalFile — WriteState persists to the configured DataDir
+// even with a nil jsManager (best-effort KV is the second half; local file
+// must succeed standalone).
+func TestDaemonWriteState_LocalFile(t *testing.T) {
+	dataDir := t.TempDir()
+	d := &Daemon{
+		config: &config.Config{DataDir: dataDir},
+		vmMgr:  vm.NewManager(),
+	}
+	d.vmMgr.Insert(&vm.VM{ID: "i-w1", InstanceType: "t3.micro"})
+
+	require.NoError(t, d.WriteState())
+
+	state, err := ReadLocalState(filepath.Join(dataDir, "state", "instance-state.json"))
+	require.NoError(t, err)
+	require.NotNil(t, state)
+	assert.Equal(t, "t3.micro", state.VMS["i-w1"].InstanceType)
+}
+
+// TestDaemonLoadState_MissingFile — fresh-install path: empty map, no error.
+func TestDaemonLoadState_MissingFile(t *testing.T) {
+	d := &Daemon{
+		config: &config.Config{DataDir: t.TempDir()},
+		vmMgr:  vm.NewManager(),
+	}
+	require.NoError(t, d.LoadState())
+	assert.Equal(t, 0, d.vmMgr.Count())
+}
+
+// TestDaemonLoadState_RoundTrip — write, then read back into a fresh daemon
+// rooted at the same DataDir.
+func TestDaemonLoadState_RoundTrip(t *testing.T) {
+	dataDir := t.TempDir()
+	writer := &Daemon{
+		config: &config.Config{DataDir: dataDir},
+		vmMgr:  vm.NewManager(),
+	}
+	writer.vmMgr.Insert(&vm.VM{ID: "i-rt1", InstanceType: "m5.large"})
+	require.NoError(t, writer.WriteState())
+
+	reader := &Daemon{
+		config: &config.Config{DataDir: dataDir},
+		vmMgr:  vm.NewManager(),
+	}
+	require.NoError(t, reader.LoadState())
+	assert.Equal(t, 1, reader.vmMgr.Count())
+	got, ok := reader.vmMgr.Get("i-rt1")
+	require.True(t, ok)
+	assert.Equal(t, "m5.large", got.InstanceType)
+}
+
+// TestDaemonLoadState_CorruptFile — corruption is fatal, daemon refuses start.
+func TestDaemonLoadState_CorruptFile(t *testing.T) {
+	dataDir := t.TempDir()
+	d := &Daemon{config: &config.Config{DataDir: dataDir}}
+
+	path := d.localStatePath()
+	require.NoError(t, writeCorruptStateFile(path))
+
+	err := d.LoadState()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "read local state")
+}
+
+func writeCorruptStateFile(path string) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
+		return err
+	}
+	return os.WriteFile(path, []byte("{not json"), 0o600)
+}
+
+// TestStartLocal_SucceedsWithoutNATS — §1d: startLocal() brings the daemon up
+// (cluster manager HTTPS, local state load, ready flag) without contacting
+// NATS. /local/status reports standalone mode immediately.
+func TestStartLocal_SucceedsWithoutNATS(t *testing.T) {
+	tmpDir := t.TempDir()
+	certPEM, keyPEM := generateTestCert(t)
+	certPath := filepath.Join(tmpDir, "server.pem")
+	keyPath := filepath.Join(tmpDir, "server.key")
+	require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
+	require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
+
+	addr, err := freeAddrForTest()
+	require.NoError(t, err)
+
+	cfg := &config.ClusterConfig{
+		Node: "node-1",
+		Nodes: map[string]config.Config{
+			"node-1": {
+				BaseDir: tmpDir,
+				DataDir: tmpDir,
+				NATS:    config.NATSConfig{Host: "nats://127.0.0.1:1"}, // unreachable
+				Daemon: config.DaemonConfig{
+					Host:    addr,
+					TLSCert: certPath,
+					TLSKey:  keyPath,
+				},
+			},
+		},
+	}
+
+	d, err := NewDaemon(cfg)
+	require.NoError(t, err)
+	d.configPath = filepath.Join(tmpDir, "spinifex.toml")
+
+	require.NoError(t, d.startLocal())
+	defer func() {
+		if d.clusterServer != nil {
+			_ = d.clusterServer.Close()
+		}
+	}()
+
+	assert.True(t, d.ready.Load(), "ready flag should be set after startLocal")
+	assert.Equal(t, DaemonModeStandalone, d.Mode())
+
+	// /local/status served via the cluster manager TLS listener.
+	client := &http.Client{
+		Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
+		Timeout:   2 * time.Second,
+	}
+	resp, err := client.Get(fmt.Sprintf("https://%s/local/status", addr))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	var body LocalStatus
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&body))
+	assert.Equal(t, DaemonModeStandalone, body.Mode)
+	assert.Equal(t, natsDisconnected, body.NATS)
+}
+
+// TestAssertNoClusterServicesInitialised_GuardsTier1Invariant — §1e-audit:
+// the invariant guard at the end of startLocal must reject any state where a
+// cluster-scoped handle has been initialised before startCluster runs. A
+// future edit that hoists JetStream/KV-touching init into startLocal would
+// re-introduce the boot-time NATS dependency that 1d removed; this test
+// pins the regression.
+func TestAssertNoClusterServicesInitialised_GuardsTier1Invariant(t *testing.T) {
+	clean := &Daemon{}
+	require.NoError(t, clean.assertNoClusterServicesInitialised(),
+		"freshly-constructed daemon must satisfy the Tier 1 invariant")
+
+	dirty := &Daemon{jsManager: &JetStreamManager{}}
+	err := dirty.assertNoClusterServicesInitialised()
+	require.Error(t, err, "non-nil cluster handle must trip the invariant")
+	assert.Contains(t, err.Error(), "jsManager")
+}
+
+// TestStartCluster_StrictRequireNATS_BoundedAndExits — §1d-strict:
+// SPINIFEX_REQUIRE_NATS=1 caps the first connect to requireNATSTimeout and
+// invokes exitFunc(1) when the bounded retry expires. Validates the dev/test
+// fail-fast path without flipping the prod default.
+func TestStartCluster_StrictRequireNATS_BoundedAndExits(t *testing.T) {
+	t.Setenv("SPINIFEX_REQUIRE_NATS", "1")
+
+	tmpDir := t.TempDir()
+	cfg := &config.ClusterConfig{
+		Node: "node-1",
+		Nodes: map[string]config.Config{
+			"node-1": {
+				BaseDir: tmpDir,
+				DataDir: tmpDir,
+				NATS:    config.NATSConfig{Host: "nats://127.0.0.1:1"}, // unreachable
+			},
+		},
+	}
+
+	d, err := NewDaemon(cfg)
+	require.NoError(t, err)
+	d.requireNATSTimeout = 200 * time.Millisecond
+	d.natsRetryOpts = []utils.RetryOption{utils.WithRetryDelay(20 * time.Millisecond)}
+
+	exitCh := make(chan int, 1)
+	d.exitFunc = func(code int) { exitCh <- code }
+
+	done := make(chan error, 1)
+	go func() { done <- d.startCluster() }()
+
+	select {
+	case code := <-exitCh:
+		assert.Equal(t, 1, code, "strict-mode timeout must request exit(1)")
+	case <-time.After(3 * time.Second):
+		t.Fatal("strict-mode startCluster did not invoke exitFunc within deadline")
+	}
+
+	select {
+	case err := <-done:
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "connect NATS (strict)")
+	case <-time.After(3 * time.Second):
+		t.Fatal("startCluster did not return after exitFunc was called")
+	}
+}
+
+// TestStartCluster_NoStrictEnv_UsesInfiniteRetry — §1d-strict:
+// without SPINIFEX_REQUIRE_NATS, startCluster keeps the Tier 1 infinite-retry
+// path. exitFunc must not fire, even when NATS is unreachable. Cancellation
+// via d.ctx is the only way out.
+func TestStartCluster_NoStrictEnv_UsesInfiniteRetry(t *testing.T) {
+	t.Setenv("SPINIFEX_REQUIRE_NATS", "")
+
+	tmpDir := t.TempDir()
+	cfg := &config.ClusterConfig{
+		Node: "node-1",
+		Nodes: map[string]config.Config{
+			"node-1": {
+				BaseDir: tmpDir,
+				DataDir: tmpDir,
+				NATS:    config.NATSConfig{Host: "nats://127.0.0.1:1"}, // unreachable
+			},
+		},
+	}
+
+	d, err := NewDaemon(cfg)
+	require.NoError(t, err)
+	d.natsRetryOpts = []utils.RetryOption{utils.WithRetryDelay(20 * time.Millisecond)}
+
+	exitCalled := make(chan int, 1)
+	d.exitFunc = func(code int) { exitCalled <- code }
+
+	done := make(chan error, 1)
+	go func() { done <- d.startCluster() }()
+
+	// Give the loop time to attempt several retries; it must not exit.
+	select {
+	case code := <-exitCalled:
+		t.Fatalf("default mode must not invoke exitFunc; got code=%d", code)
+	case <-time.After(300 * time.Millisecond):
+	}
+
+	d.cancel()
+
+	select {
+	case err := <-done:
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "connect NATS")
+		assert.NotContains(t, err.Error(), "strict")
+	case <-time.After(3 * time.Second):
+		t.Fatal("startCluster did not return after ctx cancellation")
+	}
+}
+
+// TestStartCluster_RetriesUntilContextCancelled — §1d: startCluster's NATS
+// connect loop honours d.ctx. Cancelling the daemon context unblocks an
+// otherwise-infinite retry.
+func TestStartCluster_RetriesUntilContextCancelled(t *testing.T) {
+	tmpDir := t.TempDir()
+	cfg := &config.ClusterConfig{
+		Node: "node-1",
+		Nodes: map[string]config.Config{
+			"node-1": {
+				BaseDir: tmpDir,
+				DataDir: tmpDir,
+				NATS:    config.NATSConfig{Host: "nats://127.0.0.1:1"}, // unreachable
+			},
+		},
+	}
+
+	d, err := NewDaemon(cfg)
+	require.NoError(t, err)
+	// Bound retry delay so ctx-cancel unblocks promptly.
+	d.natsRetryOpts = []utils.RetryOption{utils.WithRetryDelay(50 * time.Millisecond)}
+
+	done := make(chan error, 1)
+	go func() { done <- d.startCluster() }()
+
+	time.Sleep(200 * time.Millisecond)
+	d.cancel()
+
+	select {
+	case err := <-done:
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "connect NATS")
+	case <-time.After(3 * time.Second):
+		t.Fatal("startCluster did not return after ctx cancellation")
+	}
+}
+
+func freeAddrForTest() (string, error) {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		return "", err
+	}
+	addr := l.Addr().String()
+	_ = l.Close()
+	return addr, nil
+}

spinifex/daemon/gpu_adapter.go

@@ -0,0 +1,26 @@
+package daemon
+
+import (
+	handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
+)
+
+// daemonGPUClaimer adapts the daemon's gpu.Manager + GPUModelOverrides config to
+// the transport-neutral handlers_ec2_instance.GPUClaimer surface. Keeps the gpu
+// package out of the instance service.
+type daemonGPUClaimer struct {
+	d *Daemon
+}
+
+var _ handlers_ec2_instance.GPUClaimer = (*daemonGPUClaimer)(nil)
+
+func (g *daemonGPUClaimer) Claim(instanceID string) (string, bool, error) {
+	dev, err := g.d.gpuManager.Claim(instanceID)
+	if err != nil {
+		return "", false, err
+	}
+	return dev.PCIAddress, gpuXVGAEnabled(dev, g.d.config.Daemon.GPUModelOverrides), nil
+}
+
+func (g *daemonGPUClaimer) Release(instanceID string) error {
+	return g.d.gpuManager.Release(instanceID)
+}

spinifex/daemon/health.go

@@ -1,295 +0,0 @@
-package daemon
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log/slog"
-	"os"
-	"os/exec"
-	"syscall"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/vm"
-)
-
-const (
-	maxRestartsInWindow = 3
-	restartWindow       = 10 * time.Minute
-	restartBackoffBase  = 5 * time.Second
-	restartBackoffMax   = 2 * time.Minute
-)
-
-// restartBackoff computes the exponential backoff delay for the given
-// restart count. Pure function — no side effects.
-func restartBackoff(restartCount int) time.Duration {
-	delay := restartBackoffBase
-	for range restartCount {
-		delay *= 2
-		if delay > restartBackoffMax {
-			return restartBackoffMax
-		}
-	}
-	return delay
-}
-
-// classifyCrashReason extracts a human-readable crash reason from the error
-// returned by cmd.Wait(). Uses exec.ExitError + syscall.WaitStatus to
-// distinguish OOM kills (SIGKILL), segfaults (SIGSEGV), etc.
-func classifyCrashReason(waitErr error) string {
-	if waitErr == nil {
-		return "clean-exit"
-	}
-
-	var exitErr *exec.ExitError
-	if !errors.As(waitErr, &exitErr) {
-		return "unknown"
-	}
-
-	status, ok := exitErr.Sys().(syscall.WaitStatus)
-	if !ok {
-		return "unknown"
-	}
-
-	if status.Signaled() {
-		switch status.Signal() {
-		case syscall.SIGKILL:
-			return "oom-killed"
-		case syscall.SIGSEGV:
-			return "segfault"
-		case syscall.SIGABRT:
-			return "abort"
-		default:
-			return fmt.Sprintf("signal-%d", status.Signal())
-		}
-	}
-
-	if status.Exited() {
-		return fmt.Sprintf("exit-%d", status.ExitStatus())
-	}
-
-	return "unknown"
-}
-
-// handleInstanceCrash is called by the QEMU launch goroutine after cmd.Wait()
-// returns during runtime (after startup was confirmed successful). It detects
-// the crash reason, transitions the instance to error state, cleans up resources,
-// and schedules a restart if policy allows.
-func (d *Daemon) handleInstanceCrash(instance *vm.VM, waitErr error) {
-	// Guard: if instance is not running, this was an expected exit
-	// (stopInstance/terminateInstance set status before QEMU exits)
-	d.Instances.Mu.Lock()
-	status := instance.Status
-	d.Instances.Mu.Unlock()
-
-	if status != vm.StateRunning {
-		slog.Debug("QEMU exited but instance not in running state, skipping crash handler",
-			"instance", instance.ID, "status", status)
-		return
-	}
-
-	// Guard: coordinated shutdown in progress
-	if d.shuttingDown.Load() {
-		slog.Debug("QEMU exited during coordinated shutdown, skipping crash handler",
-			"instance", instance.ID)
-		return
-	}
-
-	reason := classifyCrashReason(waitErr)
-	slog.Error("VM process crashed", "instance", instance.ID, "reason", reason, "err", waitErr)
-
-	// Transition to error state
-	if err := d.TransitionState(instance, vm.StateError); err != nil {
-		slog.Error("Failed to transition crashed instance to error state",
-			"instance", instance.ID, "err", err)
-	}
-
-	// Update health tracking
-	now := time.Now()
-	d.Instances.Mu.Lock()
-	instance.Health.CrashCount++
-	instance.Health.LastCrashTime = now
-	instance.Health.LastCrashReason = reason
-	if instance.Health.FirstCrashTime.IsZero() {
-		instance.Health.FirstCrashTime = now
-	}
-	instance.Running = false
-	instance.PID = 0
-	d.Instances.Mu.Unlock()
-
-	// Deallocate resources to fix phantom reservation
-	instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType]
-	if ok && instanceType != nil {
-		slog.Info("Deallocating resources for crashed instance",
-			"instance", instance.ID, "type", instance.InstanceType)
-		d.resourceMgr.deallocate(instanceType)
-	}
-
-	// Clean up stale QMP socket so QEMU can rebind on restart
-	if instance.Config.QMPSocket != "" {
-		_ = os.Remove(instance.Config.QMPSocket)
-	}
-
-	// Unmount EBS volumes (same pattern as stopInstance)
-	d.unmountInstanceVolumes(instance)
-
-	// Persist state
-	if err := d.WriteState(); err != nil {
-		slog.Error("Failed to persist state after crash handling",
-			"instance", instance.ID, "err", err)
-	}
-
-	d.maybeRestartInstance(instance)
-}
-
-// unmountInstanceVolumes sends NATS unmount requests for all volumes attached
-// to the instance and updates their state to "available".
-func (d *Daemon) unmountInstanceVolumes(instance *vm.VM) {
-	instance.EBSRequests.Mu.Lock()
-	defer instance.EBSRequests.Mu.Unlock()
-
-	for _, ebsRequest := range instance.EBSRequests.Requests {
-		ebsUnMountRequest, err := json.Marshal(ebsRequest)
-		if err != nil {
-			slog.Error("Failed to marshal volume payload for crash cleanup",
-				"err", err)
-			continue
-		}
-
-		msg, err := d.natsConn.Request(d.ebsTopic("unmount"), ebsUnMountRequest, 30*time.Second)
-		if err != nil {
-			slog.Error("Failed to unmount volume after crash",
-				"name", ebsRequest.Name, "instance", instance.ID, "err", err)
-		} else {
-			slog.Info("Unmounted volume after crash",
-				"instance", instance.ID, "volume", ebsRequest.Name, "data", string(msg.Data))
-		}
-
-		// Update user-visible volume state to "available"
-		if !ebsRequest.EFI && !ebsRequest.CloudInit {
-			if err := d.volumeService.UpdateVolumeState(ebsRequest.Name, "available", "", ""); err != nil {
-				slog.Error("Failed to update volume state to available after crash",
-					"volumeId", ebsRequest.Name, "err", err)
-			}
-		}
-	}
-}
-
-// maybeRestartInstance checks restart policy and schedules a restart if allowed.
-func (d *Daemon) maybeRestartInstance(instance *vm.VM) {
-	if d.shuttingDown.Load() {
-		slog.Info("Skipping restart during shutdown", "instance", instance.ID)
-		return
-	}
-
-	now := time.Now()
-
-	d.Instances.Mu.Lock()
-	health := &instance.Health
-
-	// If crashes are outside the restart window, reset counters
-	if !health.FirstCrashTime.IsZero() && now.Sub(health.FirstCrashTime) > restartWindow {
-		slog.Info("Crash window expired, resetting counters", "instance", instance.ID)
-		health.CrashCount = 1
-		health.FirstCrashTime = now
-		health.RestartCount = 0
-	}
-
-	// Check if we've exceeded the max restarts in the window
-	if health.CrashCount > maxRestartsInWindow {
-		d.Instances.Mu.Unlock()
-		slog.Error("Instance exceeded max restarts in window, leaving in error state",
-			"instance", instance.ID,
-			"crashes", health.CrashCount,
-			"window", restartWindow,
-			"max", maxRestartsInWindow)
-		return
-	}
-
-	restartCount := health.RestartCount
-	d.Instances.Mu.Unlock()
-
-	// Check resource availability
-	instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType]
-	if !ok || instanceType == nil {
-		slog.Error("Unknown instance type, cannot restart",
-			"instance", instance.ID, "type", instance.InstanceType)
-		return
-	}
-
-	if d.resourceMgr.canAllocate(instanceType, 1) < 1 {
-		slog.Error("Insufficient resources to restart instance",
-			"instance", instance.ID, "type", instance.InstanceType)
-		return
-	}
-
-	delay := restartBackoff(restartCount)
-
-	slog.Info("Scheduling instance restart",
-		"instance", instance.ID,
-		"delay", delay,
-		"restartCount", restartCount+1)
-
-	time.AfterFunc(delay, func() {
-		d.restartCrashedInstance(instance)
-	})
-}
-
-// restartCrashedInstance re-verifies the instance is still in error state
-// and relaunches it via LaunchInstance.
-func (d *Daemon) restartCrashedInstance(instance *vm.VM) {
-	d.Instances.Mu.Lock()
-	if instance.Status != vm.StateError {
-		d.Instances.Mu.Unlock()
-		slog.Info("Instance no longer in error state, skipping restart",
-			"instance", instance.ID, "status", instance.Status)
-		return
-	}
-
-	if d.shuttingDown.Load() {
-		d.Instances.Mu.Unlock()
-		slog.Info("Daemon shutting down, skipping restart", "instance", instance.ID)
-		return
-	}
-
-	instance.Health.RestartCount++
-	d.Instances.Mu.Unlock()
-
-	slog.Info("Restarting crashed instance",
-		"instance", instance.ID,
-		"restartCount", instance.Health.RestartCount)
-
-	// Re-allocate resources before relaunch — handleCrashedInstance deallocates
-	// them, so we must re-reserve before starting the VM again.
-	instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType]
-	if !ok || instanceType == nil {
-		slog.Error("Unknown instance type during restart, cannot re-allocate resources",
-			"instance", instance.ID, "type", instance.InstanceType)
-		return
-	}
-	if err := d.resourceMgr.allocate(instanceType); err != nil {
-		slog.Error("Insufficient resources to restart crashed instance",
-			"instance", instance.ID, "type", instance.InstanceType, "err", err)
-		return
-	}
-
-	// Transition Error → Pending (valid now that we added it to the transitions map)
-	if err := d.TransitionState(instance, vm.StatePending); err != nil {
-		slog.Error("Failed to transition instance to pending for restart",
-			"instance", instance.ID, "err", err)
-		d.resourceMgr.deallocate(instanceType)
-		return
-	}
-
-	// LaunchInstance handles the full relaunch flow:
-	// check PID → MountVolumes → StartInstance → CreateQMPClient → NATS subscribe → Running
-	if err := d.LaunchInstance(instance); err != nil {
-		slog.Error("Failed to restart crashed instance",
-			"instance", instance.ID, "err", err)
-		d.resourceMgr.deallocate(instanceType)
-		if err := d.TransitionState(instance, vm.StateError); err != nil {
-			slog.Error("Failed to transition instance back to error after restart failure",
-				"instance", instance.ID, "err", err)
-		}
-	}
-}

spinifex/daemon/health_test.go

@@ -1,481 +0,0 @@
-package daemon
-
-import (
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/config"
-	"github.com/mulgadc/spinifex/spinifex/vm"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestClassifyCrashReason_CleanExit(t *testing.T) {
-	assert.Equal(t, "clean-exit", classifyCrashReason(nil))
-}
-
-func TestClassifyCrashReason_OOMKill(t *testing.T) {
-	// Start a process and kill it with SIGKILL to get a real ExitError
-	cmd := exec.Command("sleep", "60")
-	require.NoError(t, cmd.Start())
-
-	require.NoError(t, cmd.Process.Kill()) // sends SIGKILL
-	waitErr := cmd.Wait()
-	require.Error(t, waitErr)
-
-	assert.Equal(t, "oom-killed", classifyCrashReason(waitErr))
-}
-
-func TestClassifyCrashReason_ExitCode(t *testing.T) {
-	// Run a command that exits with a non-zero code
-	cmd := exec.Command("sh", "-c", "exit 42")
-	waitErr := cmd.Run()
-	require.Error(t, waitErr)
-
-	assert.Equal(t, "exit-42", classifyCrashReason(waitErr))
-}
-
-func TestClassifyCrashReason_Unknown(t *testing.T) {
-	// A non-ExitError error
-	err := fmt.Errorf("some random error")
-	assert.Equal(t, "unknown", classifyCrashReason(err))
-}
-
-func TestHandleInstanceCrash_SkipsNonRunning(t *testing.T) {
-	nc, err := nats.Connect(sharedNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	rm, err := NewResourceManager()
-	require.NoError(t, err)
-
-	d := &Daemon{
-		natsConn:    nc,
-		resourceMgr: rm,
-		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
-	}
-
-	instance := &vm.VM{
-		ID:     "i-test-stopped",
-		Status: vm.StateStopped,
-	}
-	d.Instances.VMS[instance.ID] = instance
-
-	// Should return without action (no panic, no state change)
-	d.handleInstanceCrash(instance, fmt.Errorf("test error"))
-	assert.Equal(t, vm.StateStopped, instance.Status)
-}
-
-func TestHandleInstanceCrash_SkipsShuttingDown(t *testing.T) {
-	nc, err := nats.Connect(sharedNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	rm, err := NewResourceManager()
-	require.NoError(t, err)
-
-	d := &Daemon{
-		natsConn:    nc,
-		resourceMgr: rm,
-		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
-	}
-	d.shuttingDown.Store(true)
-
-	instance := &vm.VM{
-		ID:     "i-test-shutdown",
-		Status: vm.StateRunning,
-	}
-	d.Instances.VMS[instance.ID] = instance
-
-	d.handleInstanceCrash(instance, fmt.Errorf("test error"))
-
-	// Status unchanged because shuttingDown is true
-	assert.Equal(t, vm.StateRunning, instance.Status)
-}
-
-func TestMaybeRestart_ExceedsMaxInWindow(t *testing.T) {
-	nc, err := nats.Connect(sharedNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	rm, err := NewResourceManager()
-	require.NoError(t, err)
-
-	d := &Daemon{
-		node:     "test-node",
-		natsConn: nc,
-		config: &config.Config{
-			Services: []string{"daemon"},
-		},
-		resourceMgr: rm,
-		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
-	}
-
-	// Find an allocatable instance type
-	var allocType string
-	for typeName := range d.resourceMgr.instanceTypes {
-		allocType = typeName
-		break
-	}
-	require.NotEmpty(t, allocType)
-
-	instance := &vm.VM{
-		ID:           "i-test-maxcrash",
-		Status:       vm.StateError,
-		InstanceType: allocType,
-		Health: vm.InstanceHealthState{
-			CrashCount:     maxRestartsInWindow + 1, // exceeds limit
-			FirstCrashTime: time.Now(),
-			RestartCount:   maxRestartsInWindow,
-		},
-	}
-	d.Instances.VMS[instance.ID] = instance
-
-	// Should not schedule a restart (no panic, instance stays in error)
-	d.maybeRestartInstance(instance)
-
-	// Instance should remain in error state
-	assert.Equal(t, vm.StateError, instance.Status)
-}
-
-func TestMaybeRestart_ResetsAfterWindow(t *testing.T) {
-	nc, err := nats.Connect(sharedNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	rm, err := NewResourceManager()
-	require.NoError(t, err)
-
-	d := &Daemon{
-		node:     "test-node",
-		natsConn: nc,
-		config: &config.Config{
-			Services: []string{"daemon"},
-		},
-		resourceMgr: rm,
-		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
-	}
-
-	// Find an allocatable instance type
-	var allocType string
-	for typeName := range d.resourceMgr.instanceTypes {
-		allocType = typeName
-		break
-	}
-	require.NotEmpty(t, allocType)
-
-	instance := &vm.VM{
-		ID:           "i-test-windowreset",
-		Status:       vm.StateError,
-		InstanceType: allocType,
-		Health: vm.InstanceHealthState{
-			CrashCount:     5,
-			FirstCrashTime: time.Now().Add(-restartWindow - time.Minute), // outside window
-			RestartCount:   5,
-		},
-	}
-	d.Instances.VMS[instance.ID] = instance
-
-	// maybeRestartInstance should reset counters and schedule a restart
-	// (restart will fail because there's no real QEMU, but counters should reset)
-	d.maybeRestartInstance(instance)
-
-	// Counters should be reset
-	assert.Equal(t, 1, instance.Health.CrashCount)
-	assert.Equal(t, 0, instance.Health.RestartCount)
-}
-
-func TestRestartBackoff_Exponential(t *testing.T) {
-	tests := []struct {
-		restartCount int
-		want         time.Duration
-	}{
-		{0, 5 * time.Second},
-		{1, 10 * time.Second},
-		{2, 20 * time.Second},
-		{3, 40 * time.Second},
-		{4, 80 * time.Second},
-		{5, 120 * time.Second},   // capped at 2min
-		{6, 120 * time.Second},   // stays capped
-		{100, 120 * time.Second}, // large count stays capped
-	}
-
-	for _, tc := range tests {
-		got := restartBackoff(tc.restartCount)
-		assert.Equal(t, tc.want, got, "Backoff mismatch at restart count %d", tc.restartCount)
-	}
-}
-
-// newTestDaemon creates a Daemon with shared NATS and a ResourceManager for health tests.
-// jsManager is nil — WriteState will error (logged, not fatal).
-func newTestDaemon(t *testing.T) (*Daemon, func()) {
-	t.Helper()
-	nc, err := nats.Connect(sharedNATSURL)
-	require.NoError(t, err)
-
-	rm, err := NewResourceManager()
-	require.NoError(t, err)
-
-	d := &Daemon{
-		node:     "test-node",
-		natsConn: nc,
-		config: &config.Config{
-			Services: []string{"daemon"},
-		},
-		resourceMgr: rm,
-		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
-	}
-	return d, func() { nc.Close() }
-}
-
-// smallestAllocType returns the name of the smallest allocatable instance type
-// in the resource manager, considering both vCPU and memory to ensure it fits
-// on the current host (important for CI runners with limited resources).
-func smallestAllocType(t *testing.T, rm *ResourceManager) string {
-	t.Helper()
-	var smallest string
-	var smallestScore int64 = 1<<63 - 1
-	for name, it := range rm.instanceTypes {
-		vcpu := instanceTypeVCPUs(it)
-		memMiB := instanceTypeMemoryMiB(it)
-		// Score by total resource footprint so we pick the type smallest in both dimensions
-		score := vcpu*1024 + memMiB
-		if score < smallestScore {
-			smallestScore = score
-			smallest = name
-		}
-	}
-	if smallest == "" {
-		t.Fatal("resource manager has no instance types")
-	}
-	return smallest
-}
-
-func TestHandleInstanceCrash_CoreFlow(t *testing.T) {
-	d, cleanup := newTestDaemon(t)
-	defer cleanup()
-
-	allocType := smallestAllocType(t, d.resourceMgr)
-	instanceType := d.resourceMgr.instanceTypes[allocType]
-	require.NoError(t, d.resourceMgr.allocate(instanceType))
-
-	// Record resource levels before crash
-	d.resourceMgr.mu.RLock()
-	allocVCPUBefore := d.resourceMgr.allocatedVCPU
-	allocMemBefore := d.resourceMgr.allocatedMem
-	d.resourceMgr.mu.RUnlock()
-	require.Greater(t, allocVCPUBefore, 0)
-
-	// Create a temp QMP socket file to verify removal
-	tmpDir := t.TempDir()
-	qmpPath := filepath.Join(tmpDir, "qmp.sock")
-	require.NoError(t, os.WriteFile(qmpPath, []byte("dummy"), 0o600))
-
-	instance := &vm.VM{
-		ID:           "i-test-core",
-		Status:       vm.StateRunning,
-		Running:      true,
-		PID:          12345,
-		InstanceType: allocType,
-		Config:       vm.Config{QMPSocket: qmpPath},
-	}
-	d.Instances.VMS[instance.ID] = instance
-
-	d.handleInstanceCrash(instance, fmt.Errorf("test crash"))
-
-	// Status transitioned to error
-	assert.Equal(t, vm.StateError, instance.Status)
-
-	// Health tracking updated
-	assert.Equal(t, 1, instance.Health.CrashCount)
-	assert.False(t, instance.Health.LastCrashTime.IsZero())
-	assert.Equal(t, "unknown", instance.Health.LastCrashReason) // fmt.Errorf is not *exec.ExitError
-	assert.False(t, instance.Health.FirstCrashTime.IsZero())
-
-	// Running and PID cleared
-	assert.False(t, instance.Running)
-	assert.Equal(t, 0, instance.PID)
-
-	// Resources deallocated
-	d.resourceMgr.mu.RLock()
-	assert.Less(t, d.resourceMgr.allocatedVCPU, allocVCPUBefore)
-	assert.Less(t, d.resourceMgr.allocatedMem, allocMemBefore)
-	d.resourceMgr.mu.RUnlock()
-
-	// QMP socket file removed
-	_, err := os.Stat(qmpPath)
-	assert.True(t, os.IsNotExist(err), "QMP socket should be removed")
-}
-
-func TestHandleInstanceCrash_FirstCrashSetsTime(t *testing.T) {
-	d, cleanup := newTestDaemon(t)
-	defer cleanup()
-
-	allocType := smallestAllocType(t, d.resourceMgr)
-
-	instance := &vm.VM{
-		ID:           "i-test-firstcrash",
-		Status:       vm.StateRunning,
-		Running:      true,
-		PID:          111,
-		InstanceType: allocType,
-	}
-	d.Instances.VMS[instance.ID] = instance
-
-	// First crash
-	d.handleInstanceCrash(instance, fmt.Errorf("crash 1"))
-	firstTime := instance.Health.FirstCrashTime
-	assert.False(t, firstTime.IsZero())
-	assert.Equal(t, 1, instance.Health.CrashCount)
-
-	// Reset to running for a second crash
-	d.Instances.Mu.Lock()
-	instance.Status = vm.StateRunning
-	instance.Running = true
-	instance.PID = 222
-	d.Instances.Mu.Unlock()
-
-	time.Sleep(time.Millisecond) // ensure time advances
-	d.handleInstanceCrash(instance, fmt.Errorf("crash 2"))
-
-	// FirstCrashTime should be unchanged from the first crash
-	assert.Equal(t, firstTime, instance.Health.FirstCrashTime)
-	assert.Equal(t, 2, instance.Health.CrashCount)
-}
-
-func TestHandleInstanceCrash_UnknownInstanceType(t *testing.T) {
-	d, cleanup := newTestDaemon(t)
-	defer cleanup()
-
-	instance := &vm.VM{
-		ID:           "i-test-unknown-type",
-		Status:       vm.StateRunning,
-		Running:      true,
-		PID:          333,
-		InstanceType: "z99.nonexistent",
-	}
-	d.Instances.VMS[instance.ID] = instance
-
-	// Should not panic even though instance type is not in resourceMgr
-	d.handleInstanceCrash(instance, fmt.Errorf("crash"))
-
-	assert.Equal(t, vm.StateError, instance.Status)
-	assert.Equal(t, 1, instance.Health.CrashCount)
-	assert.False(t, instance.Running)
-	assert.Equal(t, 0, instance.PID)
-}
-
-func TestMaybeRestart_SkipsShuttingDown(t *testing.T) {
-	d, cleanup := newTestDaemon(t)
-	defer cleanup()
-	d.shuttingDown.Store(true)
-
-	allocType := smallestAllocType(t, d.resourceMgr)
-
-	instance := &vm.VM{
-		ID:           "i-test-restart-shutdown",
-		Status:       vm.StateError,
-		InstanceType: allocType,
-		Health: vm.InstanceHealthState{
-			CrashCount:     1,
-			FirstCrashTime: time.Now(),
-		},
-	}
-	d.Instances.VMS[instance.ID] = instance
-
-	d.maybeRestartInstance(instance)
-
-	// RestartCount should not have changed — restart was skipped
-	assert.Equal(t, 0, instance.Health.RestartCount)
-}
-
-func TestMaybeRestart_UnknownInstanceType(t *testing.T) {
-	d, cleanup := newTestDaemon(t)
-	defer cleanup()
-
-	instance := &vm.VM{
-		ID:           "i-test-restart-unknown",
-		Status:       vm.StateError,
-		InstanceType: "z99.nonexistent",
-		Health: vm.InstanceHealthState{
-			CrashCount:     1,
-			FirstCrashTime: time.Now(),
-		},
-	}
-	d.Instances.VMS[instance.ID] = instance
-
-	// Should not panic — just log and return
-	d.maybeRestartInstance(instance)
-
-	// No restart scheduled
-	assert.Equal(t, 0, instance.Health.RestartCount)
-}
-
-func TestMaybeRestart_InsufficientResources(t *testing.T) {
-	d, cleanup := newTestDaemon(t)
-	defer cleanup()
-
-	allocType := smallestAllocType(t, d.resourceMgr)
-
-	// Exhaust all schedulable resources (host - reserved) so canAllocate
-	// returns 0, matching what's actually achievable at runtime.
-	d.resourceMgr.mu.Lock()
-	d.resourceMgr.allocatedVCPU = d.resourceMgr.hostVCPU - d.resourceMgr.reservedVCPU
-	d.resourceMgr.allocatedMem = d.resourceMgr.hostMemGB - d.resourceMgr.reservedMem
-	d.resourceMgr.mu.Unlock()
-
-	instance := &vm.VM{
-		ID:           "i-test-restart-nores",
-		Status:       vm.StateError,
-		InstanceType: allocType,
-		Health: vm.InstanceHealthState{
-			CrashCount:     1,
-			FirstCrashTime: time.Now(),
-		},
-	}
-	d.Instances.VMS[instance.ID] = instance
-
-	d.maybeRestartInstance(instance)
-
-	// No restart scheduled
-	assert.Equal(t, 0, instance.Health.RestartCount)
-}
-
-func TestMaybeRestart_SchedulesRestart(t *testing.T) {
-	d, cleanup := newTestDaemon(t)
-	defer cleanup()
-
-	allocType := smallestAllocType(t, d.resourceMgr)
-
-	instance := &vm.VM{
-		ID:           "i-test-restart-schedule",
-		Status:       vm.StateError,
-		InstanceType: allocType,
-		Health: vm.InstanceHealthState{
-			CrashCount:     1,
-			FirstCrashTime: time.Now(),
-			RestartCount:   0,
-		},
-	}
-	d.Instances.VMS[instance.ID] = instance
-
-	// maybeRestartInstance should pass all guards and call time.AfterFunc.
-	// The scheduled restartCrashedInstance will increment RestartCount and
-	// attempt LaunchInstance (which will fail without real infra — that's fine).
-	d.maybeRestartInstance(instance)
-
-	// The restart is scheduled via time.AfterFunc with 5s delay.
-	// We verify that the scheduling code path was reached by checking that
-	// the function didn't bail out early (all early returns leave RestartCount at 0
-	// and don't call time.AfterFunc). The counter reset branch does not apply since
-	// FirstCrashTime is recent. If we got here without panic, the scheduling path ran.
-	// We can't easily assert the timer fired without waiting 5s, but we can
-	// verify the counters were not reset (proving we didn't take the window-expired path).
-	assert.Equal(t, 1, instance.Health.CrashCount)
-	assert.Equal(t, 0, instance.Health.RestartCount) // incremented by restartCrashedInstance, not maybeRestart
-}

spinifex/daemon/heartbeat.go

@@ -54,9 +54,7 @@ func (d *Daemon) publishHeartbeat() {
 func (d *Daemon) buildHeartbeat() *Heartbeat {
 	totalVCPU, totalMem, reservedVCPU, reservedMem, allocVCPU, allocMem, _ := d.resourceMgr.GetResourceStats()
 
-	d.Instances.Mu.Lock()
-	vmCount := len(d.Instances.VMS)
-	d.Instances.Mu.Unlock()
+	vmCount := d.vmMgr.Count()
 
 	return &Heartbeat{
 		Node:          d.node,

spinifex/daemon/heartbeat_test.go

@@ -13,7 +13,7 @@ import (
 
 // TestBuildHeartbeat verifies that buildHeartbeat populates the struct from daemon state.
 func TestBuildHeartbeat(t *testing.T) {
-	rm, err := NewResourceManager()
+	rm, err := NewResourceManager(nil, nil)
 	require.NoError(t, err)
 
 	d := &Daemon{
@@ -25,7 +25,7 @@ func TestBuildHeartbeat(t *testing.T) {
 			Services: []string{"daemon", "nats", "viperblock"},
 		},
 		resourceMgr: rm,
-		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
+		vmMgr:       vm.NewManager(),
 	}
 
 	h := d.buildHeartbeat()
@@ -46,7 +46,7 @@ func TestBuildHeartbeat(t *testing.T) {
 
 // TestHeartbeatReflectsAllocation verifies that allocating resources changes the heartbeat values.
 func TestHeartbeatReflectsAllocation(t *testing.T) {
-	rm, err := NewResourceManager()
+	rm, err := NewResourceManager(nil, nil)
 	require.NoError(t, err)
 
 	d := &Daemon{
@@ -58,7 +58,7 @@ func TestHeartbeatReflectsAllocation(t *testing.T) {
 			Services: []string{"daemon"},
 		},
 		resourceMgr: rm,
-		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
+		vmMgr:       vm.NewManager(),
 	}
 
 	// Take a heartbeat before allocation
@@ -79,13 +79,11 @@ func TestHeartbeatReflectsAllocation(t *testing.T) {
 	require.NoError(t, err)
 
 	// Add a VM to the instance map
-	d.Instances.Mu.Lock()
-	d.Instances.VMS["i-test-001"] = &vm.VM{
+	d.vmMgr.Insert(&vm.VM{
 		ID:           "i-test-001",
 		Status:       vm.StateRunning,
 		InstanceType: allocType,
-	}
-	d.Instances.Mu.Unlock()
+	})
 
 	// Take a heartbeat after allocation
 	after := d.buildHeartbeat()

spinifex/daemon/jetstream.go

@@ -35,6 +35,15 @@ const (
 	TerminatedInstanceBucketVersion = 1
 )
 
+// KVSyncObserver receives best-effort KV sync outcomes from
+// WriteStateBytesBestEffort. Implementations must be safe for concurrent use
+// and must not block — callbacks run in the same goroutine that performed the
+// Put. nil observer is allowed.
+type KVSyncObserver interface {
+	RecordKVSyncSuccess(bucket string)
+	RecordKVSyncFailure(bucket string, err error)
+}
+
 // JetStreamManager manages JetStream KV store operations for instance state
 type JetStreamManager struct {
 	js           nats.JetStreamContext
@@ -43,6 +52,13 @@ type JetStreamManager struct {
 	terminatedKV nats.KeyValue // spinifex-terminated-instances
 	replicas     int
 	kvMu         sync.Mutex // protects kv during recovery
+	obs          KVSyncObserver
+}
+
+// SetSyncObserver registers obs to receive best-effort KV sync outcomes. Pass
+// nil to clear. Safe to call before or after Init*Bucket.
+func (m *JetStreamManager) SetSyncObserver(obs KVSyncObserver) {
+	m.obs = obs
 }
 
 // NewJetStreamManager creates a new JetStreamManager from a NATS connection.
@@ -390,23 +406,13 @@ func (m *JetStreamManager) WriteServiceManifest(nodeID string, services []string
 }
 
 // WriteState writes the instance state to the KV store for the given node.
-// It acquires instances.Mu internally.
-func (m *JetStreamManager) WriteState(nodeID string, instances *vm.Instances) error {
-	instances.Mu.Lock()
-	defer instances.Mu.Unlock()
-
+// vms must be a snapshot owned by the caller — JetStreamManager does not lock.
+func (m *JetStreamManager) WriteState(nodeID string, vms map[string]*vm.VM) error {
 	if m.kv == nil {
 		return errors.New("KV bucket not initialized")
 	}
 
-	// Create a struct without the mutex to avoid copying the lock
-	state := struct {
-		VMS map[string]*vm.VM `json:"vms"`
-	}{
-		VMS: instances.VMS,
-	}
-
-	jsonData, err := json.Marshal(state)
+	jsonData, err := marshalInstanceState(vms)
 	if err != nil {
 		return err
 	}
@@ -423,18 +429,91 @@ func (m *JetStreamManager) WriteState(nodeID string, instances *vm.Instances) er
 			if _, retryErr := kv.Put(key, jsonData); retryErr != nil {
 				return retryErr
 			}
-			slog.Debug("Wrote state to JetStream KV (after recovery)", "key", key, "instances", len(instances.VMS))
+			slog.Debug("Wrote state to JetStream KV (after recovery)", "key", key, "instances", len(vms))
 			return nil
 		}
 		return err
 	}
 
-	slog.Debug("Wrote state to JetStream KV", "key", key, "instances", len(instances.VMS))
+	slog.Debug("Wrote state to JetStream KV", "key", key, "instances", len(vms))
 	return nil
 }
 
-// LoadState loads the instance state from the KV store for the given node
-func (m *JetStreamManager) LoadState(nodeID string) (*vm.Instances, error) {
+// WriteStateBestEffort attempts to push instance state to KV with a deadline.
+// On timeout or error, it logs a warning and returns — never blocks the caller
+// past `timeout` and never returns an error. Used when the local state file is
+// the source of truth and KV is a best-effort cache. vms must be a snapshot
+// owned by the caller (e.g. from vm.Manager.SnapshotMap).
+//
+// Note: kv.Put has no context API. On timeout, the in-flight Put goroutine
+// continues and completes (or fails) on its own. This leaks at most one
+// goroutine per write; under sustained partition the leak is bounded by
+// WriteState call cadence (per-state-transition, not a tight loop).
+func (m *JetStreamManager) WriteStateBestEffort(nodeID string, vms map[string]*vm.VM, timeout time.Duration) {
+	if m.kv == nil {
+		slog.Debug("KV bucket not initialized, skipping cluster sync", "node", nodeID)
+		return
+	}
+
+	jsonData, err := marshalInstanceState(vms)
+	if err != nil {
+		slog.Warn("KV sync skipped: marshal failed", "node", nodeID, "err", err)
+		return
+	}
+
+	m.WriteStateBytesBestEffort(nodeID, jsonData, timeout)
+}
+
+// WriteStateBytesBestEffort behaves like WriteStateBestEffort but accepts
+// pre-marshalled JSON. Used by hot paths that marshal under a short-lived lock
+// and commit lock-free.
+func (m *JetStreamManager) WriteStateBytesBestEffort(nodeID string, jsonData []byte, timeout time.Duration) {
+	if m.kv == nil {
+		slog.Debug("KV bucket not initialized, skipping cluster sync", "node", nodeID)
+		return
+	}
+
+	key := InstanceStatePrefix + nodeID
+	done := make(chan error, 1)
+	go func() {
+		_, putErr := m.kv.Put(key, jsonData)
+		done <- putErr
+	}()
+
+	select {
+	case putErr := <-done:
+		if putErr != nil {
+			if m.obs != nil {
+				m.obs.RecordKVSyncFailure(InstanceStateBucket, putErr)
+			}
+			slog.Warn("KV sync failed (best-effort)", "key", key, "err", putErr)
+			return
+		}
+		if m.obs != nil {
+			m.obs.RecordKVSyncSuccess(InstanceStateBucket)
+		}
+		slog.Debug("Wrote state to KV (best-effort)", "key", key, "bytes", len(jsonData))
+	case <-time.After(timeout):
+		if m.obs != nil {
+			m.obs.RecordKVSyncFailure(InstanceStateBucket, fmt.Errorf("kv sync timeout after %s", timeout))
+		}
+		slog.Warn("KV sync timed out (best-effort)", "key", key, "timeout", timeout)
+	}
+}
+
+// marshalInstanceState produces the JSON wire form of vms.
+func marshalInstanceState(vms map[string]*vm.VM) ([]byte, error) {
+	state := struct {
+		VMS map[string]*vm.VM `json:"vms"`
+	}{
+		VMS: vms,
+	}
+	return json.Marshal(state)
+}
+
+// LoadState loads the instance state from the KV store for the given node.
+// Returns an empty (non-nil) map when no state exists for the node.
+func (m *JetStreamManager) LoadState(nodeID string) (map[string]*vm.VM, error) {
 	if m.kv == nil {
 		return nil, errors.New("KV bucket not initialized")
 	}
@@ -444,7 +523,7 @@ func (m *JetStreamManager) LoadState(nodeID string) (*vm.Instances, error) {
 	if err != nil {
 		if errors.Is(err, nats.ErrKeyNotFound) {
 			slog.Debug("No existing state in JetStream KV, returning empty state", "key", key)
-			return &vm.Instances{VMS: make(map[string]*vm.VM)}, nil
+			return make(map[string]*vm.VM), nil
 		}
 		if isStreamUnavailable(err) {
 			slog.Warn("KV stream unavailable, attempting recovery", "operation", "LoadState", "key", key, "err", err)
@@ -457,7 +536,7 @@ func (m *JetStreamManager) LoadState(nodeID string) (*vm.Instances, error) {
 			if err != nil {
 				if errors.Is(err, nats.ErrKeyNotFound) {
 					slog.Warn("No state found after KV recovery", "key", key)
-					return &vm.Instances{VMS: make(map[string]*vm.VM)}, nil
+					return make(map[string]*vm.VM), nil
 				}
 				return nil, err
 			}
@@ -467,18 +546,18 @@ func (m *JetStreamManager) LoadState(nodeID string) (*vm.Instances, error) {
 		}
 	}
 
-	var instances vm.Instances
-	if err := json.Unmarshal(entry.Value(), &instances); err != nil {
+	var state struct {
+		VMS map[string]*vm.VM `json:"vms"`
+	}
+	if err := json.Unmarshal(entry.Value(), &state); err != nil {
 		return nil, err
 	}
-
-	// Ensure the VMS map is initialized
-	if instances.VMS == nil {
-		instances.VMS = make(map[string]*vm.VM)
+	if state.VMS == nil {
+		state.VMS = make(map[string]*vm.VM)
 	}
 
-	slog.Debug("Loaded state from JetStream KV", "key", key, "instances", len(instances.VMS))
-	return &instances, nil
+	slog.Debug("Loaded state from JetStream KV", "key", key, "instances", len(state.VMS))
+	return state.VMS, nil
 }
 
 // DeleteState removes the instance state from the KV store for the given node

spinifex/daemon/jetstream_test.go

@@ -3,7 +3,9 @@ package daemon
 import (
 	"encoding/json"
 	"errors"
+	"sync"
 	"testing"
+	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/spinifex/spinifex/vm"
@@ -31,18 +33,16 @@ func TestJetStreamManager_WriteAndLoadState(t *testing.T) {
 
 	// Create test instances
 	testNodeID := "test-node-1"
-	testInstances := &vm.Instances{
-		VMS: map[string]*vm.VM{
-			"i-test-001": {
-				ID:           "i-test-001",
-				Status:       vm.StateRunning,
-				InstanceType: "t3.micro",
-			},
-			"i-test-002": {
-				ID:           "i-test-002",
-				Status:       vm.StateStopped,
-				InstanceType: "t3.small",
-			},
+	testInstances := map[string]*vm.VM{
+		"i-test-001": {
+			ID:           "i-test-001",
+			Status:       vm.StateRunning,
+			InstanceType: "t3.micro",
+		},
+		"i-test-002": {
+			ID:           "i-test-002",
+			Status:       vm.StateStopped,
+			InstanceType: "t3.small",
 		},
 	}
 
@@ -56,13 +56,13 @@ func TestJetStreamManager_WriteAndLoadState(t *testing.T) {
 	require.NotNil(t, loadedInstances, "Loaded instances should not be nil")
 
 	// Verify the loaded state matches
-	assert.Len(t, loadedInstances.VMS, 2, "Should have 2 instances")
-	assert.NotNil(t, loadedInstances.VMS["i-test-001"], "Should have i-test-001")
-	assert.NotNil(t, loadedInstances.VMS["i-test-002"], "Should have i-test-002")
-	assert.Equal(t, vm.StateRunning, loadedInstances.VMS["i-test-001"].Status)
-	assert.Equal(t, vm.StateStopped, loadedInstances.VMS["i-test-002"].Status)
-	assert.Equal(t, "t3.micro", loadedInstances.VMS["i-test-001"].InstanceType)
-	assert.Equal(t, "t3.small", loadedInstances.VMS["i-test-002"].InstanceType)
+	assert.Len(t, loadedInstances, 2, "Should have 2 instances")
+	assert.NotNil(t, loadedInstances["i-test-001"], "Should have i-test-001")
+	assert.NotNil(t, loadedInstances["i-test-002"], "Should have i-test-002")
+	assert.Equal(t, vm.StateRunning, loadedInstances["i-test-001"].Status)
+	assert.Equal(t, vm.StateStopped, loadedInstances["i-test-002"].Status)
+	assert.Equal(t, "t3.micro", loadedInstances["i-test-001"].InstanceType)
+	assert.Equal(t, "t3.small", loadedInstances["i-test-002"].InstanceType)
 }
 
 // TestJetStreamManager_LoadState_KeyNotFound tests that LoadState returns empty state when key doesn't exist
@@ -83,7 +83,7 @@ func TestJetStreamManager_LoadState_KeyNotFound(t *testing.T) {
 	instances, err := jsm.LoadState("non-existent-node")
 	require.NoError(t, err, "Should not error when key not found")
 	require.NotNil(t, instances, "Should return non-nil instances")
-	assert.Empty(t, instances.VMS, "Should return empty VMS map")
+	assert.Empty(t, instances, "Should return empty VMS map")
 }
 
 // TestJetStreamManager_BucketCreation tests that InitKVBucket creates the bucket when it doesn't exist
@@ -120,12 +120,10 @@ func TestJetStreamManager_BucketReconnection(t *testing.T) {
 	require.NoError(t, err, "First InitKVBucket should succeed")
 
 	// Write some test data
-	testInstances := &vm.Instances{
-		VMS: map[string]*vm.VM{
-			"i-persist": {
-				ID:     "i-persist",
-				Status: vm.StateRunning,
-			},
+	testInstances := map[string]*vm.VM{
+		"i-persist": {
+			ID:     "i-persist",
+			Status: vm.StateRunning,
 		},
 	}
 	err = jsm1.WriteState("persist-node", testInstances)
@@ -147,9 +145,9 @@ func TestJetStreamManager_BucketReconnection(t *testing.T) {
 	// Verify data persisted
 	loadedInstances, err := jsm2.LoadState("persist-node")
 	require.NoError(t, err)
-	assert.NotEmpty(t, loadedInstances.VMS, "Should have persisted instances")
-	assert.NotNil(t, loadedInstances.VMS["i-persist"], "Should have i-persist")
-	assert.Equal(t, vm.StateRunning, loadedInstances.VMS["i-persist"].Status)
+	assert.NotEmpty(t, loadedInstances, "Should have persisted instances")
+	assert.NotNil(t, loadedInstances["i-persist"], "Should have i-persist")
+	assert.Equal(t, vm.StateRunning, loadedInstances["i-persist"].Status)
 }
 
 // TestJetStreamManager_DeleteState tests deleting state from the KV store
@@ -168,12 +166,10 @@ func TestJetStreamManager_DeleteState(t *testing.T) {
 
 	// Write state
 	testNodeID := "delete-test-node"
-	testInstances := &vm.Instances{
-		VMS: map[string]*vm.VM{
-			"i-delete-me": {
-				ID:     "i-delete-me",
-				Status: vm.StateRunning,
-			},
+	testInstances := map[string]*vm.VM{
+		"i-delete-me": {
+			ID:     "i-delete-me",
+			Status: vm.StateRunning,
 		},
 	}
 	err = jsm.WriteState(testNodeID, testInstances)
@@ -182,7 +178,7 @@ func TestJetStreamManager_DeleteState(t *testing.T) {
 	// Verify state exists
 	loadedInstances, err := jsm.LoadState(testNodeID)
 	require.NoError(t, err)
-	assert.NotEmpty(t, loadedInstances.VMS)
+	assert.NotEmpty(t, loadedInstances)
 
 	// Delete state
 	err = jsm.DeleteState(testNodeID)
@@ -191,7 +187,7 @@ func TestJetStreamManager_DeleteState(t *testing.T) {
 	// Verify state is gone (should return empty state)
 	loadedInstances, err = jsm.LoadState(testNodeID)
 	require.NoError(t, err)
-	assert.Empty(t, loadedInstances.VMS, "Should return empty state after deletion")
+	assert.Empty(t, loadedInstances, "Should return empty state after deletion")
 }
 
 // TestJetStreamManager_DeleteState_NonExistent tests deleting state that doesn't exist
@@ -230,28 +226,24 @@ func TestJetStreamManager_WriteState_UpdateExisting(t *testing.T) {
 	testNodeID := "update-test-node"
 
 	// Write initial state
-	initialInstances := &vm.Instances{
-		VMS: map[string]*vm.VM{
-			"i-initial": {
-				ID:     "i-initial",
-				Status: vm.StateRunning,
-			},
+	initialInstances := map[string]*vm.VM{
+		"i-initial": {
+			ID:     "i-initial",
+			Status: vm.StateRunning,
 		},
 	}
 	err = jsm.WriteState(testNodeID, initialInstances)
 	require.NoError(t, err)
 
 	// Update state with different instances
-	updatedInstances := &vm.Instances{
-		VMS: map[string]*vm.VM{
-			"i-initial": {
-				ID:     "i-initial",
-				Status: vm.StateStopped, // Changed status
-			},
-			"i-new": { // Added new instance
-				ID:     "i-new",
-				Status: vm.StateRunning,
-			},
+	updatedInstances := map[string]*vm.VM{
+		"i-initial": {
+			ID:     "i-initial",
+			Status: vm.StateStopped, // Changed status
+		},
+		"i-new": { // Added new instance
+			ID:     "i-new",
+			Status: vm.StateRunning,
 		},
 	}
 	err = jsm.WriteState(testNodeID, updatedInstances)
@@ -260,9 +252,9 @@ func TestJetStreamManager_WriteState_UpdateExisting(t *testing.T) {
 	// Load and verify updated state
 	loadedInstances, err := jsm.LoadState(testNodeID)
 	require.NoError(t, err)
-	assert.Len(t, loadedInstances.VMS, 2, "Should have 2 instances")
-	assert.Equal(t, vm.StateStopped, loadedInstances.VMS["i-initial"].Status, "Status should be updated")
-	assert.NotNil(t, loadedInstances.VMS["i-new"], "Should have new instance")
+	assert.Len(t, loadedInstances, 2, "Should have 2 instances")
+	assert.Equal(t, vm.StateStopped, loadedInstances["i-initial"].Status, "Status should be updated")
+	assert.NotNil(t, loadedInstances["i-new"], "Should have new instance")
 }
 
 // TestJetStreamManager_MultipleNodes tests storing state for multiple nodes
@@ -280,20 +272,16 @@ func TestJetStreamManager_MultipleNodes(t *testing.T) {
 	require.NoError(t, err)
 
 	// Write state for node-1
-	node1Instances := &vm.Instances{
-		VMS: map[string]*vm.VM{
-			"i-node1-001": {ID: "i-node1-001", Status: vm.StateRunning},
-		},
+	node1Instances := map[string]*vm.VM{
+		"i-node1-001": {ID: "i-node1-001", Status: vm.StateRunning},
 	}
 	err = jsm.WriteState("node-1", node1Instances)
 	require.NoError(t, err)
 
 	// Write state for node-2
-	node2Instances := &vm.Instances{
-		VMS: map[string]*vm.VM{
-			"i-node2-001": {ID: "i-node2-001", Status: vm.StateStopped},
-			"i-node2-002": {ID: "i-node2-002", Status: vm.StateRunning},
-		},
+	node2Instances := map[string]*vm.VM{
+		"i-node2-001": {ID: "i-node2-001", Status: vm.StateStopped},
+		"i-node2-002": {ID: "i-node2-002", Status: vm.StateRunning},
 	}
 	err = jsm.WriteState("node-2", node2Instances)
 	require.NoError(t, err)
@@ -301,18 +289,18 @@ func TestJetStreamManager_MultipleNodes(t *testing.T) {
 	// Load and verify node-1 state
 	loadedNode1, err := jsm.LoadState("node-1")
 	require.NoError(t, err)
-	assert.Len(t, loadedNode1.VMS, 1)
-	assert.NotNil(t, loadedNode1.VMS["i-node1-001"])
+	assert.Len(t, loadedNode1, 1)
+	assert.NotNil(t, loadedNode1["i-node1-001"])
 
 	// Load and verify node-2 state
 	loadedNode2, err := jsm.LoadState("node-2")
 	require.NoError(t, err)
-	assert.Len(t, loadedNode2.VMS, 2)
-	assert.NotNil(t, loadedNode2.VMS["i-node2-001"])
-	assert.NotNil(t, loadedNode2.VMS["i-node2-002"])
+	assert.Len(t, loadedNode2, 2)
+	assert.NotNil(t, loadedNode2["i-node2-001"])
+	assert.NotNil(t, loadedNode2["i-node2-002"])
 
 	// Verify node isolation - node-1 doesn't have node-2's instances
-	_, exists := loadedNode1.VMS["i-node2-001"]
+	_, exists := loadedNode1["i-node2-001"]
 	assert.False(t, exists, "Node-1 should not have node-2's instances")
 }
 
@@ -328,7 +316,7 @@ func TestJetStreamManager_KVNotInitialized(t *testing.T) {
 	jsm, err := NewJetStreamManager(nc, 1)
 	require.NoError(t, err)
 
-	testInstances := &vm.Instances{VMS: make(map[string]*vm.VM)}
+	testInstances := make(map[string]*vm.VM)
 	err = jsm.WriteState("test-node", testInstances)
 	assert.Error(t, err, "WriteState should error when KV not initialized")
 
@@ -522,10 +510,8 @@ func TestJetStreamManager_StoppedInstances_NoInterference(t *testing.T) {
 	require.NoError(t, err)
 
 	// Write per-node state
-	nodeInstances := &vm.Instances{
-		VMS: map[string]*vm.VM{
-			"i-running-001": {ID: "i-running-001", Status: vm.StateRunning},
-		},
+	nodeInstances := map[string]*vm.VM{
+		"i-running-001": {ID: "i-running-001", Status: vm.StateRunning},
 	}
 	err = jsm.WriteState("interference-test-node", nodeInstances)
 	require.NoError(t, err)
@@ -538,8 +524,8 @@ func TestJetStreamManager_StoppedInstances_NoInterference(t *testing.T) {
 	// Verify per-node state is unaffected
 	loaded, err := jsm.LoadState("interference-test-node")
 	require.NoError(t, err)
-	assert.Len(t, loaded.VMS, 1)
-	assert.NotNil(t, loaded.VMS["i-running-001"])
+	assert.Len(t, loaded, 1)
+	assert.NotNil(t, loaded["i-running-001"])
 
 	// Verify stopped instance loads independently
 	loadedStopped, err := jsm.LoadStoppedInstance(stoppedVM.ID)
@@ -684,10 +670,8 @@ func TestJetStreamManager_WriteState_RecoverAfterStreamLost(t *testing.T) {
 	deleteInstanceStateBucket(t, nc)
 
 	// WriteState should recover by recreating the bucket
-	testInstances := &vm.Instances{
-		VMS: map[string]*vm.VM{
-			"i-recover-001": {ID: "i-recover-001", Status: vm.StateRunning},
-		},
+	testInstances := map[string]*vm.VM{
+		"i-recover-001": {ID: "i-recover-001", Status: vm.StateRunning},
 	}
 	err = jsm.WriteState("recovery-node", testInstances)
 	require.NoError(t, err, "WriteState should recover after stream loss")
@@ -695,8 +679,8 @@ func TestJetStreamManager_WriteState_RecoverAfterStreamLost(t *testing.T) {
 	// Verify data was written
 	loaded, err := jsm.LoadState("recovery-node")
 	require.NoError(t, err)
-	assert.Len(t, loaded.VMS, 1)
-	assert.NotNil(t, loaded.VMS["i-recover-001"])
+	assert.Len(t, loaded, 1)
+	assert.NotNil(t, loaded["i-recover-001"])
 }
 
 func TestJetStreamManager_LoadState_RecoverAfterStreamLost(t *testing.T) {
@@ -714,7 +698,7 @@ func TestJetStreamManager_LoadState_RecoverAfterStreamLost(t *testing.T) {
 	// LoadState should recover and return empty state
 	loaded, err := jsm.LoadState("any-node")
 	require.NoError(t, err, "LoadState should recover after stream loss")
-	assert.Empty(t, loaded.VMS)
+	assert.Empty(t, loaded)
 }
 
 func TestJetStreamManager_DeleteState_RecoverAfterStreamLost(t *testing.T) {
@@ -843,7 +827,7 @@ func TestJetStreamManager_WriteState_RecoveryFailure(t *testing.T) {
 	deleteInstanceStateBucket(t, nc)
 	swapToNonJSContext(t, jsm)
 
-	testInstances := &vm.Instances{VMS: map[string]*vm.VM{"i-fail": {ID: "i-fail"}}}
+	testInstances := map[string]*vm.VM{"i-fail": {ID: "i-fail"}}
 	err = jsm.WriteState("fail-node", testInstances)
 	assert.Error(t, err, "WriteState should return error when recovery fails")
 }
@@ -1264,3 +1248,91 @@ func TestJetStreamManager_ListTerminatedInstances_SkipsVersionKey(t *testing.T)
 	require.NoError(t, err)
 	assert.Empty(t, instances)
 }
+
+// fakeKVObserver records observer callbacks for tests.
+type fakeKVObserver struct {
+	mu        sync.Mutex
+	successes []string
+	failures  []fakeKVObserverFailure
+}
+
+type fakeKVObserverFailure struct {
+	bucket string
+	err    error
+}
+
+func (f *fakeKVObserver) RecordKVSyncSuccess(bucket string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.successes = append(f.successes, bucket)
+}
+
+func (f *fakeKVObserver) RecordKVSyncFailure(bucket string, err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.failures = append(f.failures, fakeKVObserverFailure{bucket: bucket, err: err})
+}
+
+func (f *fakeKVObserver) snapshot() ([]string, []fakeKVObserverFailure) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	successes := append([]string(nil), f.successes...)
+	failures := append([]fakeKVObserverFailure(nil), f.failures...)
+	return successes, failures
+}
+
+func TestJetStreamManager_BestEffort_Success_NotifiesObserver(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitKVBucket())
+
+	obs := &fakeKVObserver{}
+	jsm.SetSyncObserver(obs)
+
+	jsm.WriteStateBytesBestEffort("obs-success", []byte(`{"vms":{}}`), 5*time.Second)
+
+	successes, failures := obs.snapshot()
+	require.Len(t, successes, 1)
+	assert.Equal(t, InstanceStateBucket, successes[0])
+	assert.Empty(t, failures)
+}
+
+func TestJetStreamManager_BestEffort_PutError_NotifiesObserver(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitKVBucket())
+
+	obs := &fakeKVObserver{}
+	jsm.SetSyncObserver(obs)
+
+	// Closing the connection forces the inflight Put to fail without timing out.
+	nc.Close()
+
+	jsm.WriteStateBytesBestEffort("obs-fail", []byte(`{"vms":{}}`), 5*time.Second)
+
+	successes, failures := obs.snapshot()
+	assert.Empty(t, successes)
+	require.Len(t, failures, 1)
+	assert.Equal(t, InstanceStateBucket, failures[0].bucket)
+	assert.Error(t, failures[0].err)
+}
+
+func TestJetStreamManager_BestEffort_NilObserver_NoPanic(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitKVBucket())
+
+	// No observer set — must not panic on success or failure paths.
+	jsm.WriteStateBytesBestEffort("obs-nil", []byte(`{"vms":{}}`), 5*time.Second)
+}

spinifex/daemon/local_api.go

@@ -0,0 +1,128 @@
+package daemon
+
+import (
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"sort"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/mulgadc/spinifex/spinifex/vm"
+)
+
+// NATS connectivity strings reported by /local/status.
+const (
+	natsConnected    = "connected"
+	natsDisconnected = "disconnected"
+)
+
+// LocalInstance is one entry of GET /local/instances. Mirrors the harness shape
+// in spinifex/tests/e2e/ddil/harness/daemon_client.go.
+type LocalInstance struct {
+	InstanceID string `json:"instance_id"`
+	State      string `json:"state"`
+	PID        int    `json:"pid,omitempty"`
+}
+
+// LocalStatus is the response shape for GET /local/status. Reports the
+// daemon's connectivity mode plus enough context for an operator to detect
+// flapping (nats_retry_count), stale state (revision), or prolonged divergence
+// between the local file and JetStream KV (kv_sync_failures, last_kv_sync_at,
+// last_kv_sync_error).
+type LocalStatus struct {
+	Node            string `json:"node"`
+	Mode            string `json:"mode"`
+	NATS            string `json:"nats"`
+	NATSRetryCount  int64  `json:"nats_retry_count"`
+	Revision        uint64 `json:"revision"`
+	KVSyncFailures  int64  `json:"kv_sync_failures"`
+	LastKVSyncAt    string `json:"last_kv_sync_at,omitempty"`
+	LastKVSyncError string `json:"last_kv_sync_error,omitempty"`
+}
+
+// registerLocalRoutes wires the read-only /local/* endpoints onto r. Called
+// from ClusterManager so the routes share the existing TLS listener and stay
+// reachable while NATS is down.
+func (d *Daemon) registerLocalRoutes(r chi.Router) {
+	r.Get("/local/instances", d.handleLocalInstances)
+	r.Get("/local/instances/{id}", d.handleLocalInstance)
+	r.Get("/local/status", d.handleLocalStatus)
+}
+
+func (d *Daemon) handleLocalInstances(w http.ResponseWriter, _ *http.Request) {
+	out := []LocalInstance{}
+	d.vmMgr.View(func(vms map[string]*vm.VM) {
+		for _, v := range vms {
+			out = append(out, vmToLocalInstance(v))
+		}
+	})
+	sort.Slice(out, func(i, j int) bool { return out[i].InstanceID < out[j].InstanceID })
+	writeJSON(w, http.StatusOK, out)
+}
+
+func (d *Daemon) handleLocalInstance(w http.ResponseWriter, r *http.Request) {
+	id := chi.URLParam(r, "id")
+	v, ok := d.vmMgr.Get(id)
+	if !ok {
+		writeJSONError(w, http.StatusNotFound, "instance not found")
+		return
+	}
+	var li LocalInstance
+	d.vmMgr.Inspect(v, func(v *vm.VM) {
+		li = vmToLocalInstance(v)
+	})
+	writeJSON(w, http.StatusOK, li)
+}
+
+func (d *Daemon) handleLocalStatus(w http.ResponseWriter, _ *http.Request) {
+	status := LocalStatus{
+		Node:            d.node,
+		Mode:            d.Mode(),
+		NATS:            d.natsConnectivity(),
+		NATSRetryCount:  d.NATSRetryCount(),
+		Revision:        d.Revision(),
+		KVSyncFailures:  d.KVSyncFailures(),
+		LastKVSyncError: d.LastKVSyncError(),
+	}
+	if t := d.LastKVSyncAt(); !t.IsZero() {
+		status.LastKVSyncAt = t.UTC().Format(time.RFC3339)
+	}
+	writeJSON(w, http.StatusOK, status)
+}
+
+// natsConnectivity reports the live NATS link state. Mirrors the check used by
+// the /health handler so /local/status agrees with /health on the same node.
+func (d *Daemon) natsConnectivity() string {
+	if d.natsConn != nil && d.natsConn.IsConnected() {
+		return natsConnected
+	}
+	return natsDisconnected
+}
+
+func vmToLocalInstance(v *vm.VM) LocalInstance {
+	li := LocalInstance{
+		InstanceID: v.ID,
+		State:      string(v.Status),
+	}
+	// PID is read from the QEMU pidfile each request; missing file or unreadable
+	// value just omits the field (json:"omitempty"). Read failures are not
+	// surfaced — /local/instances must keep working even when QEMU is gone.
+	if pid, err := utils.ReadPidFile(v.ID); err == nil {
+		li.PID = pid
+	}
+	return li
+}
+
+func writeJSON(w http.ResponseWriter, status int, body any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	if err := json.NewEncoder(w).Encode(body); err != nil {
+		slog.Error("local API encode failed", "error", err)
+	}
+}
+
+func writeJSONError(w http.ResponseWriter, status int, msg string) {
+	writeJSON(w, status, map[string]string{"error": msg})
+}

spinifex/daemon/local_api_test.go

@@ -0,0 +1,193 @@
+package daemon
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newLocalAPITestDaemon returns a Daemon with just enough wiring to exercise
+// the read-only /local/* handlers: a vm.Manager, a node name, and a Config so
+// any path-resolving code (revision bump via WriteState) has a temp DataDir.
+// XDG_RUNTIME_DIR is pinned to a temp dir so utils.WritePidFile / ReadPidFile
+// (used by vmToLocalInstance to surface PIDs) stay scoped to the test.
+func newLocalAPITestDaemon(t *testing.T) *Daemon {
+	t.Helper()
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	d := &Daemon{
+		node:   "node-1",
+		vmMgr:  vm.NewManager(),
+		config: &config.Config{DataDir: t.TempDir()},
+	}
+	// Zero values give natsConnected=false and peersReachable=false ⇒
+	// Mode() reports standalone, which is what the /local/status tests
+	// below assert on by default.
+	return d
+}
+
+func newLocalAPIRouter(d *Daemon) chi.Router {
+	r := chi.NewRouter()
+	d.registerLocalRoutes(r)
+	return r
+}
+
+func doGET(t *testing.T, r chi.Router, path string) *httptest.ResponseRecorder {
+	t.Helper()
+	req := httptest.NewRequest(http.MethodGet, path, nil)
+	rec := httptest.NewRecorder()
+	r.ServeHTTP(rec, req)
+	return rec
+}
+
+func TestLocalAPI_Instances_Empty(t *testing.T) {
+	d := newLocalAPITestDaemon(t)
+	rec := doGET(t, newLocalAPIRouter(d), "/local/instances")
+	require.Equal(t, http.StatusOK, rec.Code)
+
+	var got []LocalInstance
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got))
+	assert.Equal(t, []LocalInstance{}, got)
+}
+
+func TestLocalAPI_Instances_ListsVMs(t *testing.T) {
+	d := newLocalAPITestDaemon(t)
+	d.vmMgr.Insert(&vm.VM{ID: "i-b", Status: vm.StateRunning})
+	d.vmMgr.Insert(&vm.VM{ID: "i-a", Status: vm.StateStopped})
+	require.NoError(t, utils.WritePidFile("i-b", 4242))
+
+	rec := doGET(t, newLocalAPIRouter(d), "/local/instances")
+	require.Equal(t, http.StatusOK, rec.Code)
+
+	var got []LocalInstance
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got))
+	require.Len(t, got, 2)
+	// Sorted by InstanceID for stable output.
+	assert.Equal(t, "i-a", got[0].InstanceID)
+	assert.Equal(t, "stopped", got[0].State)
+	assert.Zero(t, got[0].PID)
+	assert.Equal(t, "i-b", got[1].InstanceID)
+	assert.Equal(t, "running", got[1].State)
+	assert.Equal(t, 4242, got[1].PID)
+}
+
+func TestLocalAPI_Instance_Found(t *testing.T) {
+	d := newLocalAPITestDaemon(t)
+	d.vmMgr.Insert(&vm.VM{ID: "i-find", Status: vm.StateRunning})
+	require.NoError(t, utils.WritePidFile("i-find", 7))
+
+	rec := doGET(t, newLocalAPIRouter(d), "/local/instances/i-find")
+	require.Equal(t, http.StatusOK, rec.Code)
+
+	var got LocalInstance
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got))
+	assert.Equal(t, LocalInstance{InstanceID: "i-find", State: "running", PID: 7}, got)
+}
+
+func TestLocalAPI_Instance_NotFound(t *testing.T) {
+	d := newLocalAPITestDaemon(t)
+	rec := doGET(t, newLocalAPIRouter(d), "/local/instances/i-missing")
+	assert.Equal(t, http.StatusNotFound, rec.Code)
+}
+
+func TestLocalAPI_Status_Standalone_NoNATS(t *testing.T) {
+	d := newLocalAPITestDaemon(t)
+
+	rec := doGET(t, newLocalAPIRouter(d), "/local/status")
+	require.Equal(t, http.StatusOK, rec.Code)
+
+	var got LocalStatus
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got))
+	assert.Equal(t, LocalStatus{
+		Node:           "node-1",
+		Mode:           DaemonModeStandalone,
+		NATS:           natsDisconnected,
+		NATSRetryCount: 0,
+		Revision:       0,
+	}, got)
+}
+
+func TestLocalAPI_Status_RetryCountAndRevisionPropagate(t *testing.T) {
+	d := newLocalAPITestDaemon(t)
+	// Mode() is derived from natsConnected + peersReachable; flip both true
+	// to assert the cluster-mode propagation through /local/status.
+	d.natsConnected.Store(true)
+	d.peersReachable.Store(true)
+	d.natsRetryCount.Store(3)
+	require.NoError(t, d.WriteState())
+	require.NoError(t, d.WriteState())
+
+	rec := doGET(t, newLocalAPIRouter(d), "/local/status")
+	require.Equal(t, http.StatusOK, rec.Code)
+
+	var got LocalStatus
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got))
+	assert.Equal(t, DaemonModeCluster, got.Mode)
+	assert.Equal(t, int64(3), got.NATSRetryCount)
+	assert.Equal(t, uint64(2), got.Revision)
+}
+
+func TestLocalAPI_Status_ContentType(t *testing.T) {
+	d := newLocalAPITestDaemon(t)
+	rec := doGET(t, newLocalAPIRouter(d), "/local/status")
+	assert.Equal(t, "application/json", rec.Header().Get("Content-Type"))
+}
+
+func TestLocalAPI_Status_KVSync_DefaultsAreZero(t *testing.T) {
+	d := newLocalAPITestDaemon(t)
+
+	rec := doGET(t, newLocalAPIRouter(d), "/local/status")
+	require.Equal(t, http.StatusOK, rec.Code)
+
+	var got LocalStatus
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got))
+	assert.Zero(t, got.KVSyncFailures)
+	assert.Empty(t, got.LastKVSyncAt)
+	assert.Empty(t, got.LastKVSyncError)
+}
+
+func TestLocalAPI_Status_KVSyncFailure_PropagatesFields(t *testing.T) {
+	d := newLocalAPITestDaemon(t)
+
+	d.RecordKVSyncFailure(InstanceStateBucket, errors.New("kv timeout"))
+	d.RecordKVSyncFailure(InstanceStateBucket, errors.New("kv put rejected"))
+
+	rec := doGET(t, newLocalAPIRouter(d), "/local/status")
+	require.Equal(t, http.StatusOK, rec.Code)
+
+	var got LocalStatus
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got))
+	assert.Equal(t, int64(2), got.KVSyncFailures)
+	assert.Equal(t, "kv put rejected", got.LastKVSyncError)
+	assert.Empty(t, got.LastKVSyncAt, "no successful sync yet")
+}
+
+func TestLocalAPI_Status_KVSyncSuccess_ClearsErrorAndStampsTime(t *testing.T) {
+	d := newLocalAPITestDaemon(t)
+
+	d.RecordKVSyncFailure(InstanceStateBucket, errors.New("transient"))
+	before := time.Now()
+	d.RecordKVSyncSuccess(InstanceStateBucket)
+
+	rec := doGET(t, newLocalAPIRouter(d), "/local/status")
+	require.Equal(t, http.StatusOK, rec.Code)
+
+	var got LocalStatus
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got))
+	assert.Equal(t, int64(1), got.KVSyncFailures, "counter is monotonic")
+	assert.Empty(t, got.LastKVSyncError, "success clears the error string")
+
+	require.NotEmpty(t, got.LastKVSyncAt)
+	parsed, err := time.Parse(time.RFC3339, got.LastKVSyncAt)
+	require.NoError(t, err)
+	assert.WithinDuration(t, before, parsed, 5*time.Second)
+}

spinifex/daemon/local_state.go

@@ -0,0 +1,155 @@
+package daemon
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+
+	"github.com/mulgadc/spinifex/spinifex/vm"
+)
+
+const (
+	// LocalStateSchemaVersion is the on-disk schema version for instance-state.json.
+	// Bump on any breaking change to the LocalState shape; daemon refuses to start
+	// on an unknown version rather than silently losing data.
+	LocalStateSchemaVersion = 1
+
+	// DefaultLocalStateDir is the default directory under DataDir for local state files.
+	DefaultLocalStateDir = "state"
+
+	// LocalStateFileName is the on-disk filename for per-node instance state.
+	LocalStateFileName = "instance-state.json"
+
+	// DefaultDataDir is the fallback DataDir when config.DataDir is empty.
+	DefaultDataDir = "/var/lib/spinifex"
+)
+
+// LocalState is the on-disk representation of a node's instance state.
+// SchemaVersion gates compatibility; unknown versions are rejected.
+type LocalState struct {
+	SchemaVersion int               `json:"schema_version"`
+	VMS           map[string]*vm.VM `json:"vms"`
+}
+
+// LocalStatePath returns the absolute path to the per-node instance state file
+// rooted at dataDir. Empty dataDir falls back to DefaultDataDir
+// (/var/lib/spinifex), the platform-shared root. The state directory sits at
+// the platform root so it lives next to other shared state (predastore,
+// viperblock data dirs) rather than nested under the daemon's own subdir.
+func LocalStatePath(dataDir string) string {
+	if dataDir == "" {
+		dataDir = DefaultDataDir
+	}
+	return filepath.Join(dataDir, DefaultLocalStateDir, LocalStateFileName)
+}
+
+// MarshalLocalState produces the JSON wire form of vms wrapped with the schema
+// version. Callers that already hold a stable view of vms (e.g. inside a
+// vm.Manager.View callback) can marshal under the lock and pass the bytes to
+// WriteLocalStateBytes — that avoids racing json.Marshal against concurrent
+// VM-field mutations.
+func MarshalLocalState(vms map[string]*vm.VM) ([]byte, error) {
+	state := LocalState{
+		SchemaVersion: LocalStateSchemaVersion,
+		VMS:           vms,
+	}
+	data, err := json.Marshal(state)
+	if err != nil {
+		return nil, fmt.Errorf("marshal local state: %w", err)
+	}
+	return data, nil
+}
+
+// WriteLocalState atomically writes the instance state to path. vms must be a
+// snapshot owned by the caller (e.g. from vm.Manager.SnapshotMap) — this
+// function does not lock. Convenience wrapper around MarshalLocalState +
+// WriteLocalStateBytes for callers that own an exclusive snapshot.
+//
+// Atomicity: marshal → write to <path>.tmp → fsync → rename. The rename is
+// atomic on POSIX so a concurrent reader sees either the old file or the new
+// one, never a half-written file.
+func WriteLocalState(path string, vms map[string]*vm.VM) error {
+	data, err := MarshalLocalState(vms)
+	if err != nil {
+		return err
+	}
+	return WriteLocalStateBytes(path, data)
+}
+
+// WriteLocalStateBytes atomically writes pre-marshalled state JSON to path.
+// Used by hot paths that marshal under a short-lived lock and then commit to
+// disk lock-free.
+func WriteLocalStateBytes(path string, data []byte) error {
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0o750); err != nil {
+		return fmt.Errorf("mkdir %s: %w", dir, err)
+	}
+
+	tmp := path + ".tmp"
+	f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
+	if err != nil {
+		return fmt.Errorf("open tmp %s: %w", tmp, err)
+	}
+
+	if _, err := f.Write(data); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return fmt.Errorf("write tmp %s: %w", tmp, err)
+	}
+
+	if err := f.Sync(); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return fmt.Errorf("fsync tmp %s: %w", tmp, err)
+	}
+
+	if err := f.Close(); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("close tmp %s: %w", tmp, err)
+	}
+
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("rename %s -> %s: %w", tmp, path, err)
+	}
+
+	return nil
+}
+
+// ReadLocalState reads the per-node instance state from path.
+//
+// Returns (nil, nil) if the file does not exist — fresh-install signal,
+// caller should start with an empty instance map.
+//
+// Returns an error on:
+//   - JSON parse failure (corruption — caller refuses start)
+//   - Unknown SchemaVersion (caller refuses start; safer than silent migration)
+func ReadLocalState(path string) (*LocalState, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			slog.Debug("Local state file missing, treating as fresh install", "path", path)
+			return nil, nil
+		}
+		return nil, fmt.Errorf("read local state %s: %w", path, err)
+	}
+
+	var state LocalState
+	if err := json.Unmarshal(data, &state); err != nil {
+		return nil, fmt.Errorf("parse local state %s: %w", path, err)
+	}
+
+	if state.SchemaVersion != LocalStateSchemaVersion {
+		return nil, fmt.Errorf("local state %s: unknown schema_version %d (expected %d)",
+			path, state.SchemaVersion, LocalStateSchemaVersion)
+	}
+
+	if state.VMS == nil {
+		state.VMS = make(map[string]*vm.VM)
+	}
+
+	return &state, nil
+}

spinifex/daemon/local_state_test.go

@@ -0,0 +1,115 @@
+package daemon
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLocalStatePath_UsesDataDir(t *testing.T) {
+	got := LocalStatePath("/data")
+	assert.Equal(t, "/data/state/instance-state.json", got)
+}
+
+func TestLocalStatePath_DefaultWhenEmpty(t *testing.T) {
+	got := LocalStatePath("")
+	assert.Equal(t, "/var/lib/spinifex/state/instance-state.json", got)
+}
+
+func TestWriteLocalState_RoundTrip(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "state", "instance-state.json")
+
+	vms := map[string]*vm.VM{
+		"i-aaa": {ID: "i-aaa", InstanceType: "t3.micro"},
+		"i-bbb": {ID: "i-bbb", InstanceType: "m5.large"},
+	}
+
+	require.NoError(t, WriteLocalState(path, vms))
+
+	loaded, err := ReadLocalState(path)
+	require.NoError(t, err)
+	require.NotNil(t, loaded)
+	assert.Equal(t, LocalStateSchemaVersion, loaded.SchemaVersion)
+	assert.Len(t, loaded.VMS, 2)
+	assert.Equal(t, "t3.micro", loaded.VMS["i-aaa"].InstanceType)
+	assert.Equal(t, "m5.large", loaded.VMS["i-bbb"].InstanceType)
+}
+
+func TestWriteLocalState_MkdirParent(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "deep", "nested", "state", "instance-state.json")
+	require.NoError(t, WriteLocalState(path, map[string]*vm.VM{}))
+	_, err := os.Stat(path)
+	require.NoError(t, err)
+}
+
+func TestWriteLocalState_NoTmpLeftBehind(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "instance-state.json")
+	require.NoError(t, WriteLocalState(path, map[string]*vm.VM{}))
+
+	entries, err := os.ReadDir(dir)
+	require.NoError(t, err)
+	for _, e := range entries {
+		assert.NotContains(t, e.Name(), ".tmp", "stale tmp file left behind")
+	}
+}
+
+func TestWriteLocalState_AtomicReplace(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "instance-state.json")
+
+	require.NoError(t, WriteLocalState(path, map[string]*vm.VM{
+		"i-1": {ID: "i-1"},
+	}))
+
+	require.NoError(t, WriteLocalState(path, map[string]*vm.VM{
+		"i-2": {ID: "i-2"},
+	}))
+
+	loaded, err := ReadLocalState(path)
+	require.NoError(t, err)
+	require.NotNil(t, loaded)
+	assert.Len(t, loaded.VMS, 1)
+	_, ok := loaded.VMS["i-2"]
+	assert.True(t, ok, "second write should replace, not merge")
+}
+
+func TestReadLocalState_Missing(t *testing.T) {
+	state, err := ReadLocalState(filepath.Join(t.TempDir(), "absent.json"))
+	require.NoError(t, err)
+	assert.Nil(t, state, "missing file is the fresh-install signal")
+}
+
+func TestReadLocalState_CorruptJSON(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "instance-state.json")
+	require.NoError(t, os.WriteFile(path, []byte("{not json"), 0o600))
+
+	_, err := ReadLocalState(path)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "parse local state")
+}
+
+func TestReadLocalState_UnknownSchema(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "instance-state.json")
+	data, _ := json.Marshal(map[string]any{"schema_version": 99, "vms": map[string]any{}})
+	require.NoError(t, os.WriteFile(path, data, 0o600))
+
+	_, err := ReadLocalState(path)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "unknown schema_version 99")
+}
+
+func TestReadLocalState_NilVMSMapInitialized(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "instance-state.json")
+	data, _ := json.Marshal(map[string]any{"schema_version": LocalStateSchemaVersion})
+	require.NoError(t, os.WriteFile(path, data, 0o600))
+
+	state, err := ReadLocalState(path)
+	require.NoError(t, err)
+	require.NotNil(t, state)
+	assert.NotNil(t, state.VMS)
+}

spinifex/daemon/network.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"sort"
 	"strings"
 
 	"github.com/mulgadc/spinifex/spinifex/utils"
@@ -26,151 +27,83 @@ var sudoCommand = func(name string, args ...string) *exec.Cmd {
 	return exec.Command("sudo", append([]string{name}, args...)...)
 }
 
-// NetworkPlumber handles tap device and OVS bridge operations for VM networking.
-// The live implementation runs system commands (ip, ovs-vsctl); tests use a mock.
-type NetworkPlumber interface {
-	// SetupTapDevice creates a tap device and adds it to the OVS br-int bridge
-	// with the correct iface-id for OVN port binding.
-	SetupTapDevice(eniId, mac string) error
-
-	// CleanupTapDevice removes the tap device from br-int and deletes it.
-	CleanupTapDevice(eniId string) error
-}
-
-// OVSNetworkPlumber implements NetworkPlumber using system commands.
+// OVSNetworkPlumber implements vm.NetworkPlumber using system commands
+// (ip, ovs-vsctl); tests use a mock satisfying the same interface.
 type OVSNetworkPlumber struct{}
 
-var _ NetworkPlumber = (*OVSNetworkPlumber)(nil)
+var _ vm.NetworkPlumber = (*OVSNetworkPlumber)(nil)
 
-func (p *OVSNetworkPlumber) SetupTapDevice(eniId, mac string) error {
-	tapName := TapDeviceName(eniId)
-	ifaceID := OVSIfaceID(eniId)
-
-	// 0. Clear any prior state on both the OVS and kernel sides. They have
-	// independent persistence — OVS conf.db survives reboot, kernel taps
-	// don't — so a /sys/class/net check alone misses orphan OVS ports left
-	// over after a host reboot or a terminate-during-pending race. The
-	// --if-exists flag makes del-port a no-op when the port is absent, so
-	// unconditional invocation is safe. Reject --may-exist add-port: it
-	// would silently keep stale external_ids:iface-id/attached-mac from a
-	// prior launch with a different ENI.
-	if err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", "br-int", tapName).Run(); err != nil {
-		slog.Warn("Pre-create del-port failed (continuing)", "tap", tapName, "err", err)
+// SetupTap creates the kernel tap, brings it up, and attaches it to spec.Bridge.
+// The pre-create del-port is unconditional because OVS conf.db survives reboot
+// while kernel taps don't, so a /sys/class/net check would miss orphan ports.
+// Rejected alternative: `--may-exist add-port` — it would silently keep stale
+// external_ids from a prior launch with a different ENI/MAC.
+func (p *OVSNetworkPlumber) SetupTap(spec vm.TapSpec) error {
+	if err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", spec.Bridge, spec.Name).Run(); err != nil {
+		slog.Warn("Pre-create del-port failed (continuing)", "tap", spec.Name, "bridge", spec.Bridge, "err", err)
 	}
-	if _, err := os.Stat("/sys/class/net/" + tapName); err == nil {
-		if err := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run(); err != nil {
-			slog.Warn("Pre-create tap del failed (continuing)", "tap", tapName, "err", err)
+	if _, err := os.Stat("/sys/class/net/" + spec.Name); err == nil {
+		if err := sudoCommand("ip", "tuntap", "del", "dev", spec.Name, "mode", "tap").Run(); err != nil {
+			slog.Warn("Pre-create tap del failed (continuing)", "tap", spec.Name, "err", err)
 		}
 	}
 
-	// 1. Create tap device
-	if out, err := sudoCommand("ip", "tuntap", "add", "dev", tapName, "mode", "tap").CombinedOutput(); err != nil {
-		return fmt.Errorf("create tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err)
+	if out, err := sudoCommand("ip", "tuntap", "add", "dev", spec.Name, "mode", "tap").CombinedOutput(); err != nil {
+		return fmt.Errorf("create tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err)
 	}
 
-	// 2. Bring tap up
-	if out, err := sudoCommand("ip", "link", "set", tapName, "up").CombinedOutput(); err != nil {
-		if cleanErr := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run(); cleanErr != nil {
-			slog.Warn("Failed to clean up tap device after bring-up failure", "tap", tapName, "err", cleanErr)
+	if out, err := sudoCommand("ip", "link", "set", spec.Name, "up").CombinedOutput(); err != nil {
+		if cleanErr := sudoCommand("ip", "tuntap", "del", "dev", spec.Name, "mode", "tap").Run(); cleanErr != nil {
+			slog.Warn("Failed to clean up tap after bring-up failure", "tap", spec.Name, "err", cleanErr)
 		}
-		return fmt.Errorf("bring up tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err)
+		return fmt.Errorf("bring up tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err)
 	}
 
-	// 3. Add to br-int with iface-id for OVN port binding
-	if out, err := sudoCommand("ovs-vsctl",
-		"add-port", "br-int", tapName,
-		"--", "set", "Interface", tapName,
-		fmt.Sprintf("external_ids:iface-id=%s", ifaceID),
-		fmt.Sprintf("external_ids:attached-mac=%s", mac),
-	).CombinedOutput(); err != nil {
-		if cleanErr := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run(); cleanErr != nil {
-			slog.Warn("Failed to clean up tap device after OVS failure", "tap", tapName, "err", cleanErr)
+	addPortArgs := []string{"add-port", spec.Bridge, spec.Name}
+	if len(spec.ExternalIDs) > 0 {
+		addPortArgs = append(addPortArgs, "--", "set", "Interface", spec.Name)
+		// Sort keys for deterministic command construction (test assertions, log readability).
+		keys := make([]string, 0, len(spec.ExternalIDs))
+		for k := range spec.ExternalIDs {
+			keys = append(keys, k)
 		}
-		return fmt.Errorf("add tap to br-int: %s: %w", strings.TrimSpace(string(out)), err)
+		sort.Strings(keys)
+		for _, k := range keys {
+			addPortArgs = append(addPortArgs, fmt.Sprintf("external_ids:%s=%s", k, spec.ExternalIDs[k]))
+		}
+	}
+	if out, err := sudoCommand("ovs-vsctl", addPortArgs...).CombinedOutput(); err != nil {
+		if cleanErr := sudoCommand("ip", "tuntap", "del", "dev", spec.Name, "mode", "tap").Run(); cleanErr != nil {
+			slog.Warn("Failed to clean up tap after OVS failure", "tap", spec.Name, "err", cleanErr)
+		}
+		return fmt.Errorf("add tap %s to %s: %s: %w", spec.Name, spec.Bridge, strings.TrimSpace(string(out)), err)
 	}
 
-	slog.Info("Network plumbing complete", "tap", tapName, "iface-id", ifaceID, "mac", mac)
+	slog.Info("Tap plumbing complete", "tap", spec.Name, "bridge", spec.Bridge, "external_ids", spec.ExternalIDs)
 	return nil
 }
 
-func (p *OVSNetworkPlumber) CleanupTapDevice(eniId string) error {
-	tapName := TapDeviceName(eniId)
-
-	// 1. Remove from br-int (--if-exists avoids error if already gone)
-	if out, err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", "br-int", tapName).CombinedOutput(); err != nil {
-		slog.Warn("Failed to remove tap from br-int", "tap", tapName, "err", err, "out", strings.TrimSpace(string(out)))
+// CleanupTap removes the named tap from OVS (any bridge) and the kernel.
+// Idempotent: callers may invoke for an instance that never reached SetupTap
+// (e.g. terminate that races mid-launch).
+func (p *OVSNetworkPlumber) CleanupTap(name string) error {
+	if out, err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", name).CombinedOutput(); err != nil {
+		slog.Warn("Failed to remove tap from OVS", "tap", name, "err", err, "out", strings.TrimSpace(string(out)))
 	}
 
-	// 2. Delete tap device
-	if out, err := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").CombinedOutput(); err != nil {
-		return fmt.Errorf("delete tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err)
+	if _, err := os.Stat("/sys/class/net/" + name); os.IsNotExist(err) {
+		slog.Info("Tap already absent", "tap", name)
+		return nil
 	}
 
-	slog.Info("Network cleanup complete", "tap", tapName)
+	if out, err := sudoCommand("ip", "tuntap", "del", "dev", name, "mode", "tap").CombinedOutput(); err != nil {
+		return fmt.Errorf("delete tap %s: %s: %w", name, strings.TrimSpace(string(out)), err)
+	}
+
+	slog.Info("Tap cleanup complete", "tap", name)
 	return nil
 }
 
-// setupExtraENINICs creates tap devices on br-int and appends matching QEMU
-// virtio-net device entries to instance.Config for each additional ENI a
-// system VM spans. The primary ENI (instance.ENIId) is handled separately by
-// the LaunchInstance caller. Cloud-init brings the guest interfaces up via
-// per-MAC DHCP blocks written by generateNetworkConfig.
-func (d *Daemon) setupExtraENINICs(instance *vm.VM) error {
-	for idx, extra := range instance.ExtraENIs {
-		if err := d.networkPlumber.SetupTapDevice(extra.ENIID, extra.ENIMac); err != nil {
-			slog.Error("Failed to set up tap device for extra ENI", "eni", extra.ENIID, "err", err)
-			return fmt.Errorf("setup tap device for extra ENI %s: %w", extra.ENIID, err)
-		}
-		extraTapName := TapDeviceName(extra.ENIID)
-		netID := fmt.Sprintf("net%d", idx+1)
-		instance.Config.NetDevs = append(instance.Config.NetDevs, vm.NetDev{
-			Value: fmt.Sprintf("tap,id=%s,ifname=%s,script=no,downscript=no", netID, extraTapName),
-		})
-		instance.Config.Devices = append(instance.Config.Devices, vm.Device{
-			Value: fmt.Sprintf("virtio-net-pci,netdev=%s,mac=%s", netID, extra.ENIMac),
-		})
-		slog.Info("Extra VPC NIC configured",
-			"tap", extraTapName, "eni", extra.ENIID, "mac", extra.ENIMac, "subnet", extra.SubnetID)
-	}
-	return nil
-}
-
-// cleanupExtraENITaps removes tap devices for every extra ENI attached to a
-// system VM. Errors are logged but not returned so a partial cleanup still
-// frees as many resources as possible.
-func (d *Daemon) cleanupExtraENITaps(instance *vm.VM) {
-	for _, extra := range instance.ExtraENIs {
-		if err := d.networkPlumber.CleanupTapDevice(extra.ENIID); err != nil {
-			slog.Warn("Failed to clean up extra ENI tap device", "eni", extra.ENIID, "err", err)
-		}
-	}
-}
-
-// TapDeviceName returns the Linux tap device name for an ENI.
-// Linux IFNAMSIZ limits interface names to 15 characters.
-// ENI IDs like "eni-abc123def456789" are too long, so we truncate.
-func TapDeviceName(eniId string) string {
-	id := strings.TrimPrefix(eniId, "eni-")
-	name := "tap" + id
-	if len(name) > 15 {
-		name = name[:15]
-	}
-	return name
-}
-
-// OVSIfaceID returns the OVS external_ids:iface-id value for an ENI.
-// This must match the OVN LogicalSwitchPort name for ovn-controller binding.
-func OVSIfaceID(eniId string) string {
-	return "port-" + eniId
-}
-
-// generateDevMAC creates a locally-administered unicast MAC for the
-// dev/hostfwd NIC. The "dev:" tag disambiguates from the mgmt NIC of the
-// same instance (which shares instanceId).
-func generateDevMAC(instanceId string) string {
-	return utils.HashMAC("dev:" + instanceId)
-}
-
 // generateMgmtMAC creates a locally-administered unicast MAC for the
 // management NIC. The "mgmt:" tag disambiguates from the dev NIC of the
 // same instance (which shares instanceId).
@@ -178,17 +111,6 @@ func generateMgmtMAC(instanceId string) string {
 	return utils.HashMAC("mgmt:" + instanceId)
 }
 
-// MgmtTapName returns the Linux TAP device name for a management NIC.
-// Uses "mg" prefix + truncated instance ID to stay within 15-char IFNAMSIZ.
-func MgmtTapName(instanceID string) string {
-	id := strings.TrimPrefix(instanceID, "i-")
-	name := "mg" + id
-	if len(name) > 15 {
-		name = name[:15]
-	}
-	return name
-}
-
 // GetBridgeIPv4 returns the first IPv4 address on the named Linux bridge.
 // Returns "", nil if the bridge does not exist.
 func GetBridgeIPv4(bridgeName string) (string, error) {
@@ -217,74 +139,6 @@ func GetBridgeIPv4(bridgeName string) (string, error) {
 	return "", fmt.Errorf("no IPv4 address on %s", bridgeName)
 }
 
-// SetupMgmtTapDevice creates a TAP device and attaches it to the management
-// OVS bridge (provisioned by scripts/setup-ovn.sh). br-mgmt uses
-// fail-mode=standalone and is excluded from ovn-bridge-mappings, so it
-// behaves as a plain L2 switch outside the OVN overlay.
-func SetupMgmtTapDevice(instanceID, mac, bridge string) (string, error) {
-	tapName := MgmtTapName(instanceID)
-
-	// Clean up stale TAP if present
-	if _, err := os.Stat("/sys/class/net/" + tapName); err == nil {
-		slog.Warn("Stale mgmt tap found, cleaning up", "tap", tapName)
-		if err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", bridge, tapName).Run(); err != nil {
-			slog.Warn("Failed to remove stale mgmt tap from bridge", "tap", tapName, "err", err)
-		}
-		if err := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run(); err != nil {
-			slog.Warn("Failed to delete stale mgmt tap", "tap", tapName, "err", err)
-		}
-	}
-
-	// Create TAP
-	if out, err := sudoCommand("ip", "tuntap", "add", "dev", tapName, "mode", "tap").CombinedOutput(); err != nil {
-		return "", fmt.Errorf("create mgmt tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err)
-	}
-
-	// Bring up
-	if out, err := sudoCommand("ip", "link", "set", tapName, "up").CombinedOutput(); err != nil {
-		_ = sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run()
-		return "", fmt.Errorf("bring up mgmt tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err)
-	}
-
-	// Attach to OVS mgmt bridge
-	if out, err := sudoCommand("ovs-vsctl", "--may-exist", "add-port", bridge, tapName).CombinedOutput(); err != nil {
-		_ = sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run()
-		return "", fmt.Errorf("add mgmt tap to %s: %s: %w", bridge, strings.TrimSpace(string(out)), err)
-	}
-
-	slog.Info("Management TAP created", "tap", tapName, "bridge", bridge, "mac", mac)
-	return tapName, nil
-}
-
-// CleanupMgmtTapDevice removes a management TAP device from the mgmt OVS
-// bridge and deletes it. Tolerant of missing state on either side: OVS port
-// removal is gated by --if-exists, kernel tap deletion is gated on
-// /sys/class/net/<tap> presence so callers may invoke this for an instance
-// that never reached the SetupMgmtTapDevice step.
-func CleanupMgmtTapDevice(tapName string) error {
-	// Detach from whichever OVS bridge holds this tap (usually br-mgmt)
-	if out, err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", tapName).CombinedOutput(); err != nil {
-		slog.Warn("Failed to remove mgmt tap from bridge", "tap", tapName, "err", err, "out", strings.TrimSpace(string(out)))
-	}
-
-	// Skip kernel-side delete if the tap is already gone (or was never
-	// created) — avoids a misleading "Device does not exist" error log
-	// when finalizeTermination cleans up an instance whose mgmt tap
-	// setup never completed.
-	if _, err := os.Stat("/sys/class/net/" + tapName); os.IsNotExist(err) {
-		slog.Info("Management TAP already absent", "tap", tapName)
-		return nil
-	}
-
-	// Delete TAP
-	if out, err := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").CombinedOutput(); err != nil {
-		return fmt.Errorf("delete mgmt tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err)
-	}
-
-	slog.Info("Management TAP cleaned up", "tap", tapName)
-	return nil
-}
-
 // OVNHealthStatus reports the readiness of OVN networking on this compute node.
 type OVNHealthStatus struct {
 	BrIntExists     bool   `json:"br_int_exists"`

spinifex/daemon/network_test.go

@@ -1,14 +1,15 @@
 package daemon
 
 import (
-	"fmt"
 	"net"
 	"os"
 	"os/exec"
+	"slices"
 	"strings"
 	"testing"
 
 	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/stretchr/testify/require"
 )
 
 func TestTapDeviceName(t *testing.T) {
@@ -25,144 +26,17 @@ func TestTapDeviceName(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.eniId, func(t *testing.T) {
-			got := TapDeviceName(tt.eniId)
+			got := vm.TapDeviceName(tt.eniId)
 			if got != tt.expected {
-				t.Errorf("TapDeviceName(%q) = %q, want %q", tt.eniId, got, tt.expected)
+				t.Errorf("vm.TapDeviceName(%q) = %q, want %q", tt.eniId, got, tt.expected)
 			}
 			if len(got) > 15 {
-				t.Errorf("TapDeviceName(%q) = %q (len %d), exceeds IFNAMSIZ limit of 15", tt.eniId, got, len(got))
+				t.Errorf("vm.TapDeviceName(%q) = %q (len %d), exceeds IFNAMSIZ limit of 15", tt.eniId, got, len(got))
 			}
 		})
 	}
 }
 
-// MockNetworkPlumber records calls for testing.
-type MockNetworkPlumber struct {
-	SetupCalls   []mockSetupCall
-	CleanupCalls []string
-	SetupErr     error
-	CleanupErr   error
-}
-
-var _ NetworkPlumber = (*MockNetworkPlumber)(nil)
-
-type mockSetupCall struct {
-	ENIId string
-	MAC   string
-}
-
-func (m *MockNetworkPlumber) SetupTapDevice(eniId, mac string) error {
-	m.SetupCalls = append(m.SetupCalls, mockSetupCall{ENIId: eniId, MAC: mac})
-	return m.SetupErr
-}
-
-func (m *MockNetworkPlumber) CleanupTapDevice(eniId string) error {
-	m.CleanupCalls = append(m.CleanupCalls, eniId)
-	return m.CleanupErr
-}
-
-func TestStartInstance_VPCNetworking(t *testing.T) {
-	instance := &vm.VM{
-		ID:           "i-test123",
-		InstanceType: "t3.micro",
-		ENIId:        "eni-abc123def456789",
-		ENIMac:       "02:00:00:11:22:33",
-	}
-
-	// When ENI is set, config should use tap networking with MAC
-	instance.Config = vm.Config{
-		CPUCount:     1,
-		Memory:       512,
-		Architecture: "x86_64",
-	}
-
-	// Simulate what StartInstance does for VPC mode
-	tapName := TapDeviceName(instance.ENIId)
-	instance.Config.NetDevs = append(instance.Config.NetDevs, vm.NetDev{
-		Value: "tap,id=net0,ifname=" + tapName + ",script=no,downscript=no",
-	})
-	instance.Config.Devices = append(instance.Config.Devices, vm.Device{
-		Value: "virtio-net-pci,netdev=net0,mac=" + instance.ENIMac,
-	})
-
-	// Verify QEMU args
-	if len(instance.Config.NetDevs) != 1 {
-		t.Fatalf("expected 1 netdev, got %d", len(instance.Config.NetDevs))
-	}
-
-	expected := "tap,id=net0,ifname=tapabc123def456,script=no,downscript=no"
-	if instance.Config.NetDevs[0].Value != expected {
-		t.Errorf("netdev = %q, want %q", instance.Config.NetDevs[0].Value, expected)
-	}
-
-	expectedDev := "virtio-net-pci,netdev=net0,mac=02:00:00:11:22:33"
-	if instance.Config.Devices[0].Value != expectedDev {
-		t.Errorf("device = %q, want %q", instance.Config.Devices[0].Value, expectedDev)
-	}
-}
-
-func TestStartInstance_FallbackNetworking(t *testing.T) {
-	instance := &vm.VM{
-		ID:           "i-test456",
-		InstanceType: "t3.micro",
-		// No ENI — should use user-mode networking
-	}
-
-	instance.Config = vm.Config{
-		CPUCount:     1,
-		Memory:       512,
-		Architecture: "x86_64",
-	}
-
-	// Simulate what StartInstance does for non-VPC mode
-	instance.Config.NetDevs = append(instance.Config.NetDevs, vm.NetDev{
-		Value: "user,id=net0,hostfwd=tcp:127.0.0.1:22222-:22",
-	})
-	instance.Config.Devices = append(instance.Config.Devices, vm.Device{
-		Value: "virtio-net-pci,netdev=net0",
-	})
-
-	// Verify no MAC is specified (QEMU auto-assigns)
-	if len(instance.Config.Devices) != 1 {
-		t.Fatalf("expected 1 device, got %d", len(instance.Config.Devices))
-	}
-
-	// User-mode networking should not include MAC
-	dev := instance.Config.Devices[0].Value
-	if dev != "virtio-net-pci,netdev=net0" {
-		t.Errorf("device = %q, want 'virtio-net-pci,netdev=net0'", dev)
-	}
-}
-
-func TestMockNetworkPlumber_SetupAndCleanup(t *testing.T) {
-	mock := &MockNetworkPlumber{}
-
-	err := mock.SetupTapDevice("eni-abc123", "02:00:00:aa:bb:cc")
-	if err != nil {
-		t.Fatalf("SetupTapDevice: %v", err)
-	}
-	if len(mock.SetupCalls) != 1 {
-		t.Fatalf("expected 1 setup call, got %d", len(mock.SetupCalls))
-	}
-	if mock.SetupCalls[0].ENIId != "eni-abc123" {
-		t.Errorf("setup eniId = %q, want 'eni-abc123'", mock.SetupCalls[0].ENIId)
-	}
-	if mock.SetupCalls[0].MAC != "02:00:00:aa:bb:cc" {
-		t.Errorf("setup mac = %q, want '02:00:00:aa:bb:cc'", mock.SetupCalls[0].MAC)
-	}
-
-	err = mock.CleanupTapDevice("eni-abc123")
-	if err != nil {
-		t.Fatalf("CleanupTapDevice: %v", err)
-	}
-	if len(mock.CleanupCalls) != 1 {
-		t.Fatalf("expected 1 cleanup call, got %d", len(mock.CleanupCalls))
-	}
-	if mock.CleanupCalls[0] != "eni-abc123" {
-		t.Errorf("cleanup eniId = %q, want 'eni-abc123'", mock.CleanupCalls[0])
-	}
-}
-
 func TestSudoCommand_NonRoot(t *testing.T) {
 	// When not root, sudoCommand should prefix with sudo
 	cmd := sudoCommand("ovs-vsctl", "br-exists", "br-int")
@@ -200,168 +74,35 @@ func TestGenerateDevMAC(t *testing.T) {
 	// octet is hash-derived (not the literal class prefix of the old impl).
 	seen := make(map[string]bool)
 	for _, tt := range tests {
-		mac := generateDevMAC(tt.instanceId)
+		mac := vm.GenerateDevMAC(tt.instanceId)
 		hw, err := net.ParseMAC(mac)
 		if err != nil {
-			t.Errorf("generateDevMAC(%q) = %q: invalid MAC: %v", tt.instanceId, mac, err)
+			t.Errorf("vm.GenerateDevMAC(%q) = %q: invalid MAC: %v", tt.instanceId, mac, err)
 			continue
 		}
 		if hw[0]&0x03 != 0x02 {
-			t.Errorf("generateDevMAC(%q) = %q: expected unicast+LAA bits, got %#x",
+			t.Errorf("vm.GenerateDevMAC(%q) = %q: expected unicast+LAA bits, got %#x",
 				tt.instanceId, mac, hw[0])
 		}
 		if seen[mac] {
-			t.Errorf("generateDevMAC(%q) = %q, duplicate MAC", tt.instanceId, mac)
+			t.Errorf("vm.GenerateDevMAC(%q) = %q, duplicate MAC", tt.instanceId, mac)
 		}
 		seen[mac] = true
 	}
 
 	// Class separation: dev and mgmt MACs for the same instance must differ.
-	if generateDevMAC("i-abc123") == generateMgmtMAC("i-abc123") {
+	if vm.GenerateDevMAC("i-abc123") == generateMgmtMAC("i-abc123") {
 		t.Error("expected dev and mgmt MACs for same instance to differ")
 	}
 
 	// Same input should produce same output (deterministic)
-	mac1 := generateDevMAC("i-test123")
-	mac2 := generateDevMAC("i-test123")
+	mac1 := vm.GenerateDevMAC("i-test123")
+	mac2 := vm.GenerateDevMAC("i-test123")
 	if mac1 != mac2 {
 		t.Errorf("generateDevMAC not deterministic: %q != %q", mac1, mac2)
 	}
 }
 
-func TestSetupExtraENINICs_AppendsOnePerExtra(t *testing.T) {
-	mock := &MockNetworkPlumber{}
-	d := &Daemon{networkPlumber: mock}
-	instance := &vm.VM{
-		ID: "i-multi",
-		ExtraENIs: []vm.ExtraENI{
-			{ENIID: "eni-aaa", ENIMac: "02:00:00:aa:aa:aa", ENIIP: "10.0.1.4", SubnetID: "subnet-a"},
-			{ENIID: "eni-bbb", ENIMac: "02:00:00:bb:bb:bb", ENIIP: "10.0.2.4", SubnetID: "subnet-b"},
-		},
-	}
-
-	if err := d.setupExtraENINICs(instance); err != nil {
-		t.Fatalf("setupExtraENINICs failed: %v", err)
-	}
-
-	// One SetupTapDevice call per extra ENI.
-	if len(mock.SetupCalls) != 2 {
-		t.Fatalf("expected 2 SetupTapDevice calls, got %d", len(mock.SetupCalls))
-	}
-	if mock.SetupCalls[0].ENIId != "eni-aaa" || mock.SetupCalls[0].MAC != "02:00:00:aa:aa:aa" {
-		t.Errorf("first setup call = %+v, want eni-aaa/02:00:00:aa:aa:aa", mock.SetupCalls[0])
-	}
-	if mock.SetupCalls[1].ENIId != "eni-bbb" || mock.SetupCalls[1].MAC != "02:00:00:bb:bb:bb" {
-		t.Errorf("second setup call = %+v, want eni-bbb/02:00:00:bb:bb:bb", mock.SetupCalls[1])
-	}
-
-	// NetDevs and Devices each get one entry per extra ENI, named net1/net2.
-	if len(instance.Config.NetDevs) != 2 || len(instance.Config.Devices) != 2 {
-		t.Fatalf("expected 2 netdevs + 2 devices, got %d + %d",
-			len(instance.Config.NetDevs), len(instance.Config.Devices))
-	}
-	if !strings.Contains(instance.Config.NetDevs[0].Value, "id=net1") {
-		t.Errorf("netdev[0] = %q, want id=net1", instance.Config.NetDevs[0].Value)
-	}
-	if !strings.Contains(instance.Config.NetDevs[1].Value, "id=net2") {
-		t.Errorf("netdev[1] = %q, want id=net2", instance.Config.NetDevs[1].Value)
-	}
-	if !strings.Contains(instance.Config.Devices[0].Value, "mac=02:00:00:aa:aa:aa") {
-		t.Errorf("device[0] = %q, missing primary MAC", instance.Config.Devices[0].Value)
-	}
-	if !strings.Contains(instance.Config.Devices[1].Value, "mac=02:00:00:bb:bb:bb") {
-		t.Errorf("device[1] = %q, missing second MAC", instance.Config.Devices[1].Value)
-	}
-}
-
-func TestSetupExtraENINICs_NoExtras_NoOp(t *testing.T) {
-	mock := &MockNetworkPlumber{}
-	d := &Daemon{networkPlumber: mock}
-	instance := &vm.VM{ID: "i-single"}
-
-	if err := d.setupExtraENINICs(instance); err != nil {
-		t.Fatalf("setupExtraENINICs failed: %v", err)
-	}
-	if len(mock.SetupCalls) != 0 {
-		t.Errorf("expected zero setup calls for no extras, got %d", len(mock.SetupCalls))
-	}
-	if len(instance.Config.NetDevs) != 0 || len(instance.Config.Devices) != 0 {
-		t.Errorf("expected no netdevs/devices, got %d/%d",
-			len(instance.Config.NetDevs), len(instance.Config.Devices))
-	}
-}
-
-func TestSetupExtraENINICs_TapSetupErrorReturns(t *testing.T) {
-	mock := &MockNetworkPlumber{SetupErr: fmt.Errorf("simulated tap failure")}
-	d := &Daemon{networkPlumber: mock}
-	instance := &vm.VM{
-		ID: "i-multi-err",
-		ExtraENIs: []vm.ExtraENI{
-			{ENIID: "eni-aaa", ENIMac: "02:00:00:aa:aa:aa"},
-			{ENIID: "eni-bbb", ENIMac: "02:00:00:bb:bb:bb"},
-		},
-	}
-
-	err := d.setupExtraENINICs(instance)
-	if err == nil {
-		t.Fatal("expected error from failing tap setup, got nil")
-	}
-	if !strings.Contains(err.Error(), "eni-aaa") {
-		t.Errorf("error = %v, want it to mention the failing ENI", err)
-	}
-	// Must bail on first failure — second ENI should not be touched.
-	if len(mock.SetupCalls) != 1 {
-		t.Errorf("expected 1 setup call before bailout, got %d", len(mock.SetupCalls))
-	}
-	// No NIC config appended for a failed setup.
-	if len(instance.Config.NetDevs) != 0 {
-		t.Errorf("expected no netdevs on failure, got %d", len(instance.Config.NetDevs))
-	}
-}
-
-func TestCleanupExtraENITaps_CallsCleanupPerExtra(t *testing.T) {
-	mock := &MockNetworkPlumber{}
-	d := &Daemon{networkPlumber: mock}
-	instance := &vm.VM{
-		ID: "i-multi-clean",
-		ExtraENIs: []vm.ExtraENI{
-			{ENIID: "eni-111"},
-			{ENIID: "eni-222"},
-			{ENIID: "eni-333"},
-		},
-	}
-
-	d.cleanupExtraENITaps(instance)
-
-	if len(mock.CleanupCalls) != 3 {
-		t.Fatalf("expected 3 cleanup calls, got %d", len(mock.CleanupCalls))
-	}
-	for i, want := range []string{"eni-111", "eni-222", "eni-333"} {
-		if mock.CleanupCalls[i] != want {
-			t.Errorf("cleanup[%d] = %q, want %q", i, mock.CleanupCalls[i], want)
-		}
-	}
-}
-
-func TestCleanupExtraENITaps_ErrorsAreLogged(t *testing.T) {
-	mock := &MockNetworkPlumber{CleanupErr: fmt.Errorf("simulated cleanup failure")}
-	d := &Daemon{networkPlumber: mock}
-	instance := &vm.VM{
-		ID: "i-multi-clean-err",
-		ExtraENIs: []vm.ExtraENI{
-			{ENIID: "eni-111"},
-			{ENIID: "eni-222"},
-		},
-	}
-
-	// Must not panic or return — errors are swallowed by design so partial
-	// cleanup still frees later entries.
-	d.cleanupExtraENITaps(instance)
-	if len(mock.CleanupCalls) != 2 {
-		t.Errorf("expected both extras to be attempted, got %d cleanup calls", len(mock.CleanupCalls))
-	}
-}
-
 func TestFindInterfaceByIP_InvalidIP(t *testing.T) {
 	_, err := findInterfaceByIP("not-an-ip")
 	if err == nil {
@@ -393,26 +134,256 @@ func TestFindInterfaceByIP_NotFound(t *testing.T) {
 	}
 }
 
-// TestCleanupMgmtTapDevice_MissingKernelTap verifies the nil-safe branch
-// added to support finalizeTermination: a terminate that races mid-launch
-// (or an instance that never created a mgmt tap) must not log a misleading
-// "Device does not exist" error from `ip tuntap del`.
-func TestCleanupMgmtTapDevice_MissingKernelTap(t *testing.T) {
+// TestOVSNetworkPlumber_SetupTap_AddPortArgs captures the ovs-vsctl invocations
+// for both VPC-style (populated ExternalIDs → `set Interface external_ids:k=v`)
+// and management-style (empty ExternalIDs → bare `add-port`) calls. This is the
+// functional difference that previously lived in two diverged setup functions.
+func TestOVSNetworkPlumber_SetupTap_AddPortArgs(t *testing.T) {
+	cases := []struct {
+		name        string
+		spec        vm.TapSpec
+		wantAddPort []string // expected tail of ovs-vsctl args (after add-port <bridge> <name>)
+	}{
+		{
+			name: "vpc style with external_ids",
+			spec: vm.TapSpec{
+				Name:   "tapeni-test",
+				Bridge: "br-int",
+				ExternalIDs: map[string]string{
+					"iface-id":     "port-eni-test",
+					"attached-mac": "02:00:00:aa:bb:cc",
+				},
+			},
+			wantAddPort: []string{
+				"add-port", "br-int", "tapeni-test",
+				"--", "set", "Interface", "tapeni-test",
+				"external_ids:attached-mac=02:00:00:aa:bb:cc",
+				"external_ids:iface-id=port-eni-test",
+			},
+		},
+		{
+			name: "mgmt style without external_ids",
+			spec: vm.TapSpec{
+				Name:   "mg-test",
+				Bridge: "br-mgmt",
+			},
+			wantAddPort: []string{"add-port", "br-mgmt", "mg-test"},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			orig := sudoCommand
+			t.Cleanup(func() { sudoCommand = orig })
+
+			var calls [][]string
+			sudoCommand = func(name string, args ...string) *exec.Cmd {
+				call := append([]string{name}, args...)
+				calls = append(calls, call)
+				return exec.Command("/bin/true")
+			}
+
+			p := &OVSNetworkPlumber{}
+			if err := p.SetupTap(tc.spec); err != nil {
+				t.Fatalf("SetupTap: %v", err)
+			}
+
+			// Find the add-port invocation (last ovs-vsctl call).
+			var addPort []string
+			for _, c := range calls {
+				if len(c) >= 2 && c[0] == "ovs-vsctl" && c[1] != "--if-exists" {
+					addPort = c[1:]
+				}
+			}
+			if addPort == nil {
+				t.Fatalf("no add-port invocation captured; calls=%v", calls)
+			}
+			if !slices.Equal(addPort, tc.wantAddPort) {
+				t.Errorf("add-port args = %v, want %v", addPort, tc.wantAddPort)
+			}
+		})
+	}
+}
+
+// TestOVSNetworkPlumber_SetupTap_PreExistingKernelTap exercises the
+// pre-create kernel-side cleanup branch (lines guarded by /sys/class/net/<name>
+// stat). Uses "lo" as a name that's always present so os.Stat returns success.
+func TestOVSNetworkPlumber_SetupTap_PreExistingKernelTap(t *testing.T) {
+	orig := sudoCommand
+	t.Cleanup(func() { sudoCommand = orig })
+
+	failTuntapDel := false
+	sudoCommand = func(name string, args ...string) *exec.Cmd {
+		if name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "del" && failTuntapDel {
+			failTuntapDel = false
+			return exec.Command("/bin/false")
+		}
+		return exec.Command("/bin/true")
+	}
+
+	failTuntapDel = true
+	p := &OVSNetworkPlumber{}
+	if err := p.SetupTap(vm.TapSpec{Name: "lo", Bridge: "br-int"}); err != nil {
+		t.Fatalf("SetupTap with stubs should succeed: %v", err)
+	}
+}
+
+// TestOVSNetworkPlumber_SetupTap_ErrorBranches stubs sudoCommand to fail at
+// each step in turn and asserts the corresponding error is returned. Covers
+// the cleanup-on-failure paths that the success-path tests miss.
+func TestOVSNetworkPlumber_SetupTap_ErrorBranches(t *testing.T) {
+	cases := []struct {
+		name      string
+		failMatch func(name string, args []string) bool
+		wantErr   string
+	}{
+		{
+			name: "ovs del-port failure is logged-warn (continues)",
+			failMatch: func(name string, args []string) bool {
+				return name == "ovs-vsctl" && len(args) >= 1 && args[0] == "--if-exists"
+			},
+			wantErr: "", // pre-clear failure is swallowed; SetupTap returns nil
+		},
+		{
+			name: "tuntap add failure surfaces",
+			failMatch: func(name string, args []string) bool {
+				return name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "add"
+			},
+			wantErr: "create tap",
+		},
+		{
+			name: "ip link set up failure surfaces and triggers tap cleanup",
+			failMatch: func(name string, args []string) bool {
+				return name == "ip" && len(args) >= 1 && args[0] == "link"
+			},
+			wantErr: "bring up tap",
+		},
+		{
+			name: "ovs add-port failure surfaces and triggers tap cleanup",
+			failMatch: func(name string, args []string) bool {
+				return name == "ovs-vsctl" && len(args) >= 1 && args[0] == "add-port"
+			},
+			wantErr: "add tap",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			orig := sudoCommand
+			t.Cleanup(func() { sudoCommand = orig })
+
+			sudoCommand = func(name string, args ...string) *exec.Cmd {
+				if tc.failMatch(name, args) {
+					return exec.Command("/bin/false")
+				}
+				return exec.Command("/bin/true")
+			}
+
+			p := &OVSNetworkPlumber{}
+			err := p.SetupTap(vm.TapSpec{Name: "tap-test-noexist", Bridge: "br-int"})
+			if tc.wantErr == "" {
+				if err != nil {
+					t.Fatalf("expected nil, got %v", err)
+				}
+				return
+			}
+			if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
+				t.Fatalf("expected error containing %q, got %v", tc.wantErr, err)
+			}
+		})
+	}
+}
+
+// TestOVSNetworkPlumber_CleanupTap_PresentKernelTap exercises the success
+// path where the kernel device is present (uses "lo" so /sys/class/net/lo
+// stat succeeds). Asserts no error and that ip tuntap del was attempted.
+func TestOVSNetworkPlumber_CleanupTap_PresentKernelTap(t *testing.T) {
+	orig := sudoCommand
+	t.Cleanup(func() { sudoCommand = orig })
+
+	var sawTuntapDel bool
+	sudoCommand = func(name string, args ...string) *exec.Cmd {
+		if name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "del" {
+			sawTuntapDel = true
+		}
+		return exec.Command("/bin/true")
+	}
+
+	p := &OVSNetworkPlumber{}
+	if err := p.CleanupTap("lo"); err != nil {
+		t.Fatalf("CleanupTap: %v", err)
+	}
+	if !sawTuntapDel {
+		t.Errorf("expected ip tuntap del to be invoked, got no call")
+	}
+}
+
+// TestOVSNetworkPlumber_CleanupTap_DelPortLoggedWarn covers the OVS del-port
+// failure branch (logged warn, continues to kernel-side cleanup).
+func TestOVSNetworkPlumber_CleanupTap_DelPortLoggedWarn(t *testing.T) {
+	orig := sudoCommand
+	t.Cleanup(func() { sudoCommand = orig })
+
+	sudoCommand = func(name string, args ...string) *exec.Cmd {
+		if name == "ovs-vsctl" {
+			return exec.Command("/bin/false")
+		}
+		return exec.Command("/bin/true")
+	}
+
+	p := &OVSNetworkPlumber{}
+	// Tap doesn't exist on this test system → kernel-presence gate short-circuits
+	// after the OVS warn, so CleanupTap returns nil.
+	if err := p.CleanupTap("tap-test-noexist"); err != nil {
+		t.Fatalf("CleanupTap: %v", err)
+	}
+}
+
+// TestOVSNetworkPlumber_CleanupTap_TuntapDelFailure exercises the kernel-side
+// failure branch (real kernel device present + ip tuntap del returns nonzero).
+func TestOVSNetworkPlumber_CleanupTap_TuntapDelFailure(t *testing.T) {
+	orig := sudoCommand
+	t.Cleanup(func() { sudoCommand = orig })
+
+	sudoCommand = func(name string, args ...string) *exec.Cmd {
+		if name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "del" {
+			return exec.Command("/bin/false")
+		}
+		return exec.Command("/bin/true")
+	}
+
+	p := &OVSNetworkPlumber{}
+	err := p.CleanupTap("lo") // /sys/class/net/lo is present, so we reach tuntap del
+	if err == nil || !strings.Contains(err.Error(), "delete tap") {
+		t.Fatalf("expected 'delete tap' error, got %v", err)
+	}
+}
+
+// TestOVSNetworkPlumber_CleanupTap_MissingKernelTap verifies the nil-safe branch:
+// callers may invoke CleanupTap on a name that has neither an OVS port nor a
+// kernel tap (e.g. a terminate that races mid-launch, or an instance that
+// never reached SetupTap) without producing a misleading "Device does not
+// exist" error from `ip tuntap del`.
+func TestOVSNetworkPlumber_CleanupTap_MissingKernelTap(t *testing.T) {
 	// 15-char IFNAMSIZ-compliant name that does not exist in /sys/class/net.
 	// The OVS del-port call may fail on CI without ovs-vsctl, but is
 	// best-effort + logged-warn; the kernel-presence gate short-circuits
 	// before `ip tuntap del` runs.
-	if err := CleanupMgmtTapDevice("mg-test-noexist"); err != nil {
+	p := &OVSNetworkPlumber{}
+	if err := p.CleanupTap("mg-test-noexist"); err != nil {
 		t.Fatalf("expected nil for missing kernel tap, got: %v", err)
 	}
 }
 
-func TestEnsureDataRoute_NoOVS(t *testing.T) {
-	// EnsureDataRoute requires ip commands which may not work in CI.
-	// On loopback, there's no kernel subnet route, so it should return an error.
+func TestEnsureDataRoute_NoKernelRoute(t *testing.T) {
+	// 127.0.0.1 resolves to "lo", which has no kernel/scope-link subnet route.
+	// EnsureDataRoute must surface this as an error rather than silently
+	// succeed against a non-existent route — a regression that drops the
+	// error would leave Geneve traffic egressing the wrong NIC.
 	err := EnsureDataRoute("127.0.0.1")
-	// We expect an error (no kernel route for lo), but no panic.
-	_ = err
+	require.Error(t, err, "expected error for IP without a kernel subnet route")
+	require.ErrorContains(t, err, "no kernel route found",
+		"error must identify the missing-route condition, not a generic interface lookup failure")
 }
 
 func TestSetupComputeNode_ValidatesArgs(t *testing.T) {
@@ -431,39 +402,6 @@ func TestSetupComputeNode_ValidatesArgs(t *testing.T) {
 	}
 }
 
-func TestMockNetworkPlumber_SetupError(t *testing.T) {
-	mock := &MockNetworkPlumber{
-		SetupErr: fmt.Errorf("simulated setup failure"),
-	}
-	err := mock.SetupTapDevice("eni-abc123", "02:00:00:aa:bb:cc")
-	if err == nil {
-		t.Fatal("expected error from SetupTapDevice")
-	}
-	if err.Error() != "simulated setup failure" {
-		t.Errorf("unexpected error: %v", err)
-	}
-	// Call should still be recorded
-	if len(mock.SetupCalls) != 1 {
-		t.Fatalf("expected 1 setup call, got %d", len(mock.SetupCalls))
-	}
-}
-
-func TestMockNetworkPlumber_CleanupError(t *testing.T) {
-	mock := &MockNetworkPlumber{
-		CleanupErr: fmt.Errorf("simulated cleanup failure"),
-	}
-	err := mock.CleanupTapDevice("eni-abc123")
-	if err == nil {
-		t.Fatal("expected error from CleanupTapDevice")
-	}
-	if err.Error() != "simulated cleanup failure" {
-		t.Errorf("unexpected error: %v", err)
-	}
-	if len(mock.CleanupCalls) != 1 {
-		t.Fatalf("expected 1 cleanup call, got %d", len(mock.CleanupCalls))
-	}
-}
-
 func TestGenerateDevMAC_Format(t *testing.T) {
 	tests := []string{
 		"i-abc123",
@@ -474,14 +412,14 @@ func TestGenerateDevMAC_Format(t *testing.T) {
 		"i-very-long-instance-id-with-many-characters",
 	}
 	for _, id := range tests {
-		mac := generateDevMAC(id)
+		mac := vm.GenerateDevMAC(id)
 		hw, err := net.ParseMAC(mac)
 		if err != nil {
-			t.Errorf("generateDevMAC(%q) = %q: invalid MAC: %v", id, mac, err)
+			t.Errorf("vm.GenerateDevMAC(%q) = %q: invalid MAC: %v", id, mac, err)
 			continue
 		}
 		if hw[0]&0x03 != 0x02 {
-			t.Errorf("generateDevMAC(%q) = %q: expected unicast+LAA bits, got %#x",
+			t.Errorf("vm.GenerateDevMAC(%q) = %q: expected unicast+LAA bits, got %#x",
 				id, mac, hw[0])
 		}
 	}
@@ -489,9 +427,9 @@ func TestGenerateDevMAC_Format(t *testing.T) {
 
 func TestTapDeviceName_EmptyInput(t *testing.T) {
 	// Even with empty string (no eni- prefix), should not panic
-	name := TapDeviceName("")
+	name := vm.TapDeviceName("")
 	if name != "tap" {
-		t.Errorf("TapDeviceName('') = %q, want 'tap'", name)
+		t.Errorf("vm.TapDeviceName('') = %q, want 'tap'", name)
 	}
 }
 
@@ -507,9 +445,9 @@ func TestOVSIfaceID_Format(t *testing.T) {
 		{"", "port-"},
 	}
 	for _, tt := range tests {
-		got := OVSIfaceID(tt.eniId)
+		got := vm.OVSIfaceID(tt.eniId)
 		if got != tt.expected {
-			t.Errorf("OVSIfaceID(%q) = %q, want %q", tt.eniId, got, tt.expected)
+			t.Errorf("vm.OVSIfaceID(%q) = %q, want %q", tt.eniId, got, tt.expected)
 		}
 	}
 }
@@ -547,7 +485,7 @@ func TestGenerateMgmtMAC(t *testing.T) {
 	}
 
 	// Different from dev MAC for same instance
-	devMAC := generateDevMAC("i-test123")
+	devMAC := vm.GenerateDevMAC("i-test123")
 	mgmtMAC := generateMgmtMAC("i-test123")
 	if devMAC == mgmtMAC {
 		t.Errorf("dev and mgmt MACs should differ for same instance: both %q", devMAC)
@@ -567,12 +505,12 @@ func TestMgmtTapName(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.instanceID, func(t *testing.T) {
-			got := MgmtTapName(tt.instanceID)
+			got := vm.MgmtTapName(tt.instanceID)
 			if got != tt.expected {
-				t.Errorf("MgmtTapName(%q) = %q, want %q", tt.instanceID, got, tt.expected)
+				t.Errorf("vm.MgmtTapName(%q) = %q, want %q", tt.instanceID, got, tt.expected)
 			}
 			if len(got) > 15 {
-				t.Errorf("MgmtTapName(%q) = %q (len %d), exceeds IFNAMSIZ limit of 15", tt.instanceID, got, len(got))
+				t.Errorf("vm.MgmtTapName(%q) = %q (len %d), exceeds IFNAMSIZ limit of 15", tt.instanceID, got, len(got))
 			}
 		})
 	}

spinifex/daemon/ownership.go

@@ -10,14 +10,15 @@ import (
 
 // checkInstanceOwnership verifies the caller owns the instance.
 // Returns true if access is allowed, false if denied (error already sent).
-// Pre-Phase4 instances (empty AccountID) are only visible to root (GlobalAccountID).
+// Instances with an empty AccountID (legacy/migration data) are only
+// visible to root (GlobalAccountID).
 func checkInstanceOwnership(msg *nats.Msg, instanceID, ownerAccountID string) bool {
 	callerAccountID := utils.AccountIDFromMsg(msg)
 
-	// Pre-Phase4 instance: only root can access
+	// Untenanted instance: only root can access.
 	if ownerAccountID == "" {
 		if callerAccountID != utils.GlobalAccountID {
-			slog.Warn("Pre-Phase4 instance access denied (not root)",
+			slog.Warn("Untenanted instance access denied (not root)",
 				"instanceId", instanceID, "callerAccount", callerAccountID)
 			respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound)
 			return false
@@ -36,7 +37,8 @@ func checkInstanceOwnership(msg *nats.Msg, instanceID, ownerAccountID string) bo
 }
 
 // isInstanceVisible checks if the caller can see this instance (for Describe operations).
-// Pre-Phase4 instances (empty AccountID) are only visible to root (GlobalAccountID).
+// Instances with an empty AccountID (legacy/migration data) are only
+// visible to root (GlobalAccountID).
 func isInstanceVisible(callerAccountID, ownerAccountID string) bool {
 	if ownerAccountID == "" {
 		return callerAccountID == utils.GlobalAccountID
@@ -45,10 +47,10 @@ func isInstanceVisible(callerAccountID, ownerAccountID string) bool {
 }
 
 // volumeVisibleTo reports whether callerAccountID may operate on a volume with
-// the given tenantID. Pre-Phase4 volumes (empty tenantID) are root-only —
-// without this, the legacy short-circuit `tenantID != "" && tenantID != caller`
-// matched every caller and let any tenant attach a legacy/migration volume by
-// ID alone.
+// the given tenantID. Volumes with an empty tenantID (legacy/migration
+// data) are root-only — without this, the short-circuit
+// `tenantID != "" && tenantID != caller` matches every caller and lets any
+// tenant attach a legacy/migration volume by ID alone.
 func volumeVisibleTo(tenantID, callerAccountID string) bool {
 	if tenantID == "" {
 		return callerAccountID == utils.GlobalAccountID

spinifex/daemon/ownership_test.go

@@ -26,13 +26,13 @@ func TestIsInstanceVisible(t *testing.T) {
 			expected:        false,
 		},
 		{
-			name:            "pre-Phase4 instance visible to root",
+			name:            "untenanted instance visible to root",
 			callerAccountID: utils.GlobalAccountID,
 			ownerAccountID:  "",
 			expected:        true,
 		},
 		{
-			name:            "pre-Phase4 instance hidden from non-root",
+			name:            "untenanted instance hidden from non-root",
 			callerAccountID: "123456789012",
 			ownerAccountID:  "",
 			expected:        false,
@@ -82,19 +82,19 @@ func TestVolumeVisibleTo(t *testing.T) {
 			expected:        false,
 		},
 		{
-			name:            "pre-Phase4 volume visible to root",
+			name:            "untenanted volume visible to root",
 			tenantID:        "",
 			callerAccountID: utils.GlobalAccountID,
 			expected:        true,
 		},
 		{
-			name:            "pre-Phase4 volume hidden from non-root tenant",
+			name:            "untenanted volume hidden from non-root tenant",
 			tenantID:        "",
 			callerAccountID: "123456789012",
 			expected:        false,
 		},
 		{
-			name:            "pre-Phase4 volume denied to empty caller",
+			name:            "untenanted volume denied to empty caller",
 			tenantID:        "",
 			callerAccountID: "",
 			expected:        false,

spinifex/daemon/peer_health.go

@@ -0,0 +1,164 @@
+package daemon
+
+import (
+	"context"
+	"crypto/tls"
+	"log/slog"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/config"
+)
+
+// Peer-health probe cadence. Worst-case detection of total partition is
+// peerProbeInterval + (peers × peerProbeTimeout); on a 3-node cluster with
+// the current settings that lands ~6s after the partition installs — well
+// inside the 30s budget DDIL Scenario C asserts on.
+const (
+	peerProbeInterval = 2 * time.Second
+	peerProbeTimeout  = 2 * time.Second
+)
+
+// peerCount returns the number of peers (other nodes) in the cluster config.
+// Zero when the daemon is single-node or the cluster config is nil.
+func (d *Daemon) peerCount() int {
+	if d.clusterConfig == nil {
+		return 0
+	}
+	n := len(d.clusterConfig.Nodes)
+	if _, self := d.clusterConfig.Nodes[d.clusterConfig.Node]; self && n > 0 {
+		n--
+	}
+	return n
+}
+
+// peerNodes returns every node in the cluster config except this one. Order
+// is map-iteration order — callers must not depend on stability.
+func (d *Daemon) peerNodes() []config.Config {
+	if d.clusterConfig == nil {
+		return nil
+	}
+	out := make([]config.Config, 0, len(d.clusterConfig.Nodes))
+	for name, n := range d.clusterConfig.Nodes {
+		if name == d.clusterConfig.Node {
+			continue
+		}
+		out = append(out, n)
+	}
+	return out
+}
+
+// monitorPeerReachability polls every peer's /health endpoint over the
+// cluster network and updates peersReachable. Runs until d.ctx is cancelled.
+// No-op on single-node clusters: peersReachable is pinned true in NewDaemon
+// and there is nothing to probe.
+//
+// Probes are issued serially per tick because (a) typical cluster size is 2–3
+// peers, (b) any peer responding is enough to declare reachability, and
+// (c) serial dialling caps in-flight HTTPS connections during steady state.
+func (d *Daemon) monitorPeerReachability() {
+	peers := d.peerNodes()
+	if len(peers) == 0 {
+		return
+	}
+
+	client := &http.Client{
+		Timeout: peerProbeTimeout,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // self-signed per-node certs (matches harness DaemonClient)
+		},
+	}
+	defer client.CloseIdleConnections()
+
+	t := time.NewTicker(peerProbeInterval)
+	defer t.Stop()
+
+	d.probePeersOnce(client, peers)
+	for {
+		select {
+		case <-d.ctx.Done():
+			return
+		case <-t.C:
+			d.probePeersOnce(client, peers)
+		}
+	}
+}
+
+// probePeersOnce probes peers serially, short-circuiting on the first healthy
+// response. Sets peersReachable to the outcome.
+func (d *Daemon) probePeersOnce(client *http.Client, peers []config.Config) {
+	reachable := false
+	for _, p := range peers {
+		if d.probePeerHealth(client, p) {
+			reachable = true
+			break
+		}
+	}
+	prev := d.peersReachable.Swap(reachable)
+	if prev != reachable {
+		slog.Info("peer reachability changed", "reachable", reachable, "peers", len(peers))
+		if reachable {
+			// Scenario C heal edge: local NATS client stayed connected
+			// to its local server throughout the partition so
+			// onNATSReconnect never fires. The peer-probe flip is the
+			// only signal that the cluster is back. Goroutine keeps
+			// the probe ticker non-blocking; reconcileOnHeal coalesces
+			// with any concurrent NATS-reconnect path.
+			go d.reconcileOnHeal("peer-probe-heal")
+		}
+	}
+}
+
+// peerDaemonPort returns the TCP port to dial on peer p. The toml template
+// only renders [nodes.<self>.daemon] for the local node — remote-node blocks
+// carry just `host = "<ip>"` — so a peer's own DaemonConfig is typically
+// empty. Clusters are symmetric, so this falls back to the local daemon's
+// bound port. Returns "" only when neither the peer nor self has a parseable
+// host:port.
+func (d *Daemon) peerDaemonPort(p config.Config) string {
+	if p.Daemon.Host != "" {
+		if _, port, err := net.SplitHostPort(p.Daemon.Host); err == nil && port != "" {
+			return port
+		}
+	}
+	if d.config == nil || d.config.Daemon.Host == "" {
+		return ""
+	}
+	_, port, err := net.SplitHostPort(d.config.Daemon.Host)
+	if err != nil {
+		return ""
+	}
+	return port
+}
+
+// probePeerHealth issues a single short-timeout GET /health against peer p.
+// Returns true iff the peer responded with a 2xx status. Connection errors,
+// TLS failures, timeouts, and non-2xx responses all count as unreachable.
+func (d *Daemon) probePeerHealth(client *http.Client, p config.Config) bool {
+	addr := p.AdvertiseIP
+	if addr == "" {
+		addr = p.Host
+	}
+	if addr == "" {
+		return false
+	}
+	port := d.peerDaemonPort(p)
+	if port == "" {
+		return false
+	}
+	url := "https://" + net.JoinHostPort(addr, port) + "/health"
+
+	ctx, cancel := context.WithTimeout(d.ctx, peerProbeTimeout)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return false
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return false
+	}
+	defer func() { _ = resp.Body.Close() }()
+	return resp.StatusCode >= 200 && resp.StatusCode < 300
+}