release: update go.mod for v1.6.0

After PR #270 the VPC-LR-wide 0.0.0.0/0 -> IGW static route is no longer
installed implicitly at attach-internet-gateway. Subnets whose effective
route table lacks an internet-bound route get an OVN drop policy at
SubnetEgressPriorityDrop (1100), which kills LB target reply traffic
before un-DNAT can fire and stalls the LB internal-client probe until
job timeout.

Add the standard AWS pattern: describe-route-tables -> create-route
0.0.0.0/0 -> IGW. This fires vpc.add-igw-route + vpc.ungate-subnet-egress,
removing the drop policy and installing a reroute policy at
SubnetEgressPriorityIGW (1000).

.github/CODEOWNERS

@@ -0,0 +1,21 @@
+# CODEOWNERS — GitHub PR review enforcement
+#
+# Format: <path pattern>  <owner> [<owner> …]
+# Order matters — last matching pattern wins.
+#
+# Set the matching branch protection rule on `dev` and `main` to
+# "Require review from Code Owners" or this file is advisory only.
+
+# Fall-through: any change needs a team review.
+*                           @mulgadc/mulgadc-engineers
+
+# Network layer is governed by ADR-0006 invariant tests.
+# Changes here must respect those clauses — see
+# spinifex/network/invariants/ and
+# docs/development/proposals/az-local-architecture/0006-spinifex-network-layer-contract.md
+/spinifex/network/          @joshsiv-mulga
+/spinifex/vpcd/             @joshsiv-mulga
+
+# Invariants suite is the executable ADR-0006 fence — any widening or
+# narrowing of the scan roots needs an explicit owner review.
+/spinifex/network/invariants/ @joshsiv-mulga

.github/scripts/capture-net-diag.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# capture-net-diag.sh — write a snapshot of network / OVN / OVS / IPsec
+# state into $1 (a directory the caller already created on the local host).
+# Designed to run on a spinifex node after an E2E suite, before tofu destroy
+# pulls the rug out. Every command is independent — `|| true` so a missing
+# binary on one path doesn't abort the rest of the capture.
+#
+# Output layout (under $1/net-diag/):
+#   openvswitch-ipsec.status         systemctl status (output-only)
+#   openvswitch-ipsec.journal        last 200 journal lines
+#   ipsec.openvswitch_conf           ovs-vsctl get Open_vSwitch . other_config
+#   ipsec.xfrm_state                 ip xfrm state list
+#   ipsec.xfrm_policy                ip xfrm policy list
+#   ovn.sb_show                      ovn-sbctl show
+#   ovn.nb_show                      ovn-nbctl show
+#   ovs.show                         ovs-vsctl show
+#   ovs.flows_br_int                 ovs-ofctl dump-flows br-int
+#   ovs.tunnels                      ovs-appctl ofproto/list-tunnels br-int
+#   net.links                        ip -s link show
+#   net.routes                       ip route show table all
+#   net.addrs                        ip -br addr show
+#   systemd.failed_units             systemctl list-units --state=failed
+set +e
+OUT="${1:?usage: capture-net-diag.sh <output-dir>}"
+mkdir -p "$OUT"
+
+run() {
+    local label="$1"; shift
+    {
+        echo "# $*"
+        "$@" 2>&1
+    } > "$OUT/$label" || true
+}
+
+run openvswitch-ipsec.status            sudo systemctl status openvswitch-ipsec.service --no-pager --full --lines=50
+run openvswitch-ipsec.journal           sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso -n 200
+run ipsec.openvswitch_conf              sudo ovs-vsctl get Open_vSwitch . other_config
+run ipsec.xfrm_state                    sudo ip xfrm state list
+run ipsec.xfrm_policy                   sudo ip xfrm policy list
+
+run ovn.sb_show                         sudo ovn-sbctl show
+run ovn.nb_show                         sudo ovn-nbctl show
+run ovn.sb_chassis                      sudo ovn-sbctl list chassis
+
+run_per_lr() {
+    local label="$1"; shift
+    local subcmd="$1"; shift
+    {
+        echo "# ovn-nbctl $subcmd <each-logical-router>"
+        local lrs
+        lrs="$(sudo ovn-nbctl --bare --columns=name list Logical_Router 2>/dev/null)"
+        for lr in $lrs; do
+            echo
+            echo "=== $lr ==="
+            sudo ovn-nbctl "$subcmd" "$lr" 2>&1
+        done
+    } > "$OUT/$label" || true
+}
+
+run_per_lr ovn.lr_routes                lr-route-list
+run_per_lr ovn.lr_policies              lr-policy-list
+run_per_lr ovn.lr_nat                   lr-nat-list
+
+run ovs.show                            sudo ovs-vsctl show
+run ovs.flows_br_int                    sudo ovs-ofctl dump-flows br-int
+run ovs.tunnels                         sudo ovs-appctl ofproto/list-tunnels br-int
+
+run net.links                           ip -s link show
+run net.routes                          ip route show table all
+run net.addrs                           ip -br addr show
+run net.neigh                           ip neigh show
+run net.bridge_fdb                      bridge fdb show
+run net.conntrack                       sudo conntrack -L
+
+run systemd.failed_units                sudo systemctl list-units --state=failed --no-pager
+
+# Background pcap (started by start-net-capture.sh, if present).
+if [ -f /tmp/spinifex-e2e-tcpdump.pid ]; then
+    PID="$(cat /tmp/spinifex-e2e-tcpdump.pid 2>/dev/null)"
+    sudo kill -INT "$PID" 2>/dev/null || true
+    sleep 1
+    if [ -f /tmp/spinifex-e2e-wan.pcap ]; then
+        sudo cp /tmp/spinifex-e2e-wan.pcap "$OUT/wan.pcap" 2>/dev/null || true
+        sudo chmod 0644 "$OUT/wan.pcap" 2>/dev/null || true
+    fi
+fi
+
+exit 0

.github/scripts/start-net-capture.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# start-net-capture.sh — kick off background tcpdump on br-wan.
+# Pair: capture-net-diag.sh stops it and copies the pcap into net-diag/.
+#
+# Filter: anything on br-wan involving the EIP pool (192.168.0.0/23) plus
+# ARP for the same range. Single ring buffer (-W 1 -G 0) → grows until stop;
+# capped at 64 MiB via -C 64. Snaplen 200 bytes (headers + a bit) to keep
+# size sane while preserving L2/L3/L4 + first payload bytes.
+set +e
+
+PCAP=/tmp/spinifex-e2e-wan.pcap
+PIDFILE=/tmp/spinifex-e2e-tcpdump.pid
+
+if [ -f "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE" 2>/dev/null)" 2>/dev/null; then
+    echo "[start-net-capture] tcpdump already running pid=$(cat "$PIDFILE")"
+    exit 0
+fi
+
+sudo rm -f "$PCAP" "$PIDFILE"
+
+if ! command -v tcpdump >/dev/null 2>&1; then
+    echo "[start-net-capture] tcpdump not installed — skipping" >&2
+    exit 0
+fi
+if ! ip link show br-wan >/dev/null 2>&1; then
+    echo "[start-net-capture] br-wan absent — skipping" >&2
+    exit 0
+fi
+
+sudo nohup tcpdump -i br-wan -s 200 -C 64 -W 1 -U \
+    -w "$PCAP" \
+    '(net 192.168.0.0/23) or (arp and (arp[24:4]&0xfffffe00=0xc0a80000))' \
+    >/tmp/spinifex-e2e-tcpdump.log 2>&1 &
+PID=$!
+echo "$PID" | sudo tee "$PIDFILE" >/dev/null
+disown
+
+sleep 1
+if ! kill -0 "$PID" 2>/dev/null; then
+    echo "[start-net-capture] tcpdump died — see /tmp/spinifex-e2e-tcpdump.log" >&2
+    cat /tmp/spinifex-e2e-tcpdump.log >&2 || true
+    sudo rm -f "$PIDFILE"
+    exit 1
+fi
+
+echo "[start-net-capture] tcpdump pid=$PID pcap=$PCAP"
+exit 0

.github/workflows/e2e-go.yml

@@ -4,8 +4,8 @@ name: E2E (Go)
 # Mirrors e2e.yml's single-node provisioning chain but runs `go test` via
 # gotestsum so we get JUnit XML + per-test artifact directories surfaced to
 # GitHub Actions. Manual-dispatch only while the conversion lands — push
-# trigger lifts once the single-node Go port (mulga-siv-89) is green and
-# replaces run-e2e.sh; matches e2e-ddil.yml's Phase 1 trigger model.
+# trigger lifts once the single-node Go port is green and replaces
+# run-e2e.sh; matches e2e-ddil.yml's Phase 1 trigger model.
 
 on:
   workflow_dispatch:
@@ -17,6 +17,10 @@ on:
         description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb')"
         required: false
         default: "e2e-single e2e-multinode e2e-cert e2e-lb"
+      test_run:
+        description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
+        required: false
+        default: ""
 
 permissions:
   contents: read
@@ -32,6 +36,7 @@ concurrency:
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }}
+  TEST_RUN: ${{ inputs.test_run || '' }}
 
 jobs:
   e2e_go_single:
@@ -210,6 +215,16 @@ jobs:
           tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
           ok "shipped binaries to $VM"
 
+      - name: Start packet capture on VM br-wan
+        run: |
+          set +e
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          scp "${SSH_OPTS[@]}" \
+            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
+            "$VM":/tmp/start-net-capture.sh
+          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true
+
       - name: Run Go E2E suites
         id: e2e
         run: |
@@ -226,6 +241,13 @@ jobs:
             echo "rc=0" >> "$GITHUB_OUTPUT"
             exit 0
           fi
+          # Optional regex passed through to `go test -test.run`. Empty input
+          # means run all tests in the suite (default behaviour).
+          if [ -n "${TEST_RUN}" ]; then
+            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          else
+            TEST_RUN_FLAG=""
+          fi
           VM_HOST="tf-user@${{ steps.vm.outputs.wan_ip }}"
           SSH_KEY="${{ steps.vm.outputs.ssh_key }}"
           # On cancel / runner SIGTERM / step timeout, force-kill any test
@@ -258,7 +280,7 @@ jobs:
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \
@@ -277,6 +299,23 @@ jobs:
           echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
           exit 0  # don't fail the step — artifact upload + final gate happen below
 
+      - name: Capture network/IPsec diagnostics on VM
+        if: always()
+        run: |
+          set +e
+          # shellcheck source=/dev/null
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          # Ship the diag script, run it into the VM's ARTIFACT_DIR, leave the
+          # output for the next step's tar to pull down. Captures OVN/OVS/IPsec
+          # state at failure time — before teardown destroys the VM.
+          scp "${SSH_OPTS[@]}" \
+            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
+            "$VM":/tmp/capture-net-diag.sh
+          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'"
+          ok "net diagnostics captured"
+
       - name: Collect artifacts from VM
         if: always()
         run: |
@@ -552,6 +591,19 @@ jobs:
           tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
           ok "shipped binaries to $VM"
 
+      - name: Start packet capture on all nodes br-wan
+        run: |
+          set +e
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
+          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
+          for ip in "${NODE_LIST[@]}"; do
+              [ -z "$ip" ] && continue
+              scp "${SSH_OPTS[@]}" \
+                  "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
+                  tf-user@"$ip":/tmp/start-net-capture.sh
+              ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
+          done
+
       - name: Pre-flight diagnostics on node1
         # Captures cluster + tf-user state directly via SSH so the post-mortem
         # in run 26163994815-style spx-NATS failures has structured data
@@ -605,6 +657,11 @@ jobs:
             echo "rc=0" >> "$GITHUB_OUTPUT"
             exit 0
           fi
+          if [ -n "${TEST_RUN}" ]; then
+            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          else
+            TEST_RUN_FLAG=""
+          fi
           VM_HOST="tf-user@${{ steps.cluster.outputs.node1_ip }}"
           SSH_KEY="${{ steps.cluster.outputs.ssh_key }}"
           NODES_CSV="${{ steps.cluster.outputs.nodes_csv }}"
@@ -634,7 +691,7 @@ jobs:
                  SPINIFEX_SSH_USER=tf-user \
                  SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \
@@ -653,6 +710,42 @@ jobs:
           echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
           exit 0
 
+      - name: Capture network/IPsec diagnostics on all nodes
+        if: always()
+        run: |
+          set +e
+          # shellcheck source=/dev/null
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
+          # Capture per-node into the node1 ARTIFACT_DIR (where Collect pulls
+          # from). One subdir per node so xfrm SA state is comparable across
+          # peers — critical for diagnosing the IPsec wiring.
+          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
+          NODE1_IP="${NODE_LIST[0]}"
+          scp "${SSH_OPTS[@]}" \
+            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
+            tf-user@"$NODE1_IP":/tmp/capture-net-diag.sh
+          for i in "${!NODE_LIST[@]}"; do
+            idx=$((i + 1))
+            ip="${NODE_LIST[$i]}"
+            if [ "$idx" -ne 1 ]; then
+              scp "${SSH_OPTS[@]}" \
+                "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
+                tf-user@"$ip":/tmp/capture-net-diag.sh
+            fi
+            ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
+              "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag-node${idx}'"
+            # node1 keeps its diag locally; pull node2..N back into node1's
+            # ARTIFACT_DIR so the existing tar-pull below catches everything.
+            if [ "$idx" -ne 1 ]; then
+              ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
+                "tar -C '${ARTIFACT_DIR}' -cf - 'net-diag-node${idx}' 2>/dev/null || true" \
+                | ssh "${SSH_OPTS[@]}" tf-user@"$NODE1_IP" \
+                  "tar -C '${ARTIFACT_DIR}' -xf - 2>/dev/null"
+            fi
+            ok "net diagnostics node${idx} captured"
+          done
+
       - name: Collect artifacts from node1
         if: always()
         run: |
@@ -675,6 +768,15 @@ jobs:
               > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
             ok "journal node${idx}: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null || echo 0) lines"
           done
+          # openvswitch-ipsec journal per node (separate file).
+          # Debian unit name; the binary inside is /usr/share/openvswitch/scripts/ovs-ctl
+          # which spawns ovs-monitor-ipsec as a child.
+          for i in "${!NODE_LIST[@]}"; do
+            idx=$((i + 1))
+            ip="${NODE_LIST[$i]}"
+            ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
+              > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
+          done
 
       - name: Convert test logs to JUnit XML
         if: always()
@@ -919,6 +1021,11 @@ jobs:
           R=$'\033[0m'; B=$'\033[1m'; C=$'\033[36m'
           printf '\n%s%s━━ suite reboot ━━%s\n' "$B" "$C" "$R"
 
+          if [ -n "${TEST_RUN}" ]; then
+            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          else
+            TEST_RUN_FLAG=""
+          fi
           GITHUB_ACTIONS=true FORCE_COLOR=1 \
             SPINIFEX_E2E=1 SPINIFEX_MODE=single \
             SPINIFEX_WAN_IP="${{ steps.vm.outputs.wan_ip }}" \
@@ -927,7 +1034,7 @@ jobs:
             SPINIFEX_AWS_ACCESS_KEY_ID="${{ steps.creds.outputs.ak }}" \
             SPINIFEX_AWS_SECRET_ACCESS_KEY="${{ steps.creds.outputs.sk }}" \
             ARTIFACT_DIR="${ARTIFACT_DIR}" \
-            ./spinifex/tests/e2e/_bin/reboot.test -test.v -test.timeout=30m \
+            ./spinifex/tests/e2e/_bin/reboot.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
             2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log" \
                  | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'
           rc=${PIPESTATUS[0]}
@@ -1029,7 +1136,7 @@ jobs:
     # dispatched ref, always present. The previous `&& inputs.sha` gate
     # plus `statuses/${{ inputs.sha }}` produced an 'invalid hash' error
     # whenever the workflow was dispatched without the optional input
-    # (tests passed, status step silently failed). Refs mulga-siv-112.
+    # (tests passed, status step silently failed).
     if: always()
     needs: [ e2e_go_single, e2e_go_multinode, e2e_go_reboot ]
     runs-on: [ self-hosted, ci-single ]

.github/workflows/e2e.yml

@@ -77,6 +77,37 @@ jobs:
             "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
             --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
 
+      - name: Start Packet Capture (siv-167)
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' <<'REMOTE'
+          set -x
+          sudo mkdir -p /tmp/spinifex-diag
+          sudo chmod 0777 /tmp/spinifex-diag
+          sudo rm -f /tmp/spinifex-diag/br-wan.pcap /tmp/spinifex-diag/tcpdump.pid /tmp/spinifex-diag/tcpdump.log
+          sudo bash -c '
+            nohup tcpdump -i br-wan -nn -s 0 -U \
+              -w /tmp/spinifex-diag/br-wan.pcap \
+              "(tcp port 22 or tcp port 80 or tcp port 443 or tcp port 9999) or arp" \
+              </dev/null >/tmp/spinifex-diag/tcpdump.log 2>&1 &
+            echo $! > /tmp/spinifex-diag/tcpdump.pid
+          '
+          sleep 2
+          PID=$(cat /tmp/spinifex-diag/tcpdump.pid 2>/dev/null || echo "")
+          if [ -n "$PID" ] && sudo kill -0 "$PID" 2>/dev/null; then
+            echo "tcpdump running PID=$PID"
+            ls -la /tmp/spinifex-diag/
+          else
+            echo "tcpdump failed to start"
+            cat /tmp/spinifex-diag/tcpdump.log 2>&1 || true
+            exit 1
+          fi
+          REMOTE
+
       - name: Run E2E Tests
         working-directory: mulga/scripts/tofu-cluster
         run: |
@@ -87,6 +118,73 @@ jobs:
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
             "cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh"
 
+      - name: Collect Diagnostics
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          DIAG_DIR="${RUNNER_TEMP}/diag-single"
+          mkdir -p "$DIAG_DIR"
+          ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' > "$DIAG_DIR/collect.log" 2>&1 <<'REMOTE' || true
+          set -x
+          DIAG=/tmp/spinifex-diag
+          sudo mkdir -p "$DIAG" && cd "$DIAG"
+          if [ -f tcpdump.pid ]; then
+            PID=$(cat tcpdump.pid)
+            sudo kill -INT "$PID" 2>/dev/null || true
+            for _ in 1 2 3 4 5; do sudo kill -0 "$PID" 2>/dev/null || break; sleep 1; done
+            sudo kill -KILL "$PID" 2>/dev/null || true
+          fi
+          sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
+          sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
+          sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
+          sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
+          sudo ovs-vsctl show > ovs-vsctl-show.txt 2>&1 || true
+          sudo ovs-ofctl dump-flows br-int > br-int-flows.txt 2>&1 || true
+          sudo ovs-ofctl dump-flows br-ex > br-ex-flows.txt 2>&1 || true
+          sudo ovs-appctl dpctl/dump-flows > dpctl-flows.txt 2>&1 || true
+          sudo ovs-appctl dpctl/dump-conntrack -m > dpctl-conntrack.txt 2>&1 || true
+          sudo ovn-nbctl show > ovn-nbctl-show.txt 2>&1 || true
+          sudo ovn-sbctl show > ovn-sbctl-show.txt 2>&1 || true
+          sudo ovn-sbctl list Port_Binding > sbdb-port-binding.txt 2>&1 || true
+          sudo ovn-sbctl list Chassis > sbdb-chassis.txt 2>&1 || true
+          sudo ovn-nbctl list NAT > nbdb-nat.txt 2>&1 || true
+          sudo ovn-nbctl list Logical_Router > nbdb-lr.txt 2>&1 || true
+          sudo ovn-nbctl list Logical_Router_Policy > nbdb-lr-policy.txt 2>&1 || true
+          sudo ovn-nbctl list Logical_Router_Static_Route > nbdb-lr-static-route.txt 2>&1 || true
+          sudo ovn-appctl -t ovn-controller ct-zone-list > ovn-ctzones.txt 2>&1 || true
+          sudo ip -d link > ip-link.txt 2>&1 || true
+          sudo ip addr > ip-addr.txt 2>&1 || true
+          sudo ip route > ip-route.txt 2>&1 || true
+          sudo ip neigh > ip-neigh.txt 2>&1 || true
+          sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
+          sudo bridge link > bridge-link.txt 2>&1 || true
+          for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
+            [ -e "$f" ] && echo "$f $(cat $f)"
+          done > hairpin-modes.txt 2>&1 || true
+          sudo iptables-save > iptables.txt 2>&1 || true
+          sudo nft list ruleset > nft.txt 2>&1 || true
+          sudo dmesg --ctime > dmesg.txt 2>&1 || true
+          sudo virsh list --all > virsh-list.txt 2>&1 || true
+          ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true
+          sudo chmod -R a+r /tmp/spinifex-diag
+          tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag
+          REMOTE
+          scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \
+            tf-user@${WAN_IP}:/tmp/spinifex-diag.tar.gz "$DIAG_DIR/spinifex-diag.tar.gz" || true
+
+      - name: Upload Diagnostics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-diag-single-${{ github.run_id }}
+          path: ${{ runner.temp }}/diag-single/
+          if-no-files-found: warn
+          retention-days: 14
+
       - name: Teardown
         if: always()
         working-directory: mulga/scripts/tofu-cluster
@@ -151,6 +249,66 @@ jobs:
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
             "cd ~/Development/mulga/spinifex && SPX_EXTERNAL_POOL=\"${SPINIFEX_CI_POOL}\" ./tests/e2e/run-pseudo-multinode-e2e.sh"
 
+      - name: Collect Diagnostics
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          DIAG_DIR="${RUNNER_TEMP}/diag-pseudo"
+          mkdir -p "$DIAG_DIR"
+          ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' > "$DIAG_DIR/collect.log" 2>&1 <<'REMOTE' || true
+          set -x
+          DIAG=/tmp/spinifex-diag
+          sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG"
+          sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
+          sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
+          sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
+          sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
+          sudo ovs-vsctl show > ovs-vsctl-show.txt 2>&1 || true
+          sudo ovs-ofctl dump-flows br-int > br-int-flows.txt 2>&1 || true
+          sudo ovs-ofctl dump-flows br-ex > br-ex-flows.txt 2>&1 || true
+          sudo ovs-appctl dpctl/dump-flows > dpctl-flows.txt 2>&1 || true
+          sudo ovs-appctl dpctl/dump-conntrack -m > dpctl-conntrack.txt 2>&1 || true
+          sudo ovn-nbctl show > ovn-nbctl-show.txt 2>&1 || true
+          sudo ovn-sbctl show > ovn-sbctl-show.txt 2>&1 || true
+          sudo ovn-sbctl list Port_Binding > sbdb-port-binding.txt 2>&1 || true
+          sudo ovn-sbctl list Chassis > sbdb-chassis.txt 2>&1 || true
+          sudo ovn-nbctl list NAT > nbdb-nat.txt 2>&1 || true
+          sudo ovn-nbctl list Logical_Router > nbdb-lr.txt 2>&1 || true
+          sudo ovn-nbctl list Logical_Router_Policy > nbdb-lr-policy.txt 2>&1 || true
+          sudo ovn-nbctl list Logical_Router_Static_Route > nbdb-lr-static-route.txt 2>&1 || true
+          sudo ovn-appctl -t ovn-controller ct-zone-list > ovn-ctzones.txt 2>&1 || true
+          sudo ip -d link > ip-link.txt 2>&1 || true
+          sudo ip addr > ip-addr.txt 2>&1 || true
+          sudo ip route > ip-route.txt 2>&1 || true
+          sudo ip neigh > ip-neigh.txt 2>&1 || true
+          sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
+          sudo bridge link > bridge-link.txt 2>&1 || true
+          for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
+            [ -e "$f" ] && echo "$f $(cat $f)"
+          done > hairpin-modes.txt 2>&1 || true
+          sudo iptables-save > iptables.txt 2>&1 || true
+          sudo nft list ruleset > nft.txt 2>&1 || true
+          sudo dmesg --ctime > dmesg.txt 2>&1 || true
+          sudo virsh list --all > virsh-list.txt 2>&1 || true
+          ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true
+          tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag
+          REMOTE
+          scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \
+            tf-user@${WAN_IP}:/tmp/spinifex-diag.tar.gz "$DIAG_DIR/spinifex-diag.tar.gz" || true
+
+      - name: Upload Diagnostics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-diag-pseudo-${{ github.run_id }}
+          path: ${{ runner.temp }}/diag-pseudo/
+          if-no-files-found: warn
+          retention-days: 14
+
       - name: Teardown
         if: always()
         working-directory: mulga/scripts/tofu-cluster
@@ -230,6 +388,72 @@ jobs:
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
             "cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}"
 
+      - name: Collect Diagnostics
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          DIAG_DIR="${RUNNER_TEMP}/diag-multi"
+          mkdir -p "$DIAG_DIR"
+          cat > "${RUNNER_TEMP}/collect-diag.sh" <<'REMOTE'
+          set -x
+          DIAG=/tmp/spinifex-diag
+          sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG"
+          sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
+          sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
+          sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
+          sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
+          sudo ovs-vsctl show > ovs-vsctl-show.txt 2>&1 || true
+          sudo ovs-ofctl dump-flows br-int > br-int-flows.txt 2>&1 || true
+          sudo ovs-ofctl dump-flows br-ex > br-ex-flows.txt 2>&1 || true
+          sudo ovs-appctl dpctl/dump-flows > dpctl-flows.txt 2>&1 || true
+          sudo ovs-appctl dpctl/dump-conntrack -m > dpctl-conntrack.txt 2>&1 || true
+          sudo ovn-nbctl show > ovn-nbctl-show.txt 2>&1 || true
+          sudo ovn-sbctl show > ovn-sbctl-show.txt 2>&1 || true
+          sudo ovn-sbctl list Port_Binding > sbdb-port-binding.txt 2>&1 || true
+          sudo ovn-sbctl list Chassis > sbdb-chassis.txt 2>&1 || true
+          sudo ovn-nbctl list NAT > nbdb-nat.txt 2>&1 || true
+          sudo ovn-nbctl list Logical_Router > nbdb-lr.txt 2>&1 || true
+          sudo ovn-nbctl list Logical_Router_Policy > nbdb-lr-policy.txt 2>&1 || true
+          sudo ovn-nbctl list Logical_Router_Static_Route > nbdb-lr-static-route.txt 2>&1 || true
+          sudo ovn-appctl -t ovn-controller ct-zone-list > ovn-ctzones.txt 2>&1 || true
+          sudo ip -d link > ip-link.txt 2>&1 || true
+          sudo ip addr > ip-addr.txt 2>&1 || true
+          sudo ip route > ip-route.txt 2>&1 || true
+          sudo ip neigh > ip-neigh.txt 2>&1 || true
+          sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
+          sudo bridge link > bridge-link.txt 2>&1 || true
+          for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
+            [ -e "$f" ] && echo "$f $(cat $f)"
+          done > hairpin-modes.txt 2>&1 || true
+          sudo iptables-save > iptables.txt 2>&1 || true
+          sudo nft list ruleset > nft.txt 2>&1 || true
+          sudo dmesg --ctime > dmesg.txt 2>&1 || true
+          sudo virsh list --all > virsh-list.txt 2>&1 || true
+          ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true
+          tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag
+          REMOTE
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            NODE_IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            NODE_DIR="$DIAG_DIR/node${i}-${NODE_IP}"
+            mkdir -p "$NODE_DIR"
+            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${NODE_IP} 'bash -s' < "${RUNNER_TEMP}/collect-diag.sh" > "$NODE_DIR/collect.log" 2>&1 || true
+            scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \
+              tf-user@${NODE_IP}:/tmp/spinifex-diag.tar.gz "$NODE_DIR/spinifex-diag.tar.gz" || true
+          done
+
+      - name: Upload Diagnostics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-diag-multi-${{ github.run_id }}
+          path: ${{ runner.temp }}/diag-multi/
+          if-no-files-found: warn
+          retention-days: 14
+
       - name: Teardown
         if: always()
         working-directory: mulga/scripts/tofu-cluster

.github/workflows/release.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v6
         with:
-          go-version: '1.26.3'
+          go-version: '1.26.4'
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image

Makefile

@@ -73,11 +73,24 @@ build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
 	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
 
-build-system-image: ## Build a system image from manifest (use IMAGE=lb)
+build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-server)
 ifndef IMAGE
 	$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
 endif
-	./scripts/build-system-image.sh scripts/images/$(IMAGE).conf
+	@if [ -f scripts/images/$(IMAGE).conf ]; then \
+		./scripts/build-system-image.sh scripts/images/$(IMAGE).conf; \
+	elif [ -f scripts/images/$(IMAGE)/manifest.conf ]; then \
+		./scripts/build-system-image.sh scripts/images/$(IMAGE)/manifest.conf; \
+	else \
+		echo "ERROR: no manifest at scripts/images/$(IMAGE).conf or scripts/images/$(IMAGE)/manifest.conf"; \
+		exit 1; \
+	fi
+
+build-eks-server-image: ## Build the eks-server AMI (K3s control-plane variant)
+	$(MAKE) build-system-image IMAGE=eks-server
+
+build-eks-agent-image: ## Build the eks-agent AMI (K3s worker variant)
+	$(MAKE) build-system-image IMAGE=eks-agent
 
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
@@ -196,9 +209,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.3 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.4 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.4.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -281,24 +294,53 @@ distro-clean:
 
 # Ansible dev lifecycle (experimental, parallel to dev-*.sh / reset-dev-env.sh).
 # See scripts/ansible/README.md and docs/development/improvements/ansible-dev-lifecycle.md.
+#
+# Variable overrides: pass EXTRA_VARS="key=val key2=val2" to forward as
+# `--extra-vars` to ansible-playbook. Plain `-e` on the make command line is
+# make's own flag and does NOT reach ansible.
+#   make ansible-dev-install EXTRA_VARS="spinifex_external_pool_start=192.168.1.90 spinifex_external_pool_end=192.168.1.99"
+EXTRA_VARS ?=
+_ANSIBLE_EXTRA = $(if $(strip $(EXTRA_VARS)),--extra-vars "$(EXTRA_VARS)",)
+
 ansible-dev-preflight:
-	cd scripts/ansible && ansible-playbook playbooks/dev-preflight.yml
+	cd scripts/ansible && ansible-playbook playbooks/dev-preflight.yml $(_ANSIBLE_EXTRA)
 
 ansible-dev-teardown:
-	cd scripts/ansible && ansible-playbook playbooks/dev-teardown.yml
+	cd scripts/ansible && ansible-playbook playbooks/dev-teardown.yml $(_ANSIBLE_EXTRA)
 
 ansible-dev-install:
-	cd scripts/ansible && ansible-playbook playbooks/dev-install.yml
+	cd scripts/ansible && ansible-playbook playbooks/dev-install.yml $(_ANSIBLE_EXTRA)
 
 ansible-dev-reset:
-	cd scripts/ansible && ansible-playbook playbooks/dev-reset.yml
+	cd scripts/ansible && ansible-playbook playbooks/dev-reset.yml $(_ANSIBLE_EXTRA)
 
 ansible-dev-deploy:
-	cd scripts/ansible && ansible-playbook playbooks/dev-deploy.yml
+	cd scripts/ansible && ansible-playbook playbooks/dev-deploy.yml $(_ANSIBLE_EXTRA)
 
-.PHONY: build build-ui build-installer build-lb-agent build-system-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race test-actions test-harness manifest-check diff-coverage bench run \
+ansible-dev-status:
+	cd scripts/ansible && ansible-playbook playbooks/dev-status.yml
+
+ansible-dev-logs:
+	cd scripts/ansible && ansible-playbook playbooks/dev-logs.yml
+
+# Snapshot / restore require an explicit -e snapshot_name=<name>. The
+# wrapper passes ANSIBLE_EXTRA through so devs can run
+# `make ansible-dev-snapshot ANSIBLE_EXTRA='-e snapshot_name=before-merge'`.
+ansible-dev-snapshot:
+	cd scripts/ansible && ansible-playbook playbooks/dev-snapshot.yml $(ANSIBLE_EXTRA)
+
+ansible-dev-restore:
+	cd scripts/ansible && ansible-playbook playbooks/dev-restore.yml $(ANSIBLE_EXTRA)
+
+ansible-dev-version:
+	cd scripts/ansible && ansible-playbook playbooks/dev-version.yml
+
+ansible-dev-vpc:
+	cd scripts/ansible && ansible-playbook playbooks/dev-vpc.yml $(ANSIBLE_EXTRA)
+
+.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-server-image build-eks-agent-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck \
 	distro distro-amd64 distro-arm64 distro-clean \
-	ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset ansible-dev-deploy
+	ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset ansible-dev-deploy ansible-dev-status ansible-dev-logs ansible-dev-snapshot ansible-dev-restore ansible-dev-version ansible-dev-vpc

build/systemd/spinifex-daemon.service

@@ -1,7 +1,14 @@
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
-After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service
+After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service
+# Wants=openvswitch-ipsec.service so systemd brings the IPsec userspace
+# daemon up before spinifex flips other_config:ipsec_encapsulation=true.
+# Without that daemon, OVS holds the cert pointers unused and strongSwan
+# never gets IKE config — tunnel traffic is encrypted with no SAs and
+# silently dropped. Wants= (not Requires=) keeps daemon bootable on
+# single-node / lab hosts where the package is intentionally absent.
+Wants=openvswitch-ipsec.service
 # No Wants=spinifex-nats.service: Wants= reactivates a stopped unit during
 # `systemctl restart spinifex-daemon`, which would mask the standalone-startup
 # path the Tier 1 DDIL Scenario B exercises. spinifex.target Wants= covers
@@ -23,7 +30,7 @@ Group=spinifex
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5
-# DDIL Tier 1 (mulga-siv-58): KillMode=process so systemd only signals the
+# DDIL Tier 1: KillMode=process so systemd only signals the
 # daemon's main PID, not the cgroup. QEMU children stay alive across
 # `systemctl restart spinifex-daemon` and the new daemon reattaches to them
 # via the persisted local state file + pidfile + QMP socket. With the
@@ -38,7 +45,7 @@ OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
-# DDIL Tier 1 (mulga-siv-58): QEMU pidfiles, QMP sockets, and the
+# DDIL Tier 1: QEMU pidfiles, QMP sockets, and the
 # shared qemu.log all resolve via utils.pidPath() — which falls back
 # to os.TempDir() (=/tmp) when $HOME/spinifex doesn't exist. With
 # PrivateTmp=yes, each daemon start gets a fresh /tmp namespace, so

cmd/dhcptest/main.go

@@ -89,7 +89,7 @@ func main() {
 		}
 	}
 
-	// Build nclient4 options — identical to spinifex/services/vpcd/dhcp/nclient4.go.
+	// Build nclient4 options — standalone DHCP probe for upstream-router debugging.
 	opts := []nclient4.ClientOpt{
 		nclient4.WithHWAddr(hwAddr),
 		nclient4.WithTimeout(*timeout),
@@ -105,7 +105,7 @@ func main() {
 	}
 	defer client.Close()
 
-	// Identity modifiers — identical to identityModifiers() in vpcd dhcp package.
+	// Identity modifiers — client-id / hostname / vendor-class for upstream lease tagging.
 	var mods []dhcpv4.Modifier
 	if cid != "" {
 		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(cid))))

cmd/eks-token-webhook/main.go

@@ -0,0 +1,38 @@
+// eks-token-webhook is the K8s TokenReview webhook authenticator baked into
+// the eks-server AMI. It decodes the SigV4-presigned GetCallerIdentity URL
+// produced by `aws eks get-token`, enforces the X-K8s-Aws-Id cross-cluster
+// pin, calls Mulga STS over NATS, and resolves the principal to a TokenReview
+// response via the cluster's AccessEntry KV.
+//
+// This is a placeholder build that returns 503 Service Unavailable on every
+// path. It exists so the eks-server AMI build succeeds (INSTALL_BINARIES is
+// satisfied) and the K3s server VM boots cleanly. Real implementation lands
+// with bead mulga-cs-eks-6c.
+package main
+
+import (
+	"flag"
+	"log/slog"
+	"net/http"
+	"os"
+)
+
+func main() {
+	addr := flag.String("addr", "127.0.0.1:8443", "listen address")
+	flag.Parse()
+
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})))
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Retry-After", "30")
+		w.Header().Set("Content-Type", "application/json")
+		http.Error(w, `{"error":"eks-token-webhook stub: real implementation not yet deployed"}`, http.StatusServiceUnavailable)
+	})
+
+	slog.Info("eks-token-webhook stub starting", "addr", *addr)
+	if err := http.ListenAndServe(*addr, mux); err != nil {
+		slog.Error("listen failed", "err", err)
+		os.Exit(1)
+	}
+}

cmd/installer/install/install.go

@@ -304,9 +304,10 @@ func installSpinifex(cfg *Config) error {
 		return err
 	}
 
-	// dhcpcd-base is present on the installed system (used by setup-ovn.sh for
-	// macvlan mode). Mask the standalone dhcpcd.service so it never auto-starts
-	// and races with systemd-networkd's built-in DHCP client on br-wan.
+	// dhcpcd-base is present on the installed system (used by setup-ovn.sh
+	// for WAN DHCP acquisition). Mask the standalone dhcpcd.service so it
+	// never auto-starts and races with systemd-networkd's built-in DHCP
+	// client on br-wan.
 	if err := maskSystemdUnit(mountRoot, "dhcpcd.service"); err != nil {
 		slog.Warn("installSpinifex: failed to mask dhcpcd.service", "err", err)
 	}

cmd/spinifex/cmd/admin.go

@@ -294,12 +294,14 @@ func init() {
 	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (default when WAN detected) or '' (disabled)")
 	adminInitCmd.Flags().String("external-iface", "", "WAN NIC for br-external (auto-detected from default route)")
 	adminInitCmd.Flags().String("external-source", "", "Pool IP source: 'dhcp' (default when no --external-pool) or 'static' (uses --external-pool range)")
+	adminInitCmd.Flags().String("external-bind-bridge", "", "Linux bridge for upstream DHCP DORA (default 'br-wan' when --external-source=dhcp)")
 	adminInitCmd.Flags().String("external-pool", "", "External IP pool range as start-end (e.g., 192.168.1.150-192.168.1.250)")
 	adminInitCmd.Flags().String("external-gateway", "", "WAN gateway IP (auto-detected from default route)")
 	adminInitCmd.Flags().String("gateway-ip", "", "OVN gateway router's external IP for SNAT (default: pool range_start for pool mode, required for nat mode without DHCP)")
 	adminInitCmd.Flags().Int("external-prefix-len", 24, "External pool subnet prefix length (auto-detected)")
 	adminInitCmd.Flags().Bool("no-external", false, "Disable external networking (overlay-only, no internet for VMs)")
 	adminInitCmd.Flags().Bool("gpu-passthrough", false, "Enable VFIO GPU passthrough (sets gpu_passthrough = true in daemon config)")
+	adminInitCmd.Flags().Bool("ipsec", true, "Encrypt intra-AZ Geneve via OVN native IPsec (cluster-wide); disable only for trusted single-rack lab")
 
 	// Flags for admin join
 	adminJoinCmd.Flags().String("region", "ap-southeast-2", "Region for this node")
@@ -601,6 +603,12 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		Host:       appConfig.Nodes[appConfig.Node].Predastore.Host,
 	}
 
+	mkey, err := utils.LoadViperblockMasterKey(appConfig.Nodes[appConfig.Node].Viperblock.EncryptionKeyFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err)
+		os.Exit(1)
+	}
+
 	vbConfig := viperblock.VB{
 		VolumeName: volumeId,
 		VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()),
@@ -610,7 +618,9 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 				Size: 0,
 			},
 		},
-		VolumeConfig: manifest,
+		VolumeConfig:      manifest,
+		MasterKey:         mkey,
+		EncryptionEnabled: mkey != nil,
 	}
 
 	err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath)
@@ -838,12 +848,14 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	externalMode, _ := cmd.Flags().GetString("external-mode")
 	externalIface, _ := cmd.Flags().GetString("external-iface")
 	externalSource, _ := cmd.Flags().GetString("external-source")
+	externalBindBridge, _ := cmd.Flags().GetString("external-bind-bridge")
 	externalPool, _ := cmd.Flags().GetString("external-pool")
 	externalGateway, _ := cmd.Flags().GetString("external-gateway")
 	externalPrefixLen, _ := cmd.Flags().GetInt("external-prefix-len")
 	gatewayIP, _ := cmd.Flags().GetString("gateway-ip")
 	noExternal, _ := cmd.Flags().GetBool("no-external")
 	gpuPassthrough, _ := cmd.Flags().GetBool("gpu-passthrough")
+	ipsecEnabled, _ := cmd.Flags().GetBool("ipsec")
 
 	// Fire telemetry in background (completes during init work, waited at end)
 	noTelemetry, _ := cmd.Flags().GetBool("no-telemetry")
@@ -893,9 +905,9 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 				fmt.Printf("  %-14s %-18s %-20s %-16s %s\n", iface.Name, iface.IP, iface.Subnet, gw, strings.ToUpper(iface.Role))
 			}
 			if detected.LANCount == 0 {
-				fmt.Println("\n  Mode: single-NIC (macvlan for external bridge)")
+				fmt.Println("\n  Mode: single-NIC (veth-bridged external)")
 			} else {
-				fmt.Printf("\n  Mode: %d LAN + 1 WAN (macvlan for external bridge)\n", detected.LANCount)
+				fmt.Printf("\n  Mode: %d LAN + 1 WAN (veth-bridged external)\n", detected.LANCount)
 			}
 
 			// Apply auto-detected values when flags not explicitly set
@@ -910,13 +922,11 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 					externalPrefixLen = detected.WAN.PrefixLen
 				}
 
-				// Default mode: always "pool" — with static range if --external-pool
-				// is given, otherwise with source=dhcp (gateway IP from router DHCP)
+				// Default mode: always "pool". Source defaults to "static"; if
+				// --external-pool is omitted the validator below will error with
+				// a SuggestPoolRange hint.
 				if externalMode == "" && !cmd.Flags().Changed("external-mode") {
 					externalMode = "pool"
-					if externalPool == "" && externalSource == "" {
-						externalSource = "dhcp"
-					}
 				}
 			}
 		}
@@ -928,25 +938,30 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 	if externalMode == "pool" {
-		// Resolve source: if not explicitly set, infer from whether a pool range was given
+		// Default source: dhcp when no --external-pool, else static.
 		if externalSource == "" {
-			if externalPool != "" {
-				externalSource = "static"
-			} else {
+			if externalPool == "" {
 				externalSource = "dhcp"
+			} else {
+				externalSource = "static"
 			}
 		}
-		if externalSource == "dhcp" {
-			// DHCP source: no static range needed, just gateway
-			if externalGateway == "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required when --external-mode=pool\n")
+		switch externalSource {
+		case "dhcp":
+			if externalPool != "" {
+				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool not allowed with --external-source=dhcp (addresses come from upstream DHCP server)\n")
+				os.Exit(1)
+			}
+			if externalBindBridge == "" {
+				externalBindBridge = "br-wan"
+			}
+		case "static":
+			if externalBindBridge != "" {
+				fmt.Fprintf(os.Stderr, "❌ Error: --external-bind-bridge only valid with --external-source=dhcp\n")
 				os.Exit(1)
 			}
-		} else {
-			// Static source: need pool range
 			if externalPool == "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool is required when --external-mode=pool (e.g., 192.168.1.150-192.168.1.250)\n")
-				fmt.Fprintf(os.Stderr, "   Or use --external-source=dhcp to get IPs from router DHCP\n")
+				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool is required with --external-source=static (e.g., 192.168.1.150-192.168.1.250)\n")
 				if detectedNet != nil && detectedNet.WAN != nil {
 					sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN)
 					fmt.Fprintf(os.Stderr, "   Suggested: --external-pool=%s-%s\n", sugStart, sugEnd)
@@ -954,7 +969,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 				os.Exit(1)
 			}
 			if externalGateway == "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required when --external-mode=pool\n")
+				fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required with --external-source=static\n")
 				os.Exit(1)
 			}
 			parts := strings.SplitN(externalPool, "-", 2)
@@ -963,6 +978,9 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 				os.Exit(1)
 			}
 			poolStart, poolEnd = parts[0], parts[1]
+		default:
+			fmt.Fprintf(os.Stderr, "❌ Error: --external-source must be 'static' or 'dhcp', got: %s\n", externalSource)
+			os.Exit(1)
 		}
 	}
 	if externalGateway != "" && net.ParseIP(externalGateway) == nil {
@@ -983,10 +1001,6 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	// For pool/dhcp source, the gateway IP is obtained via DHCP on the
-	// macvlan/bridge interface (no static pool range or explicit gateway-ip).
-	useExternalDHCP := externalMode == "pool" && externalSource == "dhcp" && gatewayIP == ""
-
 	// Validate IP address format
 	if net.ParseIP(bindIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP address for --bind: %s\n", bindIP)
@@ -1104,6 +1118,18 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	// Generate SSL certificates (with bind IP in SANs for multi-node support)
 	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP)
 
+	// Generate per-node IPsec peer cert when cluster-wide IPsec is enabled
+	// (default true). Reuses the cluster CA — no intermediate strongSwan PKI.
+	if ipsecEnabled {
+		caCertPath := filepath.Join(configDir, "ca.pem")
+		caKeyPath := filepath.Join(configDir, "ca.key")
+		if err := admin.GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, node, bindIP); err != nil {
+			fmt.Fprintf(os.Stderr, "Error generating IPsec peer certificate: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("🔐 IPsec peer certificate generated (intra-AZ Geneve encryption ON)")
+	}
+
 	// Install CA certificate into system trust store
 	installCACertificate(filepath.Join(configDir, "ca.pem"))
 
@@ -1125,30 +1151,32 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	isMultiNode := nodes >= 2
 
 	if isMultiNode {
-		// Build cluster-wide network config for propagation to joining nodes
-		var networkConfig *formation.NetworkConfig
+		// Build cluster-wide network config for propagation to joining nodes.
+		// Always emit so the IPSecEnabled flag reaches joiners even when
+		// external networking is disabled.
+		networkConfig := &formation.NetworkConfig{
+			IPSecEnabled: ipsecEnabled,
+		}
 		if externalMode != "" {
 			bootstrapVpcId := utils.GenerateResourceID("vpc")
 			bootstrapSubnetId := utils.GenerateResourceID("subnet")
 			bootstrapIgwId := utils.GenerateResourceID("igw")
-			networkConfig = &formation.NetworkConfig{
-				ExternalMode:        externalMode,
-				ExternalDHCP:        useExternalDHCP,
-				PoolName:            "wan",
-				PoolSource:          externalSource,
-				PoolStart:           poolStart,
-				PoolEnd:             poolEnd,
-				PoolGateway:         externalGateway,
-				PoolGatewayIP:       gatewayIP,
-				PoolPrefixLen:       externalPrefixLen,
-				PoolDNSServers:      dnsServers,
-				BootstrapAccountId:  admin.DefaultAccountID(),
-				BootstrapVpcId:      bootstrapVpcId,
-				BootstrapSubnetId:   bootstrapSubnetId,
-				BootstrapIgwId:      bootstrapIgwId,
-				BootstrapCidr:       handlers_ec2_vpc.DefaultVPCCidr,
-				BootstrapSubnetCidr: handlers_ec2_vpc.DefaultSubnetCidr,
-			}
+			networkConfig.ExternalMode = externalMode
+			networkConfig.PoolName = "wan"
+			networkConfig.PoolSource = externalSource
+			networkConfig.PoolBindBridge = externalBindBridge
+			networkConfig.PoolStart = poolStart
+			networkConfig.PoolEnd = poolEnd
+			networkConfig.PoolGateway = externalGateway
+			networkConfig.PoolGatewayIP = gatewayIP
+			networkConfig.PoolPrefixLen = externalPrefixLen
+			networkConfig.PoolDNSServers = dnsServers
+			networkConfig.BootstrapAccountId = admin.DefaultAccountID()
+			networkConfig.BootstrapVpcId = bootstrapVpcId
+			networkConfig.BootstrapSubnetId = bootstrapSubnetId
+			networkConfig.BootstrapIgwId = bootstrapIgwId
+			networkConfig.BootstrapCidr = handlers_ec2_vpc.DefaultVPCCidr
+			networkConfig.BootstrapSubnetCidr = handlers_ec2_vpc.DefaultSubnetCidr
 		}
 
 		runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, natsToken, clusterName,
@@ -1248,10 +1276,9 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 
 		ExternalMode:   externalMode,
 		ExternalIface:  externalIface,
-		DhcpBindBridge: detectedDhcpBindBridge(detectedNet),
-		ExternalDHCP:   useExternalDHCP,
 		PoolName:       "wan",
 		PoolSource:     externalSource,
+		PoolBindBridge: externalBindBridge,
 		PoolStart:      poolStart,
 		PoolEnd:        poolEnd,
 		PoolGateway:    externalGateway,
@@ -1268,20 +1295,23 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		BootstrapSubnetCidr: handlers_ec2_vpc.DefaultSubnetCidr,
 
 		GPUPassthrough: gpuPassthrough,
+		IPSecEnabled:   ipsecEnabled,
 	}
 
 	// Print external networking summary
 	if externalMode != "" {
 		fmt.Printf("\n📡 External networking: %s\n", externalMode)
 		fmt.Printf("  WAN interface: %s\n", externalIface)
-		if poolStart != "" {
+		switch externalSource {
+		case "static":
 			fmt.Printf("  Source:        static (IP range)\n")
 			fmt.Printf("  IP pool:       %s - %s\n", poolStart, poolEnd)
 			fmt.Printf("  ⚠️  Ensure %s-%s is excluded from your router's DHCP range.\n", poolStart, poolEnd)
-		} else if useExternalDHCP {
-			fmt.Printf("  Source:        dhcp (from router DHCP)\n")
-			fmt.Println("  Gateway IP:    DHCP (obtained during OVN setup)")
-		} else if gatewayIP != "" {
+		case "dhcp":
+			fmt.Printf("  Source:        dhcp (upstream DHCP server)\n")
+			fmt.Printf("  Bind bridge:   %s\n", externalBindBridge)
+		}
+		if gatewayIP != "" {
 			fmt.Printf("  Gateway IP:    %s (static)\n", gatewayIP)
 		}
 	}
@@ -1861,6 +1891,24 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}
 	fmt.Printf("✅ Server certificate generated with bind IP: %s\n\n", bindIP)
 
+	// Match the leader's intra-AZ IPsec posture: NetworkConfig.IPSecEnabled is
+	// authoritative. When the formation response omits NetworkConfig entirely
+	// (no current code path; defensive guard against future regressions) fall
+	// back to the AWS-parity default of ON.
+	ipsecEnabled := true
+	if statusResp.NetworkConfig != nil {
+		ipsecEnabled = statusResp.NetworkConfig.IPSecEnabled
+	}
+	if ipsecEnabled {
+		caCertPath := filepath.Join(configDir, "ca.pem")
+		caKeyPath := filepath.Join(configDir, "ca.key")
+		if err := admin.GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, node, bindIP); err != nil {
+			fmt.Fprintf(os.Stderr, "Error generating IPsec peer certificate: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("🔐 IPsec peer certificate generated (intra-AZ Geneve encryption ON)")
+	}
+
 	// Build cluster topology from formation data
 	clusterRoutes := formation.BuildClusterRoutes(statusResp.Nodes)
 	predastoreNodes := formation.BuildPredastoreNodes(statusResp.Nodes)
@@ -2270,10 +2318,11 @@ func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region
 // applyNetworkConfig copies cluster-wide network settings from a formation
 // NetworkConfig into ConfigSettings and auto-detects the local WAN interface.
 func applyNetworkConfig(settings *admin.ConfigSettings, nc *formation.NetworkConfig) {
+	settings.IPSecEnabled = nc.IPSecEnabled
 	settings.ExternalMode = nc.ExternalMode
-	settings.ExternalDHCP = nc.ExternalDHCP
 	settings.PoolName = nc.PoolName
 	settings.PoolSource = nc.PoolSource
+	settings.PoolBindBridge = nc.PoolBindBridge
 	settings.PoolStart = nc.PoolStart
 	settings.PoolEnd = nc.PoolEnd
 	settings.PoolGateway = nc.PoolGateway
@@ -2292,7 +2341,6 @@ func applyNetworkConfig(settings *admin.ConfigSettings, nc *formation.NetworkCon
 		detected, err := admin.DetectNetwork()
 		if err == nil && detected.WAN != nil {
 			settings.ExternalIface = detected.WAN.Name
-			settings.DhcpBindBridge = detectedDhcpBindBridge(detected)
 		}
 	}
 }
@@ -2401,29 +2449,6 @@ func writeBootstrapFilesWithAdmin(configDir, bootstrapDir string, masterKey []by
 	return handlers_iam.SaveBootstrapData(filepath.Join(bootstrapDir, "bootstrap.json"), bd)
 }
 
-// detectedDhcpBindBridge returns the bridge name where the vpcd DHCP client
-// should bind its AF_PACKET socket — i.e. the interface that physically sees
-// LAN DHCP traffic.
-//
-//   - Default route on a bridge (Linux or OVS, `br-*` prefix): return the
-//     bridge name verbatim. In veth mode this is the Linux bridge holding the
-//     WAN NIC; in direct mode this is the OVS bridge holding the WAN NIC.
-//   - Default route on a bare physical NIC: return "br-wan" as the name the
-//     installer will create.
-//
-// Never returns "br-ext". br-ext is the OVN-side bridge owned by
-// setup-ovn.sh's ovn-bridge-mappings and never sees LAN DHCP frames.
-func detectedDhcpBindBridge(detected *admin.DetectedNetwork) string {
-	if detected == nil || detected.WAN == nil {
-		return ""
-	}
-	name := detected.WAN.Name
-	if strings.HasPrefix(name, "br-") {
-		return name
-	}
-	return "br-wan"
-}
-
 // detectDNSServers auto-detects DNS servers from the host for the specified
 // interface. Uses resolvectl (systemd-resolved) first, then falls back to
 // /etc/resolv.conf. Returns up to 3 servers. Falls back to public DNS if none found.

cmd/spinifex/cmd/admin_gpu.go

@@ -52,16 +52,62 @@ Must be run as root.`,
 	Run: runAdminGpuSetup,
 }
 
+var adminGpuMigCmd = &cobra.Command{
+	Use:   "mig",
+	Short: "Manage NVIDIA MIG partitioning on this node",
+}
+
+var adminGpuMigStatusCmd = &cobra.Command{
+	Use:   "status",
+	Short: "Show MIG capability, mode, and slice state for all GPUs on this node",
+	Run:   runAdminGpuMigStatus,
+}
+
+var adminGpuMigEnableCmd = &cobra.Command{
+	Use:   "enable",
+	Short: "Enable MIG mode on all MIG-capable GPUs (or a specific one)",
+	Long: `Enable NVIDIA MIG mode on all MIG-capable GPUs (or a specific one).
+
+Without --profile the daemon handles slice creation dynamically: each
+RunInstances call for a mig.* instance type carves the right profile on
+demand and destroys the slices when the last instance stops.
+
+With --profile, slices are pre-carved immediately. The daemon recovers
+these slices on restart and serves them as a static pool for that profile.
+
+Must be run as root.`,
+	Run: runAdminGpuMigEnable,
+}
+
+var adminGpuMigDisableCmd = &cobra.Command{
+	Use:   "disable",
+	Short: "Destroy all MIG instances and disable MIG mode",
+	Long: `Destroy all GPU instances and disable NVIDIA MIG mode on all MIG-capable
+GPUs (or a specific one). Blocked while GPU instances are running.
+
+Must be run as root.`,
+	Run: runAdminGpuMigDisable,
+}
+
 func init() {
 	adminCmd.AddCommand(adminGpuCmd)
 	adminGpuCmd.AddCommand(adminGpuStatusCmd)
 	adminGpuCmd.AddCommand(adminGpuEnableCmd)
 	adminGpuCmd.AddCommand(adminGpuDisableCmd)
 	adminGpuCmd.AddCommand(adminGpuSetupCmd)
+	adminGpuCmd.AddCommand(adminGpuMigCmd)
+	adminGpuMigCmd.AddCommand(adminGpuMigStatusCmd)
+	adminGpuMigCmd.AddCommand(adminGpuMigEnableCmd)
+	adminGpuMigCmd.AddCommand(adminGpuMigDisableCmd)
 
 	adminGpuStatusCmd.Flags().String("node", "", "Target node name (default: local node)")
-	// enable, disable, and setup write to the local spinifex.toml and signal the
-	// local daemon — they must be run directly on the target host.
+
+	adminGpuMigEnableCmd.Flags().String("profile", "", "MIG profile to pre-carve (e.g. \"1g.10gb\"); omit for fully dynamic slice management")
+	adminGpuMigEnableCmd.Flags().String("gpu", "", "PCI address of GPU to configure (default: all MIG-capable GPUs)")
+
+	adminGpuMigDisableCmd.Flags().String("gpu", "", "PCI address of GPU to configure (default: all MIG-capable GPUs)")
+	// enable, disable, setup, and mig subcommands write to the local spinifex.toml
+	// and signal the local daemon — they must be run directly on the target host.
 }
 
 // gpuNodeStatus queries NATS and returns the NodeStatusResponse for the target node.
@@ -417,6 +463,277 @@ func runAdminGpuSetup(_ *cobra.Command, _ []string) {
 	gpuToggle(nil, true)
 }
 
+func runAdminGpuMigStatus(_ *cobra.Command, _ []string) {
+	devices, err := gpu.Discover()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "GPU discovery failed: %v\n", err)
+		os.Exit(1)
+	}
+	if len(devices) == 0 {
+		fmt.Println("No GPUs detected on this node.")
+		return
+	}
+
+	anyMIG := false
+	for _, d := range devices {
+		if d.MIGCapable {
+			anyMIG = true
+		}
+		migMode := "N/A"
+		if d.MIGCapable {
+			if d.MIGEnabled {
+				migMode = "enabled"
+			} else {
+				migMode = "disabled"
+			}
+		}
+		fmt.Printf("GPU:           %s\n", d.PCIAddress)
+		fmt.Printf("  Model:       %s\n", d.Model)
+		fmt.Printf("  MIG capable: %v\n", d.MIGCapable)
+		fmt.Printf("  MIG mode:    %s\n", migMode)
+
+		if d.MIGEnabled {
+			instances, listErr := gpu.ListInstances(d.PCIAddress)
+			if listErr != nil {
+				fmt.Printf("  Slices:      (error: %v)\n", listErr)
+			} else {
+				fmt.Printf("  Slices:      %d\n", len(instances))
+				for _, inst := range instances {
+					fmt.Printf("    GI %d: %-12s  mdev: %s\n",
+						inst.GIID, inst.Profile.Name, inst.MdevPath)
+				}
+			}
+		}
+		fmt.Println()
+	}
+
+	if !anyMIG {
+		fmt.Println("No MIG-capable GPUs found. MIG requires an NVIDIA A100, A30, or H100 GPU.")
+	}
+}
+
+func runAdminGpuMigEnable(cmd *cobra.Command, _ []string) {
+	if os.Getuid() != 0 {
+		fmt.Fprintln(os.Stderr, "Error: spx admin gpu mig enable must be run as root.")
+		os.Exit(1)
+	}
+
+	profileName, _ := cmd.Flags().GetString("profile")
+	targetPCI, _ := cmd.Flags().GetString("gpu")
+
+	cfgPath := viper.GetString("config")
+	if cfgPath == "" {
+		cfgPath = DefaultConfigFile()
+	}
+	cfg, err := config.LoadConfig(cfgPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error loading config: %v\n", err)
+		os.Exit(1)
+	}
+
+	resp, err := gpuNodeStatus(cfg.Node)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	if resp.AllocGPUs > 0 {
+		fmt.Fprintf(os.Stderr, "Error: %d GPU instance(s) are running. Terminate them first.\n", resp.AllocGPUs)
+		os.Exit(1)
+	}
+
+	devices, err := gpu.Discover()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "GPU discovery failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	targets := migFilterTargets(devices, targetPCI)
+	if len(targets) == 0 {
+		if targetPCI != "" {
+			fmt.Fprintf(os.Stderr, "Error: no MIG-capable GPU found at PCI address %s\n", targetPCI)
+		} else {
+			fmt.Fprintln(os.Stderr, "Error: no MIG-capable GPUs found. MIG requires an NVIDIA A100, A30, or H100 GPU.")
+		}
+		os.Exit(1)
+	}
+
+	for _, d := range targets {
+		fmt.Printf("==> Configuring MIG on %s (%s)\n", d.PCIAddress, d.Model)
+
+		if !d.MIGEnabled {
+			fmt.Println("    Enabling MIG mode...")
+			if err := gpu.EnableMIGMode(d.PCIAddress); err != nil {
+				fmt.Fprintf(os.Stderr, "    Error: %v\n", err)
+				os.Exit(1)
+			}
+		} else {
+			fmt.Println("    MIG mode already enabled")
+		}
+
+		if profileName == "" {
+			// Dynamic mode: the daemon carves slices on demand at RunInstances time.
+			// Destroy any pre-carved slices so the daemon starts with a free GPU.
+			existing, _ := gpu.ListInstances(d.PCIAddress)
+			if len(existing) > 0 {
+				fmt.Printf("    Destroying %d pre-carved instance(s) so daemon can manage slices dynamically...\n", len(existing))
+				if err := gpu.DestroyAllInstances(d.PCIAddress); err != nil {
+					fmt.Fprintf(os.Stderr, "    Error: %v\n", err)
+					os.Exit(1)
+				}
+			}
+			fmt.Println("    Dynamic slice management enabled — use 'mig.*' instance types at RunInstances time.")
+			continue
+		}
+
+		// Static pre-carve: create a fixed set of slices for the given profile.
+		profiles, err := gpu.ListProfiles(d.PCIAddress)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "    Error listing profiles: %v\n", err)
+			os.Exit(1)
+		}
+
+		var selected *gpu.MIGProfile
+		for i, p := range profiles {
+			if p.Name == profileName {
+				selected = &profiles[i]
+				break
+			}
+		}
+		if selected == nil {
+			names := make([]string, len(profiles))
+			for i, p := range profiles {
+				names[i] = p.Name
+			}
+			fmt.Fprintf(os.Stderr, "    Error: profile %q not found. Available: %s\n",
+				profileName, strings.Join(names, ", "))
+			os.Exit(1)
+		}
+
+		existing, _ := gpu.ListInstances(d.PCIAddress)
+		if len(existing) > 0 {
+			fmt.Printf("    Destroying %d existing instance(s)...\n", len(existing))
+			if err := gpu.DestroyAllInstances(d.PCIAddress); err != nil {
+				fmt.Fprintf(os.Stderr, "    Error: %v\n", err)
+				os.Exit(1)
+			}
+		}
+
+		fmt.Printf("    Creating MIG instances (profile: %s)...\n", selected.Name)
+		instances, err := gpu.CreateInstances(d.PCIAddress, *selected)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "    Error: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("    Created %d instance(s)\n", len(instances))
+		for _, inst := range instances {
+			fmt.Printf("      GI %d: %s  mdev: %s\n", inst.GIID, inst.Profile.Name, inst.MdevPath)
+		}
+	}
+
+	migReloadDaemon()
+	fmt.Println("MIG enabled. Run 'spx admin gpu mig status' to verify.")
+}
+
+func runAdminGpuMigDisable(cmd *cobra.Command, _ []string) {
+	if os.Getuid() != 0 {
+		fmt.Fprintln(os.Stderr, "Error: spx admin gpu mig disable must be run as root.")
+		os.Exit(1)
+	}
+
+	targetPCI, _ := cmd.Flags().GetString("gpu")
+
+	cfgPath := viper.GetString("config")
+	if cfgPath == "" {
+		cfgPath = DefaultConfigFile()
+	}
+	cfg, err := config.LoadConfig(cfgPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error loading config: %v\n", err)
+		os.Exit(1)
+	}
+
+	resp, err := gpuNodeStatus(cfg.Node)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	if resp.AllocGPUs > 0 {
+		fmt.Fprintf(os.Stderr, "Error: %d GPU instance(s) are running. Terminate them first.\n", resp.AllocGPUs)
+		os.Exit(1)
+	}
+
+	devices, err := gpu.Discover()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "GPU discovery failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	targets := migFilterTargets(devices, targetPCI)
+	if len(targets) == 0 {
+		if targetPCI != "" {
+			fmt.Fprintf(os.Stderr, "Error: no MIG-capable GPU found at PCI address %s\n", targetPCI)
+		} else {
+			fmt.Fprintln(os.Stderr, "Error: no MIG-capable GPUs found on this node.")
+		}
+		os.Exit(1)
+	}
+
+	for _, d := range targets {
+		fmt.Printf("==> Disabling MIG on %s (%s)\n", d.PCIAddress, d.Model)
+		if !d.MIGEnabled {
+			fmt.Println("    MIG mode not active — skipping")
+			continue
+		}
+		fmt.Println("    Destroying all GPU instances...")
+		if err := gpu.DestroyAllInstances(d.PCIAddress); err != nil {
+			fmt.Fprintf(os.Stderr, "    Error: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("    Disabling MIG mode...")
+		if err := gpu.DisableMIGMode(d.PCIAddress); err != nil {
+			fmt.Fprintf(os.Stderr, "    Error: %v\n", err)
+			os.Exit(1)
+		}
+	}
+
+	fmt.Println("==> Clearing mig_profile from config...")
+	if err := admin.SetMIGProfile(cfgPath, cfg.Node, ""); err != nil {
+		fmt.Fprintf(os.Stderr, "Error writing config: %v\n", err)
+		os.Exit(1)
+	}
+
+	migReloadDaemon()
+	fmt.Println("MIG disabled. GPUs will operate in whole-GPU passthrough mode.")
+}
+
+// migFilterTargets returns all MIG-capable GPUs, optionally filtered to a
+// specific PCI address.
+func migFilterTargets(devices []gpu.GPUDevice, targetPCI string) []gpu.GPUDevice {
+	var out []gpu.GPUDevice
+	for _, d := range devices {
+		if !d.MIGCapable {
+			continue
+		}
+		if targetPCI != "" && d.PCIAddress != targetPCI {
+			continue
+		}
+		out = append(out, d)
+	}
+	return out
+}
+
+// migReloadDaemon sends SIGHUP to spinifex-daemon so it picks up the updated
+// MIG configuration without a full restart.
+func migReloadDaemon() {
+	out, err := exec.Command("systemctl", "kill", "-s", "HUP", "spinifex-daemon").CombinedOutput()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: failed to signal daemon: %v\n%s\n", err, out)
+		fmt.Fprintln(os.Stderr, "Restart the daemon manually: sudo systemctl restart spinifex-daemon")
+		return
+	}
+	fmt.Println("==> Daemon signalled (SIGHUP) — MIG pool will be rebuilt on next request.")
+}
+
 // gpuSetupCollectVFIOIDs collects vendor:device PCI ID pairs for each GPU and
 // all sibling PCI functions on the same device slot (e.g. GPU + HDMI audio).
 // All siblings must bind to vfio-pci together for IOMMU group isolation.

cmd/spinifex/cmd/admin_test.go

@@ -142,49 +142,6 @@ func TestSpinifexTomlTemplate_AdvertiseOmittedWhenEmpty(t *testing.T) {
 	assert.NotContains(t, string(data), "advertise =")
 }
 
-// detectedDhcpBindBridge must return the default-route interface name verbatim
-// when it's a bridge (Linux or OVS, br-* prefix). The old detectedWanBridge()
-// returned hardcoded "br-ext" for Linux bridges — that value broke DHCP on
-// consumer-router LANs because br-ext never sees LAN DHCP traffic (mulga-998).
-func TestDetectedDhcpBindBridge(t *testing.T) {
-	tests := []struct {
-		name     string
-		detected *admin.DetectedNetwork
-		want     string
-	}{
-		{
-			name:     "Linux bridge default route returns bridge name (not 'br-ext')",
-			detected: &admin.DetectedNetwork{WAN: &admin.DetectedInterface{Name: "br-wan"}},
-			want:     "br-wan",
-		},
-		{
-			name:     "OVS bridge default route returned verbatim",
-			detected: &admin.DetectedNetwork{WAN: &admin.DetectedInterface{Name: "br-ext"}},
-			want:     "br-ext",
-		},
-		{
-			name:     "physical NIC defaults to 'br-wan' (bridge the installer creates)",
-			detected: &admin.DetectedNetwork{WAN: &admin.DetectedInterface{Name: "enp0s3"}},
-			want:     "br-wan",
-		},
-		{
-			name:     "nil DetectedNetwork",
-			detected: nil,
-			want:     "",
-		},
-		{
-			name:     "DetectedNetwork with nil WAN",
-			detected: &admin.DetectedNetwork{},
-			want:     "",
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, tt.want, detectedDhcpBindBridge(tt.detected))
-		})
-	}
-}
-
 // Legacy `wan_bridge` TOML key must fail-start vpcd with guidance, not silently
 // alias (per mulga-998 D3). Prevents the footgun where operators inherited the
 // old key pointing at 'br-ext' and got broken DHCP on veth-mode hosts.
@@ -206,8 +163,8 @@ wan_bridge = "br-ext"
 
 	err := checkLegacyWanBridgeKey("node1", cfgPath)
 	require.Error(t, err)
-	assert.Contains(t, err.Error(), "dhcp_bind_bridge")
 	assert.Contains(t, err.Error(), "wan_bridge")
+	assert.Contains(t, err.Error(), "Remove")
 }
 
 func TestCheckLegacyWanBridgeKey_EnvVarRejected(t *testing.T) {
@@ -218,7 +175,7 @@ func TestCheckLegacyWanBridgeKey_EnvVarRejected(t *testing.T) {
 	err := checkLegacyWanBridgeKey("node1", "/tmp/unused.toml")
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "SPINIFEX_VPCD_WAN_BRIDGE")
-	assert.Contains(t, err.Error(), "dhcp_bind_bridge")
+	assert.Contains(t, err.Error(), "Remove")
 }
 
 func TestCheckLegacyWanBridgeKey_CleanConfigPasses(t *testing.T) {
@@ -228,7 +185,7 @@ func TestCheckLegacyWanBridgeKey_CleanConfigPasses(t *testing.T) {
 node = "node1"
 [nodes.node1.vpcd]
 ovn_nb_addr = "tcp:127.0.0.1:6641"
-dhcp_bind_bridge = "br-wan"
+external_interface = "enp0s3"
 `), 0o644))
 
 	viper.Reset()
@@ -331,3 +288,101 @@ func TestImagesRemoveCmd_FlagSchema(t *testing.T) {
 
 // cobraRequiredAnnotation is the annotation key cobra uses to mark required flags.
 const cobraRequiredAnnotation = "cobra_annotation_bash_completion_one_required_flag"
+
+// `spx admin init` must expose --external-source and --external-bind-bridge so
+// operators can pick the DHCP path for [[network.external_pools]].
+func TestAdminInitCmd_ExternalDHCPFlagSchema(t *testing.T) {
+	sourceFlag := adminInitCmd.Flags().Lookup("external-source")
+	require.NotNil(t, sourceFlag, "--external-source must be defined")
+	assert.Equal(t, "", sourceFlag.DefValue)
+
+	bindFlag := adminInitCmd.Flags().Lookup("external-bind-bridge")
+	require.NotNil(t, bindFlag, "--external-bind-bridge must be defined")
+	assert.Equal(t, "", bindFlag.DefValue)
+
+	poolFlag := adminInitCmd.Flags().Lookup("external-pool")
+	require.NotNil(t, poolFlag)
+	assert.Equal(t, "", poolFlag.DefValue)
+}
+
+// DHCP-sourced pool must emit source + bind_bridge into spinifex.toml so the
+// daemon's validator and vpcd's DHCPManager find the bridge they DORA on.
+func TestSpinifexTomlTemplate_ExternalPoolDHCPSource(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:           "node1",
+		Az:             "ap-southeast-2a",
+		Port:           "4432",
+		Region:         "ap-southeast-2",
+		BindIP:         "10.11.12.1",
+		ConfigDir:      dir,
+		OVNNBAddr:      "tcp:127.0.0.1:6641",
+		OVNSBAddr:      "tcp:127.0.0.1:6642",
+		ExternalMode:   "pool",
+		ExternalIface:  "eth0",
+		PoolName:       "wan",
+		PoolSource:     "dhcp",
+		PoolBindBridge: "br-wan",
+		PoolPrefixLen:  24,
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `source      = "dhcp"`)
+	assert.Contains(t, content, `bind_bridge = "br-wan"`)
+	assert.NotContains(t, content, "range_start", "dhcp pool must not emit range_start")
+	assert.NotContains(t, content, "range_end", "dhcp pool must not emit range_end")
+	assert.NotContains(t, content, "gateway     =", "gateway omitted for dhcp pool (discovered from OFFER)")
+}
+
+// Static-sourced pool must not emit bind_bridge — the validator rejects it.
+func TestSpinifexTomlTemplate_ExternalPoolStaticSource(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:          "node1",
+		Az:            "ap-southeast-2a",
+		Port:          "4432",
+		Region:        "ap-southeast-2",
+		BindIP:        "10.11.12.1",
+		ConfigDir:     dir,
+		OVNNBAddr:     "tcp:127.0.0.1:6641",
+		OVNSBAddr:     "tcp:127.0.0.1:6642",
+		ExternalMode:  "pool",
+		ExternalIface: "eth0",
+		PoolName:      "wan",
+		PoolSource:    "static",
+		PoolStart:     "192.168.1.150",
+		PoolEnd:       "192.168.1.250",
+		PoolGateway:   "192.168.1.1",
+		PoolPrefixLen: 24,
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `source      = "static"`)
+	assert.NotContains(t, content, "bind_bridge", "static pool must not emit bind_bridge")
+	assert.Contains(t, content, `range_start = "192.168.1.150"`)
+	assert.Contains(t, content, `gateway     = "192.168.1.1"`)
+}
+
+// Formation joiners must receive PoolBindBridge from the init node so the
+// cluster-wide config reaches every chassis intact.
+func TestApplyNetworkConfig_PropagatesPoolBindBridge(t *testing.T) {
+	settings := &admin.ConfigSettings{}
+	nc := &formation.NetworkConfig{
+		ExternalMode:   "pool",
+		PoolName:       "wan",
+		PoolSource:     "dhcp",
+		PoolBindBridge: "br-wan",
+		PoolPrefixLen:  24,
+	}
+	applyNetworkConfig(settings, nc)
+	assert.Equal(t, "dhcp", settings.PoolSource)
+	assert.Equal(t, "br-wan", settings.PoolBindBridge)
+}

cmd/spinifex/cmd/service.go

@@ -29,7 +29,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/services/predastore"
 	"github.com/mulgadc/spinifex/spinifex/services/spinifexui"
 	"github.com/mulgadc/spinifex/spinifex/services/viperblockd"
-	"github.com/mulgadc/spinifex/spinifex/services/vpcd"
+	"github.com/mulgadc/spinifex/spinifex/vpcd"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -283,19 +283,25 @@ var viperblockStartCmd = &cobra.Command{
 			shardWAL = *nodeConfig.Viperblock.ShardWAL
 		}
 
+		encryptionKeyFile := nodeConfig.Viperblock.EncryptionKeyFile
+		if envKey := viper.GetString("viperblock-encryption-key-file"); envKey != "" {
+			encryptionKeyFile = envKey
+		}
+
 		service, err := service.New("viperblock", &viperblockd.Config{
-			NatsHost:   nodeConfig.NATS.Host,
-			NatsToken:  nodeConfig.NATS.ACL.Token,
-			NatsCACert: nodeConfig.NATS.CACert,
-			PluginPath: pluginPath,
-			S3Host:     nodeConfig.Predastore.Host,
-			Bucket:     nodeConfig.Predastore.Bucket,
-			Region:     nodeConfig.Predastore.Region,
-			AccessKey:  nodeConfig.Predastore.AccessKey,
-			SecretKey:  nodeConfig.Predastore.SecretKey,
-			BaseDir:    nodeConfig.Predastore.BaseDir,
-			NodeName:   clusterConfig.Node,
-			ShardWAL:   shardWAL,
+			NatsHost:          nodeConfig.NATS.Host,
+			NatsToken:         nodeConfig.NATS.ACL.Token,
+			NatsCACert:        nodeConfig.NATS.CACert,
+			PluginPath:        pluginPath,
+			S3Host:            nodeConfig.Predastore.Host,
+			Bucket:            nodeConfig.Predastore.Bucket,
+			Region:            nodeConfig.Predastore.Region,
+			AccessKey:         nodeConfig.Predastore.AccessKey,
+			SecretKey:         nodeConfig.Predastore.SecretKey,
+			BaseDir:           nodeConfig.Predastore.BaseDir,
+			NodeName:          clusterConfig.Node,
+			ShardWAL:          shardWAL,
+			EncryptionKeyFile: encryptionKeyFile,
 		})
 
 		if err != nil {
@@ -678,11 +684,8 @@ var spinifexUIStatusCmd = &cobra.Command{
 }
 
 // checkLegacyWanBridgeKey fails vpcd startup if the deprecated `wan_bridge`
-// TOML key or `SPINIFEX_VPCD_WAN_BRIDGE` env-var is present. The key was
-// renamed to `dhcp_bind_bridge` (see mulga-998.a) because the old name misled
-// operators into pointing at the OVN-side bridge ("br-ext"), which never sees
-// LAN DHCP traffic. Per D3: no silent alias, no auto-rewrite — operator must
-// rename the key before vpcd will start.
+// TOML key or `SPINIFEX_VPCD_WAN_BRIDGE` env-var is present. Per D3: no silent
+// alias, no auto-rewrite — operator must remove the key before vpcd will start.
 func checkLegacyWanBridgeKey(node, cfgFile string) error {
 	legacyInTOML := viper.IsSet("nodes." + node + ".vpcd.wan_bridge")
 	legacyInEnv := os.Getenv("SPINIFEX_VPCD_WAN_BRIDGE") != ""
@@ -695,9 +698,7 @@ func checkLegacyWanBridgeKey(node, cfgFile string) error {
 	}
 	return fmt.Errorf(
 		"vpcd: deprecated 'wan_bridge' key found in %s. "+
-			"Rename 'wan_bridge' to 'dhcp_bind_bridge'. The value may also be wrong — verify it is "+
-			"the Linux bridge holding your WAN NIC (veth mode) or the OVS bridge holding your WAN NIC (direct mode); "+
-			"never 'br-ext'. Typical value on a consumer-router LAN: 'br-wan'. "+
+			"Remove the key entirely; vpcd auto-detects the WAN bridge (br-wan). "+
 			"Then: sudo systemctl restart spinifex-vpcd",
 		source,
 	)
@@ -734,6 +735,7 @@ var vpcdStartCmd = &cobra.Command{
 			extPools = append(extPools, vpcd.ExternalPoolConfig{
 				Name:            p.Name,
 				Source:          p.Source,
+				BindBridge:      p.BindBridge,
 				RangeStart:      p.RangeStart,
 				RangeEnd:        p.RangeEnd,
 				Gateway:         p.Gateway,
@@ -742,7 +744,6 @@ var vpcdStartCmd = &cobra.Command{
 				DNSServers:      p.DNSServers,
 				Region:          p.Region,
 				AZ:              p.AZ,
-				DhcpBindBridge:  nodeConfig.VPCD.DhcpBindBridge,
 				GwLrpRangeStart: p.GwLrpRangeStart,
 				GwLrpRangeEnd:   p.GwLrpRangeEnd,
 			})
@@ -778,8 +779,8 @@ var vpcdStartCmd = &cobra.Command{
 			ExternalPools:     extPools,
 			Bootstrap:         bootstrap,
 			ExternalInterface: nodeConfig.VPCD.ExternalInterface,
-			DhcpBindBridge:    nodeConfig.VPCD.DhcpBindBridge,
 			BridgeMode:        nodeConfig.VPCD.BridgeMode,
+			AZ:                nodeConfig.AZ,
 		})
 		if err != nil {
 			fmt.Println("Error starting vpcd service:", err)
@@ -916,6 +917,14 @@ func init() {
 	viper.BindEnv("plugin-path", "SPINIFEX_VIPERBLOCK_PLUGIN_PATH")
 	viper.BindPFlag("plugin-path", predastoreCmd.PersistentFlags().Lookup("plugin-path"))
 
+	// Viperblock at-rest encryption master key (shared with other on-node
+	// services via group ownership; mode 0640 or stricter). Distinct viper
+	// key from predastore's encryption-key-file so the two BindPFlag calls
+	// don't collide globally.
+	viperblockCmd.PersistentFlags().String("encryption-key-file", "", "Path to the 32-byte AES-256 master key file for at-rest encryption (empty disables encryption)")
+	viper.BindEnv("viperblock-encryption-key-file", "SPINIFEX_VIPERBLOCK_ENCRYPTION_KEY_FILE")
+	viper.BindPFlag("viperblock-encryption-key-file", viperblockCmd.PersistentFlags().Lookup("encryption-key-file"))
+
 	viperblockCmd.AddCommand(viperblockStartCmd)
 	viperblockCmd.AddCommand(viperblockStopCmd)
 	viperblockCmd.AddCommand(viperblockStatusCmd)

cmd/spinifex/cmd/templates/spinifex.toml

@@ -1,5 +1,5 @@
 # Domain configuration file in TOML format.
-version = "2"
+version = "3"
 epoch = 1
 node = "{{.Node}}"
 {{- if .OperatorEmail}}
@@ -7,24 +7,27 @@ node = "{{.Node}}"
 [operator]
 email = "{{.OperatorEmail}}"
 {{- end}}
-{{- if .ExternalMode}}
 
 [network]
+ipsec_enabled = {{.IPSecEnabled}}
+{{- if .ExternalMode}}
 external_mode = "{{.ExternalMode}}"
-{{- if .ExternalDHCP}}
-external_dhcp = true
-{{- end}}
 
 [[network.external_pools]]
 name        = "{{.PoolName}}"
 {{- if .PoolSource}}
 source      = "{{.PoolSource}}"
 {{- end}}
+{{- if .PoolBindBridge}}
+bind_bridge = "{{.PoolBindBridge}}"
+{{- end}}
 {{- if .PoolStart}}
 range_start = "{{.PoolStart}}"
 range_end   = "{{.PoolEnd}}"
 {{- end}}
+{{- if .PoolGateway}}
 gateway     = "{{.PoolGateway}}"
+{{- end}}
 {{- if .PoolGatewayIP}}
 gateway_ip  = "{{.PoolGatewayIP}}"
 {{- end}}
@@ -97,15 +100,6 @@ ovn_sb_addr = "{{.OVNSBAddr}}"
 {{- if .ExternalIface}}
 external_interface = "{{.ExternalIface}}"
 {{- end}}
-{{- if .DhcpBindBridge}}
-# dhcp_bind_bridge: the bridge where vpcd's DHCP client binds its AF_PACKET
-# socket. On Linux-bridge-WAN hosts (netplan default with the WAN NIC enslaved),
-# this is the Linux bridge name (e.g. "br-wan"). On direct-OVS hosts, this is
-# the OVS bridge holding the WAN NIC.
-# NEVER set this to "br-ext" — that is the OVN-side bridge and never sees LAN
-# DHCP traffic.
-dhcp_bind_bridge = "{{.DhcpBindBridge}}"
-{{- end}}
 
 [nodes.{{.Node}}.predastore]
 host = "{{.BindIP}}:8443"

docs/compute/gpu-passthrough/README.md

@@ -7,6 +7,11 @@ tags:
   - passthrough
   - vfio
   - ec2
+sections:
+  - overview
+  - prerequisites
+  - instructions
+  - troubleshooting
 resources:
   - title: "Spinifex Repository"
     url: "https://github.com/mulgadc/spinifex"
@@ -14,26 +19,6 @@ resources:
     url: "https://docs.aws.amazon.com/ec2/latest/instancetypes/ac.html"
 ---
 
-# GPU Passthrough
-
-> Configure VFIO GPU passthrough on a Spinifex node to expose NVIDIA or AMD GPUs to EC2 instances.
-
-> **Note:** GPU passthrough is supported on x86_64 hosts only.
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Prerequisites](#prerequisites)
-- [Host Setup](#host-setup)
-- [Enable Passthrough](#enable-passthrough)
-- [Status and Monitoring](#status-and-monitoring)
-- [GPU Instance Types](#gpu-instance-types)
-- [Launching GPU Instances](#launching-gpu-instances)
-- [Disable Passthrough](#disable-passthrough)
-- [Troubleshooting](#troubleshooting)
-
----
-
 ## Overview
 
 Spinifex supports VFIO-based GPU passthrough, binding NVIDIA and AMD GPUs directly to guest VMs via the `vfio-pci` kernel driver. Once configured, the node exposes GPU-enabled EC2 instance types and the GPU is allocated exclusively to individual instances.
@@ -42,9 +27,9 @@ At startup, Spinifex always probes for GPU hardware and surfaces the result in t
 
 The passthrough state is reflected in the login banner:
 
-![GPU not yet enabled](../../../.github/assets/images/gpu-banner1.png)
+<img src="../../../.github/assets/images/gpu-banner1.png" alt="GPU not yet enabled">
 
----
+> **Note:** GPU passthrough is supported on x86_64 hosts only.
 
 ## Prerequisites
 
@@ -52,9 +37,9 @@ The passthrough state is reflected in the login banner:
 - An NVIDIA or AMD GPU must be physically installed.
 - The Spinifex node must be running on bare metal (not inside a VM).
 
----
+## Instructions
 
-## Host Setup
+### Host Setup
 
 Run the one-time host configuration as root. This command is idempotent — safe to re-run after a reboot.
 
@@ -79,9 +64,7 @@ Setup complete — reboot required.
   Then run: sudo spx admin gpu enable
 ```
 
----
-
-## Enable Passthrough
+### Enable Passthrough
 
 After setup and reboot, enable GPU passthrough:
 
@@ -91,14 +74,11 @@ sudo spx admin gpu enable
 
 The command writes the configuration, signals the daemon (`SIGHUP`), and waits up to 30 seconds for the daemon to confirm the new state. On success it prints the current GPU status.
 
+<img src="../../../.github/assets/images/gpu-enabled.png" alt="GPU enabled">
 
-![GPU enabled](../../../.github/assets/images/gpu-enabled.png)
+### Status and Monitoring
 
----
-
-## Status and Monitoring
-
-### Per-node status
+**Per-node status:**
 
 ```bash
 spx admin gpu status
@@ -108,7 +88,7 @@ spx admin gpu status --node <node-name>
 
 Output includes hardware detected, IOMMU state, vfio-pci binding, passthrough enabled/disabled, GPU pool allocation (`allocated/total`), and the GPU-capable EC2 instance types available on that node.
 
-### Cluster view
+**Cluster view:**
 
 `spx top nodes` includes a `GPU (used/total)` column:
 
@@ -122,30 +102,30 @@ Output includes hardware detected, IOMMU state, vfio-pci binding, passthrough en
 spx top nodes
 ```
 
----
+### GPU Instance Types
 
-## GPU Instance Types
-
-GPU instance types are derived from detected hardware.
-
-To list available GPU instance types on the current node:
+GPU instance types are derived from detected hardware. To list available GPU instance types on the current node:
 
 ```bash
 aws ec2 describe-instance-types \
   --filters Name=instance-type,Values=g*
 ```
 
----
+### Launching GPU Instances
 
-## Launching GPU Instances
-
-Import the GPU-enabled AMI before launching:
+Spinifex ships two pre-built GPU guest images. Import the one that matches your host hardware before launching:
 
 ```bash
-spx admin images import --name ubuntu-gpu-nvidia   # NVIDIA hosts
-spx admin images import --name ubuntu-gpu-amd      # AMD hosts
+spx admin images import --name ubuntu-26.04-nvidia-gpu-x86_64   # NVIDIA hosts
+spx admin images import --name ubuntu-26.04-amd-gpu-x86_64      # AMD hosts
 ```
 
+These images are built on Ubuntu 26.04 and serve as a base for GPU workloads. Both include Docker CE, Python 3, and common utilities (`git`, `curl`, `htop`, `tmux`, `ffmpeg`).
+
+**NVIDIA** (`ubuntu-26.04-nvidia-gpu-x86_64`): NVIDIA server driver with DKMS pre-built against the image kernel, `nvidia-smi`, nvidia-container-toolkit wired into Docker (`--gpus all` works on first boot). CUDA toolkit and cuDNN are not included — use an NGC container or install inside the instance.
+
+**AMD** (`ubuntu-26.04-amd-gpu-x86_64`): `linux-firmware` (amdgpu firmware blobs), ROCm 7.2 CLI tools (`rocm-smi`, `rocminfo`, `amd-smi`). Full ROCm compute libraries (`rocblas`, etc.) are not included — install inside the instance as needed.
+
 Launch a GPU instance:
 
 ```bash
@@ -165,9 +145,7 @@ nvidia-smi
 rocm-smi
 ```
 
----
-
-## Disable Passthrough
+### Disable Passthrough
 
 Passthrough cannot be disabled while GPU instances are running. Terminate all GPU instances first, then:
 
@@ -177,8 +155,6 @@ sudo spx admin gpu disable
 
 The command signals the daemon and waits for confirmation. The GPU is released back to the host kernel on the next reboot (the vfio-pci binding persists until `setup` is re-run without the blacklists).
 
----
-
 ## Troubleshooting
 
 ### IOMMU not active after reboot

docs/getting-started/install-source/README.md

@@ -32,7 +32,7 @@ This guide builds Spinifex from source. For production deployments, the [binary
 **Requirements:**
 
 - Ubuntu 26.04 / Debian 13
-- Go 1.26.3+
+- Go 1.26.4+
 - GCC, make, pkg-config
 - QEMU/KVM, OVN/Open vSwitch, AWS CLI v2, dhcpcd-base
 

docs/reference-architectures/SMCI-H14-orchestration/README.md

@@ -0,0 +1,266 @@
+---
+title: "Multi-Tenant AI on Supermicro H14 with AMD MI350X"
+description: "Provision isolated GPU VMs for simultaneous AI workloads on a Supermicro H14 bare-metal node using Spinifex's EC2-compatible API."
+category: "Reference Architectures"
+tags:
+  - supermicro
+  - amd
+  - mi350x
+  - gpu-passthrough
+  - multi-tenant
+  - vllm
+resources:
+  - title: "GPU Passthrough"
+    url: "/docs/gpu-passthrough"
+  - title: "Launching Instances"
+    url: "/docs/launching-instances"
+  - title: "AMD Instinct MI350X"
+    url: "https://www.amd.com/en/products/accelerators/instinct/mi300/mi350x.html"
+  - title: "vLLM Documentation"
+    url: "https://docs.vllm.ai"
+sections:
+  - overview
+  - prerequisites
+  - instructions
+---
+
+## Overview
+
+This document serves as a tutorial for setting up multi-tenant AI workloads with Spinifex. The reference architecture for this tutorial is the Supermicro H14 platform - specifications listed below.
+
+The H14 chassis comes equipped with **8× AMD Instinct MI350X** (288 GB HBM3e each, 2.3 TB total). We used Spinifex to provision three isolated VMs, each assigned 2× MI350X via direct PCIe passthrough, and ran three computationally intensive workloads simultaneously: a YOLO11x vision model, a Qwen3-VL 235B FP8 vision language model, and a Llama 3.3 70B large language model.
+
+<p><video src="https://iso.mulgadc.com/clipped-demo.mp4" controls width="100%" style="border-radius:6px"></video></p>
+
+### Platform
+
+| Component | Specification |
+|---|---|
+| **Bare-metal host** | Supermicro H14 |
+| **Host OS** | Ubuntu 24.04 LTS or Debian 13 (minimum) |
+| **Orchestration** | Spinifex — EC2-compatible bare-metal API |
+| **Guest OS** | Ubuntu 26.04 LTS |
+| **GPUs** | 8× AMD Instinct MI350X (288 GB HBM3e each, 2.3 TB total) |
+| **GPU passthrough** | PCIe passthrough via vfio-pci |
+| **Instance type** | `g7e.12xlarge` — 2× MI350X per VM |
+| **Container runtime** | Docker (with ROCm device access) |
+| **Inference runtime** | vLLM (`rocm/vllm` image) |
+| **VM network** | Internal VPC, 192.168.10.0/24 |
+| **Block storage** | Viperblock — EBS-compatible, local NVMe-backed |
+
+Spinifex runs on the bare-metal host and exposes an EC2-compatible API to any standard `aws ec2` tooling. Guest VMs receive their GPU allocation at the hardware level — each VM's OS sees the MI350Xs as native PCIe devices with no virtualisation layer in the data path. Workloads across VMs share no GPU memory, no address space, and no network segment except through the VPC.
+
+GPU passthrough requires a host kernel and OS that supports vfio-pci. Ubuntu 24.04 LTS (kernel 6.8+) and Debian 13 (kernel 6.12+) are the tested minimum baselines for the host. Guest VMs run Ubuntu 26.04 LTS.
+
+## Prerequisites
+
+- Supermicro H14 with 8× AMD Instinct MI350X installed
+- Host OS: **Ubuntu 24.04 LTS** (kernel 6.8+) or **Debian 13** (kernel 6.12+) — minimum for vfio-pci support
+- Spinifex installed and all services running (`systemctl status spinifex.target`)
+- AMD GPU AMI registered (`ami-ubuntu-amd-gpu`) — Ubuntu 26.04 LTS with ROCm-compatible kernel
+- AWS CLI configured with `AWS_PROFILE=spinifex` pointing at the Spinifex endpoint
+- SSH key pair imported into Spinifex, VPC and security group created (see [Launching Instances](/docs/launching-instances))
+
+GPU passthrough must be configured before launching GPU instances. This is a one-time step per host.
+
+```bash
+# Detect GPUs and bind to vfio-pci (requires reboot)
+sudo spx admin gpu setup
+
+# After rebooting — confirm passthrough is active and signal the daemon
+sudo spx admin gpu enable
+```
+
+`spx admin gpu setup` blacklists the AMD driver on the host and binds each GPU to `vfio-pci`. After reboot, `spx admin gpu enable` verifies the binding and makes the GPU pool available to `RunInstances`.
+
+<img src="../../../.github/assets/images/gpu-enabled.png" alt="Spinifex console confirming PCIe passthrough is active for each GPU">
+
+To check available GPU instance types:
+
+```bash
+export AWS_PROFILE=spinifex
+aws ec2 describe-instance-types \
+    --query 'InstanceTypes[?GpuInfo].[InstanceType,GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Name]' \
+    --output table
+```
+
+## Instructions
+
+### 1. Provision the VMs
+
+Each VM was provisioned using standard AWS EC2 CLI commands — the only change from a normal AWS workflow is `AWS_PROFILE=spinifex`, which redirects the CLI to Spinifex's local EC2-compatible endpoint instead of AWS:
+
+| VM | Instance type | GPUs | Role | Model |
+|---|---|---|---|---|
+| `vm-yolo` | g7e.12xlarge | 2× MI350X | Real-time object detection | YOLO11x |
+| `vm-vlm`  | g7e.12xlarge | 2× MI350X | Multi-modal scene analysis | Qwen3-VL 235B FP8 |
+| `vm-chat` | g7e.12xlarge | 2× MI350X | Conversational LLM | Llama 3.3 70B |
+
+Six of the eight available MI350Xs are claimed here; the remaining two stay idle on the host, available for a fourth tenant without touching the existing three.
+
+```bash
+export AWS_PROFILE=spinifex
+
+# vm-yolo — YOLO11x object detection, 200 GB disk
+YOLO_ID=$(aws ec2 run-instances \
+    --image-id ami-ubuntu-amd-gpu \
+    --instance-type g7e.12xlarge \
+    --key-name spinifex-key \
+    --subnet-id <subnet-id> \
+    --security-group-ids <sg-id> \
+    --block-device-mappings 'DeviceName=/dev/sda1,Ebs={VolumeSize=200,DeleteOnTermination=true}' \
+    --count 1 \
+    --query 'Instances[0].InstanceId' --output text)
+echo "vm-yolo launched: $YOLO_ID"
+
+# vm-vlm — Qwen3-VL 235B FP8, 600 GB disk for weights
+VLM_ID=$(aws ec2 run-instances \
+    --image-id ami-ubuntu-amd-gpu \
+    --instance-type g7e.12xlarge \
+    --key-name spinifex-key \
+    --subnet-id <subnet-id> \
+    --security-group-ids <sg-id> \
+    --block-device-mappings 'DeviceName=/dev/sda1,Ebs={VolumeSize=600,DeleteOnTermination=true}' \
+    --count 1 \
+    --query 'Instances[0].InstanceId' --output text)
+echo "vm-vlm launched: $VLM_ID"
+
+# vm-chat — Llama 3.3 70B, 300 GB disk
+CHAT_ID=$(aws ec2 run-instances \
+    --image-id ami-ubuntu-amd-gpu \
+    --instance-type g7e.12xlarge \
+    --key-name spinifex-key \
+    --subnet-id <subnet-id> \
+    --security-group-ids <sg-id> \
+    --block-device-mappings 'DeviceName=/dev/sda1,Ebs={VolumeSize=300,DeleteOnTermination=true}' \
+    --count 1 \
+    --query 'Instances[0].InstanceId' --output text)
+echo "vm-chat launched: $CHAT_ID"
+
+# Wait for all three to reach running state
+for ID in "$YOLO_ID" "$VLM_ID" "$CHAT_ID"; do
+    aws ec2 wait instance-running --instance-ids "$ID"
+    echo "$ID is running"
+done
+
+# Retrieve IPs (Spinifex assigns addresses from the internal VPC)
+YOLO_IP=$(aws ec2 describe-instances --instance-ids "$YOLO_ID" \
+    --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)
+VLM_IP=$(aws ec2 describe-instances --instance-ids "$VLM_ID" \
+    --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)
+CHAT_IP=$(aws ec2 describe-instances --instance-ids "$CHAT_ID" \
+    --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)
+
+echo "vm-yolo: $YOLO_IP"
+echo "vm-vlm:  $VLM_IP"
+echo "vm-chat: $CHAT_IP"
+```
+
+### 2. Verify GPU access
+
+Once SSH is available, confirm both GPUs are visible inside each VM:
+
+```bash
+ssh -i ~/.ssh/spinifex-key ubuntu@$CHAT_IP 'lspci | grep -i amd'
+```
+
+Or install and run `amd-smi`:
+
+```bash
+ssh -i ~/.ssh/spinifex-key ubuntu@$CHAT_IP \
+    'curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg >/dev/null && \
+     echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.3 noble main" | sudo tee /etc/apt/sources.list.d/rocm.list && \
+     sudo apt-get update -qq && sudo apt-get install -y -q amd-smi-lib && \
+     amd-smi list'
+```
+
+<img src="../../../.github/assets/images/2gpus.png" alt="amd-smi inside vm-chat confirming two MI350Xs are directly attached, each with a unique UUID">
+
+Two MI350X entries with distinct UUIDs confirm direct PCIe passthrough is working.
+
+### 3. The orchestration layer
+
+Spinifex provides an EC2-compatible API on bare metal. The relevant capabilities for this demo:
+
+- **Multi-GPU claim per VM.** Each `g7e.12xlarge` requires two PCIe addresses claimed as a unit. Spinifex queries the available GPU pool on the host and assigns them to the guest VM at creation time, rolling back if the full allocation cannot be satisfied.
+- **PCIe passthrough via vfio-pci.** All 8 MI350Xs are bound to `vfio-pci` on the bare-metal host. Each VM gets its GPUs at the hardware level.
+- **Internal VPC (192.168.10.0/24).** VMs communicate directly without traversing the host's public network.
+
+The result: three completely independent EC2 instances, each running their own separate workloads using only the GPU resources they were allocated.
+
+### 4. Test methodology
+
+Two capture runs were made against the same workload script: **30 s baseline** (YOLO and VLM models running, no chat load) → **25 scripted Llama chat prompts** at 15 s intervals → **30 s settle tail**.
+
+The only difference between the two runs was tensor parallelism:
+
+- **Demo 1 (TP=1):** each VM uses one of its two allocated GPUs. The second sits idle.
+- **Demo 2 (TP=2):** each VM shards its model symmetrically across both GPUs using AMD Infinity Fabric (XGMI).
+
+| Metric | TP=1 baseline (Demo 1) | TP=2 (Demo 2) |
+|---|---|---|
+| Wall clock | 613 s | 600 s |
+| YOLO avg fps | 16.91 (min 15.3 / max 18.0) | 17.09 (min 15.6 / max 18.1) |
+| Qwen peak tok/s | 61.7 | 53.0 |
+| Llama peak tok/s | 36.4 | **53.1** |
+| Both GPUs active (chat VM) | no — GPU 1 idle | **yes — symmetric** |
+| VRAM per GPU (Qwen) | 259 GB on one, 0.3 GB on other | 241 GB on **both** |
+| VRAM per GPU (Llama) | 261 GB on one, 0.3 GB on other | 248 GB on **both** |
+
+### 5. Results
+
+### Inter-tenant isolation
+
+The charts below show the three independent workloads spiking activity on their associated GPUs, measured in GFX utilisation and power draw. YOLO average FPS stays at ~17 despite utilisation in the YOLO VM never exceeding 20%, indicating a bottleneck in networking or encoding rather than inference. Each chart highlights workload independence — pulses of GPU activity share no correlation across VMs.
+
+<img src="../../../.github/assets/images/smci/smci-demo1-gpu-utilisation.png" alt="GPU utilisation across all three VMs — Demo 1 TP=1">
+
+GPU utilisation per VM across the full Demo 1 capture.
+
+<img src="../../../.github/assets/images/smci/smci-demo2-gpu-power.png" alt="GPU power draw per workload — Demo 2 TP=2">
+
+Power draw per VM during Demo 2.
+
+### Single vs double GPU
+
+When TP=2 is enabled, each VM shards its model symmetrically across both GPUs. Inside the chat VM, both GPUs pulse together on every request, each holding exactly **248 GB of VRAM** (weights split + KV cache shards). With TP=1, GPU 1 sat at 0% for the entire run.
+
+<img src="../../../.github/assets/images/smci/smci-demo2-per-gpu-chat.png" alt="Llama 3.3 70B per-GPU activity — TP=2">
+
+Per-GPU utilisation inside `vm-chat` with TP=2.
+
+<img src="../../../.github/assets/images/smci/smci-demo1-per-gpu-chat.png" alt="Llama 3.3 70B per-GPU activity — TP=1">
+
+Same VM with TP=1.
+
+### vLLM generation throughput
+
+With TP=1, Qwen peaks at ~60 tok/s and Llama caps around 35 tok/s — both compute-bound on a single MI350X. Enabling TP=2 lifts Llama's peak to ~50 tok/s (+47%) as generation is distributed symmetrically across both GPUs. Qwen's throughput is slightly lower under TP=2 due to collective overhead on the shared-memory transport.
+
+<img src="../../../.github/assets/images/smci/smci-demo1-vllm-throughput.png" alt="vLLM generation throughput — TP=1">
+
+Generation throughput during Demo 1 (TP=1).
+
+<img src="../../../.github/assets/images/smci/smci-demo2-vllm-throughput.png" alt="vLLM generation throughput — TP=2">
+
+Generation throughput during Demo 2 (TP=2).
+
+### 6. Teardown
+
+```bash
+# Stop workloads on each VM
+for IP in "$YOLO_IP" "$VLM_IP" "$CHAT_IP"; do
+    ssh -i ~/.ssh/spinifex-key ubuntu@$IP 'docker stop $(docker ps -q)' || true
+done
+
+# Terminate all three instances — releases 6× MI350X back to the Spinifex pool
+aws ec2 terminate-instances --instance-ids "$YOLO_ID" "$VLM_ID" "$CHAT_ID"
+```
+
+The six MI350Xs are immediately returned to the pool once the instances terminate, ready for a new allocation without touching the host.
+
+### 7. Conclusion
+
+This tutorial demonstrates how Spinifex can turn a single bare-metal chassis into a multi-tenant AI serving platform. Spinifex utilises the flexibility of PCIe passthrough to allocate GPU resources in whichever configuration is required by the workload/s.
+
+Importantly, it does so with standard `aws ec2` CLI calls — `run-instances`, `describe-instances`, `terminate-instances` — against Spinifex's EC2-compatible endpoint. Teams already operating AWS infrastructure can point their existing tooling at a Spinifex node with a single profile change, against GPUs that sit in their own rack.

docs/reference-architectures/arcee-trinity-mi350x/README.md

@@ -0,0 +1,319 @@
+---
+title: "Arcee Trinity 400B FP8 on AMD MI350X with Spinifex"
+description: "Provision, load, and benchmark arcee-ai/Trinity-Large-Preview-FP8 across two AMD MI350X GPUs on a Supermicro H14 using Spinifex's EC2-compatible API."
+category: "Reference Architectures"
+tags:
+  - supermicro
+  - amd
+  - mi350x
+  - gpu-passthrough
+  - vllm
+  - llm
+  - arcee
+resources:
+  - title: "GPU Passthrough"
+    url: "/docs/gpu-passthrough"
+  - title: "Launching Instances"
+    url: "/docs/launching-instances"
+  - title: "Arcee AI Trinity"
+    url: "https://www.arcee.ai/trinity"
+  - title: "Trinity on HuggingFace"
+    url: "https://huggingface.co/arcee-ai/Trinity-Large-Preview-FP8"
+  - title: "AMD Instinct MI350X"
+    url: "https://www.amd.com/en/products/accelerators/instinct/mi300/mi350x.html"
+  - title: "vLLM Documentation"
+    url: "https://docs.vllm.ai"
+sections:
+  - overview
+  - prerequisites
+  - instructions
+---
+
+## Overview
+
+### About Trinity
+
+[Trinity](https://www.arcee.ai/trinity) is a family of open-weight language models from [Arcee AI](https://www.arcee.ai), built specifically for production agentic workloads: reliable tool calling, structured JSON output, coherent multi-turn conversations, and long-context reasoning. The family spans four sizes — Nano (6B), Mini (26B), Large Preview (400B), and Large Thinking (400B reasoning-optimised), with consistent capabilities and API surface across all of them. A workflow validated on Nano can be promoted to Large with no prompt changes.
+
+Trinity Large Preview is a **sparse Mixture-of-Experts (MoE)** model. The headline number is 400B total parameters, but only **13B parameters activate per token** — the MoE routing selects a subset of experts for each forward pass, so inference cost is closer to a much smaller dense model than the raw parameter count suggests. The result is a 400B-class model with a **512K token context window** that fits on two high-memory GPUs and generates at competitive throughput.
+
+Arcee has trained Trinity with a heavy focus on agent reliability — function selection accuracy, valid parameter generation, graceful failure recovery, and schema adherence for structured outputs. It is available as open weights on HuggingFace and is natively compatible with vLLM, SGLang, and llama.cpp.
+
+### This guide
+
+This writeup covers running Trinity Large Preview FP8 on a Supermicro H14 bare-metal node via Spinifex — our EC2-compatible orchestration layer. Two AMD Instinct MI350X GPUs (288 GB HBM3e each, 576 GB combined) provide just enough headroom for the FP8 weights plus active KV cache, provisioned and managed with standard `aws ec2` CLI commands.
+
+At that scale the FP8 weights require around 400 GB of GPU memory — more than a single MI350X can hold. This guide covers splitting the model symmetrically across two MI350Xs using vLLM tensor parallelism (TP=2), running on a `g7e.12xlarge` instance provisioned via Spinifex on the H14.
+
+The complete stack:
+
+| Layer | Detail |
+|---|---|
+| Bare-metal host | Supermicro H14, 8× AMD Instinct MI350X |
+| Orchestration | Spinifex (EC2-compatible API) |
+| Instance type | `g7e.12xlarge` — 2× MI350X via PCIe passthrough |
+| Disk | 800 GB (model weights alone are ~400 GB) |
+| Inference runtime | vLLM (`rocm/vllm` Docker image) |
+| Tensor parallelism | TP=2 — model sharded symmetrically across GPU 0 and GPU 1 |
+| Model | `arcee-ai/Trinity-Large-Preview-FP8` |
+
+### Platform
+
+| Component | Specification |
+|---|---|
+| **Bare-metal host** | Supermicro H14 |
+| **Host OS** | Ubuntu 24.04 LTS or Debian 13 (minimum) |
+| **Orchestration** | Spinifex — EC2-compatible bare-metal API |
+| **Guest OS** | Ubuntu 26.04 LTS |
+| **GPUs** | 2× AMD Instinct MI350X (288 GB HBM3e each, 576 GB combined) |
+| **GPU passthrough** | PCIe passthrough via vfio-pci |
+| **Instance type** | `g7e.12xlarge` |
+| **Container runtime** | Docker (with ROCm device access) |
+| **Inference runtime** | vLLM (`rocm/vllm` image), tensor parallelism TP=2 |
+| **Block storage** | Viperblock — EBS-compatible, 800 GB |
+| **Model** | `arcee-ai/Trinity-Large-Preview-FP8` (400B MoE, ~13B active/token) |
+
+Spinifex runs on the bare-metal host and presents an EC2-compatible API endpoint. Launching a `g7e.12xlarge` atomically claims two MI350Xs from the host's GPU pool and binds them to the guest VM via PCIe passthrough — the guest OS communicates with the hardware directly, with no software virtualisation layer in the data path. When the instance terminates, those two GPUs are immediately returned to the pool.
+
+GPU passthrough requires a host kernel and OS that supports vfio-pci. Ubuntu 24.04 LTS (kernel 6.8+) and Debian 13 (kernel 6.12+) are the tested minimum baselines for the host. Guest VMs run Ubuntu 26.04 LTS, which ships with the ROCm-compatible kernel and userspace expected by the AMD GPU AMI.
+
+## Prerequisites
+
+- Supermicro H14 with at least 2× AMD Instinct MI350X installed
+- Host OS: **Ubuntu 24.04 LTS** (kernel 6.8+) or **Debian 13** (kernel 6.12+) — minimum for vfio-pci support
+- Spinifex installed and all services running (`systemctl status spinifex.target`)
+- GPU passthrough configured — `spx admin gpu setup` (reboot required) then `spx admin gpu enable`
+- AMD GPU AMI registered (`ami-ubuntu-amd-gpu`) — Ubuntu 26.04 LTS with ROCm-compatible kernel
+- AWS CLI configured with `AWS_PROFILE=spinifex` pointing at the Spinifex endpoint
+- SSH key pair imported, VPC and security group created (see [Launching Instances](/docs/launching-instances))
+- Docker installed in the guest VM (included in the AMD GPU AMI)
+- HuggingFace account with access to `arcee-ai/Trinity-Large-Preview-FP8`
+
+GPU passthrough must be configured before launching GPU instances:
+
+```bash
+sudo spx admin gpu setup   # binds GPUs to vfio-pci — requires reboot
+# ... reboot ...
+sudo spx admin gpu enable  # confirms passthrough and makes GPU pool available
+```
+
+## Instructions
+
+### 1. Provision the instance
+
+Spinifex exposes a standard EC2 API on the bare-metal host. The only difference from a real AWS workflow is `AWS_PROFILE=spinifex`.
+
+```bash
+export AWS_PROFILE=spinifex
+
+# Check available GPU instance types
+aws ec2 describe-instance-types \
+    --query 'InstanceTypes[?GpuInfo].[InstanceType,GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Name]' \
+    --output table
+
+# Launch a g7e.12xlarge with 800 GB disk
+# Spinifex atomically claims 2× MI350X via PCIe passthrough for this instance type
+INSTANCE_ID=$(aws ec2 run-instances \
+    --image-id ami-ubuntu-amd-gpu \
+    --instance-type g7e.12xlarge \
+    --key-name spinifex-key \
+    --subnet-id <subnet-id> \
+    --security-group-ids <sg-id> \
+    --block-device-mappings 'DeviceName=/dev/sda1,Ebs={VolumeSize=800,DeleteOnTermination=true}' \
+    --count 1 \
+    --query 'Instances[0].InstanceId' --output text)
+
+echo "Launched: $INSTANCE_ID"
+
+# Wait for running state
+aws ec2 wait instance-running --instance-ids "$INSTANCE_ID"
+
+# Get the IP
+INSTANCE_IP=$(aws ec2 describe-instances \
+    --instance-ids "$INSTANCE_ID" \
+    --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)
+
+echo "IP: $INSTANCE_IP"
+```
+
+Once SSH is available, confirm both GPUs are visible:
+
+```bash
+ssh -i ~/.ssh/spinifex-key ubuntu@$INSTANCE_IP 'lspci | grep -i amd'
+```
+
+Or install and run `amd-smi` on the instance:
+
+<img src="../../../.github/assets/images/2gpus.png" alt="amd-smi inside the guest VM confirming two MI350Xs are directly attached, each with a unique UUID">
+
+Two MI350X entries with distinct UUIDs confirm direct PCIe passthrough.
+
+### 2. Pull the vLLM Docker image
+
+```bash
+ssh -i ~/.ssh/spinifex-key ubuntu@$INSTANCE_IP
+
+# On the VM:
+docker pull rocm/vllm:latest
+```
+
+> The `rocm/vllm` image is large (~20 GB). Pull it while the model is downloading in parallel if bandwidth allows.
+
+### 3. Pull the model
+
+Trinity Large Preview FP8 weighs roughly 400 GB, so expect the model download to take some time.
+
+```bash
+# On the VM:
+pip install 'huggingface-hub[hf_xet]'
+hf download 'arcee-ai/Trinity-Large-Preview-FP8' --repo-type model
+
+# Monitor progress — looking for ~400 GB total
+du -sh ~/.cache/huggingface/hub/
+```
+
+The model lands in `~/.cache/huggingface/hub/`. vLLM picks it up automatically from there on serve.
+
+### 4. Run vLLM
+
+Trinity needs TP=2 to fit across both MI350Xs:
+
+```bash
+# On the VM:
+docker run --rm -d \
+    --name trinity \
+    --device /dev/kfd \
+    --device /dev/dri \
+    --group-add video \
+    --ipc host \
+    --network host \
+    -v ~/.cache/huggingface:/root/.cache/huggingface \
+    rocm/vllm:latest \
+    vllm serve arcee-ai/Trinity-Large-Preview-FP8 \
+        --tensor-parallel-size 2 \
+        --dtype auto \
+        --max-model-len 8192 \
+        --gpu-memory-utilization 0.90 \
+        --host 0.0.0.0 \
+        --port 8003
+```
+
+Watch the startup log:
+
+```bash
+docker logs -f trinity
+```
+
+Loading a 400B FP8 model across two GPUs takes **2–4 minutes**. Wait for:
+
+```
+INFO:     Application startup complete.
+```
+
+Then verify:
+
+```bash
+curl http://localhost:8003/health
+curl http://localhost:8003/v1/models | python3 -m json.tool
+```
+
+### 5. Test it
+
+A quick sanity check via the OpenAI-compatible API before running benchmarks:
+
+```bash
+curl http://localhost:8003/v1/chat/completions \
+    -H "Content-Type: application/json" \
+    -d '{
+        "model": "arcee-ai/Trinity-Large-Preview-FP8",
+        "messages": [{"role": "user", "content": "Explain tensor parallelism in three sentences."}],
+        "max_tokens": 256
+    }'
+```
+
+The first request will be slowest — KV cache is cold. Subsequent requests warm up noticeably.
+
+### 6. Dashboard
+
+We created a quick dashboard for a live view of the model's telemetry — throughput, latency, GPU utilisation, VRAM, and KV cache — streamed via SSE from the vLLM metrics endpoint and the GPU stats sidecar.
+
+<p><video src="https://iso.mulgadc.com/trinity-workload.mp4" controls width="100%" style="border-radius:6px"></video></p>
+
+### 7. Benchmark results
+
+Two captures were run against the live model:
+
+- **Chat session:** A short interactive session — low concurrency, variable cadence, 56 s total. Reflects real conversational use.
+- **Scripted benchmark:** 5-minute sustained load driver — rotating batch of 15 long-form technical prompts at 1.5 s cadence. Reflects peak throughput.
+
+| Metric | Chat session | Benchmark (5 min) |
+|---|---|---|
+| Peak tok/s | 67.1 | 67.0 |
+| Avg tok/s (under load) | 18.7 | **52.9** |
+| Avg TTFT | 1668 ms | 758 ms |
+| Min TTFT | 1502 ms | 538 ms |
+| Max TTFT | 1871 ms | 1165 ms |
+| GPU 0 util (avg) | 30.2% | 88.8% |
+| GPU 1 util (avg) | 30.1% | 88.7% |
+| VRAM per GPU | 259.6 GB | 259.6 GB |
+| Avg power (both GPUs) | 664 W | 788 W |
+| Peak power | 828 W | 839 W |
+| GPU temp | 65 °C | 65 °C |
+
+The lower avg utilisation in the chat session reflects idle gaps between conversational turns; peak throughput is identical at ~67 tok/s in both runs, indicating the hardware ceiling rather than a software one.
+
+### Generation throughput
+
+<img src="../../../.github/assets/images/trinity/trinity-throughput.png" alt="Trinity generation throughput — scripted benchmark">
+
+Generation and prompt-ingest tok/s over the 5-minute benchmark. The model sustains ~53 tok/s average at continuous load, peaking at 67 tok/s.
+
+### Per-GPU compute utilisation
+
+<img src="../../../.github/assets/images/trinity/trinity-per-gpu-util.png" alt="Trinity per-GPU utilisation — scripted benchmark">
+
+GPU 0 and GPU 1 track each other closely throughout the benchmark — TP=2 distributes attention and FFN layers symmetrically across both MI350Xs.
+
+### Per-GPU VRAM
+
+<img src="../../../.github/assets/images/trinity/trinity-per-gpu-vram.png" alt="Trinity per-GPU VRAM — scripted benchmark">
+
+Both GPUs hold 259.6 GB — model weights sharded evenly, plus KV cache. 90.2% of HBM3e is occupied at rest, leaving ~26 GB headroom per GPU for active KV cache during generation.
+
+### Per-GPU power
+
+<img src="../../../.github/assets/images/trinity/trinity-per-gpu-power.png" alt="Trinity per-GPU power — scripted benchmark">
+
+Power draw during active generation peaks at ~420 W per GPU, for a combined chassis draw well under the MI350X's 750 W TDP per card. Idle between prompts returns to ~300 W.
+
+### Latency
+
+<img src="../../../.github/assets/images/trinity/trinity-latency.png" alt="Trinity latency — scripted benchmark">
+
+Cumulative average TTFT and E2E latency over the benchmark. TTFT settles to ~758 ms avg once the KV cache is warm; E2E tracks request length as expected.
+
+### KV cache utilisation
+
+<img src="../../../.github/assets/images/trinity/trinity-kv-cache.png" alt="Trinity KV cache utilisation — scripted benchmark">
+
+KV cache occupancy rises as requests pile up during the sustained load phase. The queue depth stays near zero — the model keeps pace with the 1.5 s prompt cadence.
+
+### 8. Teardown
+
+```bash
+# Stop the container
+ssh -i ~/.ssh/spinifex-key ubuntu@$INSTANCE_IP 'docker stop trinity'
+
+# Terminate the instance — releases the 2× MI350X back to the Spinifex pool
+aws ec2 terminate-instances --instance-ids "$INSTANCE_ID"
+```
+
+The two MI350Xs are immediately available to Spinifex for a new workload once the instance terminates.
+
+### 9. Conclusion
+
+Trinity Large Preview demonstrates that frontier-class open-weight models are now within reach of a two-GPU bare-metal setup. The MoE architecture is crucial: 400B total parameters with only 13B active per token means the model generates at ~53 tok/s sustained and peaks at 67 tok/s on a pair of MI350Xs — throughput that would be difficult to match with a dense model of comparable quality at that memory footprint.
+
+Combining this with Spinifex as the infrastructure layer allows teams to own the entire workflow. Provisioning a `g7e.12xlarge` instance, pulling a 400 GB model, and serving it through vLLM took a handful of standard `aws ec2` commands and a single `docker run`. The same workflow runs identically on any Spinifex-managed bare-metal node, with the GPUs returned to the pool when instances terminate.
+
+For teams evaluating Trinity for production agentic workloads — tool calling, structured outputs, long-context reasoning — this represents a credible self-hosted deployment path: open weights, Apache 2.0 license, owned hardware and familiar tooling.

docs/service-interfaces.yaml

@@ -131,7 +131,7 @@ services:
       - "s3.*"
 
   vpcd:
-    path: spinifex/services/vpcd
+    path: spinifex/vpcd
     # OVN-backed VPC / subnet / SG / NAT / IGW data plane.
     subscribes:
       - "vpc.create"

go.mod

@@ -1,6 +1,6 @@
 module github.com/mulgadc/spinifex
 
-go 1.26.3
+go 1.26.4
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
@@ -8,12 +8,13 @@ require (
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/go-chi/chi/v5 v5.3.0
+	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/uuid v1.6.0
 	github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mulgadc/predastore v1.5.0
-	github.com/mulgadc/viperblock v1.5.0
+	github.com/mulgadc/predastore v1.6.0
+	github.com/mulgadc/viperblock v1.6.0
 	github.com/nats-io/nats-server/v2 v2.14.1
 	github.com/nats-io/nats.go v1.52.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
@@ -25,6 +26,7 @@ require (
 	go.uber.org/goleak v1.3.0
 	golang.org/x/crypto v0.52.0
 	golang.org/x/sync v0.20.0
+	golang.org/x/sys v0.45.0
 	gopkg.in/ini.v1 v1.67.2
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -36,8 +38,8 @@ require (
 	github.com/antithesishq/antithesis-sdk-go v0.7.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect
-	github.com/aws/smithy-go v1.25.1 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.41.9 // indirect
+	github.com/aws/smithy-go v1.26.0 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
@@ -126,7 +128,6 @@ require (
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/mod v0.35.0 // indirect
 	golang.org/x/net v0.55.0 // indirect
-	golang.org/x/sys v0.45.0 // indirect
 	golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect
 	golang.org/x/term v0.43.0 // indirect
 	golang.org/x/text v0.37.0 // indirect

go.sum

@@ -31,30 +31,30 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
-github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
-github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
-github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/aws-sdk-go-v2 v1.41.9 h1:/rYeyO2+HrMztAmxAq9++XJtFMqSIpSsNA0yDGALYq4=
+github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.19 h1:yuFzSV1U0aRNYCQGVaTY2zW2M/L93pYHnXnrJUphYhU=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.19/go.mod h1:7y63L1kGzeoDlJaQ3Z578KrnmfBut96JjvJUzGwR+YE=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 h1:Uii3frf9ztec/ABM2/FSH9/z7PLzxfpG8h4RpkUFflQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25/go.mod h1:G6kntsA2GorAxDPbap6xgB2F+amSLUF8GJTi7PUoX44=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 h1:r1+/l6m+WaUJF9HISEsNOLHSNj5EXYQxK8VX6Cz9NlA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25/go.mod h1:cKf+D+NMDK1LndD7BowHbBZPgR9V0/5HubH0PFWvA+c=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 h1:A1PmWU2zfkIm9EyFlJncFXL4W4phML+h8KjltUsCvNQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26/go.mod h1:dY4MRzXEizrD4hqtpKvWVGPX7QleSGGVY+EBolo1RmM=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 h1:d5/908OJ4bXg8lyjeMPvXetEKqoDoLi5Owy1zNue3yg=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10/go.mod h1:a57l7Hwh+FWI+we50g5NPJHYUKeJKfXbc4w8SyXu8Ig=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 h1:dD3dhHNglpd98gs72my22Ndqi1hqQGllFFg1F+twfxg=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25/go.mod h1:0yAbjPfd64gG7mj85RW+fMEYdfBgCRZw8g/oWcL1pjc=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDbiit/GcBE2K4IUpMZymaA0kOz3xK978=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0=
+github.com/aws/smithy-go v1.26.0 h1:9ouqbi+NyKP7fV3Te7UElCwdAb6Y8uk7LGwPE5tVe/s=
+github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -147,6 +147,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
 github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
+github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -286,10 +288,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/predastore v1.5.0 h1:FtL9EquoGKl3tcIRIoAXGhQStkN4uPDx0TOQZ89Mh+c=
-github.com/mulgadc/predastore v1.5.0/go.mod h1:fXIGz7yCKcHh68ybcw7aigdOdzIfyt19h/0diPF+JCU=
-github.com/mulgadc/viperblock v1.5.0 h1:2OprAI1oAtitUosS5A9LbenU/RCaw0oxUJvUhv8+tgU=
-github.com/mulgadc/viperblock v1.5.0/go.mod h1:lIuOaBO3gC0o/EwWhJPpCuhUEPQ3/y74xckGlcBKaos=
+github.com/mulgadc/predastore v1.6.0 h1:a0xJtrmAvmLlc19B5xUR9upbkl7VlDntpAKnxbkM07E=
+github.com/mulgadc/predastore v1.6.0/go.mod h1:RJh0V0TG27s0q6n0o4CIQFu7CjjPYv4yGnKpIiP9tLs=
+github.com/mulgadc/viperblock v1.6.0 h1:mFx0eVOJoT3WL+X/GPrZOI0ySqerpiCtUW4vPpUxbtI=
+github.com/mulgadc/viperblock v1.6.0/go.mod h1:cwE2FVrvxUD+QaMWnAZeKVnxF8VtTG9zxiUaP1m5oL0=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

scripts/ansible/.gitignore

@@ -0,0 +1,2 @@
+.creds/
+*.retry

scripts/ansible/README.md

@@ -5,7 +5,6 @@ Lives alongside the existing `scripts/dev-*.sh` and `scripts/reset-dev-env.sh`;
 neither set is removed. Decide which to keep after parity testing.
 
 Plan: `docs/development/improvements/ansible-dev-lifecycle.md` (mulga top-level).
-Bead: `mulga-siv-9`.
 
 ## Prerequisites
 
@@ -32,6 +31,12 @@ inventory and roles relative to CWD.
 | `playbooks/dev-install.yml` | Build + install + init + smoketest on clean box | `dev-install.sh` |
 | `playbooks/dev-reset.yml` | Capture settings → teardown → build → install → init → smoketest | `reset-dev-env.sh` (full) |
 | `playbooks/dev-deploy.yml` | Rebuild + swap binaries/microvm artifacts + restart (no setup.sh, no smoketest) | `make deploy` |
+| `playbooks/dev-status.yml` | Read-only health snapshot (units, ports, OVN/OVS, gateway, counts, drift) | none |
+| `playbooks/dev-logs.yml` | Dump journald + `/var/log/spinifex/*` + status/networking/versions into a tarball | none |
+| `playbooks/dev-snapshot.yml` | Quiesce spinifex+OVN+OVS, tar state paths to a named bundle, restart | none |
+| `playbooks/dev-restore.yml` | Stop services, wipe + untar a named bundle, refresh CA, restart | none |
+| `playbooks/dev-version.yml` | Build-vs-install drift report (spx, plugin, microvm) + sub-repo HEADs | none |
+| `playbooks/dev-vpc.yml` | Idempotent dev VPC fixture (VPC + public subnet + IGW + default route); `-e dev_vpc_state=absent` tears down | partial overlap with `scripts/iac/examples/*` |
 
 Upcoming (not yet implemented):
 
@@ -46,6 +51,12 @@ ansible-playbook playbooks/dev-teardown.yml
 ansible-playbook playbooks/dev-install.yml
 ansible-playbook playbooks/dev-reset.yml
 ansible-playbook playbooks/dev-deploy.yml
+ansible-playbook playbooks/dev-status.yml
+ansible-playbook playbooks/dev-logs.yml
+ansible-playbook playbooks/dev-snapshot.yml -e snapshot_name=before-merge
+ansible-playbook playbooks/dev-restore.yml  -e snapshot_name=before-merge
+ansible-playbook playbooks/dev-version.yml
+ansible-playbook playbooks/dev-vpc.yml
 ```
 
 Or via `make` (from `spinifex/`):
@@ -56,6 +67,13 @@ make ansible-dev-teardown
 make ansible-dev-install
 make ansible-dev-reset
 make ansible-dev-deploy
+make ansible-dev-status
+make ansible-dev-logs
+make ansible-dev-snapshot ANSIBLE_EXTRA='-e snapshot_name=before-merge'
+make ansible-dev-restore  ANSIBLE_EXTRA='-e snapshot_name=before-merge'
+make ansible-dev-version
+make ansible-dev-vpc
+make ansible-dev-vpc ANSIBLE_EXTRA='-e dev_vpc_state=absent'
 ```
 
 ### When to use which
@@ -64,6 +82,11 @@ make ansible-dev-deploy
 - Iterate on Go code, microvm initramfs, lb-agent → `ansible-dev-deploy` (fast)
 - Changed systemd units, helper scripts, logrotate, setup.sh → `ansible-dev-reset` (slow, full rebuild)
 - Need a clean slate without reinstall → `ansible-dev-teardown`
+- "Is my dev box healthy?" → `ansible-dev-status` (read-only, never mutates)
+- Filing a bug / capturing the state of a broken box → `ansible-dev-logs` (writes `/tmp/spinifex-logs-<host>-<ts>.tar.gz`; override `-e logs_since=10min` or `-e logs_include_dmesg=false`)
+- Checkpoint before a risky op (branch switch, viperblock surgery, ovn restart cycle) → `ansible-dev-snapshot ANSIBLE_EXTRA='-e snapshot_name=<name>'`; roll back via `ansible-dev-restore` instead of a full `dev-reset`. Bundles live in `/var/lib/spinifex-snapshots/<name>.tar.gz` (root-owned, survives `dev-teardown`).
+- "Do I need to `ansible-dev-deploy`?" → `ansible-dev-version` (drift-only check; faster than `dev-status`)
+- Want a known-good VPC/subnet for manual EC2 testing without writing tofu → `ansible-dev-vpc` (smoketest will then find and use it automatically). Tear down with `ANSIBLE_EXTRA='-e dev_vpc_state=absent'`.
 
 ## Variable overrides
 
@@ -83,14 +106,117 @@ Useful overrides:
 | `teardown_process_wait_seconds` | `30` | Grace period for processes to exit |
 | `spinifex_region` | `ap-southeast-2` | Region passed to `spx admin init` |
 | `spinifex_az` | `{{ region }}a` | AZ passed to `spx admin init` |
-| `spinifex_external_mode` | `pool` | `pool` \| `nat` external networking |
+| `spinifex_external_mode` | `pool` | External networking mode — `pool` or empty (NAT removed) |
+| `spinifex_external_no_external` | `false` | `true` → emit `--no-external` and skip external networking |
+| `spinifex_external_pool_start` | `""` | **Required** for `pool` mode — first IP of the static WAN pool |
+| `spinifex_external_pool_end` | `""` | **Required** for `pool` mode — last IP of the static WAN pool |
+| `spinifex_external_gateway` | `""` | WAN gateway (auto-detected when blank) |
+| `spinifex_external_prefix_len` | `""` | WAN prefix length (auto-detected when blank) |
+| `spinifex_external_gateway_ip` | `""` | OVN router external-IP override (`--gateway-ip`) |
+
+`pool` mode is mandatory unless you set `spinifex_external_no_external=true`.
+`spx admin init` rejects the run without a `--external-pool` range, so the
+init role asserts `spinifex_external_pool_start/_end` upfront. Either persist
+them in `vars/defaults.yml`, or pass them at invocation:
+
+Direct `ansible-playbook` (use `-e` here — this is ansible's flag):
+
+```
+ansible-playbook playbooks/dev-install.yml \
+  -e spinifex_external_pool_start=192.168.1.150 \
+  -e spinifex_external_pool_end=192.168.1.250
+```
+
+Via `make` (use `EXTRA_VARS=` — make eats its own `-e`):
+
+```
+make ansible-dev-install \
+  EXTRA_VARS="spinifex_external_pool_start=192.168.1.150 spinifex_external_pool_end=192.168.1.250"
+```
 
 `dev-reset.yml` automatically captures `region`, `az`, `external_mode`,
-pool range, gateway, prefix-len, nat gateway_ip from the existing
+pool range, gateway, prefix-len and `gateway_ip` from the existing
 `/etc/spinifex/spinifex.toml` before teardown and replays them into
-`init`. Use `-e` to override.
+`init`. Legacy `external_mode = "nat"` is coerced to `"pool"` with a warn.
+Use `-e` to override.
+
+## Helper scripts
+
+`scripts/aws-as-spinifex.sh` and `scripts/spx-as-spinifex.sh` wrap `aws`
+and `spx` invocations in `sg spinifex -c '...'` so they run with the
+`spinifex` supplementary group active. Both helpers preset
+`AWS_PROFILE=spinifex`. All AWS-calling tasks in the roles use these
+via `spinifex_aws_cli` / `spinifex_spx_cli` from `vars/defaults.yml`.
+
+Background: `/etc/spinifex/ca.pem` lives under `drwxr-x--- root:spinifex`,
+so the AWS CLI needs `spinifex` group membership to read the CA bundle.
+Ansible's `become_user` is a no-op when the target user equals the
+calling user (no `sudo -u` invoked → no `initgroups(3)` → supplementary
+groups inherited from the operator's shell). On a fresh login the
+operator's shell would have `spinifex` (added by `setup.sh`); on shells
+that predate the group add it does not. The helpers sidestep this
+entirely.
 
 ## Leak catalog
 
 `roles/teardown/README.md` tracks every known state source. When teardown
 misses something, add it there and in `roles/teardown/tasks/main.yml`.
+
+## Multi-node cluster bootstrap
+
+`playbooks/cluster-bootstrap.yml` provisions a 2+ node Spinifex cluster
+across a long-running tofu env (`env1`, `env2`, `bryn`, etc). Inventory
+is generated dynamically from `scripts/tofu-cluster/envs/<env>.tfvars`
+via `inventory/tofu.py` — first node in `nodes = [...]` is the init host,
+the rest are joins.
+
+```bash
+make ansible-cluster-bootstrap ENV=env1 POOL=192.168.1.150-192.168.1.159
+```
+
+`POOL` is optional; omit for DHCP fallback. Equivalent without make:
+
+```bash
+cd scripts/ansible
+CLUSTER_ENV=env1 ansible-playbook -i inventory/tofu.py \
+    playbooks/cluster-bootstrap.yml \
+    -e cluster_env=env1 \
+    -e cluster_external_pool=192.168.1.150-192.168.1.159
+```
+
+Plays sequence:
+
+1. **runner (localhost)** — `build` role produces `/tmp/spinifex-local.tar.gz`.
+2. **all env hosts (parallel)** — copies tarball, stops any dev-layout
+   services (`~/spinifex/`), runs `preflight` + `install` roles. `setup.sh`
+   creates `/etc/spinifex`, the `spinifex` user/group, systemd units, and
+   installs `/usr/local/bin/spx`.
+3. **init host only** — `cluster-init` role runs `spx admin init` in the
+   background, waits for the formation server, captures the join token.
+   CA is then copied into `/usr/local/share/ca-certificates/spinifex-ca.crt`
+   **before** `spinifex.target` starts (required — predastore enforces
+   strict TLS verify against the system trust store).
+4. **join hosts (parallel)** — `cluster-join` role runs `spx admin join`
+   against the formation server, then installs CA, then starts
+   `spinifex.target`.
+5. **all hosts** — polls `https://<ip>:4432/health` until `awsgw=ok`.
+6. **init host only** — `cluster-post` role imports the SSH key + Ubuntu
+   AMI (both idempotent: skip if already present).
+
+Re-running the playbook against an already-bootstrapped env is safe —
+`setup.sh`, `spx admin init --force`, CA copy, key/AMI import are all
+idempotent.
+
+### Migration from `bootstrap.sh`
+
+`scripts/tofu-cluster/bootstrap.sh` ran services under `~/spinifex/` as
+`tf-user` via `start-dev.sh` with no systemd supervision, and installed
+the cluster CA *after* services started — predastore would crash with
+`x509: certificate signed by unknown authority` on every first boot.
+`cluster-bootstrap.yml` replaces that flow with production-layout
+(`/etc/spinifex/` + systemd) and correct CA-before-services ordering.
+The first run on an env previously bootstrapped by `bootstrap.sh` will
+stop the dev-layout services, install the prod layout, and re-init the
+cluster from scratch via `spx admin init --force`.
+
+`bootstrap.sh` stays in-tree for CI and pseudo-multinode envs.

scripts/ansible/ansible.cfg

@@ -5,7 +5,13 @@ stdout_callback = ansible.builtin.default
 host_key_checking = False
 retry_files_enabled = False
 interpreter_python = auto_silent
-forks = 1
+forks = 10
+
+[inventory]
+# inventory/tofu.py is an executable script inventory used by
+# cluster-bootstrap.yml — opted into via `-i inventory/tofu.py` on the
+# command line; not default to keep dev-* playbooks pointing at localhost.
+enable_plugins = host_list, script, auto, yaml, ini, toml
 
 [callback_default]
 result_format = yaml

scripts/ansible/inventory/tofu.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""Dynamic ansible inventory generator from tofu-cluster tfvars.
+
+Reads scripts/tofu-cluster/envs/<env>.tfvars (HCL subset) and emits ansible
+JSON inventory. First node in the tfvars `nodes = [...]` list is the init
+host (forms the cluster); the rest are join hosts.
+
+Usage:
+    inventory/tofu.py --list --env env1
+    inventory/tofu.py --host <ip> --env env1
+
+Selecting the env:
+    CLUSTER_ENV=env1 inventory/tofu.py --list
+    inventory/tofu.py --list --env env1
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+
+def _repo_root() -> Path:
+    here = Path(__file__).resolve()
+    for parent in here.parents:
+        if (parent / "go.work").exists() or (parent / ".gitmodules").exists():
+            return parent
+    raise SystemExit(f"could not locate mulga repo root from {here}")
+
+
+def _tfvars_path(env: str) -> Path:
+    root = _repo_root()
+    candidate = root / "scripts" / "tofu-cluster" / "envs" / f"{env}.tfvars"
+    if not candidate.exists():
+        raise SystemExit(f"tfvars file not found: {candidate}")
+    return candidate
+
+
+_STRING_RE = re.compile(r'^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*"([^"]*)"\s*$')
+_NUMBER_RE = re.compile(r'^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([0-9]+)\s*$')
+_NODE_FIELD_RE = re.compile(r'(\w+)\s*=\s*"([^"]*)"')
+
+
+def _parse_tfvars(path: Path) -> dict:
+    """Minimal tfvars parser — handles the subset used by envs/*.tfvars.
+
+    Supports:
+      key = "string"
+      key = 12345
+      nodes = [
+        { name = "x", wan_ip = "1.2.3.4", ... },
+        ...
+      ]
+    """
+    out: dict = {"nodes": []}
+    text = path.read_text()
+
+    # Strip line comments
+    text = re.sub(r"#.*", "", text)
+
+    # Pull the nodes block out first, then process remaining lines.
+    nodes_match = re.search(r"nodes\s*=\s*\[(.+?)\]\s*$", text, re.DOTALL | re.MULTILINE)
+    if nodes_match:
+        nodes_body = nodes_match.group(1)
+        for entry in re.finditer(r"\{([^}]+)\}", nodes_body):
+            fields = dict(_NODE_FIELD_RE.findall(entry.group(1)))
+            if "name" in fields and "wan_ip" in fields:
+                out["nodes"].append(fields)
+        text = text.replace(nodes_match.group(0), "")
+
+    for line in text.splitlines():
+        if m := _STRING_RE.match(line):
+            out[m.group(1)] = m.group(2)
+        elif m := _NUMBER_RE.match(line):
+            out[m.group(1)] = int(m.group(2))
+
+    return out
+
+
+def _resolve_ssh_key(value: str | None) -> str:
+    if not value:
+        return os.path.expanduser("~/.ssh/tf-user-ap-southeast-2")
+    return os.path.expanduser(value)
+
+
+def _build_inventory(env: str) -> dict:
+    data = _parse_tfvars(_tfvars_path(env))
+    nodes = data.get("nodes") or []
+    if not nodes:
+        raise SystemExit(f"{env}.tfvars has no nodes")
+
+    ssh_user = data.get("ssh_user", "tf-user")
+    ssh_key = _resolve_ssh_key(data.get("ssh_public_key_path", "").replace(".pub", ""))
+
+    hostvars: dict[str, dict] = {}
+    all_hosts: list[str] = []
+    init_hosts: list[str] = []
+    join_hosts: list[str] = []
+
+    for idx, node in enumerate(nodes):
+        ip = node["wan_ip"]
+        role = "init" if idx == 0 else "join"
+        hostvars[ip] = {
+            "ansible_host": ip,
+            "ansible_user": ssh_user,
+            "ansible_ssh_private_key_file": ssh_key,
+            "ansible_ssh_common_args": "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR",
+            "spinifex_node_name": node["name"],
+            "spinifex_node_role": role,
+            "spinifex_node_index": idx,
+            "spinifex_wan_ip": ip,
+        }
+        all_hosts.append(ip)
+        (init_hosts if role == "init" else join_hosts).append(ip)
+
+    # Group-level vars derived from tfvars (operator can override with -e).
+    cluster_init_host = init_hosts[0]
+    group_vars = {
+        "cluster_env": env,
+        "cluster_init_host": cluster_init_host,
+        "cluster_init_ip": cluster_init_host,
+        "spinifex_node_count": len(nodes),
+        "spinifex_region": data.get("region", "ap-southeast-2"),
+        "spinifex_az": data.get("az", "ap-southeast-2a"),
+    }
+    if cidr := data.get("env_cidr"):
+        group_vars["cluster_env_cidr"] = cidr
+
+    inventory = {
+        "_meta": {"hostvars": hostvars},
+        env: {"hosts": all_hosts, "vars": group_vars},
+        "cluster": {"children": [env]},
+        "init": {"hosts": init_hosts},
+        "join": {"hosts": join_hosts},
+    }
+    return inventory
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--list", action="store_true", help="emit full inventory")
+    parser.add_argument("--host", help="emit hostvars for a single host (ignored — _meta carries them)")
+    parser.add_argument(
+        "--env",
+        default=os.environ.get("CLUSTER_ENV"),
+        help="tofu env name (matches scripts/tofu-cluster/envs/<env>.tfvars); also via CLUSTER_ENV env var",
+    )
+    args = parser.parse_args()
+
+    if not args.env:
+        sys.stderr.write("error: --env required (or CLUSTER_ENV)\n")
+        return 64
+
+    if args.host:
+        json.dump({}, sys.stdout)
+        sys.stdout.write("\n")
+        return 0
+
+    inventory = _build_inventory(args.env)
+    json.dump(inventory, sys.stdout, indent=2)
+    sys.stdout.write("\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/ansible/playbooks/cluster-bootstrap.yml

@@ -0,0 +1,330 @@
+---
+# cluster-bootstrap.yml — multi-node Spinifex cluster bootstrap against a
+# tofu-provisioned env. Inventory comes from inventory/tofu.py which parses
+# scripts/tofu-cluster/envs/<env>.tfvars.
+#
+# Usage:
+#   cd spinifex/scripts/ansible
+#   ansible-playbook -i inventory/tofu.py playbooks/cluster-bootstrap.yml \
+#     -e cluster_env=env1 -e cluster_external_pool=192.168.1.150-192.168.1.159
+#
+# Or via the make wrapper:
+#   make ansible-cluster-bootstrap ENV=env1 POOL=192.168.1.150-192.168.1.159
+#
+# Behaviour:
+#   - all hosts: preflight + install (setup.sh) — production layout
+#   - init host: spx admin init (backgrounded) + CA install + spinifex.target
+#   - join hosts: spx admin join + CA install + spinifex.target (parallel)
+#   - init host: import ssh key + ubuntu AMI (idempotent)
+
+- name: Cluster bootstrap — build artifact once on runner
+  hosts: localhost
+  gather_facts: true
+  vars_files:
+    - ../vars/defaults.yml
+  tasks:
+    - name: assert cluster_env supplied
+      ansible.builtin.assert:
+        that: cluster_env | default('') | length > 0
+        fail_msg: "Pass -e cluster_env=<env> (matches scripts/tofu-cluster/envs/<env>.tfvars)"
+  roles:
+    - build
+
+- name: Cluster bootstrap — install (all nodes, parallel)
+  hosts: "{{ cluster_env }}"
+  gather_facts: true
+  serial: 100%
+  vars:
+    install_run_ovn: false   # we drive setup-ovn ourselves with role-aware args
+    cluster_bootstrap_purge: true   # wipe stale single-node predastore.toml + raft data
+  vars_files:
+    - ../vars/defaults.yml
+  pre_tasks:
+    - name: copy build tarball to each node
+      ansible.builtin.copy:
+        src: "{{ build_tarball_path }}"
+        dest: "{{ build_tarball_path }}"
+        mode: "0644"
+
+    - name: stop dev-layout services if running (migration from ~/spinifex/)
+      become: true
+      # Bracket trick on the first letter prevents the pattern literal in the
+      # shell command line from matching the running shell itself (pkill -f
+      # against literal patterns is otherwise self-killing — rc=-9).
+      # `spx [a]dmin` catches lingering formation servers from prior bootstrap
+      # attempts that own port 4432 and steal joins from the new init.
+      ansible.builtin.shell:
+        cmd: |
+          pkill -9 -f "[b]in/spx service" 2>/dev/null || true
+          pkill -9 -f "spx [a]dmin" 2>/dev/null || true
+          pkill -9 -f "spinifex-[u]i" 2>/dev/null || true
+          pkill -9 -f "[n]ats-server" 2>/dev/null || true
+          pkill -9 -f "[p]redastore" 2>/dev/null || true
+          pkill -9 -f "[v]iperblock" 2>/dev/null || true
+          sleep 2
+          rm -rf "{{ ansible_env.HOME }}/spinifex" 2>/dev/null || true
+          exit 0
+      changed_when: false
+  roles:
+    - install
+
+# Wipe OVN/OVS state before setup-ovn runs. Without this, re-bootstrapping a
+# previously-built cluster leaves stale chassis rows in the SB DB, stale
+# external_ids on Open_vSwitch, stale br-int flows + ports, stale
+# systemd-networkd unit files, and stale ovs-monitor-ipsec strongSwan certs.
+# Symptom: ovn-controller flows reference dead encap endpoints; IPsec SAs
+# fail AUTHENTICATION_FAILED; new joins shadow ghosts under the same
+# node-name.
+- name: Cluster bootstrap — OVN/OVS state reset (all nodes, parallel)
+  hosts: "{{ cluster_env }}"
+  gather_facts: false
+  serial: 100%
+  vars_files:
+    - ../vars/defaults.yml
+  tasks:
+    - name: reset — stop OVN/OVS services
+      become: true
+      ansible.builtin.systemd:
+        name: "{{ item }}"
+        state: stopped
+      loop:
+        - ovn-controller
+        - ovn-central
+        - openvswitch-ipsec
+        - openvswitch-switch
+      failed_when: false
+
+    - name: reset — delete OVN NB/SB databases
+      become: true
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - "{{ ovn_nb_db }}"
+        - "{{ ovn_sb_db }}"
+
+    - name: reset — delete OVS conf.db + system-id (force fresh ovsdb)
+      become: true
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - /etc/openvswitch/conf.db
+        - /etc/openvswitch/.conf.db.~lock~
+        - /etc/openvswitch/system-id.conf
+
+    - name: reset — drop spinifex-managed systemd-networkd unit files
+      become: true
+      ansible.builtin.shell: |
+        rm -f /etc/systemd/network/14-spinifex-*.netdev \
+              /etc/systemd/network/15-spinifex-*.netdev \
+              /etc/systemd/network/15-spinifex-*.network \
+              /etc/systemd/network/16-spinifex-*.network
+        networkctl reload 2>/dev/null || true
+      changed_when: true
+
+    - name: reset — enumerate stale spinifex veth/tap netdevs
+      ansible.builtin.shell: |
+        ip -o link show | awk -F': ' '{print $2}' | cut -d'@' -f1 | \
+          grep -E '^(veth-wan-br|veth-wan-ovs|spx-ext-|spx-tap-|spx-)' || true
+      register: reset_links
+      changed_when: false
+
+    - name: reset — delete stale netdevs
+      become: true
+      ansible.builtin.command: "ip link delete {{ item }}"
+      loop: "{{ reset_links.stdout_lines }}"
+      changed_when: true
+      failed_when: false
+
+    - name: reset — clear ovs-monitor-ipsec strongSwan state
+      become: true
+      ansible.builtin.shell: |
+        rm -rf /var/lib/openvswitch/pki 2>/dev/null || true
+        rm -f /etc/ipsec.d/certs/ovs-* /etc/ipsec.d/private/ovs-* 2>/dev/null || true
+        rm -f /etc/ipsec.conf.d/ovs-* 2>/dev/null || true
+      changed_when: true
+
+    - name: reset — start openvswitch-switch with fresh conf.db
+      become: true
+      ansible.builtin.systemd:
+        name: openvswitch-switch
+        state: started
+        daemon_reload: true
+
+- name: Cluster bootstrap — OVN central (init host only)
+  hosts: init
+  gather_facts: false
+  vars_files:
+    - ../vars/defaults.yml
+  tasks:
+    - name: setup-ovn --management on init host
+      become: true
+      ansible.builtin.command:
+        cmd: >-
+          /usr/local/share/spinifex/setup-ovn.sh
+          --management
+          --encap-ip={{ spinifex_wan_ip }}
+          --node-name={{ hostvars[inventory_hostname]['spinifex_node_name'] }}
+      changed_when: true
+
+- name: Cluster bootstrap — OVN chassis (join hosts, parallel)
+  hosts: join
+  gather_facts: false
+  vars_files:
+    - ../vars/defaults.yml
+  tasks:
+    - name: setup-ovn --ovn-remote on join hosts
+      become: true
+      ansible.builtin.command:
+        cmd: >-
+          /usr/local/share/spinifex/setup-ovn.sh
+          --ovn-remote=tcp:{{ cluster_init_host }}:6642
+          --encap-ip={{ spinifex_wan_ip }}
+          --node-name={{ hostvars[inventory_hostname]['spinifex_node_name'] }}
+      changed_when: true
+
+- name: Cluster bootstrap — init host (start formation server)
+  hosts: init
+  gather_facts: false
+  vars_files:
+    - ../vars/defaults.yml
+  roles:
+    - cluster-init
+
+- name: Cluster bootstrap — join hosts run spx admin join (parallel)
+  hosts: join
+  gather_facts: false
+  serial: 100%
+  vars_files:
+    - ../vars/defaults.yml
+  roles:
+    - cluster-join
+
+- name: Cluster bootstrap — wait for init to finish writing multi-node configs
+  hosts: init
+  gather_facts: false
+  vars_files:
+    - ../vars/defaults.yml
+  tasks:
+    - name: wait for spx admin init async job to complete
+      become: true
+      ansible.builtin.async_status:
+        jid: "{{ cluster_init_async.ansible_job_id }}"
+      register: cluster_init_done
+      until: cluster_init_done.finished
+      retries: 60
+      delay: 5
+
+    - name: dump init log on failure
+      ansible.builtin.command: tail -100 /tmp/spx-admin-init.log
+      register: cluster_init_log
+      changed_when: false
+      when: cluster_init_done.rc | default(1) != 0
+
+    - name: assert init exited cleanly
+      ansible.builtin.assert:
+        that: cluster_init_done.rc == 0
+        fail_msg: |
+          spx admin init failed (rc={{ cluster_init_done.rc }})
+          {{ cluster_init_log.stdout | default('(log not captured)') }}
+
+    - name: confirm multi-node predastore.toml written
+      become: true
+      ansible.builtin.shell: |
+        if grep -q '127\.0\.0\.1' {{ spinifex_etc_dir }}/predastore/predastore.toml 2>/dev/null; then
+          echo "ERROR: predastore.toml still references loopback — multi-node config not written"
+          exit 1
+        fi
+        grep -c '^\[\[nodes\]\]' {{ spinifex_etc_dir }}/predastore/predastore.toml 2>/dev/null || echo 0
+      register: predastore_node_count
+      changed_when: false
+
+- name: Cluster bootstrap — install CA on all nodes (before service start)
+  hosts: "{{ cluster_env }}"
+  gather_facts: false
+  vars_files:
+    - ../vars/defaults.yml
+  tasks:
+    - name: install CA into system trust
+      become: true
+      ansible.builtin.copy:
+        src: "{{ spinifex_ca_src }}"
+        dest: "{{ spinifex_ca_trust_path }}"
+        mode: "0644"
+        remote_src: true
+
+    - name: rebuild CA bundle
+      become: true
+      ansible.builtin.command: update-ca-certificates
+      changed_when: true
+
+    # spinifex-ui reverse-proxies localhost:9999 (awsgw) and localhost:8443
+    # (predastore) — hardcoded in spinifex/services/spinifexui/spinifexui.go.
+    # spx admin init binds those services to BindIP only, so the loopback
+    # connection fails (BadGateway in UI). Patch to wildcard bind; cert
+    # already includes 127.0.0.1 in IP SANs.
+    - name: patch awsgw bind to 0.0.0.0 (UI proxies via localhost)
+      become: true
+      ansible.builtin.replace:
+        path: "{{ spinifex_config_file }}"
+        regexp: '^(\s*host\s*=\s*")[^"]+(:9999")'
+        replace: '\g<1>0.0.0.0\g<2>'
+
+    - name: patch predastore bind to 0.0.0.0 (UI proxies via localhost)
+      become: true
+      ansible.builtin.replace:
+        path: "{{ spinifex_config_file }}"
+        regexp: '^(\s*host\s*=\s*")[^"]+(:8443")'
+        replace: '\g<1>0.0.0.0\g<2>'
+
+- name: Cluster bootstrap — start services (init first, then joins)
+  hosts: init
+  gather_facts: false
+  vars_files:
+    - ../vars/defaults.yml
+  tasks:
+    - name: start spinifex.target on init
+      become: true
+      ansible.builtin.systemd:
+        name: spinifex.target
+        state: started
+
+- name: Cluster bootstrap — start services on join hosts
+  hosts: join
+  gather_facts: false
+  serial: 100%
+  vars_files:
+    - ../vars/defaults.yml
+  tasks:
+    - name: start spinifex.target on joiners
+      become: true
+      ansible.builtin.systemd:
+        name: spinifex.target
+        state: started
+
+- name: Cluster bootstrap — readiness check (all nodes)
+  hosts: "{{ cluster_env }}"
+  gather_facts: false
+  vars_files:
+    - ../vars/defaults.yml
+  tasks:
+    - name: wait for awsgw responsiveness
+      ansible.builtin.uri:
+        url: "https://{{ spinifex_wan_ip }}:{{ cluster_formation_port }}/health"
+        validate_certs: false
+        return_content: true
+      register: cluster_health
+      until:
+        - cluster_health.status == 200
+        - cluster_health.json.service_health.awsgw == "ok"
+      retries: 30
+      delay: 5
+
+- name: Cluster bootstrap — post-bootstrap fixtures (init host only)
+  hosts: init
+  gather_facts: true
+  vars_files:
+    - ../vars/defaults.yml
+  roles:
+    - cluster-post

scripts/ansible/playbooks/dev-logs.yml

@@ -0,0 +1,17 @@
+---
+# Capture spinifex + ovn/ovs + qemu/nbdkit logs from journald and
+# /var/log/spinifex/ into a single tarball for bug reports.
+# Tail mode is intentionally not provided — use `journalctl -u 'spinifex-*' -f`
+# directly; ansible is the wrong tool for follow.
+- name: Dev logs — dump journald + varlog + status into a bundle
+  hosts: localhost
+  gather_facts: true
+  vars_files:
+    - ../vars/defaults.yml
+  vars:
+    logs_since: "1h"
+    logs_output_dir: "/tmp"
+    logs_include_dmesg: true
+
+  roles:
+    - logs

scripts/ansible/playbooks/dev-reset.yml

@@ -23,6 +23,7 @@
       when: reset_config_stat.stat.exists
 
     # List-concat-and-take-first pattern: missing regex match falls back to current default.
+    # `gateway\s*=` deliberately does not match `gateway_ip` / `gateway_lrp_*`.
     - name: capture — extract scalars
       ansible.builtin.set_fact:
         spinifex_region: "{{ (reset_config_txt | regex_findall('region\\s*=\\s*\"([^\"]+)\"') + [spinifex_region])[0] }}"
@@ -30,11 +31,25 @@
         spinifex_external_mode: "{{ (reset_config_txt | regex_findall('external_mode\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_mode])[0] }}"
         spinifex_external_pool_start: "{{ (reset_config_txt | regex_findall('range_start\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_pool_start])[0] }}"
         spinifex_external_pool_end: "{{ (reset_config_txt | regex_findall('range_end\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_pool_end])[0] }}"
-        spinifex_external_gateway: "{{ (reset_config_txt | regex_findall('gateway\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_gateway])[0] }}"
+        spinifex_external_gateway: "{{ (reset_config_txt | regex_findall('\\bgateway\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_gateway])[0] }}"
         spinifex_external_prefix_len: "{{ (reset_config_txt | regex_findall('prefix_len\\s*=\\s*([0-9]+)') + [spinifex_external_prefix_len | string])[0] }}"
-        spinifex_nat_gateway_ip: "{{ (reset_config_txt | regex_findall('gateway_ip\\s*=\\s*\"([^\"]+)\"') + [spinifex_nat_gateway_ip])[0] }}"
+        spinifex_external_gateway_ip: "{{ (reset_config_txt | regex_findall('gateway_ip\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_gateway_ip])[0] }}"
       when: reset_config_stat.stat.exists
 
+    # Legacy: external_mode = "nat" was removed from spx. Coerce to "pool"
+    # so old configs still reset cleanly; warn so the operator notices.
+    - name: capture — coerce legacy external_mode=nat to pool
+      ansible.builtin.set_fact:
+        spinifex_external_mode: "pool"
+      when: spinifex_external_mode == 'nat'
+
+    - name: capture — warn on legacy nat coercion
+      ansible.builtin.debug:
+        msg: "⚠️  Captured external_mode=nat from legacy config; coerced to 'pool' (NAT mode removed from spx admin init)."
+      when:
+        - reset_config_stat.stat.exists
+        - (reset_config_txt | regex_findall('external_mode\\s*=\\s*\"nat\"')) | length > 0
+
     - name: capture — summary
       ansible.builtin.debug:
         msg: |
@@ -44,7 +59,7 @@
             external_mode={{ spinifex_external_mode }}
             pool={{ spinifex_external_pool_start }}-{{ spinifex_external_pool_end }}
             gateway={{ spinifex_external_gateway }}, prefix={{ spinifex_external_prefix_len }}
-            nat_gateway={{ spinifex_nat_gateway_ip }}
+            gateway_ip={{ spinifex_external_gateway_ip }}
           {% else %}
           No existing config; using defaults from vars/defaults.yml (or -e overrides).
           {% endif %}

scripts/ansible/playbooks/dev-restore.yml

@@ -0,0 +1,14 @@
+---
+# Restore a dev-snapshot bundle: stop services, wipe the snapshot path
+# set, untar the bundle, refresh CA trust, restart.
+#
+# Required: -e snapshot_name=<name>
+# Optional: -e snapshot_dir=/some/path
+- name: Dev restore — wipe + untar snapshot + restart
+  hosts: localhost
+  gather_facts: true
+  vars_files:
+    - ../vars/defaults.yml
+
+  roles:
+    - restore

scripts/ansible/playbooks/dev-snapshot.yml

@@ -0,0 +1,17 @@
+---
+# Quiesce spinifex + ovn + ovs, tar /etc/spinifex + /var/lib/spinifex +
+# /var/lib/ovn + /etc/openvswitch/conf.db into a named bundle, restart.
+# Use to checkpoint a working state before a risky operation (branch
+# switch, viperblock surgery, ovn-controller restart cycle). Roll back
+# via dev-restore without a full dev-reset.
+#
+# Required: -e snapshot_name=<name>
+# Optional: -e snapshot_dir=/some/path
+- name: Dev snapshot — quiesce + tar state + restart
+  hosts: localhost
+  gather_facts: true
+  vars_files:
+    - ../vars/defaults.yml
+
+  roles:
+    - snapshot

scripts/ansible/playbooks/dev-status.yml

@@ -0,0 +1,13 @@
+---
+# Read-only health snapshot for a Spinifex dev node. Never mutates state.
+# Run any time the operator wants to know "is my box healthy?" — pairs with
+# dev-preflight (run before install) as the post-install / running-system
+# diagnostic.
+- name: Dev status — health snapshot (read-only)
+  hosts: localhost
+  gather_facts: true
+  vars_files:
+    - ../vars/defaults.yml
+
+  roles:
+    - status

scripts/ansible/playbooks/dev-version.yml

@@ -0,0 +1,11 @@
+---
+# Detailed build-vs-install drift report. Faster than dev-status (no AWS,
+# OVN, or port probing). Answers "do I need to ansible-dev-deploy?".
+- name: Dev version — build vs install drift (read-only)
+  hosts: localhost
+  gather_facts: false
+  vars_files:
+    - ../vars/defaults.yml
+
+  roles:
+    - version

scripts/ansible/playbooks/dev-vpc.yml

@@ -0,0 +1,14 @@
+---
+# Idempotent dev-mode VPC fixture: spinifex-dev-vpc + public subnet +
+# IGW + default route. Hooks into smoketest's launch path (which finds
+# the public subnet automatically). Re-runs are no-ops once converged.
+#
+# Default state: present. Tear down with -e dev_vpc_state=absent.
+- name: Dev VPC — fixture VPC/subnet/IGW/route (idempotent)
+  hosts: localhost
+  gather_facts: false
+  vars_files:
+    - ../vars/defaults.yml
+
+  roles:
+    - vpc

scripts/ansible/roles/cluster-init/meta/main.yml

@@ -0,0 +1,9 @@
+---
+galaxy_info:
+  role_name: cluster-init
+  description: |
+    Run `spx admin init` on the cluster's init host, backgrounded so the
+    formation server accepts joins from the cluster-join role. Installs
+    cluster CA into the system trust store BEFORE starting spinifex.target
+    so predastore's strict TLS verify is satisfied at first start.
+dependencies: []

scripts/ansible/roles/cluster-init/tasks/main.yml

@@ -0,0 +1,77 @@
+---
+- name: cluster-init — assert this is the init host
+  ansible.builtin.assert:
+    that: spinifex_node_role == "init"
+    fail_msg: "cluster-init role must only run against hosts in the [init] group"
+
+# vars/defaults.yml sets spinifex_node_{name,count} for single-node dev use
+# (precedence 11). Inventory hostvars (precedence 4) lose that fight, so
+# we promote the inventory values via set_fact (precedence 15) before any
+# task templates `{{ spinifex_node_count }}`.
+- name: cluster-init — promote inventory node identity over defaults.yml
+  ansible.builtin.set_fact:
+    spinifex_node_name: "{{ hostvars[inventory_hostname]['spinifex_node_name'] }}"
+    spinifex_node_count: "{{ groups[cluster_env] | length }}"
+
+- name: cluster-init — assemble external network flags
+  ansible.builtin.set_fact:
+    cluster_external_flags: >-
+      {% if cluster_external_pool | length > 0 %}--external-mode=pool --external-pool={{ cluster_external_pool }}{% endif %}
+
+- name: cluster-init — run spx admin init (backgrounded, formation server waits for joins)
+  become: true
+  ansible.builtin.shell: |
+    set -o pipefail
+    spx admin init --force \
+      --node {{ spinifex_node_name }} \
+      --nodes {{ spinifex_node_count }} \
+      --bind {{ spinifex_wan_ip }} \
+      --cluster-bind {{ spinifex_wan_ip }} \
+      --port {{ cluster_formation_port }} \
+      --region {{ spinifex_region }} \
+      --az {{ spinifex_az }} \
+      {{ cluster_external_flags }} \
+      > /tmp/spx-admin-init.log 2>&1
+  async: 600
+  poll: 0
+  register: cluster_init_async
+
+- name: cluster-init — brief pause so init can fail fast if misconfigured
+  ansible.builtin.pause:
+    seconds: 5
+
+- name: cluster-init — check if init exited prematurely (sanity)
+  ansible.builtin.shell: |
+    if pgrep -f 'spx admin init' >/dev/null; then
+      echo "running"
+    else
+      tail -50 /tmp/spx-admin-init.log 2>/dev/null
+      exit 1
+    fi
+  register: cluster_init_alive
+  changed_when: false
+  failed_when: cluster_init_alive.rc != 0
+
+- name: cluster-init — wait for formation server health
+  ansible.builtin.uri:
+    url: "https://{{ spinifex_wan_ip }}:{{ cluster_formation_port }}/formation/health"
+    validate_certs: false
+    status_code: 200
+  register: cluster_init_health
+  until: cluster_init_health.status == 200
+  retries: "{{ cluster_formation_timeout // 2 }}"
+  delay: 2
+
+- name: cluster-init — fetch join token (delegated read for joiners)
+  become: true
+  ansible.builtin.slurp:
+    src: "{{ cluster_join_token_path }}"
+  register: cluster_join_token_raw
+
+- name: cluster-init — register join token as host fact
+  ansible.builtin.set_fact:
+    cluster_join_token: "{{ cluster_join_token_raw.content | b64decode | trim }}"
+
+- name: cluster-init — join token captured; waiting for joins handled by later play
+  ansible.builtin.debug:
+    msg: "Formation server up on {{ spinifex_wan_ip }}:{{ cluster_formation_port }}, {{ spinifex_node_count }} nodes expected"

scripts/ansible/roles/cluster-join/meta/main.yml

@@ -0,0 +1,8 @@
+---
+galaxy_info:
+  role_name: cluster-join
+  description: |
+    Run `spx admin join` against the cluster init host's formation server.
+    Installs cluster CA + starts spinifex.target after join completes
+    (CA-before-services ordering avoids predastore TLS-verify crashes).
+dependencies: []

scripts/ansible/roles/cluster-join/tasks/main.yml

@@ -0,0 +1,39 @@
+---
+- name: cluster-join — assert this is a join host
+  ansible.builtin.assert:
+    that: spinifex_node_role == "join"
+    fail_msg: "cluster-join role must only run against hosts in the [join] group"
+
+# Promote inventory hostvars over vars/defaults.yml (see cluster-init).
+- name: cluster-join — promote inventory node identity over defaults.yml
+  ansible.builtin.set_fact:
+    spinifex_node_name: "{{ hostvars[inventory_hostname]['spinifex_node_name'] }}"
+
+- name: cluster-join — pull join token from init host
+  ansible.builtin.set_fact:
+    cluster_join_token: "{{ hostvars[cluster_init_host]['cluster_join_token'] }}"
+
+- name: cluster-join — assert token present
+  ansible.builtin.assert:
+    that: cluster_join_token | length > 0
+    fail_msg: "join token missing — init host did not export cluster_join_token"
+
+- name: cluster-join — run spx admin join (retry up to 5 times)
+  become: true
+  ansible.builtin.command:
+    cmd: >-
+      spx admin join
+      --node {{ spinifex_node_name }}
+      --bind {{ spinifex_wan_ip }}
+      --cluster-bind {{ spinifex_wan_ip }}
+      --host {{ cluster_init_host }}:{{ cluster_formation_port }}
+      --token {{ cluster_join_token }}
+      --region {{ spinifex_region }}
+      --az {{ spinifex_az }}
+  register: cluster_join_result
+  retries: 5
+  delay: 3
+  until: cluster_join_result.rc == 0
+
+# CA install + service start handled by cluster-bootstrap.yml plays that
+# run after all joins complete and init has written multi-node configs.

scripts/ansible/roles/cluster-post/meta/main.yml

@@ -0,0 +1,8 @@
+---
+galaxy_info:
+  role_name: cluster-post
+  description: |
+    Post-bootstrap idempotent fixtures on the init host: import SSH key,
+    import Ubuntu AMI (dual-version detect 24.04/26.04). Skip when
+    artifacts already exist.
+dependencies: []

scripts/ansible/roles/cluster-post/tasks/main.yml

@@ -0,0 +1,205 @@
+---
+- name: cluster-post — locate ssh public key for import
+  ansible.builtin.find:
+    paths: "{{ ansible_env.HOME }}/.ssh"
+    patterns:
+      - spinifex-key.pub
+      - tf-user-*.pub
+    file_type: file
+  register: cluster_post_ssh_pubkey
+
+- name: cluster-post — fail when no public key found
+  ansible.builtin.assert:
+    that: cluster_post_ssh_pubkey.files | length > 0
+    fail_msg: "no public key in ~/.ssh — needed for spinifex-key import"
+
+- name: cluster-post — set pubkey path fact
+  ansible.builtin.set_fact:
+    cluster_post_pubkey_path: "{{ cluster_post_ssh_pubkey.files[0].path }}"
+
+- name: cluster-post — check whether spinifex-key already imported
+  ansible.builtin.command:
+    cmd: "{{ spinifex_aws_cli }} ec2 describe-key-pairs --key-names spinifex-key"
+  register: cluster_post_key_check
+  changed_when: false
+  failed_when: false
+
+- name: cluster-post — import spinifex-key (if absent)
+  ansible.builtin.command:
+    cmd: >-
+      {{ spinifex_aws_cli }} ec2 import-key-pair
+      --key-name spinifex-key
+      --public-key-material fileb://{{ cluster_post_pubkey_path }}
+  register: cluster_post_key_import
+  retries: 10
+  delay: 6
+  until: cluster_post_key_import.rc == 0
+  when: cluster_post_key_check.rc != 0
+
+- name: cluster-post — locate ubuntu cloud image (26.04 preferred, 24.04 fallback)
+  ansible.builtin.stat:
+    path: "{{ item }}"
+  register: cluster_post_ubuntu_stat
+  loop:
+    - "{{ ansible_env.HOME }}/images/ubuntu-26.04.img"
+    - "{{ ansible_env.HOME }}/images/ubuntu-24.04.img"
+
+- name: cluster-post — pick first available image
+  ansible.builtin.set_fact:
+    cluster_post_ubuntu_image: "{{ (cluster_post_ubuntu_stat.results | selectattr('stat.exists') | map(attribute='item') | list | first) | default('') }}"
+
+- name: cluster-post — fail when no ubuntu image present
+  ansible.builtin.assert:
+    that: cluster_post_ubuntu_image | length > 0
+    fail_msg: "no ubuntu-{24,26}.04.img in ~/images/ — gold image must stage it"
+
+- name: cluster-post — derive ubuntu version from image filename
+  ansible.builtin.set_fact:
+    cluster_post_ubuntu_version: "{{ cluster_post_ubuntu_image | regex_search('ubuntu-([0-9]+\\.[0-9]+)\\.img', '\\1') | first }}"
+
+- name: cluster-post — check whether ubuntu AMI already imported
+  ansible.builtin.command:
+    cmd: >-
+      {{ spinifex_aws_cli }} ec2 describe-images
+      --owners self
+      --filters Name=name,Values=ubuntu-{{ cluster_post_ubuntu_version }}
+  register: cluster_post_ami_check
+  changed_when: false
+  failed_when: false
+
+- name: cluster-post — import ubuntu AMI (if absent)
+  ansible.builtin.command:
+    cmd: >-
+      {{ spinifex_spx_cli }} admin images import
+      --file {{ cluster_post_ubuntu_image }}
+      --arch x86_64
+      --distro ubuntu
+      --version {{ cluster_post_ubuntu_version }}
+      --boot-mode uefi
+  when:
+    - cluster_post_ami_check.rc != 0
+       or "ubuntu-" + cluster_post_ubuntu_version not in cluster_post_ami_check.stdout
+
+- name: cluster-post — slurp final admin creds from ~/.aws/credentials
+  # Source of truth: SetupAWSCredentials writes the FINAL admin keys here.
+  # Init log shows obsolete keys (multi-node path silently regenerates them
+  # after the initial print — see cmd/spinifex/cmd/admin.go writeBootstrapFiles
+  # called twice in the multi-node flow).
+  ansible.builtin.slurp:
+    src: "{{ ansible_env.HOME }}/.aws/credentials"
+  register: cluster_post_aws_creds_raw
+
+- name: cluster-post — extract admin access/secret keys from spinifex profile
+  ansible.builtin.set_fact:
+    cluster_post_admin_access_key: >-
+      {{ (cluster_post_aws_creds_raw.content | b64decode
+          | regex_search('\[spinifex\][^\[]*aws_access_key_id\s*=\s*(\S+)', '\1')
+          | first) | default('') }}
+    cluster_post_admin_secret_key: >-
+      {{ (cluster_post_aws_creds_raw.content | b64decode
+          | regex_search('\[spinifex\][^\[]*aws_secret_access_key\s*=\s*(\S+)', '\1')
+          | first) | default('') }}
+
+- name: cluster-post — assert creds parsed
+  ansible.builtin.assert:
+    that:
+      - cluster_post_admin_access_key | length > 0
+      - cluster_post_admin_secret_key | length > 0
+    fail_msg: "could not parse admin keys from {{ ansible_env.HOME }}/.aws/credentials [spinifex] profile"
+
+- name: cluster-post — ensure local .creds dir exists
+  delegate_to: localhost
+  become: false
+  ansible.builtin.file:
+    path: "{{ playbook_dir }}/../.creds"
+    state: directory
+    mode: "0700"
+
+- name: cluster-post — write creds to operator workstation
+  delegate_to: localhost
+  become: false
+  ansible.builtin.copy:
+    dest: "{{ playbook_dir }}/../.creds/{{ cluster_env }}.txt"
+    mode: "0600"
+    content: |
+      # Spinifex cluster admin credentials — {{ cluster_env }}
+      # Generated: {{ ansible_facts['date_time']['iso8601'] }}
+      # Init host: {{ cluster_init_host }}
+      #
+      # UI:        https://{{ cluster_init_host }}:3000
+      # AWS API:   https://{{ cluster_init_host }}:4432
+      # Region:    {{ spinifex_region }}
+      #
+      # Login (admin user / AWS profile spinifex):
+      AWS_ACCESS_KEY_ID={{ cluster_post_admin_access_key }}
+      AWS_SECRET_ACCESS_KEY={{ cluster_post_admin_secret_key }}
+
+# Per-env AWS profile in operator's ~/.aws/credentials so tofu / aws CLI can
+# target this cluster via AWS_PROFILE=spinifex-{{ cluster_env }}. blockinfile
+# marker keeps re-runs idempotent — replaces the block in place, never
+# duplicates. Does not touch the user's existing [spinifex] profile (single-
+# node dev). Tofu side: provider "aws" { profile = "spinifex-env1" ... }.
+- name: cluster-post — ensure local ~/.aws dir exists
+  delegate_to: localhost
+  become: false
+  ansible.builtin.file:
+    path: "{{ lookup('env', 'HOME') }}/.aws"
+    state: directory
+    mode: "0700"
+
+- name: cluster-post — ensure local ~/.aws/credentials exists
+  delegate_to: localhost
+  become: false
+  ansible.builtin.copy:
+    dest: "{{ lookup('env', 'HOME') }}/.aws/credentials"
+    content: ""
+    force: false
+    mode: "0600"
+
+- name: cluster-post — upsert spinifex-{{ cluster_env }} profile in local ~/.aws/credentials
+  delegate_to: localhost
+  become: false
+  ansible.builtin.blockinfile:
+    path: "{{ lookup('env', 'HOME') }}/.aws/credentials"
+    marker: "# {mark} SPINIFEX-{{ cluster_env | upper }} MANAGED BLOCK"
+    block: |
+      [spinifex-{{ cluster_env }}]
+      aws_access_key_id = {{ cluster_post_admin_access_key }}
+      aws_secret_access_key = {{ cluster_post_admin_secret_key }}
+      region = {{ spinifex_region }}
+      ca_bundle = {{ playbook_dir }}/../.creds/{{ cluster_env }}-ca.pem
+      endpoint_url = https://{{ cluster_init_host }}:4432
+
+- name: cluster-post — slurp cluster CA from init host
+  ansible.builtin.slurp:
+    src: "{{ spinifex_ca_src }}"
+  register: cluster_post_ca_raw
+  become: true
+
+- name: cluster-post — write CA pem to operator workstation
+  delegate_to: localhost
+  become: false
+  ansible.builtin.copy:
+    dest: "{{ playbook_dir }}/../.creds/{{ cluster_env }}-ca.pem"
+    mode: "0644"
+    content: "{{ cluster_post_ca_raw.content | b64decode }}"
+
+- name: cluster-post — resolve absolute paths for summary
+  delegate_to: localhost
+  become: false
+  ansible.builtin.command: "readlink -f {{ playbook_dir }}/../.creds"
+  register: cluster_post_creds_abs
+  changed_when: false
+
+- name: cluster-post — summary
+  ansible.builtin.debug:
+    msg: |
+      Post-bootstrap done:
+        spinifex-key:    {{ 'imported now' if (cluster_post_key_import.changed | default(false)) else 'already present' }}
+        ubuntu AMI:      ubuntu-{{ cluster_post_ubuntu_version }}
+        admin creds:     {{ cluster_post_creds_abs.stdout }}/{{ cluster_env }}.txt
+        CA cert:         {{ cluster_post_creds_abs.stdout }}/{{ cluster_env }}-ca.pem
+        AWS profile:     spinifex-{{ cluster_env }} (in ~/.aws/credentials)
+        UI URL:          https://{{ cluster_init_host }}:3000
+        Access Key:      {{ cluster_post_admin_access_key }}
+        Tofu usage:      AWS_PROFILE=spinifex-{{ cluster_env }} tofu plan

scripts/ansible/roles/deploy/meta/main.yml

@@ -0,0 +1,7 @@
+---
+galaxy_info:
+  role_name: deploy
+  description: In-place binary + microvm artifact swap for a running Spinifex node (parity with `make deploy`).
+  license: BSD-3-Clause
+  min_ansible_version: "2.14"
+dependencies: []

scripts/ansible/roles/deploy/tasks/main.yml

@@ -127,11 +127,7 @@
     state: started
 
 - name: deploy — wait for awsgw to respond
-  become: true
-  become_user: "{{ spinifex_invoking_user }}"
-  environment:
-    AWS_PROFILE: spinifex
-  ansible.builtin.command: aws ec2 describe-regions
+  ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions"
   register: deploy_awsgw
   retries: 15
   delay: 2

scripts/ansible/roles/init/tasks/main.yml

@@ -8,28 +8,38 @@
       --node {{ spinifex_node_name }}
       --nodes {{ spinifex_node_count }}
 
+- name: init — no-external opt-out
+  ansible.builtin.set_fact:
+    init_external_flags: "--no-external"
+  when: spinifex_external_no_external | bool
+
+- name: init — assert pool range supplied when mode=pool
+  ansible.builtin.assert:
+    that:
+      - spinifex_external_pool_start | length > 0
+      - spinifex_external_pool_end | length > 0
+    fail_msg: |
+      spx admin init requires --external-pool when --external-mode=pool.
+      Set spinifex_external_pool_start / spinifex_external_pool_end in
+      scripts/ansible/vars/defaults.yml, or pass:
+        -e spinifex_external_pool_start=192.168.1.150
+        -e spinifex_external_pool_end=192.168.1.250
+      Pass -e spinifex_external_no_external=true to skip external networking.
+  when:
+    - not (spinifex_external_no_external | bool)
+    - spinifex_external_mode == 'pool'
+
 - name: init — pool-mode external flags
   ansible.builtin.set_fact:
     init_external_flags: >-
       --external-mode=pool
       --external-pool={{ spinifex_external_pool_start }}-{{ spinifex_external_pool_end }}
-      --external-gateway={{ spinifex_external_gateway }}
-      --external-prefix-len={{ spinifex_external_prefix_len }}
+      {% if spinifex_external_gateway | length > 0 %}--external-gateway={{ spinifex_external_gateway }}{% endif %}
+      {% if spinifex_external_prefix_len | string | length > 0 %}--external-prefix-len={{ spinifex_external_prefix_len }}{% endif %}
+      {% if spinifex_external_gateway_ip | length > 0 %}--gateway-ip={{ spinifex_external_gateway_ip }}{% endif %}
   when:
+    - not (spinifex_external_no_external | bool)
     - spinifex_external_mode == 'pool'
-    - spinifex_external_pool_start | length > 0
-    - spinifex_external_pool_end | length > 0
-
-- name: init — NAT-mode external flags
-  ansible.builtin.set_fact:
-    init_external_flags: >-
-      --external-mode=nat
-      --gateway-ip={{ spinifex_nat_gateway_ip }}
-      --external-gateway={{ spinifex_external_gateway }}
-      --external-prefix-len={{ spinifex_external_prefix_len }}
-  when:
-    - spinifex_external_mode == 'nat'
-    - spinifex_nat_gateway_ip | length > 0
 
 - name: init — assemble full flag string
   ansible.builtin.set_fact:
@@ -63,11 +73,7 @@
 # to NATS + daemons after that. Bash start-dev.sh has internal wait loops;
 # systemd path does not. Poll a benign call until gateway is fully ready.
 - name: init — wait for AWS gateway readiness
-  become: true
-  become_user: "{{ spinifex_invoking_user }}"
-  environment:
-    AWS_PROFILE: spinifex
-  ansible.builtin.command: aws ec2 describe-regions
+  ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions"
   register: init_gw_ready
   until: init_gw_ready.rc == 0
   retries: 30
@@ -79,6 +85,6 @@
     msg: |
       Init complete:
         region={{ spinifex_region }}, az={{ spinifex_az }}, node={{ spinifex_node_name }}
-        external_mode={{ spinifex_external_mode }}
+        external_mode={{ 'disabled' if (spinifex_external_no_external | bool) else spinifex_external_mode }}
         spinifex.target: started
         CA: {{ spinifex_ca_trust_path }}

scripts/ansible/roles/install/tasks/main.yml

@@ -40,6 +40,19 @@
     path: "{{ spinifex_data_dir }}/config"
     state: absent
 
+- name: install — purge stale cluster state (configs + raft + NATS KV) on bootstrap
+  become: true
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - "{{ spinifex_etc_dir }}/predastore/predastore.toml"
+    - "{{ spinifex_etc_dir }}/spinifex.toml"
+    - "{{ spinifex_data_dir }}/predastore"
+    - "{{ spinifex_data_dir }}/nats"
+    - "{{ spinifex_data_dir }}/awsgw"
+  when: cluster_bootstrap_purge | default(false) | bool
+
 - name: install — run setup.sh (creates users, dirs, systemd units)
   become: true
   ansible.builtin.command:
@@ -57,8 +70,9 @@
 - name: install — run setup-ovn.sh --management
   become: true
   ansible.builtin.command:
-    cmd: /usr/local/share/spinifex/setup-ovn.sh --management
+    cmd: "/usr/local/share/spinifex/setup-ovn.sh --management --node-name={{ spinifex_node_name }}"
   changed_when: true
+  when: install_run_ovn | default(true) | bool
 
 - name: install — summary
   ansible.builtin.debug:

scripts/ansible/roles/logs/meta/main.yml

@@ -0,0 +1,7 @@
+---
+galaxy_info:
+  role_name: logs
+  description: Dump spinifex + ovn + system logs into a tarball for bug reports.
+  license: BSD-3-Clause
+  min_ansible_version: "2.14"
+dependencies: []

scripts/ansible/roles/logs/tasks/main.yml

@@ -0,0 +1,153 @@
+---
+# Collect a self-contained log bundle for bug reports. NEVER mutates running
+# state — only reads journalctl + /var/log/spinifex/* + status output, then
+# tars into logs_output_dir. Errors on missing inputs are non-fatal so a
+# half-broken node still produces a bundle.
+
+- name: logs — resolve staging directory
+  ansible.builtin.set_fact:
+    logs_stage_dir: "/tmp/spinifex-logs-{{ ansible_facts['hostname'] }}-{{ ansible_facts['date_time']['iso8601_basic_short'] }}"
+    logs_bundle_path: "{{ logs_output_dir }}/spinifex-logs-{{ ansible_facts['hostname'] }}-{{ ansible_facts['date_time']['iso8601_basic_short'] }}.tar.gz"
+
+- name: logs — create staging tree
+  ansible.builtin.file:
+    path: "{{ logs_stage_dir }}/{{ item }}"
+    state: directory
+    mode: "0755"
+  loop:
+    - journal
+    - varlog
+    - status
+    - kernel
+
+- name: logs — capture journalctl for spinifex-* units
+  become: true
+  ansible.builtin.shell: |
+    journalctl --no-pager --output=short-iso \
+      -u 'spinifex-*' --since {{ logs_since | quote }} \
+      > {{ logs_stage_dir }}/journal/spinifex.log 2>&1 || true
+  changed_when: true
+
+- name: logs — capture journalctl for ovn-* + openvswitch-switch
+  become: true
+  ansible.builtin.shell: |
+    journalctl --no-pager --output=short-iso \
+      -u 'ovn-*' -u openvswitch-switch \
+      --since {{ logs_since | quote }} \
+      > {{ logs_stage_dir }}/journal/ovn-ovs.log 2>&1 || true
+  changed_when: true
+
+- name: logs — capture journalctl for nbdkit + qemu (best-effort match)
+  become: true
+  ansible.builtin.shell: |
+    journalctl --no-pager --output=short-iso \
+      --since {{ logs_since | quote }} \
+      _COMM=qemu-system-x86_64 + _COMM=qemu-system-aarch64 + _COMM=nbdkit \
+      > {{ logs_stage_dir }}/journal/qemu-nbdkit.log 2>&1 || true
+  changed_when: true
+
+- name: logs — copy /var/log/spinifex/*
+  become: true
+  ansible.builtin.shell: |
+    if [ -d {{ spinifex_log_dir | quote }} ]; then
+      cp -a {{ spinifex_log_dir | quote }}/. {{ logs_stage_dir | quote }}/varlog/ 2>/dev/null || true
+    fi
+  changed_when: true
+
+- name: logs — systemctl status snapshot (spinifex-*)
+  become: true
+  ansible.builtin.shell: |
+    systemctl status 'spinifex-*' --no-pager --all \
+      > {{ logs_stage_dir }}/status/spinifex-units.txt 2>&1 || true
+    systemctl status ovn-controller ovn-central openvswitch-switch \
+      --no-pager > {{ logs_stage_dir }}/status/ovn-ovs-units.txt 2>&1 || true
+    systemctl list-units 'spinifex-*' --all --no-legend --plain \
+      > {{ logs_stage_dir }}/status/spinifex-list.txt 2>&1 || true
+  changed_when: true
+
+- name: logs — networking snapshot (ip, ss, ovs)
+  become: true
+  ansible.builtin.shell: |
+    ip -d -s link show > {{ logs_stage_dir }}/status/ip-link.txt 2>&1 || true
+    ip addr show       > {{ logs_stage_dir }}/status/ip-addr.txt 2>&1 || true
+    ip route show table all > {{ logs_stage_dir }}/status/ip-route.txt 2>&1 || true
+    ss -tlnp           > {{ logs_stage_dir }}/status/ss-tlnp.txt 2>&1 || true
+    if command -v ovs-vsctl >/dev/null 2>&1; then
+      ovs-vsctl show     > {{ logs_stage_dir }}/status/ovs-show.txt 2>&1 || true
+      ovs-vsctl list-br  > {{ logs_stage_dir }}/status/ovs-br.txt 2>&1 || true
+    fi
+    if command -v ovn-nbctl >/dev/null 2>&1; then
+      ovn-nbctl show   > {{ logs_stage_dir }}/status/ovn-nb.txt 2>&1 || true
+      ovn-sbctl show   > {{ logs_stage_dir }}/status/ovn-sb.txt 2>&1 || true
+    fi
+  changed_when: true
+
+- name: logs — kernel ring buffer
+  become: true
+  ansible.builtin.shell: |
+    dmesg --time-format=iso > {{ logs_stage_dir }}/kernel/dmesg.txt 2>&1 || true
+  changed_when: true
+  when: logs_include_dmesg | bool
+
+- name: logs — capture spx + plugin versions
+  become: true
+  ansible.builtin.shell: |
+    {
+      echo '## spx version'
+      /usr/local/bin/spx version 2>&1 | grep -vE '^[0-9]{4}/' || true
+      echo
+      echo '## nbdkit plugin sha256'
+      pdir=$(nbdkit --dump-config 2>/dev/null | awk -F= '/^plugindir=/{print $2; exit}')
+      [ -n "$pdir" ] && sha256sum "$pdir/nbdkit-viperblock-plugin.so" 2>/dev/null || true
+      echo
+      echo '## microvm artifacts'
+      sha256sum /usr/share/spinifex/microvm/* 2>/dev/null || true
+      echo
+      echo '## kernel + os'
+      uname -a
+      cat /etc/os-release 2>/dev/null || true
+    } > {{ logs_stage_dir }}/status/versions.txt 2>&1 || true
+  changed_when: true
+
+- name: logs — create bundle tarball
+  become: true
+  ansible.builtin.shell: |
+    tar -C {{ logs_stage_dir | dirname | quote }} \
+      -czf {{ logs_bundle_path | quote }} \
+      {{ logs_stage_dir | basename | quote }}
+  changed_when: true
+
+- name: logs — make bundle owned by invoking user
+  become: true
+  ansible.builtin.file:
+    path: "{{ logs_bundle_path }}"
+    owner: "{{ spinifex_invoking_user }}"
+    group: "{{ spinifex_invoking_user }}"
+    mode: "0640"
+
+- name: logs — remove staging tree
+  become: true
+  ansible.builtin.file:
+    path: "{{ logs_stage_dir }}"
+    state: absent
+
+- name: logs — bundle size
+  ansible.builtin.stat:
+    path: "{{ logs_bundle_path }}"
+  register: logs_bundle_stat
+
+- name: logs — summary
+  ansible.builtin.debug:
+    msg: |
+      Log bundle written:
+        path  : {{ logs_bundle_path }}
+        size  : {{ (logs_bundle_stat.stat.size | int / 1024) | round(1) }} KiB
+        since : {{ logs_since }}
+      Contents:
+        journal/    spinifex-*, ovn-*+openvswitch-switch, qemu/nbdkit
+        varlog/     copy of {{ spinifex_log_dir }}
+        status/     systemctl status, ip/ss/ovs/ovn show, versions
+        kernel/     dmesg{{ '' if logs_include_dmesg else ' (skipped via -e logs_include_dmesg=false)' }}
+
+      Attach to bug report. For live tail use:
+        journalctl -u 'spinifex-*' -f

scripts/ansible/roles/restore/meta/main.yml

@@ -0,0 +1,7 @@
+---
+galaxy_info:
+  role_name: restore
+  description: Restore a spinifex+ovn+ovs snapshot bundle, refresh CA trust, restart.
+  license: BSD-3-Clause
+  min_ansible_version: "2.14"
+dependencies: []

scripts/ansible/roles/restore/tasks/main.yml

@@ -0,0 +1,140 @@
+---
+# Inverse of the snapshot role: stop services, wipe the same path set,
+# untar the named bundle, refresh CA trust (because /etc/spinifex/ca.pem
+# came from the snapshot but the system trust store didn't), restart.
+
+- name: restore — require snapshot_name
+  ansible.builtin.assert:
+    that:
+      - snapshot_name is defined
+      - snapshot_name | length > 0
+      - snapshot_name is match('^[A-Za-z0-9._-]+$')
+    fail_msg: "snapshot_name required (alphanumeric/._-): -e snapshot_name=<name>"
+
+- name: restore — resolve bundle path
+  ansible.builtin.set_fact:
+    snapshot_bundle_path: "{{ snapshot_dir }}/{{ snapshot_name }}.tar.gz"
+
+- name: restore — assert bundle exists
+  become: true
+  ansible.builtin.stat:
+    path: "{{ snapshot_bundle_path }}"
+  register: restore_bundle_stat
+
+- name: restore — fail if missing
+  ansible.builtin.assert:
+    that: restore_bundle_stat.stat.exists
+    fail_msg: |
+      Snapshot bundle missing: {{ snapshot_bundle_path }}.
+      Available bundles:
+        ls -lh {{ snapshot_dir }}/*.tar.gz 2>/dev/null
+
+- name: restore — stop services in dependency order
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: stopped
+  loop: "{{ snapshot_services_stop_order }}"
+  failed_when: false
+
+- name: restore — kill lingering QEMU VMs
+  become: true
+  ansible.builtin.command: "pkill -x {{ item }}"
+  loop: "{{ spinifex_qemu_processes }}"
+  changed_when: false
+  failed_when: false
+
+- name: restore — wait for QEMU exit
+  become: true
+  ansible.builtin.shell: |
+    set -e
+    for i in $(seq 1 {{ teardown_process_wait_seconds }}); do
+      alive=0
+      for proc in {{ spinifex_qemu_processes | join(' ') }}; do
+        if pgrep -x "$proc" >/dev/null 2>&1; then alive=1; break; fi
+      done
+      if [ "$alive" = "0" ]; then exit 0; fi
+      sleep 1
+    done
+    exit 1
+  changed_when: false
+
+- name: restore — wipe target paths (only those the snapshot will replace)
+  become: true
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop: "{{ snapshot_paths }}"
+
+- name: restore — recreate parent dirs that tar will refill
+  become: true
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  loop:
+    - "{{ ovn_db_dir }}"
+    - /etc/openvswitch
+
+- name: restore — extract bundle (preserve permissions/ACLs)
+  become: true
+  ansible.builtin.command:
+    cmd: >-
+      tar --acls --xattrs -xzpf {{ snapshot_bundle_path | quote }}
+      -C /
+  changed_when: true
+
+- name: restore — refresh spinifex CA in system trust
+  become: true
+  ansible.builtin.stat:
+    path: "{{ spinifex_ca_src }}"
+  register: restore_ca_stat
+
+- name: restore — copy CA back to trust path
+  become: true
+  ansible.builtin.copy:
+    src: "{{ spinifex_ca_src }}"
+    dest: "{{ spinifex_ca_trust_path }}"
+    mode: "0644"
+    remote_src: true
+  when: restore_ca_stat.stat.exists
+
+- name: restore — rebuild CA bundle
+  become: true
+  ansible.builtin.command: update-ca-certificates
+  changed_when: true
+  when: restore_ca_stat.stat.exists
+
+- name: restore — daemon-reload (covers any unit file delta in the snapshot)
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: restore — restart services in reverse order
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: started
+  loop: "{{ snapshot_services_stop_order | reverse | list }}"
+  failed_when: false
+
+- name: restore — wait for AWS gateway readiness
+  ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions"
+  register: restore_gw_ready
+  retries: 30
+  delay: 2
+  until: restore_gw_ready.rc == 0
+  changed_when: false
+  failed_when: false
+
+- name: restore — summary
+  ansible.builtin.debug:
+    msg: |
+      Restore complete:
+        snapshot : {{ snapshot_name }}
+        bundle   : {{ snapshot_bundle_path }}
+        paths    : {{ snapshot_paths | join(', ') }}
+        CA       : {{ 'refreshed' if restore_ca_stat.stat.exists else 'not in bundle — system trust unchanged' }}
+        gateway  : {{ 'ready' if restore_gw_ready.rc == 0 else 'NOT ready after restart — investigate' }}

scripts/ansible/roles/smoketest/tasks/main.yml

@@ -16,12 +16,8 @@
     creates: "{{ smoketest_user_home }}/.ssh/spinifex-key"
 
 - name: smoketest — import SSH key to spinifex
-  become: true
-  become_user: "{{ spinifex_invoking_user }}"
-  environment:
-    AWS_PROFILE: spinifex
   ansible.builtin.command: >-
-    aws ec2 import-key-pair
+    {{ spinifex_aws_cli }} ec2 import-key-pair
     --key-name spinifex-key
     --public-key-material fileb://{{ smoketest_user_home }}/.ssh/spinifex-key.pub
   register: smoketest_keypair
@@ -51,12 +47,8 @@
   register: smoketest_local_img
 
 - name: smoketest — import AMI from local file
-  become: true
-  become_user: "{{ spinifex_invoking_user }}"
-  environment:
-    AWS_PROFILE: spinifex
   ansible.builtin.command: >-
-    spx admin images import
+    {{ spinifex_spx_cli }} admin images import
     --file {{ smoketest_local_image_path }}
     --distro {{ spinifex_smoketest_distro }}
     --version {{ spinifex_smoketest_version }}
@@ -75,12 +67,8 @@
   when: smoketest_local_img.stat.exists
 
 - name: smoketest — import AMI by name (fallback)
-  become: true
-  become_user: "{{ spinifex_invoking_user }}"
-  environment:
-    AWS_PROFILE: spinifex
   ansible.builtin.command: >-
-    spx admin images import
+    {{ spinifex_spx_cli }} admin images import
     --name {{ spinifex_smoketest_distro }}-{{ spinifex_smoketest_version }}-{{ smoketest_arch }}
   register: smoketest_import_remote
   until: >-
@@ -105,11 +93,7 @@
     smoketest_instance_type: "{{ 't3a.small' if smoketest_amd.rc == 0 else 't3.small' }}"
 
 - name: smoketest — find AMI ID
-  become: true
-  become_user: "{{ spinifex_invoking_user }}"
-  environment:
-    AWS_PROFILE: spinifex
-  ansible.builtin.command: aws ec2 describe-images --query "Images[0].ImageId" --output text
+  ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-images --query Images[0].ImageId --output text"
   register: smoketest_ami
   until: smoketest_ami.rc == 0
   retries: 10
@@ -117,12 +101,8 @@
   changed_when: false
 
 - name: smoketest — find public subnet (best-effort)
-  become: true
-  become_user: "{{ spinifex_invoking_user }}"
-  environment:
-    AWS_PROFILE: spinifex
   ansible.builtin.command: >-
-    aws ec2 describe-subnets
+    {{ spinifex_aws_cli }} ec2 describe-subnets
     --filters Name=map-public-ip-on-launch,Values=true
     --query Subnets[0].SubnetId
     --output text
@@ -131,11 +111,7 @@
   failed_when: false
 
 - name: smoketest — fallback to first subnet (best-effort)
-  become: true
-  become_user: "{{ spinifex_invoking_user }}"
-  environment:
-    AWS_PROFILE: spinifex
-  ansible.builtin.command: aws ec2 describe-subnets --query "Subnets[0].SubnetId" --output text
+  ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-subnets --query Subnets[0].SubnetId --output text"
   register: smoketest_subnet_any
   changed_when: false
   failed_when: false
@@ -161,12 +137,8 @@
   when: smoketest_subnet_id | trim == ''
 
 - name: smoketest — launch instance
-  become: true
-  become_user: "{{ spinifex_invoking_user }}"
-  environment:
-    AWS_PROFILE: spinifex
   ansible.builtin.command: >-
-    aws ec2 run-instances
+    {{ spinifex_aws_cli }} ec2 run-instances
     --image-id {{ smoketest_ami.stdout }}
     --instance-type {{ smoketest_instance_type }}
     --key-name spinifex-key
@@ -182,12 +154,8 @@
   when: smoketest_subnet_id | trim != ''
 
 - name: smoketest — wait for instance running (proves launch path works)
-  become: true
-  become_user: "{{ spinifex_invoking_user }}"
-  environment:
-    AWS_PROFILE: spinifex
   ansible.builtin.command: >-
-    aws ec2 describe-instances
+    {{ spinifex_aws_cli }} ec2 describe-instances
     --instance-ids {{ smoketest_instance.stdout }}
     --query Reservations[0].Instances[0].State.Name
     --output text
@@ -203,12 +171,8 @@
     - smoketest_instance.stdout | trim != ''
 
 - name: smoketest — terminate instance (cleanup)
-  become: true
-  become_user: "{{ spinifex_invoking_user }}"
-  environment:
-    AWS_PROFILE: spinifex
   ansible.builtin.command: >-
-    aws ec2 terminate-instances
+    {{ spinifex_aws_cli }} ec2 terminate-instances
     --instance-ids {{ smoketest_instance.stdout }}
   register: smoketest_terminate
   until: smoketest_terminate.rc == 0

scripts/ansible/roles/snapshot/meta/main.yml

@@ -0,0 +1,7 @@
+---
+galaxy_info:
+  role_name: snapshot
+  description: Quiesce spinifex + ovn + ovs, tar state paths, restart.
+  license: BSD-3-Clause
+  min_ansible_version: "2.14"
+dependencies: []

scripts/ansible/roles/snapshot/tasks/main.yml

@@ -0,0 +1,154 @@
+---
+# Snapshot a quiesced spinifex node: stop spinifex.target → ovn → ovs in order,
+# tar /etc/spinifex + /var/lib/spinifex + /var/lib/ovn + /etc/openvswitch/conf.db,
+# restart in reverse. Lets devs checkpoint a working state before risky ops
+# (branch switch, viperblock surgery, ovn-controller restart cycle) and roll
+# back via dev-restore without a full ~10-min dev-reset.
+#
+# Caveat: /var/lib/spinifex contains viperblock volumes — snapshot size scales
+# with VM disks. Default gzip; switch to a faster compressor by hand if needed.
+
+- name: snapshot — require snapshot_name
+  ansible.builtin.assert:
+    that:
+      - snapshot_name is defined
+      - snapshot_name | length > 0
+      - snapshot_name is match('^[A-Za-z0-9._-]+$')
+    fail_msg: "snapshot_name required (alphanumeric/._-): -e snapshot_name=<name>"
+
+- name: snapshot — resolve bundle path
+  ansible.builtin.set_fact:
+    snapshot_bundle_path: "{{ snapshot_dir }}/{{ snapshot_name }}.tar.gz"
+
+- name: snapshot — refuse to overwrite existing bundle
+  ansible.builtin.stat:
+    path: "{{ snapshot_bundle_path }}"
+  register: snapshot_existing
+
+- name: snapshot — assert not already present
+  ansible.builtin.assert:
+    that: not snapshot_existing.stat.exists
+    fail_msg: |
+      Snapshot {{ snapshot_bundle_path }} already exists. Pick another
+      name, or `sudo rm` the existing bundle first.
+
+- name: snapshot — ensure snapshot dir exists
+  become: true
+  ansible.builtin.file:
+    path: "{{ snapshot_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0750"
+
+- name: snapshot — stop services in dependency order
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: stopped
+  loop: "{{ snapshot_services_stop_order }}"
+  failed_when: false
+
+- name: snapshot — kill lingering QEMU VMs
+  become: true
+  ansible.builtin.command: "pkill -x {{ item }}"
+  loop: "{{ spinifex_qemu_processes }}"
+  changed_when: false
+  failed_when: false
+
+- name: snapshot — wait for QEMU exit (up to {{ teardown_process_wait_seconds }}s)
+  become: true
+  ansible.builtin.shell: |
+    set -e
+    for i in $(seq 1 {{ teardown_process_wait_seconds }}); do
+      alive=0
+      for proc in {{ spinifex_qemu_processes | join(' ') }}; do
+        if pgrep -x "$proc" >/dev/null 2>&1; then alive=1; break; fi
+      done
+      if [ "$alive" = "0" ]; then exit 0; fi
+      sleep 1
+    done
+    echo "QEMU still running after {{ teardown_process_wait_seconds }}s" >&2
+    exit 1
+  changed_when: false
+
+- name: snapshot — verify which paths actually exist (skip missing silently)
+  ansible.builtin.stat:
+    path: "{{ item }}"
+  register: snapshot_path_stats
+  loop: "{{ snapshot_paths }}"
+  loop_control:
+    label: "{{ item }}"
+
+- name: snapshot — assemble list of present paths
+  ansible.builtin.set_fact:
+    snapshot_present_paths: >-
+      {{ snapshot_path_stats.results
+         | selectattr('stat.exists') | map(attribute='item') | list }}
+
+- name: snapshot — assert at least one path present
+  ansible.builtin.assert:
+    that: snapshot_present_paths | length > 0
+    fail_msg: |
+      None of the snapshot paths exist:
+      {{ snapshot_paths | join(', ') }}
+      Has spinifex ever been installed on this node?
+
+- name: snapshot — create bundle (tar czf with absolute paths)
+  become: true
+  ansible.builtin.command:
+    cmd: >-
+      tar --acls --xattrs -czpf {{ snapshot_bundle_path | quote }}
+      {{ snapshot_present_paths | map('quote') | join(' ') }}
+  changed_when: true
+
+- name: snapshot — record manifest inside the bundle dir for traceability
+  become: true
+  ansible.builtin.copy:
+    dest: "{{ snapshot_dir }}/{{ snapshot_name }}.manifest"
+    mode: "0640"
+    content: |
+      snapshot_name: {{ snapshot_name }}
+      created_at:    {{ ansible_facts['date_time']['iso8601'] }}
+      hostname:      {{ ansible_facts['hostname'] }}
+      paths:
+      {% for p in snapshot_present_paths %}  - {{ p }}
+      {% endfor %}
+      services_quiesced:
+      {% for s in snapshot_services_stop_order %}  - {{ s }}
+      {% endfor %}
+
+- name: snapshot — restart services in reverse order
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: started
+  loop: "{{ snapshot_services_stop_order | reverse | list }}"
+  failed_when: false
+
+- name: snapshot — wait for AWS gateway readiness
+  ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions"
+  register: snapshot_gw_ready
+  retries: 30
+  delay: 2
+  until: snapshot_gw_ready.rc == 0
+  changed_when: false
+  failed_when: false
+
+- name: snapshot — bundle size
+  become: true
+  ansible.builtin.stat:
+    path: "{{ snapshot_bundle_path }}"
+  register: snapshot_bundle_stat
+
+- name: snapshot — summary
+  ansible.builtin.debug:
+    msg: |
+      Snapshot complete:
+        name    : {{ snapshot_name }}
+        path    : {{ snapshot_bundle_path }}
+        size    : {{ (snapshot_bundle_stat.stat.size | int / 1024 / 1024) | round(1) }} MiB
+        paths   : {{ snapshot_present_paths | join(', ') }}
+        gateway : {{ 'ready' if snapshot_gw_ready.rc == 0 else 'NOT ready after restart — investigate' }}
+
+      Restore with: ansible-playbook playbooks/dev-restore.yml -e snapshot_name={{ snapshot_name }}

scripts/ansible/roles/status/meta/main.yml

@@ -0,0 +1,7 @@
+---
+galaxy_info:
+  role_name: status
+  description: Read-only health snapshot for a Spinifex dev node (doctor).
+  license: BSD-3-Clause
+  min_ansible_version: "2.14"
+dependencies: []

scripts/ansible/roles/status/tasks/main.yml

@@ -0,0 +1,259 @@
+---
+# Read-only health snapshot. NEVER mutates state. Every probe must use
+# `changed_when: false` and `failed_when: false` so the role always reaches
+# the summary task even when the node is half-broken — surfacing brokenness
+# is the whole point.
+
+- name: status — spinifex.target activation
+  ansible.builtin.command: systemctl is-active spinifex.target
+  register: status_target
+  changed_when: false
+  failed_when: false
+
+- name: status — list spinifex-* units (active + failed)
+  ansible.builtin.shell: |
+    systemctl list-units 'spinifex-*' --all --no-legend --plain \
+      --output=json 2>/dev/null || \
+      systemctl list-units 'spinifex-*' --all --no-legend --plain
+  register: status_units
+  changed_when: false
+  failed_when: false
+
+- name: status — failed spinifex-* units (one line per)
+  ansible.builtin.shell: |
+    systemctl list-units 'spinifex-*' --state=failed --no-legend --plain \
+      | awk '{print $1}'
+  register: status_failed_units
+  changed_when: false
+  failed_when: false
+
+- name: status — port listeners (which process holds each spinifex port)
+  ansible.builtin.shell: |
+    ss -tlnpH 'sport = :{{ item.port }}' | head -1
+  register: status_ports
+  loop: "{{ spinifex_ports }}"
+  changed_when: false
+  failed_when: false
+  loop_control:
+    label: "{{ item.service }} ({{ item.port }})"
+
+- name: status — OVS tooling present?
+  ansible.builtin.command: which ovs-vsctl
+  register: status_ovs_which
+  changed_when: false
+  failed_when: false
+
+- name: status — OVS bridges
+  become: true
+  ansible.builtin.command: ovs-vsctl list-br
+  register: status_ovs_brs
+  changed_when: false
+  failed_when: false
+  when: status_ovs_which.rc == 0
+
+- name: status — br-int exists?
+  become: true
+  ansible.builtin.command: ovs-vsctl br-exists br-int
+  register: status_brint
+  changed_when: false
+  failed_when: false
+  when: status_ovs_which.rc == 0
+
+- name: status — ovn-controller active?
+  ansible.builtin.command: systemctl is-active ovn-controller
+  register: status_ovn_ctrl
+  changed_when: false
+  failed_when: false
+
+- name: status — ovn-central active?
+  ansible.builtin.command: systemctl is-active ovn-central
+  register: status_ovn_central
+  changed_when: false
+  failed_when: false
+
+- name: status — OVN NB/SB DB files present?
+  become: true
+  ansible.builtin.stat:
+    path: "{{ item }}"
+  register: status_ovn_dbs
+  loop:
+    - "{{ ovn_nb_db }}"
+    - "{{ ovn_sb_db }}"
+  loop_control:
+    label: "{{ item }}"
+
+- name: status — spinifex.toml present?
+  become: true
+  ansible.builtin.stat:
+    path: "{{ spinifex_config_file }}"
+  register: status_config
+
+- name: status — CA trust present?
+  become: true
+  ansible.builtin.stat:
+    path: "{{ spinifex_ca_trust_path }}"
+  register: status_ca
+
+- name: status — AWS gateway round-trip (describe-regions)
+  ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions --output text"
+  register: status_awsgw
+  changed_when: false
+  failed_when: false
+
+- name: status — installed spx binary
+  become: true
+  ansible.builtin.stat:
+    path: /usr/local/bin/spx
+    checksum_algorithm: sha256
+  register: status_spx_installed
+
+- name: status — built spx binary
+  ansible.builtin.stat:
+    path: "{{ spinifex_root }}/bin/spx"
+    checksum_algorithm: sha256
+  register: status_spx_built
+
+- name: status — spx version (installed)
+  ansible.builtin.shell: |
+    /usr/local/bin/spx version 2>/dev/null | grep -vE '^[0-9]{4}/' | head -1
+  register: status_spx_version
+  changed_when: false
+  failed_when: false
+  when: status_spx_installed.stat.exists
+
+- name: status — installed nbdkit plugin
+  become: true
+  ansible.builtin.shell: |
+    set -e
+    pdir=$(nbdkit --dump-config | awk -F= '/^plugindir=/{print $2; exit}')
+    [ -f "$pdir/nbdkit-viperblock-plugin.so" ] && \
+      sha256sum "$pdir/nbdkit-viperblock-plugin.so" | awk '{print $1}'
+  register: status_plugin_installed_sha
+  changed_when: false
+  failed_when: false
+
+- name: status — built nbdkit plugin
+  ansible.builtin.stat:
+    path: "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so"
+    checksum_algorithm: sha256
+  register: status_plugin_built
+
+- name: status — microvm artifacts (built)
+  ansible.builtin.stat:
+    path: "{{ spinifex_root }}/build/microvm/{{ item }}"
+    checksum_algorithm: sha256
+  register: status_microvm_built
+  loop:
+    - vmlinuz
+    - initramfs.cpio.gz
+
+- name: status — microvm artifacts (installed)
+  become: true
+  ansible.builtin.stat:
+    path: /usr/share/spinifex/microvm/{{ item }}
+    checksum_algorithm: sha256
+  register: status_microvm_installed
+  loop:
+    - vmlinuz
+    - initramfs.cpio.gz
+
+- name: status — running QEMU VMs
+  ansible.builtin.shell: pgrep -fa 'qemu-system-' | wc -l
+  register: status_qemu_count
+  changed_when: false
+  failed_when: false
+
+- name: status — count AMIs
+  ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-images --query length(Images) --output text"
+  register: status_ami_count
+  changed_when: false
+  failed_when: false
+
+- name: status — count keypairs
+  ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-key-pairs --query length(KeyPairs) --output text"
+  register: status_key_count
+  changed_when: false
+  failed_when: false
+
+- name: status — count volumes
+  ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-volumes --query length(Volumes) --output text"
+  register: status_vol_count
+  changed_when: false
+  failed_when: false
+
+- name: status — count running instances
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 describe-instances
+    --filters Name=instance-state-name,Values=running
+    --query length(Reservations[].Instances[]) --output text
+  register: status_instance_count
+  changed_when: false
+  failed_when: false
+
+- name: status — compute drift facts
+  ansible.builtin.set_fact:
+    status_spx_drift: >-
+      {{ status_spx_installed.stat.exists and status_spx_built.stat.exists and
+         (status_spx_installed.stat.checksum | default('')) !=
+         (status_spx_built.stat.checksum  | default('')) }}
+    status_plugin_drift: >-
+      {{ status_plugin_built.stat.exists and
+         (status_plugin_installed_sha.stdout | default('') | trim) !=
+         (status_plugin_built.stat.checksum | default('')) }}
+    status_microvm_built_present: "{{ status_microvm_built.results
+      | map(attribute='stat.exists') | min }}"
+
+- name: status — render port lines
+  ansible.builtin.set_fact:
+    status_port_lines: >-
+      {{ status_port_lines | default([]) +
+         [('%-24s' | format(item.item.service ~ ' (' ~ item.item.port ~ ')'))
+          ~ ': '
+          ~ ('listening' if (item.stdout | length) > 0 else 'free')] }}
+  loop: "{{ status_ports.results }}"
+  loop_control:
+    label: "{{ item.item.service }}"
+
+- name: status — assemble summary
+  ansible.builtin.set_fact:
+    status_summary_lines: >-
+      {{
+        [
+          '── Spinifex dev-node status ──────────────────────────────────────────────',
+          'target           : ' ~ (status_target.stdout | trim),
+          'failed units     : ' ~ (status_failed_units.stdout_lines | length)
+            ~ ((' → ' ~ (status_failed_units.stdout_lines | join(', '))) if (status_failed_units.stdout_lines | length) > 0 else ''),
+          '── Ports ──'
+        ]
+        + status_port_lines
+        + [
+          '── Networking ──',
+          'ovs-vsctl        : ' ~ ('present' if status_ovs_which.rc == 0 else 'MISSING'),
+          'ovs bridges      : ' ~ ((status_ovs_brs.stdout_lines | default([]) | join(', ')) or '(none)'),
+          'br-int           : ' ~ ('present' if (status_brint.rc | default(1)) == 0 else 'MISSING'),
+          'ovn-controller   : ' ~ (status_ovn_ctrl.stdout | trim),
+          'ovn-central      : ' ~ (status_ovn_central.stdout | trim),
+          'ovn NB DB        : ' ~ ('present' if status_ovn_dbs.results[0].stat.exists else 'absent'),
+          'ovn SB DB        : ' ~ ('present' if status_ovn_dbs.results[1].stat.exists else 'absent'),
+          '── Config / trust ──',
+          'spinifex.toml    : ' ~ ('present' if status_config.stat.exists else 'MISSING') ~ ' (' ~ spinifex_config_file ~ ')',
+          'CA trust         : ' ~ ('present' if status_ca.stat.exists else 'MISSING') ~ ' (' ~ spinifex_ca_trust_path ~ ')',
+          '── AWS gateway ──',
+          'describe-regions : ' ~ ('OK' if status_awsgw.rc == 0 else 'FAIL (rc=' ~ status_awsgw.rc ~ ')'),
+          'AMIs             : ' ~ (status_ami_count.stdout | default('?') | trim),
+          'keypairs         : ' ~ (status_key_count.stdout | default('?') | trim),
+          'volumes          : ' ~ (status_vol_count.stdout | default('?') | trim),
+          'running instances: ' ~ (status_instance_count.stdout | default('?') | trim),
+          'running QEMU     : ' ~ (status_qemu_count.stdout | default('?') | trim),
+          '── Versions / drift ──',
+          'spx (installed)  : ' ~ ((status_spx_version.stdout | default('') | trim) or '(not installed)'),
+          'spx drift        : ' ~ ('YES (built ≠ installed) — run ansible-dev-deploy' if status_spx_drift else 'no'),
+          'plugin drift     : ' ~ ('YES (built ≠ installed) — run ansible-dev-deploy' if status_plugin_drift else 'no'),
+          'microvm built    : ' ~ ('present' if status_microvm_built_present else 'not built on this branch'),
+          '──────────────────────────────────────────────────────────────────────────'
+        ]
+      }}
+
+- name: status — summary
+  ansible.builtin.debug:
+    msg: "{{ status_summary_lines }}"

scripts/ansible/roles/version/meta/main.yml

@@ -0,0 +1,7 @@
+---
+galaxy_info:
+  role_name: version
+  description: Detailed build-vs-install drift report for spinifex artifacts.
+  license: BSD-3-Clause
+  min_ansible_version: "2.14"
+dependencies: []

scripts/ansible/roles/version/tasks/main.yml

@@ -0,0 +1,161 @@
+---
+# Detailed build-vs-install drift report for the four artifacts that
+# `ansible-dev-deploy` swaps: spx, the nbdkit viperblock plugin, and the
+# two microvm artifacts (kernel + initramfs). Pure read-only; faster
+# than dev-status (no AWS / OVN / port probing).
+
+- name: version — built spx
+  ansible.builtin.stat:
+    path: "{{ spinifex_root }}/bin/spx"
+    checksum_algorithm: sha256
+  register: ver_spx_built
+
+- name: version — installed spx
+  become: true
+  ansible.builtin.stat:
+    path: /usr/local/bin/spx
+    checksum_algorithm: sha256
+  register: ver_spx_installed
+
+- name: version — installed spx version string
+  ansible.builtin.shell: |
+    /usr/local/bin/spx version 2>/dev/null | grep -vE '^[0-9]{4}/' | head -1
+  register: ver_spx_version
+  changed_when: false
+  failed_when: false
+  when: ver_spx_installed.stat.exists
+
+- name: version — built plugin
+  ansible.builtin.stat:
+    path: "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so"
+    checksum_algorithm: sha256
+  register: ver_plugin_built
+
+- name: version — nbdkit plugin dir
+  ansible.builtin.shell: |
+    nbdkit --dump-config 2>/dev/null | awk -F= '/^plugindir=/{print $2; exit}'
+  register: ver_plugindir
+  changed_when: false
+  failed_when: false
+
+- name: version — installed plugin
+  become: true
+  ansible.builtin.stat:
+    path: "{{ ver_plugindir.stdout | trim }}/nbdkit-viperblock-plugin.so"
+    checksum_algorithm: sha256
+  register: ver_plugin_installed
+  when: ver_plugindir.stdout | trim | length > 0
+
+- name: version — built microvm artifacts
+  ansible.builtin.stat:
+    path: "{{ spinifex_root }}/build/microvm/{{ item }}"
+    checksum_algorithm: sha256
+  register: ver_microvm_built
+  loop:
+    - vmlinuz
+    - initramfs.cpio.gz
+
+- name: version — installed microvm artifacts
+  become: true
+  ansible.builtin.stat:
+    path: /usr/share/spinifex/microvm/{{ item }}
+    checksum_algorithm: sha256
+  register: ver_microvm_installed
+  loop:
+    - vmlinuz
+    - initramfs.cpio.gz
+
+- name: version — sub-repo HEAD short-rev (spinifex)
+  ansible.builtin.command: git -C {{ spinifex_root | quote }} rev-parse --short HEAD
+  register: ver_spinifex_head
+  changed_when: false
+  failed_when: false
+
+- name: version — sub-repo HEAD short-rev (viperblock)
+  ansible.builtin.command: git -C {{ mulga_root }}/viperblock rev-parse --short HEAD
+  register: ver_viperblock_head
+  changed_when: false
+  failed_when: false
+
+- name: version — sub-repo HEAD short-rev (predastore)
+  ansible.builtin.command: git -C {{ mulga_root }}/predastore rev-parse --short HEAD
+  register: ver_predastore_head
+  changed_when: false
+  failed_when: false
+
+- name: version — assemble per-artifact rows
+  ansible.builtin.set_fact:
+    ver_artifact_rows: >-
+      {{
+        [
+          ('%-18s built=%s  installed=%s  drift=%s' | format(
+            'spx',
+            (ver_spx_built.stat.checksum     | default('-') | truncate(13, true, ''))
+              if ver_spx_built.stat.exists     else 'MISSING',
+            (ver_spx_installed.stat.checksum | default('-') | truncate(13, true, ''))
+              if ver_spx_installed.stat.exists else 'MISSING',
+            ('YES' if ver_spx_built.stat.exists and ver_spx_installed.stat.exists
+                  and ver_spx_built.stat.checksum != ver_spx_installed.stat.checksum
+                  else 'no')
+          )),
+          ('%-18s built=%s  installed=%s  drift=%s' | format(
+            'plugin (nbdkit)',
+            (ver_plugin_built.stat.checksum    | default('-') | truncate(13, true, ''))
+              if ver_plugin_built.stat.exists    else 'MISSING',
+            (ver_plugin_installed.stat.checksum | default('-') | truncate(13, true, ''))
+              if (ver_plugin_installed.stat | default({})).get('exists', false) else 'MISSING',
+            ('YES' if ver_plugin_built.stat.exists
+                  and (ver_plugin_installed.stat | default({})).get('exists', false)
+                  and ver_plugin_built.stat.checksum != ver_plugin_installed.stat.checksum
+                  else 'no')
+          )),
+          ('%-18s built=%s  installed=%s  drift=%s' | format(
+            'microvm vmlinuz',
+            (ver_microvm_built.results[0].stat.checksum     | default('-') | truncate(13, true, ''))
+              if ver_microvm_built.results[0].stat.exists     else 'MISSING',
+            (ver_microvm_installed.results[0].stat.checksum | default('-') | truncate(13, true, ''))
+              if ver_microvm_installed.results[0].stat.exists else 'MISSING',
+            ('YES' if ver_microvm_built.results[0].stat.exists
+                  and ver_microvm_installed.results[0].stat.exists
+                  and ver_microvm_built.results[0].stat.checksum
+                      != ver_microvm_installed.results[0].stat.checksum
+                  else 'no')
+          )),
+          ('%-18s built=%s  installed=%s  drift=%s' | format(
+            'microvm initramfs',
+            (ver_microvm_built.results[1].stat.checksum     | default('-') | truncate(13, true, ''))
+              if ver_microvm_built.results[1].stat.exists     else 'MISSING',
+            (ver_microvm_installed.results[1].stat.checksum | default('-') | truncate(13, true, ''))
+              if ver_microvm_installed.results[1].stat.exists else 'MISSING',
+            ('YES' if ver_microvm_built.results[1].stat.exists
+                  and ver_microvm_installed.results[1].stat.exists
+                  and ver_microvm_built.results[1].stat.checksum
+                      != ver_microvm_installed.results[1].stat.checksum
+                  else 'no')
+          ))
+        ]
+      }}
+
+- name: version — compute any-drift flag
+  ansible.builtin.set_fact:
+    ver_any_drift: "{{ 'YES' in (ver_artifact_rows | join(' ')) }}"
+
+- name: version — summary
+  ansible.builtin.debug:
+    msg: >-
+      {{
+        [
+          '── Spinifex build vs install ─────────────────────────────────────────────',
+          'spx (installed): ' ~ ((ver_spx_version.stdout | default('') | trim) or '(not installed)')
+        ]
+        + ver_artifact_rows
+        + [
+          '── Sub-repo HEADs ──',
+          'spinifex   : ' ~ ((ver_spinifex_head.stdout | default('') | trim) or '(not a git repo)'),
+          'viperblock : ' ~ ((ver_viperblock_head.stdout | default('') | trim) or '(missing)'),
+          'predastore : ' ~ ((ver_predastore_head.stdout | default('') | trim) or '(missing)'),
+          '──────────────────────────────────────────────────────────────────────────',
+          ('DRIFT detected — run: make ansible-dev-deploy' if ver_any_drift
+           else 'No drift; built = installed for all artifacts.')
+        ]
+      }}

scripts/ansible/roles/vpc/meta/main.yml

@@ -0,0 +1,7 @@
+---
+galaxy_info:
+  role_name: vpc
+  description: Idempotent dev-mode VPC + public subnet + IGW + default route fixture.
+  license: BSD-3-Clause
+  min_ansible_version: "2.14"
+dependencies: []

scripts/ansible/roles/vpc/tasks/absent.yml

@@ -0,0 +1,128 @@
+---
+# Tear down the dev VPC fixture. Deletes in reverse dependency order:
+# route-table associations, route table, subnet, IGW (detach + delete),
+# VPC. Each step is best-effort so a partially-converged fixture still
+# converges to "gone" on re-run.
+
+- name: vpc — find VPC by Name tag
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 describe-vpcs
+    --filters Name=tag:Name,Values={{ dev_vpc_name }}
+    --query Vpcs[0].VpcId --output text
+  register: absent_vpc
+  changed_when: false
+
+- name: vpc — nothing to do
+  ansible.builtin.debug:
+    msg: "VPC tagged Name={{ dev_vpc_name }} not found; nothing to delete."
+  when: absent_vpc.stdout | trim in ['', 'None']
+
+- name: vpc — short-circuit when no VPC
+  ansible.builtin.meta: end_play
+  when: absent_vpc.stdout | trim in ['', 'None']
+
+- name: vpc — set VPC id fact
+  ansible.builtin.set_fact:
+    absent_vpc_id: "{{ absent_vpc.stdout | trim }}"
+
+- name: vpc — find subnet
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 describe-subnets
+    --filters Name=vpc-id,Values={{ absent_vpc_id }}
+    Name=tag:Name,Values={{ dev_vpc_name }}-public
+    --query Subnets[0].SubnetId --output text
+  register: absent_subnet
+  changed_when: false
+
+# Enumerate ALL non-main route tables in the VPC (main RT can't be
+# deleted; it goes when the VPC does). Catches orphans from earlier
+# runs that landed before the create-tags workaround.
+- name: vpc — find all non-main route tables in VPC
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 describe-route-tables
+    --filters Name=vpc-id,Values={{ absent_vpc_id }}
+    --query RouteTables[?!(Associations[?Main==`true`])].RouteTableId
+    --output text
+  register: absent_rt_list
+  changed_when: false
+
+- name: vpc — split RT id list
+  ansible.builtin.set_fact:
+    absent_rt_ids: "{{ absent_rt_list.stdout.split() | reject('equalto', '') | list }}"
+
+- name: vpc — find all subnet/RT associations to disassociate
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 describe-route-tables
+    --route-table-ids {{ item }}
+    --query RouteTables[0].Associations[?SubnetId!=null].RouteTableAssociationId
+    --output text
+  register: absent_assocs
+  loop: "{{ absent_rt_ids }}"
+  changed_when: false
+  failed_when: false
+
+- name: vpc — disassociate subnets from route tables
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 disassociate-route-table
+    --association-id {{ item }}
+  loop: "{{ absent_assocs.results
+            | map(attribute='stdout')
+            | map('default', '')
+            | map('split')
+            | flatten
+            | reject('equalto', '')
+            | list }}"
+  changed_when: true
+  failed_when: false
+
+- name: vpc — delete all non-main route tables
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 delete-route-table
+    --route-table-id {{ item }}
+  loop: "{{ absent_rt_ids }}"
+  changed_when: true
+  failed_when: false
+
+- name: vpc — delete subnet
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 delete-subnet
+    --subnet-id {{ absent_subnet.stdout | trim }}
+  when: absent_subnet.stdout | trim not in ['', 'None']
+  changed_when: true
+  failed_when: false
+
+- name: vpc — find IGW
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 describe-internet-gateways
+    --filters Name=attachment.vpc-id,Values={{ absent_vpc_id }}
+    --query InternetGateways[0].InternetGatewayId --output text
+  register: absent_igw
+  changed_when: false
+
+- name: vpc — detach IGW
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 detach-internet-gateway
+    --internet-gateway-id {{ absent_igw.stdout | trim }}
+    --vpc-id {{ absent_vpc_id }}
+  when: absent_igw.stdout | trim not in ['', 'None']
+  changed_when: true
+  failed_when: false
+
+- name: vpc — delete IGW
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 delete-internet-gateway
+    --internet-gateway-id {{ absent_igw.stdout | trim }}
+  when: absent_igw.stdout | trim not in ['', 'None']
+  changed_when: true
+  failed_when: false
+
+- name: vpc — delete VPC
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 delete-vpc
+    --vpc-id {{ absent_vpc_id }}
+  changed_when: true
+  failed_when: false
+
+- name: vpc — summary (absent)
+  ansible.builtin.debug:
+    msg: "Dev VPC {{ dev_vpc_name }} (was {{ absent_vpc_id }}) torn down."

scripts/ansible/roles/vpc/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- name: vpc — dispatch by state
+  ansible.builtin.include_tasks: "{{ dev_vpc_state }}.yml"

scripts/ansible/roles/vpc/tasks/present.yml

@@ -0,0 +1,209 @@
+---
+# Idempotent create. Every step describes-then-creates: re-running on an
+# already-converged VPC produces zero changes. Tag everything with
+# Name={{ dev_vpc_name }} so a single tag query finds the whole graph.
+
+- name: vpc — find existing VPC by Name tag
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 describe-vpcs
+    --filters Name=tag:Name,Values={{ dev_vpc_name }}
+    --query Vpcs[0].VpcId --output text
+  register: vpc_existing
+  changed_when: false
+
+- name: vpc — create VPC if missing
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 create-vpc
+    --cidr-block {{ dev_vpc_cidr }}
+    --tag-specifications
+    ResourceType=vpc,Tags=[{Key=Name,Value={{ dev_vpc_name }}}]
+    --query Vpc.VpcId --output text
+  register: vpc_created
+  when: vpc_existing.stdout | trim in ['', 'None']
+  changed_when: true
+
+- name: vpc — set VPC id fact
+  ansible.builtin.set_fact:
+    dev_vpc_id: >-
+      {{ vpc_created.stdout | default('') | trim
+         if (vpc_existing.stdout | trim in ['', 'None'])
+         else vpc_existing.stdout | trim }}
+
+- name: vpc — assert VPC id resolved
+  ansible.builtin.assert:
+    that: dev_vpc_id | length > 0
+    fail_msg: "VPC id could not be resolved after describe/create"
+
+- name: vpc — find existing IGW attached to this VPC
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 describe-internet-gateways
+    --filters Name=attachment.vpc-id,Values={{ dev_vpc_id }}
+    --query InternetGateways[0].InternetGatewayId --output text
+  register: vpc_igw_existing
+  changed_when: false
+
+- name: vpc — create IGW if missing
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 create-internet-gateway
+    --tag-specifications
+    ResourceType=internet-gateway,Tags=[{Key=Name,Value={{ dev_vpc_name }}-igw}]
+    --query InternetGateway.InternetGatewayId --output text
+  register: vpc_igw_created
+  when: vpc_igw_existing.stdout | trim in ['', 'None']
+  changed_when: true
+
+- name: vpc — set IGW id fact
+  ansible.builtin.set_fact:
+    dev_vpc_igw_id: >-
+      {{ vpc_igw_created.stdout | default('') | trim
+         if (vpc_igw_existing.stdout | trim in ['', 'None'])
+         else vpc_igw_existing.stdout | trim }}
+
+- name: vpc — attach IGW to VPC if needed
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 attach-internet-gateway
+    --internet-gateway-id {{ dev_vpc_igw_id }}
+    --vpc-id {{ dev_vpc_id }}
+  register: vpc_igw_attach
+  when: vpc_igw_existing.stdout | trim in ['', 'None']
+  changed_when: true
+  failed_when:
+    - vpc_igw_attach.rc | default(0) != 0
+    - "'Resource.AlreadyAssociated' not in (vpc_igw_attach.stderr | default(''))"
+
+- name: vpc — find existing subnet by Name tag
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 describe-subnets
+    --filters Name=vpc-id,Values={{ dev_vpc_id }}
+    Name=tag:Name,Values={{ dev_vpc_name }}-public
+    --query Subnets[0].SubnetId --output text
+  register: vpc_subnet_existing
+  changed_when: false
+
+- name: vpc — create subnet if missing
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 create-subnet
+    --vpc-id {{ dev_vpc_id }}
+    --cidr-block {{ dev_vpc_subnet_cidr }}
+    --availability-zone {{ dev_vpc_subnet_az }}
+    --tag-specifications
+    ResourceType=subnet,Tags=[{Key=Name,Value={{ dev_vpc_name }}-public}]
+    --query Subnet.SubnetId --output text
+  register: vpc_subnet_created
+  when: vpc_subnet_existing.stdout | trim in ['', 'None']
+  changed_when: true
+
+- name: vpc — set subnet id fact
+  ansible.builtin.set_fact:
+    dev_vpc_subnet_id: >-
+      {{ vpc_subnet_created.stdout | default('') | trim
+         if (vpc_subnet_existing.stdout | trim in ['', 'None'])
+         else vpc_subnet_existing.stdout | trim }}
+
+- name: vpc — ensure subnet has map-public-ip-on-launch
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 modify-subnet-attribute
+    --subnet-id {{ dev_vpc_subnet_id }}
+    --map-public-ip-on-launch
+  register: vpc_subnet_mapip
+  changed_when: vpc_subnet_existing.stdout | trim in ['', 'None']
+  failed_when: false
+
+# Try tag-based lookup first. If the gateway didn't tag a previously
+# created RT (spinifex gateway silently drops --tag-specifications on
+# create-route-table AND no-ops create-tags for route tables — see
+# mulga-siv-94; works correctly for subnets), fall back to "RT
+# associated with our subnet" which doesn't need tags.
+- name: vpc — find existing route table by Name tag
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 describe-route-tables
+    --filters Name=vpc-id,Values={{ dev_vpc_id }}
+    Name=tag:Name,Values={{ dev_vpc_name }}-public
+    --query RouteTables[0].RouteTableId --output text
+  register: vpc_rt_existing
+  changed_when: false
+
+- name: vpc — fallback lookup by subnet association
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 describe-route-tables
+    --filters Name=association.subnet-id,Values={{ dev_vpc_subnet_id }}
+    --query RouteTables[0].RouteTableId --output text
+  register: vpc_rt_byassoc
+  changed_when: false
+  when: vpc_rt_existing.stdout | trim in ['', 'None']
+
+- name: vpc — create route table if neither lookup found one
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 create-route-table
+    --vpc-id {{ dev_vpc_id }}
+    --query RouteTable.RouteTableId --output text
+  register: vpc_rt_created
+  when:
+    - vpc_rt_existing.stdout | trim in ['', 'None']
+    - (vpc_rt_byassoc.stdout | default('') | trim) in ['', 'None']
+  changed_when: true
+
+- name: vpc — set route-table id fact
+  ansible.builtin.set_fact:
+    dev_vpc_rt_id: >-
+      {%- if (vpc_rt_existing.stdout | trim) not in ['', 'None'] -%}
+      {{ vpc_rt_existing.stdout | trim }}
+      {%- elif (vpc_rt_byassoc.stdout | default('') | trim) not in ['', 'None'] -%}
+      {{ vpc_rt_byassoc.stdout | default('') | trim }}
+      {%- else -%}
+      {{ vpc_rt_created.stdout | default('') | trim }}
+      {%- endif -%}
+
+- name: vpc — assert route-table id resolved
+  ansible.builtin.assert:
+    that: dev_vpc_rt_id | length > 0
+    fail_msg: "Route table id could not be resolved after describe/create"
+
+# Spinifex gateway dropped --tag-specifications on the create call;
+# apply via create-tags so re-runs can find the RT via tag filter.
+- name: vpc — tag route table (workaround for create-tag-specifications gap)
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 create-tags
+    --resources {{ dev_vpc_rt_id }}
+    --tags Key=Name,Value={{ dev_vpc_name }}-public
+  changed_when: false
+  failed_when: false
+
+- name: vpc — add default route via IGW (ignore RouteAlreadyExists)
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 create-route
+    --route-table-id {{ dev_vpc_rt_id }}
+    --destination-cidr-block 0.0.0.0/0
+    --gateway-id {{ dev_vpc_igw_id }}
+  register: vpc_route_add
+  changed_when:
+    - vpc_route_add.rc == 0
+    - "'RouteAlreadyExists' not in (vpc_route_add.stderr | default(''))"
+  failed_when:
+    - vpc_route_add.rc != 0
+    - "'RouteAlreadyExists' not in (vpc_route_add.stderr | default(''))"
+
+- name: vpc — associate subnet with route table (ignore already-associated)
+  ansible.builtin.command: >-
+    {{ spinifex_aws_cli }} ec2 associate-route-table
+    --route-table-id {{ dev_vpc_rt_id }}
+    --subnet-id {{ dev_vpc_subnet_id }}
+  register: vpc_assoc
+  changed_when:
+    - vpc_assoc.rc == 0
+    - "'Resource.AlreadyAssociated' not in (vpc_assoc.stderr | default(''))"
+  failed_when:
+    - vpc_assoc.rc != 0
+    - "'Resource.AlreadyAssociated' not in (vpc_assoc.stderr | default(''))"
+
+- name: vpc — summary (present)
+  ansible.builtin.debug:
+    msg: |
+      Dev VPC ready:
+        VPC         : {{ dev_vpc_id }} ({{ dev_vpc_cidr }})
+        IGW         : {{ dev_vpc_igw_id }}
+        Subnet      : {{ dev_vpc_subnet_id }} ({{ dev_vpc_subnet_cidr }} @ {{ dev_vpc_subnet_az }}, public)
+        Route table : {{ dev_vpc_rt_id }}
+
+      Smoketest now finds this subnet via its existing
+      Name=map-public-ip-on-launch,Values=true filter.

scripts/ansible/scripts/aws-as-spinifex.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Run `aws` CLI inside a shell that has the `spinifex` supplementary group
+# active, with AWS_PROFILE=spinifex preset. Works around mulga-siv-92:
+# ansible's `become` is a no-op when the target user equals the calling
+# user, so `sudo -u` is never invoked and supplementary groups are not
+# re-initialised. Without the `spinifex` group, /etc/spinifex/ca.pem (the
+# AWS endpoint CA bundle) is unreadable and aws fails with SSL Errno 13.
+#
+# Usage from ansible tasks:
+#   ansible.builtin.command:
+#     cmd: "{{ playbook_dir }}/../scripts/aws-as-spinifex.sh ec2 describe-regions"
+#
+# All args are forwarded to `aws`. Quoting is preserved via printf %q so
+# things like `--query 'length(Images)'` reach aws intact.
+set -euo pipefail
+
+if [[ $# -eq 0 ]]; then
+    echo "usage: $0 <aws-cli-args>" >&2
+    exit 64
+fi
+
+cmd="AWS_PROFILE=spinifex aws"
+for arg in "$@"; do
+    cmd+=" $(printf '%q' "$arg")"
+done
+
+exec sg spinifex -c "$cmd"

scripts/ansible/scripts/spx-as-spinifex.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Run `spx` inside a shell that has the `spinifex` supplementary group
+# active. Works around mulga-siv-92 (see aws-as-spinifex.sh for details).
+# All args are forwarded to /usr/local/bin/spx via printf %q so flag
+# values with spaces or shell metachars survive intact.
+set -euo pipefail
+
+if [[ $# -eq 0 ]]; then
+    echo "usage: $0 <spx-args>" >&2
+    exit 64
+fi
+
+cmd="AWS_PROFILE=spinifex /usr/local/bin/spx"
+for arg in "$@"; do
+    cmd+=" $(printf '%q' "$arg")"
+done
+
+exec sg spinifex -c "$cmd"

scripts/ansible/vars/defaults.yml

@@ -23,12 +23,20 @@ spinifex_node_name: node1
 spinifex_node_count: 1
 
 # External network.
+# spx admin init contract (as of network-layer-redesign):
+#   external_mode  : "pool" or "" (NAT path removed)
+#   external_pool  : MANDATORY when mode=pool — set start/end below or pass -e.
+#   external_gw    : auto-detected from default route when blank.
+#   external_prefix: auto-detected from WAN when blank.
+#   external_gw_ip : OVN router external-IP override (was nat_gateway_ip).
+#   no_external    : true → emit --no-external (skip external networking).
 spinifex_external_mode: pool
+spinifex_external_no_external: false
 spinifex_external_pool_start: ""
 spinifex_external_pool_end: ""
 spinifex_external_gateway: ""
 spinifex_external_prefix_len: ""
-spinifex_nat_gateway_ip: ""
+spinifex_external_gateway_ip: ""
 
 # Preflight checks.
 spinifex_ports:
@@ -76,11 +84,54 @@ teardown_process_wait_seconds: 30
 spinifex_root: "{{ lookup('pipe', 'readlink -f ' ~ playbook_dir ~ '/../../..') }}"
 mulga_root: "{{ lookup('pipe', 'readlink -f ' ~ playbook_dir ~ '/../../../..') }}"
 
+# Wrappers around `aws` and `spx` that re-enter a shell with the
+# `spinifex` supplementary group active. Needed because ansible's
+# `become_user` is a no-op when target user equals current user, so
+# supplementary groups are never re-initialised and /etc/spinifex/ca.pem
+# (drwxr-x--- root:spinifex) is unreadable.
+spinifex_aws_cli: "{{ playbook_dir }}/../scripts/aws-as-spinifex.sh"
+spinifex_spx_cli: "{{ playbook_dir }}/../scripts/spx-as-spinifex.sh"
+
+# dev-vpc fixture. Deterministic VPC + public subnet + IGW + default
+# route. Idempotent: re-running detects existing resources by Name tag.
+# Used by smoketest's launch path (which looks for any
+# map-public-ip-on-launch=true subnet).
+dev_vpc_name: spinifex-dev-vpc
+dev_vpc_cidr: 10.99.0.0/16
+dev_vpc_subnet_cidr: 10.99.1.0/24
+dev_vpc_subnet_az: "{{ spinifex_az }}"
+dev_vpc_state: present  # present | absent
+
 # Build output.
 build_tarball_path: /tmp/spinifex-local.tar.gz
 
+# Snapshot / restore. Snapshots quiesce the full spinifex+ovn+ovs stack,
+# tar these paths, then restart services. Restore wipes + untars + restarts.
+# Default snapshot dir is root-owned so it survives `dev-teardown`
+# (teardown only removes the four spinifex_*_dir paths above).
+snapshot_dir: /var/lib/spinifex-snapshots
+snapshot_paths:
+    - "{{ spinifex_etc_dir }}"
+    - "{{ spinifex_data_dir }}"
+    - "{{ ovn_db_dir }}"
+    - /etc/openvswitch/conf.db
+# Order matters: stop spinifex first (depends on OVN/OVS), then OVN, then OVS.
+# Restart in reverse.
+snapshot_services_stop_order:
+    - spinifex.target
+    - ovn-controller
+    - ovn-central
+    - openvswitch-switch
+
 # Smoketest image — parameterised so tofu examples can share the same base.
 spinifex_smoketest_distro: debian
 spinifex_smoketest_version: "13"
 # Local image file path (if present, imported directly; otherwise fall back to --name).
 spinifex_smoketest_local_image: "~/images/{{ spinifex_smoketest_distro }}-{{ spinifex_smoketest_version }}.img"
+
+# Multi-node cluster bootstrap (cluster-bootstrap.yml).
+# cluster_env, cluster_init_host, spinifex_node_count come from inventory/tofu.py.
+cluster_external_pool: ""               # "<start>-<end>" or empty for DHCP fallback
+cluster_join_token_path: /etc/spinifex/join-token
+cluster_formation_port: 4432
+cluster_formation_timeout: 60           # seconds (uri retries = value // 2)

scripts/build-system-image.sh

@@ -134,7 +134,7 @@ if ! command -v qemu-img &>/dev/null; then
     exit 1
 fi
 
-if [[ "$DISTRO" == "ubuntu" ]] && ! command -v parted &>/dev/null; then
+if [[ "$DISTRO" == "ubuntu" ]] && ! command -v parted &>/dev/null && ! [[ -x /usr/sbin/parted ]]; then
     echo "ERROR: parted not found. Install parted."
     exit 1
 fi
@@ -326,7 +326,65 @@ apt-get install -y --no-install-recommends ${APT_PACKAGES}
 "
 fi
 
-# Enable services
+# Step 5: Copy binaries into the image (before setup script, which may reference them)
+if [[ -n "${INSTALL_BINARIES:-}" ]]; then
+    echo "Installing binaries..."
+    IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
+    for pair in "${BINARY_PAIRS[@]}"; do
+        src="${pair%%:*}"
+        dst="${pair#*:}"
+        src_path="${PROJECT_DIR}/${src}"
+        echo "  ${src} -> ${dst}"
+        sudo cp "$src_path" "${MOUNT_DIR}${dst}"
+        sudo chmod 755 "${MOUNT_DIR}${dst}"
+    done
+fi
+
+# Copy auxiliary files (systemd units, OpenRC initd scripts, cron entries, configs).
+# Same src:dst pair grammar as INSTALL_BINARIES; mode 0644, parent dirs auto-created.
+# Optional — manifests without INSTALL_FILES (e.g. existing GPU images) skip this step.
+if [[ -n "${INSTALL_FILES:-}" ]]; then
+    echo "Installing files..."
+    IFS=' ' read -ra FILE_PAIRS <<< "$INSTALL_FILES"
+    for pair in "${FILE_PAIRS[@]}"; do
+        src="${pair%%:*}"
+        dst="${pair#*:}"
+        src_path="${PROJECT_DIR}/${src}"
+        if [[ ! -f "$src_path" ]]; then
+            echo "ERROR: INSTALL_FILES source not found: $src_path"
+            exit 1
+        fi
+        echo "  ${src} -> ${dst}"
+        sudo mkdir -p "${MOUNT_DIR}$(dirname "$dst")"
+        sudo cp "$src_path" "${MOUNT_DIR}${dst}"
+        sudo chmod 0644 "${MOUNT_DIR}${dst}"
+    done
+fi
+
+# Run custom setup script inside chroot (after binaries are installed)
+if [[ -n "${SETUP_SCRIPT:-}" ]]; then
+    setup_path="${PROJECT_DIR}/${SETUP_SCRIPT}"
+    if [[ ! -f "$setup_path" ]]; then
+        echo "ERROR: Setup script not found: $setup_path"
+        exit 1
+    fi
+    echo "Running setup script: ${SETUP_SCRIPT}..."
+    sudo cp "$setup_path" "${MOUNT_DIR}/tmp/setup.sh"
+    sudo chmod 755 "${MOUNT_DIR}/tmp/setup.sh"
+    if [[ "$DISTRO" == "ubuntu" ]]; then
+        if [[ ! -x "${MOUNT_DIR}/bin/bash" ]]; then
+            echo "ERROR: /bin/bash not found or not executable in chroot"
+            exit 1
+        fi
+        sudo chroot "$MOUNT_DIR" /bin/bash -x /tmp/setup.sh
+    else
+        sudo chroot "$MOUNT_DIR" /tmp/setup.sh
+    fi
+    sudo rm -f "${MOUNT_DIR}/tmp/setup.sh"
+fi
+
+# Enable services (runs after INSTALL_FILES + SETUP_SCRIPT so init scripts the
+# manifest references are already on disk).
 if [[ -n "${ENABLE_SERVICES:-}" ]]; then
     echo "Enabling services: ${ENABLE_SERVICES}..."
     IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES"
@@ -345,38 +403,6 @@ if [[ -n "${ENABLE_SERVICES:-}" ]]; then
     done
 fi
 
-# Step 5: Copy binaries into the image (before setup script, which may reference them)
-if [[ -n "${INSTALL_BINARIES:-}" ]]; then
-    echo "Installing binaries..."
-    IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
-    for pair in "${BINARY_PAIRS[@]}"; do
-        src="${pair%%:*}"
-        dst="${pair#*:}"
-        src_path="${PROJECT_DIR}/${src}"
-        echo "  ${src} -> ${dst}"
-        sudo cp "$src_path" "${MOUNT_DIR}${dst}"
-        sudo chmod 755 "${MOUNT_DIR}${dst}"
-    done
-fi
-
-# Run custom setup script inside chroot (after binaries are installed)
-if [[ -n "${SETUP_SCRIPT:-}" ]]; then
-    setup_path="${PROJECT_DIR}/${SETUP_SCRIPT}"
-    if [[ ! -f "$setup_path" ]]; then
-        echo "ERROR: Setup script not found: $setup_path"
-        exit 1
-    fi
-    echo "Running setup script: ${SETUP_SCRIPT}..."
-    sudo cp "$setup_path" "${MOUNT_DIR}/tmp/setup.sh"
-    sudo chmod 755 "${MOUNT_DIR}/tmp/setup.sh"
-    if [[ "$DISTRO" == "ubuntu" ]]; then
-        sudo chroot "$MOUNT_DIR" /bin/bash /tmp/setup.sh
-    else
-        sudo chroot "$MOUNT_DIR" /tmp/setup.sh
-    fi
-    sudo rm -f "${MOUNT_DIR}/tmp/setup.sh"
-fi
-
 # Step 6: Clean up and unmount
 echo "Cleaning up image..."
 if [[ "$DISTRO" == "alpine" ]]; then
@@ -426,6 +452,9 @@ if [[ "$DO_IMPORT" == true ]]; then
     echo "Importing as AMI..."
     rm -f "$OUTPUT_IMAGE"
     IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64 --boot-mode "${BOOT_MODE}")
+    if [[ -n "${AMI_NAME:-}" ]]; then
+        IMPORT_ARGS+=(--name "$AMI_NAME")
+    fi
     if [[ -n "${SYSTEM_TAG:-}" ]]; then
         IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
     fi
@@ -434,10 +463,14 @@ else
     echo "To import as AMI, run:"
     echo "  spx admin images import \\"
     echo "    --file $OUTPUT_RAW \\"
+    NAME_HINT=""
+    if [[ -n "${AMI_NAME:-}" ]]; then
+        NAME_HINT=" \\\n    --name ${AMI_NAME}"
+    fi
     if [[ -n "${SYSTEM_TAG:-}" ]]; then
-        echo "    --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 --boot-mode ${BOOT_MODE} \\"
+        echo -e "    --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 --boot-mode ${BOOT_MODE}${NAME_HINT} \\"
         echo "    --tag ${SYSTEM_TAG}"
     else
-        echo "    --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 --boot-mode ${BOOT_MODE}"
+        echo -e "    --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 --boot-mode ${BOOT_MODE}${NAME_HINT}"
     fi
 fi

scripts/check-coverage.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # Usage: scripts/check-coverage.sh <coverprofile> [quiet]
 
 PROFILE="${1:?Usage: check-coverage.sh <coverprofile> [quiet]}"
-MIN=70.0
+MIN=68.0
 QUIET="${2:-}"
 
 if [[ ! -s "$PROFILE" ]]; then

scripts/iac/proxmox-template/README.md

@@ -7,7 +7,7 @@ Builds a Proxmox VM template with all Spinifex dependencies pre-installed. Cloni
 | Component | Details |
 |---|---|
 | System packages | QEMU, nbdkit, libvirt, gcc, jq, curl, etc. (`make install-system`) |
-| Go | 1.26.3 (`make install-go`) |
+| Go | 1.26.4 (`make install-go`) |
 | AWS CLI | v2 (`make install-aws`) |
 | Repositories | spinifex, viperblock, predastore (with `go.work` configured) |
 | Module cache | `go mod download` completed for all repos |

scripts/images/eks-agent/k3s-agent.initd

@@ -0,0 +1,38 @@
+#!/sbin/openrc-run
+# K3s agent (worker node). Env vars K3S_URL + K3S_TOKEN come from cloud-init
+# user-data — the orchestrator embeds them at RunInstances time per the
+# Nodegroup bootstrap flow (eks-v1.md Q14).
+#
+# K3S_URL points at the cluster's NLB endpoint (https://{cluster}.{account}
+# .eks.{region}.{suffix}:443) so the agent join path stays stable across the
+# v2 HA migration (single-VM v1 → 3-node etcd quorum v2). No AMI rebuild
+# needed when the control plane expands.
+
+description="K3s Kubernetes agent (Mulga EKS worker)"
+command="/usr/local/bin/k3s"
+command_args="agent"
+command_background="yes"
+pidfile="/run/k3s-agent.pid"
+output_log="/var/log/k3s-agent.log"
+error_log="/var/log/k3s-agent.log"
+
+start_stop_daemon_args="--wait 5000"
+
+depend() {
+    need net cgroups
+    after firewall
+}
+
+start_pre() {
+    # Cloud-init drops these into /etc/spinifex-eks/agent.env. We source them
+    # here so OpenRC injects them into k3s' environment.
+    if [ -f /etc/spinifex-eks/agent.env ]; then
+        # shellcheck disable=SC1091
+        . /etc/spinifex-eks/agent.env
+        export K3S_URL K3S_TOKEN K3S_NODE_NAME K3S_NODE_LABEL
+    fi
+    if [ -z "${K3S_URL:-}" ] || [ -z "${K3S_TOKEN:-}" ]; then
+        eerror "K3S_URL and K3S_TOKEN must be set in /etc/spinifex-eks/agent.env"
+        return 1
+    fi
+}

scripts/images/eks-agent/manifest.conf

@@ -0,0 +1,25 @@
+IMAGE_NAME="eks-agent"
+IMAGE_DESCRIPTION="EKS K3s agent VM — Alpine + K3s v1.32.5+k3s1 (worker node)"
+DISTRO="alpine"
+ALPINE_VERSION="3.21.7"
+
+# 2G for the worker — kubelet + containerd + node images. Customer workload
+# storage attaches via separate EBS volumes (managed by viperblock); this is
+# the OS root only.
+IMAGE_SIZE="2G"
+
+APK_PACKAGES="ca-certificates curl iptables ip6tables conntrack-tools cgroup-tools openrc"
+
+# K3s binary downloaded by setup.sh (same pinned version + SHA path as
+# eks-server) to keep server / agent in lock-step.
+
+INSTALL_FILES="\
+scripts/images/eks-agent/k3s-agent.initd:/etc/init.d/k3s-agent \
+scripts/images/eks-agent/mulga-eks-state-report.sh:/etc/periodic/15min/mulga-eks-state-report"
+
+ENABLE_SERVICES="crond k3s-agent"
+
+SETUP_SCRIPT="scripts/images/eks-agent/setup.sh"
+
+AMI_NAME="spinifex-eks-agent"
+SYSTEM_TAG="spinifex:managed-by=eks"

scripts/images/eks-agent/mulga-eks-state-report.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+set -eu
+
+# mulga-eks-state-report (agent variant) — periodic NATS publish of agent
+# liveness signal. Subject: eks.state.{accountID}.{clusterName}.agent.{nodeName}
+# Payload: JSON { kubelet: "ok"|"fail", ts: <unix> }
+#
+# The agent has no API server to query for its own status; we shell-check the
+# local kubelet socket via crictl, and fall back to a process check.
+
+ENVFILE=/etc/spinifex-eks/state-report.env
+[ -f "${ENVFILE}" ] || { logger -t mulga-eks-state-report "${ENVFILE} missing"; exit 0; }
+# shellcheck disable=SC1090
+. "${ENVFILE}"
+
+: "${SPINIFEX_NATS_URL:?}"
+: "${EKS_ACCOUNT_ID:?}"
+: "${EKS_CLUSTER_NAME:?}"
+: "${EKS_NODE_NAME:?}"
+
+if pgrep -f 'k3s agent' >/dev/null 2>&1; then
+    KUBELET=ok
+else
+    KUBELET=fail
+fi
+
+TS=$(date +%s)
+PAYLOAD=$(printf '{"kubelet":"%s","ts":%s}' "${KUBELET}" "${TS}")
+SUBJ="eks.state.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.agent.${EKS_NODE_NAME}"
+
+NATS_ARGS="-s ${SPINIFEX_NATS_URL}"
+if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then
+    NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}"
+fi
+
+# shellcheck disable=SC2086
+printf '%s' "${PAYLOAD}" | nats ${NATS_ARGS} pub "${SUBJ}" --stdin 2>&1 \
+    | logger -t mulga-eks-state-report

scripts/images/eks-agent/setup.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -eu
+
+# setup.sh — chroot customisation for the eks-agent AMI.
+# Pulls the same K3s release as eks-server (lock-step versioning per Q12) and
+# wires the k3s-agent OpenRC service.
+
+K3S_VERSION="v1.32.5+k3s1"
+K3S_URL_BASE="https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}"
+
+echo "[eks-agent-setup] fetching k3s checksums ${K3S_VERSION}"
+curl -fsSL -o /tmp/k3s-checksums.txt "${K3S_URL_BASE}/sha256sum-amd64.txt"
+K3S_SHA256=$(awk '/[ \t]k3s$/{print $1; exit}' /tmp/k3s-checksums.txt)
+if [ -z "${K3S_SHA256}" ]; then
+    echo "[eks-agent-setup] could not parse k3s sha256 from upstream checksums"
+    cat /tmp/k3s-checksums.txt
+    exit 1
+fi
+
+echo "[eks-agent-setup] downloading k3s ${K3S_VERSION} (sha256=${K3S_SHA256})"
+curl -fsSL -o /usr/local/bin/k3s "${K3S_URL_BASE}/k3s"
+echo "${K3S_SHA256}  /usr/local/bin/k3s" > /tmp/k3s.sha256
+if ! sha256sum -c /tmp/k3s.sha256; then
+    echo "[eks-agent-setup] k3s SHA256 mismatch — refusing to bake AMI"
+    exit 1
+fi
+chmod 0755 /usr/local/bin/k3s
+ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl
+ln -sf /usr/local/bin/k3s /usr/local/bin/crictl
+ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
+
+# nats-cli — see eks-server setup.sh for rationale.
+NATS_CLI_VERSION="0.4.0"
+NATS_CLI_URL="https://github.com/nats-io/natscli/releases/download/v${NATS_CLI_VERSION}/nats-${NATS_CLI_VERSION}-linux-amd64.zip"
+echo "[eks-agent-setup] downloading nats-cli ${NATS_CLI_VERSION}"
+curl -fsSL -o /tmp/nats-cli.zip "${NATS_CLI_URL}"
+apk add --no-cache unzip
+unzip -q -d /tmp/nats-cli /tmp/nats-cli.zip
+install -m 0755 /tmp/nats-cli/nats-${NATS_CLI_VERSION}-linux-amd64/nats /usr/local/bin/nats
+rm -rf /tmp/nats-cli /tmp/nats-cli.zip
+apk del --no-cache unzip
+
+chmod 0755 /etc/init.d/k3s-agent
+chmod 0755 /etc/periodic/15min/mulga-eks-state-report
+
+mkdir -p /etc/spinifex-eks
+echo "[eks-agent-setup] done"

scripts/images/eks-server/eks-token-webhook.initd

@@ -0,0 +1,17 @@
+#!/sbin/openrc-run
+# eks-token-webhook — K8s TokenReview authenticator for IAM-backed kubectl.
+# Binds 127.0.0.1:8443 only; kube-apiserver is the sole client (loopback path).
+# Stub binary returns 503 on every request until cs-eks-6c lands.
+
+description="EKS Token Review webhook (Mulga)"
+command="/usr/local/bin/eks-token-webhook"
+command_args="--addr=127.0.0.1:8443"
+command_background="yes"
+pidfile="/run/eks-token-webhook.pid"
+output_log="/var/log/eks-token-webhook.log"
+error_log="/var/log/eks-token-webhook.log"
+
+depend() {
+    need net
+    before k3s
+}

scripts/images/eks-server/k3s-first-boot.initd

@@ -0,0 +1,23 @@
+#!/sbin/openrc-run
+# k3s-first-boot — one-shot OpenRC service that runs the first-boot publisher
+# after K3s is up. Self-disables by removing itself from the default runlevel
+# once the sentinel file is gone.
+
+description="K3s first-boot publisher (node-token + admin kubeconfig)"
+command="/usr/local/sbin/k3s-first-boot"
+command_background="no"
+pidfile="/run/k3s-first-boot.pid"
+output_log="/var/log/k3s-first-boot.log"
+error_log="/var/log/k3s-first-boot.log"
+
+depend() {
+    need k3s
+    after k3s
+}
+
+start_pre() {
+    if [ ! -f /var/lib/spinifex-eks/first-boot.pending ]; then
+        einfo "first-boot already complete; nothing to do"
+        return 0
+    fi
+}

scripts/images/eks-server/k3s-first-boot.sh

@@ -0,0 +1,91 @@
+#!/bin/sh
+set -eu
+
+# k3s-first-boot — runs once after the K3s server reaches a healthy state.
+# Reads the bootstrap node-token and admin kubeconfig that K3s writes during
+# `--cluster-init`, rewrites the kubeconfig server address to the cluster's
+# NLB endpoint (so workers and external kubectl can use it), and publishes
+# both as one-shot NATS messages for the spinifex cluster reconciler to
+# consume into KV.
+#
+# Required env (from cloud-init user-data /etc/spinifex-eks/first-boot.env):
+#   SPINIFEX_NATS_URL          nats://...
+#   SPINIFEX_NATS_CREDS_FILE   /etc/spinifex-eks/nats.creds
+#   EKS_ACCOUNT_ID
+#   EKS_CLUSTER_NAME
+#   EKS_NLB_ENDPOINT           https://{cluster}.{accountID}.eks.{region}.{suffix}
+#
+# Idempotent: a sentinel file at /var/lib/spinifex-eks/first-boot.pending gates
+# execution. On success the sentinel is removed and the OpenRC service is
+# pulled from the default runlevel so it does not retry on subsequent boots.
+
+SENTINEL=/var/lib/spinifex-eks/first-boot.pending
+ENVFILE=/etc/spinifex-eks/first-boot.env
+LOGTAG="k3s-first-boot"
+
+log() { echo "[${LOGTAG}] $*"; }
+die() { log "ERROR: $*"; exit 1; }
+
+if [ ! -f "${SENTINEL}" ]; then
+    log "sentinel missing — first boot already complete"
+    exit 0
+fi
+
+if [ ! -f "${ENVFILE}" ]; then
+    die "${ENVFILE} not found — cloud-init did not seed first-boot env"
+fi
+
+# shellcheck disable=SC1090
+. "${ENVFILE}"
+
+for v in SPINIFEX_NATS_URL EKS_ACCOUNT_ID EKS_CLUSTER_NAME EKS_NLB_ENDPOINT; do
+    eval "val=\${$v:-}"
+    [ -n "${val}" ] || die "env ${v} not set"
+done
+
+# 1. Wait for K3s /healthz. K3s self-signs initially; --insecure-skip is fine
+#    here, we are on loopback.
+log "waiting for K3s /healthz on 127.0.0.1:6443"
+i=0
+while [ "${i}" -lt 300 ]; do
+    if curl -sk --max-time 2 https://127.0.0.1:6443/healthz | grep -q '^ok$'; then
+        log "K3s healthz ok after ${i}s"
+        break
+    fi
+    i=$((i + 2))
+    sleep 2
+done
+[ "${i}" -lt 300 ] || die "K3s did not become healthy within 5 minutes"
+
+# 2. Read node-token + admin kubeconfig.
+TOKEN_FILE=/var/lib/rancher/k3s/server/node-token
+KUBECONFIG_FILE=/etc/rancher/k3s/k3s.yaml
+[ -r "${TOKEN_FILE}" ] || die "${TOKEN_FILE} unreadable"
+[ -r "${KUBECONFIG_FILE}" ] || die "${KUBECONFIG_FILE} unreadable"
+
+NODE_TOKEN=$(cat "${TOKEN_FILE}")
+# K3s ships kubeconfig with server: https://127.0.0.1:6443 — rewrite to the
+# NLB endpoint so it works from outside the control plane VM.
+KUBECONFIG_REWRITTEN=$(sed "s|server: https://127\.0\.0\.1:6443|server: ${EKS_NLB_ENDPOINT}|" "${KUBECONFIG_FILE}")
+
+# 3. Publish one-shot NATS messages. Subjects per eks-v1.md Q14 + Q15.
+TOKEN_SUBJ="eks.bus.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.k3s-bootstrap-token"
+KUBE_SUBJ="eks.bus.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.k3s-admin-kubeconfig"
+
+NATS_ARGS="-s ${SPINIFEX_NATS_URL}"
+if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then
+    NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}"
+fi
+
+log "publishing node-token to ${TOKEN_SUBJ}"
+# shellcheck disable=SC2086
+printf '%s' "${NODE_TOKEN}" | nats ${NATS_ARGS} pub "${TOKEN_SUBJ}" --stdin
+
+log "publishing admin kubeconfig to ${KUBE_SUBJ}"
+# shellcheck disable=SC2086
+printf '%s' "${KUBECONFIG_REWRITTEN}" | nats ${NATS_ARGS} pub "${KUBE_SUBJ}" --stdin
+
+# 4. Self-disable. Remove sentinel, pull from runlevel.
+rm -f "${SENTINEL}"
+rc-update del k3s-first-boot default 2>/dev/null || true
+log "first boot complete"

scripts/images/eks-server/k3s.initd

@@ -0,0 +1,36 @@
+#!/sbin/openrc-run
+# K3s server control plane. v1 = single-node embedded etcd (eks-v1.md Q6).
+# --cluster-init bootstraps a new etcd quorum on first start; subsequent
+# starts pick up the existing data dir at /var/lib/rancher/k3s. v2 HA grows
+# the cluster by launching additional eks-server VMs with --server pointing
+# at the existing NLB endpoint (Q17) — no AMI change required.
+
+description="K3s Kubernetes server (Mulga EKS control plane)"
+command="/usr/local/bin/k3s"
+command_args="server --config /etc/rancher/k3s/config.yaml"
+command_background="yes"
+pidfile="/run/k3s.pid"
+output_log="/var/log/k3s.log"
+error_log="/var/log/k3s.log"
+
+# Generous start timeout — K3s bootstraps containerd, runs etcd cluster-init,
+# pulls system images. First boot can take 2-3 minutes on cold cache.
+start_stop_daemon_args="--wait 5000"
+
+depend() {
+    need net cgroups
+    after firewall
+}
+
+start_pre() {
+    # config.yaml is written by k3s-first-boot from the skeleton + cloud-init
+    # data. Refuse to start if neither exists.
+    if [ ! -f /etc/rancher/k3s/config.yaml ]; then
+        if [ -f /etc/rancher/k3s/config.yaml.skel ]; then
+            cp /etc/rancher/k3s/config.yaml.skel /etc/rancher/k3s/config.yaml
+        else
+            eerror "no /etc/rancher/k3s/config.yaml and no .skel to derive from"
+            return 1
+        fi
+    fi
+}

scripts/images/eks-server/manifest.conf

@@ -0,0 +1,34 @@
+IMAGE_NAME="eks-server"
+IMAGE_DESCRIPTION="EKS K3s server VM — Alpine + K3s v1.32.5+k3s1 + eks-token-webhook + state-report + etcd-snapshot"
+DISTRO="alpine"
+ALPINE_VERSION="3.21.7"
+
+# 4G fits Alpine + K3s server binary (~80MB) + etcd snapshot working set.
+# K3s embedded etcd lives on the data volume the orchestrator attaches at
+# LaunchSystemInstance time; root disk only carries the image + logs.
+IMAGE_SIZE="4G"
+
+# Packages required at runtime (K3s pulls cgroups + iptables; nats-cli used by
+# first-boot publisher + state-report; etcd-client used by snapshot script).
+APK_PACKAGES="ca-certificates curl iptables ip6tables conntrack-tools cgroup-tools etcd-ctl openrc"
+
+# k3s binary downloaded by setup.sh (network access inside chroot); SHA256
+# verified at install time per spec Q12.
+BUILD_COMMANDS="CGO_ENABLED=0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook"
+
+INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook"
+
+INSTALL_FILES="\
+scripts/images/eks-server/k3s.initd:/etc/init.d/k3s \
+scripts/images/eks-server/eks-token-webhook.initd:/etc/init.d/eks-token-webhook \
+scripts/images/eks-server/k3s-first-boot.initd:/etc/init.d/k3s-first-boot \
+scripts/images/eks-server/k3s-first-boot.sh:/usr/local/sbin/k3s-first-boot \
+scripts/images/eks-server/mulga-eks-state-report.sh:/etc/periodic/15min/mulga-eks-state-report \
+scripts/images/eks-server/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot"
+
+ENABLE_SERVICES="crond k3s eks-token-webhook k3s-first-boot"
+
+SETUP_SCRIPT="scripts/images/eks-server/setup.sh"
+
+AMI_NAME="spinifex-eks-server"
+SYSTEM_TAG="spinifex:managed-by=eks"

scripts/images/eks-server/mulga-eks-etcd-snapshot.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+set -eu
+
+# mulga-eks-etcd-snapshot — nightly K3s embedded etcd snapshot to predastore.
+# Run from /etc/periodic/daily (Alpine crond).
+# K3s has a built-in `k3s etcd-snapshot` command that handles the snapshot
+# semantics; this wrapper exposes it on cron and uploads to the system-bucket
+# (see eks-v1.md Q6 + Q17).
+#
+# Bucket layout (Q17):
+#   eks-backups-system/{accountID}/{clusterName}/etcd-{timestamp}.snap
+# Retention: managed via S3 lifecycle policy on the bucket; this script does
+# not prune.
+
+ENVFILE=/etc/spinifex-eks/etcd-snapshot.env
+[ -f "${ENVFILE}" ] || { logger -t mulga-eks-etcd-snapshot "${ENVFILE} missing"; exit 0; }
+# shellcheck disable=SC1090
+. "${ENVFILE}"
+
+: "${EKS_ACCOUNT_ID:?}"
+: "${EKS_CLUSTER_NAME:?}"
+: "${SPINIFEX_PREDASTORE_ENDPOINT:?}"
+: "${SPINIFEX_PREDASTORE_AKID:?}"
+: "${SPINIFEX_PREDASTORE_SECRET:?}"
+
+SNAPSHOT_DIR=/var/lib/rancher/k3s/server/db/snapshots
+TS=$(date -u +%Y%m%dT%H%M%SZ)
+NAME="etcd-${TS}"
+BUCKET="eks-backups-system"
+KEY="${EKS_ACCOUNT_ID}/${EKS_CLUSTER_NAME}/${NAME}.snap"
+
+/usr/local/bin/k3s etcd-snapshot save --name "${NAME}" 2>&1 | logger -t mulga-eks-etcd-snapshot
+
+SNAPSHOT_FILE=$(ls -t "${SNAPSHOT_DIR}/${NAME}"* 2>/dev/null | head -1 || true)
+if [ -z "${SNAPSHOT_FILE}" ]; then
+    logger -t mulga-eks-etcd-snapshot "snapshot file for ${NAME} not found in ${SNAPSHOT_DIR}"
+    exit 1
+fi
+
+# Upload via curl SigV4. predastore implements the S3 v4 wire so the standard
+# AWS PUT request signs cleanly. aws-cli is not in the image (kept small) — we
+# hand-roll the signed request with curl --aws-sigv4.
+#
+# Note: --aws-sigv4 requires curl >= 7.75 (Alpine 3.21 ships 8.x).
+URL="${SPINIFEX_PREDASTORE_ENDPOINT%/}/${BUCKET}/${KEY}"
+curl -fsSL \
+    --aws-sigv4 "aws:amz:${AWS_REGION:-au-mel-1}:s3" \
+    --user "${SPINIFEX_PREDASTORE_AKID}:${SPINIFEX_PREDASTORE_SECRET}" \
+    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
+    --upload-file "${SNAPSHOT_FILE}" \
+    "${URL}" 2>&1 | logger -t mulga-eks-etcd-snapshot
+
+# Remove the local snapshot to bound disk usage; the canonical copy lives in
+# predastore. Restore path (`spx admin eks restore-snapshot`) pulls from there.
+rm -f "${SNAPSHOT_FILE}"
+logger -t mulga-eks-etcd-snapshot "uploaded ${KEY}"

scripts/images/eks-server/mulga-eks-state-report.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -eu
+
+# mulga-eks-state-report — periodic NATS publish of K3s server health.
+# Invoked every 15 minutes via /etc/periodic/15min (Alpine crond). One-shot
+# best-effort; failures log but do not retry — the spinifex reconciler also
+# does its own NLB healthcheck so this is supplementary signal.
+#
+# Subject: eks.state.{accountID}.{clusterName}.server
+# Payload: JSON { healthz: "ok"|"fail", node_count: N, ts: <unix-seconds> }
+
+ENVFILE=/etc/spinifex-eks/state-report.env
+[ -f "${ENVFILE}" ] || { logger -t mulga-eks-state-report "${ENVFILE} missing — exiting"; exit 0; }
+# shellcheck disable=SC1090
+. "${ENVFILE}"
+
+: "${SPINIFEX_NATS_URL:?}"
+: "${EKS_ACCOUNT_ID:?}"
+: "${EKS_CLUSTER_NAME:?}"
+
+KUBECONFIG=${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}
+export KUBECONFIG
+
+if curl -sk --max-time 3 https://127.0.0.1:6443/healthz | grep -q '^ok$'; then
+    HEALTH=ok
+else
+    HEALTH=fail
+fi
+
+if [ "${HEALTH}" = "ok" ]; then
+    NODE_COUNT=$(kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ')
+else
+    NODE_COUNT=0
+fi
+
+TS=$(date +%s)
+PAYLOAD=$(printf '{"healthz":"%s","node_count":%s,"ts":%s}' "${HEALTH}" "${NODE_COUNT}" "${TS}")
+SUBJ="eks.state.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.server"
+
+NATS_ARGS="-s ${SPINIFEX_NATS_URL}"
+if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then
+    NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}"
+fi
+
+# shellcheck disable=SC2086
+printf '%s' "${PAYLOAD}" | nats ${NATS_ARGS} pub "${SUBJ}" --stdin 2>&1 \
+    | logger -t mulga-eks-state-report

scripts/images/eks-server/setup.sh

@@ -0,0 +1,85 @@
+#!/bin/sh
+set -eu
+
+# setup.sh — chroot customisation for the eks-server AMI.
+#
+# Runs inside an Alpine 3.21 chroot under build-system-image.sh after packages
+# and binaries are installed. Downloads the pinned K3s binary, verifies its
+# SHA256, drops it into /usr/local/bin, sets executable bits on init scripts +
+# cron entries, and writes the K3s server config skeleton.
+#
+# Network access inside the chroot is provided by the host's /etc/resolv.conf
+# (build-system-image.sh copies it in). curl is in APK_PACKAGES.
+
+K3S_VERSION="v1.32.5+k3s1"
+K3S_URL_BASE="https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}"
+
+# Pull the upstream signed checksums file and pin the amd64 line. A tampered
+# release replacing both files would still produce a self-consistent download,
+# so the SHA file URL is the trust anchor — anyone forging a Mulga AMI build
+# would need to compromise the k3s-io GitHub release artefacts.
+echo "[eks-server-setup] fetching k3s checksums ${K3S_VERSION}"
+curl -fsSL -o /tmp/k3s-checksums.txt "${K3S_URL_BASE}/sha256sum-amd64.txt"
+K3S_SHA256=$(awk '/[ \t]k3s$/{print $1; exit}' /tmp/k3s-checksums.txt)
+if [ -z "${K3S_SHA256}" ]; then
+    echo "[eks-server-setup] could not parse k3s sha256 from upstream checksums"
+    cat /tmp/k3s-checksums.txt
+    exit 1
+fi
+echo "[eks-server-setup] downloading k3s ${K3S_VERSION} (sha256=${K3S_SHA256})"
+curl -fsSL -o /usr/local/bin/k3s "${K3S_URL_BASE}/k3s"
+echo "${K3S_SHA256}  /usr/local/bin/k3s" > /tmp/k3s.sha256
+if ! sha256sum -c /tmp/k3s.sha256; then
+    echo "[eks-server-setup] k3s SHA256 mismatch — refusing to bake AMI"
+    exit 1
+fi
+chmod 0755 /usr/local/bin/k3s
+ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl
+ln -sf /usr/local/bin/k3s /usr/local/bin/crictl
+ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
+
+# nats-cli is not in Alpine main/community — pull the upstream release binary.
+# Used by k3s-first-boot.sh (one-shot publishes) and mulga-eks-state-report.sh
+# (periodic publishes).
+NATS_CLI_VERSION="0.4.0"
+NATS_CLI_URL="https://github.com/nats-io/natscli/releases/download/v${NATS_CLI_VERSION}/nats-${NATS_CLI_VERSION}-linux-amd64.zip"
+echo "[eks-server-setup] downloading nats-cli ${NATS_CLI_VERSION}"
+curl -fsSL -o /tmp/nats-cli.zip "${NATS_CLI_URL}"
+apk add --no-cache unzip
+unzip -q -d /tmp/nats-cli /tmp/nats-cli.zip
+install -m 0755 /tmp/nats-cli/nats-${NATS_CLI_VERSION}-linux-amd64/nats /usr/local/bin/nats
+rm -rf /tmp/nats-cli /tmp/nats-cli.zip
+apk del --no-cache unzip
+
+# Init scripts ship as 0644 from INSTALL_FILES; OpenRC requires 0755.
+chmod 0755 /etc/init.d/k3s /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot
+chmod 0755 /usr/local/sbin/k3s-first-boot
+chmod 0755 /etc/periodic/15min/mulga-eks-state-report
+chmod 0755 /etc/periodic/daily/mulga-eks-etcd-snapshot
+
+# K3s server config — empty skeleton; cloud-init / first-boot fills in the
+# per-cluster fields (cluster-cidr, service-cidr, token-file, etc).
+# Webhook-token-auth is NOT wired in the skeleton: enabling it before the
+# eks-token-webhook binary is the real implementation (currently a 503 stub,
+# replaced by cs-eks-6c) would block /healthz from anonymous callers. The
+# Sprint 6c orchestrator drops a webhook-config.yaml + restarts k3s once
+# the binary is the real one.
+mkdir -p /etc/rancher/k3s
+cat > /etc/rancher/k3s/config.yaml.skel <<'EOF'
+# Populated at first boot by cloud-init user-data via k3s-first-boot.sh.
+# Fields documented at https://docs.k3s.io/installation/configuration.
+cluster-init: true
+disable:
+  - traefik
+  - servicelb
+  - local-storage
+EOF
+
+# Sentinel file marker — k3s-first-boot self-disables after first success by
+# checking for this path. Initial state is "not yet run".
+touch /var/lib/spinifex-eks/first-boot.pending 2>/dev/null || {
+    mkdir -p /var/lib/spinifex-eks
+    touch /var/lib/spinifex-eks/first-boot.pending
+}
+
+echo "[eks-server-setup] done"

scripts/images/ubuntu-gpu-amd-setup.sh

@@ -17,34 +17,85 @@ export DEBIAN_FRONTEND=noninteractive
 apt-get update -qq
 apt-get install -y --no-install-recommends curl gnupg ca-certificates
 
-# AMD ROCm apt repository — rocminfo and rocm-smi-lib are not in the Ubuntu
-# default repos, so we add AMD's signed repo before the main package install.
 UBUNTU_CODENAME=$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME}}")
+
+# AMD ROCm 7.2 repo — pinned to "noble" (24.04) since AMD hasn't published a
+# "resolute" (26.04) dist yet; the noble packages are ABI-compatible.
 mkdir -p /etc/apt/keyrings
 curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key \
     | gpg --dearmor -o /etc/apt/keyrings/rocm.gpg
-echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.3 ${UBUNTU_CODENAME} main" \
+echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/7.2.3 noble main" \
     > /etc/apt/sources.list.d/rocm.list
-apt-get update -qq
+
+# The Ubuntu minimal cloud image already ships with a kernel. Detect it and
+# install matching headers — do NOT install linux-image-generic, which pulls a
+# newer ABI the bootloader won't know about (grub-install can't run in a chroot).
+KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||') || true
+if [[ -z "${KVER}" ]]; then
+    # Ubuntu 26.04 minimal cloud image keeps the kernel on the ESP (not mounted
+    # in the chroot), so /boot is empty — fall back to /lib/modules.
+    KVER=$(ls /lib/modules/ 2>/dev/null | sort -V | tail -1)
+fi
+if [[ -z "${KVER}" ]]; then
+    echo "ERROR: No kernel found in /boot or /lib/modules — base image may be missing a kernel"
+    exit 1
+fi
+echo "Target kernel: ${KVER}"
+
+# Prevent kernel postinst hooks from attempting grub-install against the host disk.
+cat > /etc/kernel-img.conf <<'EOF'
+do_symlinks = yes
+do_bootloader = no
+do_initrd = yes
+link_in_boot = yes
+EOF
 
 # linux-firmware carries the AMD GPU firmware blobs loaded by the amdgpu
 # kernel module at boot. Without them the GPU is invisible to guest userland.
 apt-get install -y -o Acquire::Retries=3 --no-install-recommends \
-    linux-image-generic \
-    linux-headers-generic \
+    "linux-headers-${KVER}" \
     linux-firmware \
     initramfs-tools \
     pciutils \
-    rocminfo rocm-smi-lib \
     python3 python3-venv python3-pip \
     git curl wget htop tmux \
     ffmpeg libgl1 libglib2.0-0
 
-KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||')
-if [ -z "${KVER}" ]; then
-    echo "ERROR: No kernel found under /boot"
-    exit 1
+apt-get update -qq
+apt-get install -y --no-install-recommends rocminfo rocm-smi-lib amd-smi-lib
+
+# Create /opt/rocm symlink (normally done by rocm-core) and add to PATH.
+ROCM_DIR=$(ls -d /opt/rocm-* 2>/dev/null | sort -V | tail -1)
+if [[ -n "${ROCM_DIR}" ]]; then
+    ln -sfn "${ROCM_DIR}" /opt/rocm
+    echo "export PATH=\"/opt/rocm/bin:\$PATH\"" > /etc/profile.d/rocm.sh
+    chmod 644 /etc/profile.d/rocm.sh
 fi
+
+# ── Docker CE ─────────────────────────────────────────────────────────────────
+# AMD GPU containers use device passthrough — no separate container runtime
+# is needed. Users pass --device=/dev/kfd --device=/dev/dri at run time.
+mkdir -p /usr/share/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg \
+    | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${UBUNTU_CODENAME} stable" \
+    > /etc/apt/sources.list.d/docker.list
+apt-get update -qq
+apt-get install -y --no-install-recommends \
+    docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+systemctl enable docker
+
+# Ensure render + video groups exist for /dev/kfd and /dev/dri access.
+groupadd -f render
+groupadd -f video
+
+# Add the cloud-init default user (UID 1000) to docker/render/video at first boot.
+mkdir -p /etc/cloud/cloud.cfg.d
+cat > /etc/cloud/cloud.cfg.d/99-gpu-groups.cfg <<'EOF'
+runcmd:
+  - usermod -aG docker,render,video $(id -un 1000) || true
+EOF
+
 echo "Rebuilding initramfs for kernel: ${KVER}"
 update-initramfs -u -k "${KVER}"
 
@@ -56,7 +107,9 @@ Unattended-Upgrade::Package-Blacklist {
     "linux-firmware";
     "linux-image-";
     "linux-headers-";
+    "docker-";
+    "containerd";
 };
 EOF
 
-echo "AMD GPU image setup complete: kernel=${KVER}, linux-firmware and ROCm userland installed"
+echo "AMD GPU image setup complete: kernel=${KVER}, linux-firmware and ROCm userland installed, Docker ready"

scripts/images/ubuntu-gpu-amd.conf

@@ -4,9 +4,11 @@ DISTRO="ubuntu"
 UBUNTU_VERSION="26.04"
 UBUNTU_CODENAME="resolute"
 
-# 12G provides headroom for kernel headers, linux-firmware, and ROCm packages
-IMAGE_SIZE="12G"
+# 16G provides headroom for kernel headers, linux-firmware, ROCm packages, and Docker CE.
+IMAGE_SIZE="16G"
 
 SETUP_SCRIPT="scripts/images/ubuntu-gpu-amd-setup.sh"
 
+AMI_NAME="ubuntu-amd-gpu"
+
 SYSTEM_TAG="gpu-vendor=amd"

scripts/images/ubuntu-gpu-nvidia-setup.sh

@@ -19,31 +19,61 @@ set -euo pipefail
 
 export DEBIAN_FRONTEND=noninteractive
 
-apt-get update -qq
+apt-get update
 
-# Detect the cloud image's existing kernel BEFORE installing anything.
-# Installing linux-image-generic would upgrade to a newer kernel version;
-# DKMS would then build for that version but the VM boots the original
-# cloud image kernel, causing a version mismatch at modprobe time.
-KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||')
+# The Ubuntu minimal cloud image already ships with a kernel. We detect it and
+# build DKMS against it — do NOT install linux-image-generic, which pulls a
+# newer kernel ABI that won't boot (grub-install can't run in a chroot, so the
+# bootloader never learns about the new kernel and keeps booting the original).
+apt-get install -y --no-install-recommends initramfs-tools
+
+echo "=== /boot ==="
+ls -la /boot/ || true
+echo "=== /lib/modules ==="
+ls /lib/modules/ || true
+echo "=== installed kernel packages ==="
+dpkg -l 'linux-image-*' | grep '^ii' || true
+
+KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||') || true
 if [[ -z "${KVER}" ]]; then
-    echo "ERROR: No kernel found under /boot"
+    # Ubuntu 26.04 minimal cloud image keeps the kernel on the ESP (not mounted
+    # in the chroot), so /boot is empty — fall back to /lib/modules.
+    KVER=$(ls /lib/modules/ 2>/dev/null | sort -V | tail -1)
+fi
+if [[ -z "${KVER}" ]]; then
+    echo "ERROR: No kernel found in /boot or /lib/modules — base image may be missing a kernel"
     exit 1
 fi
 echo "Target kernel: ${KVER}"
 
-# Install headers for the exact kernel already present, plus initramfs-tools.
-apt-get install -y --no-install-recommends \
-    "linux-headers-${KVER}" \
-    initramfs-tools
+# Prevent kernel postinst hooks from attempting grub-install against the host
+# disk — this fails silently in a chroot and can abort the postinst, leaving
+# /boot incomplete.
+cat > /etc/kernel-img.conf <<'EOF'
+do_symlinks = yes
+do_bootloader = no
+do_initrd = yes
+link_in_boot = yes
+EOF
 
-# Install the NVIDIA server DKMS driver. Use the distro default version (no
-# hardcoded suffix) so Ubuntu 26.04's 580 driver is used instead of 550.
-# nvidia-dkms-server is a meta package that resolves to the current recommended
-# version; nvidia-utils-server brings nvidia-smi and related tools.
+# Install headers for the exact pre-installed kernel so DKMS builds against it.
 apt-get install -y --no-install-recommends \
-    nvidia-dkms-server \
-    nvidia-utils-server
+    "linux-headers-${KVER}"
+
+# Detect the latest available versioned NVIDIA server driver. Ubuntu 26.04+
+# no longer ships unversioned meta-packages (nvidia-dkms-server); the packages
+# are now versioned (e.g. nvidia-dkms-570-server).
+NVIDIA_VER=$(apt-cache search '^nvidia-dkms-[0-9]+-server$' 2>/dev/null \
+    | grep -oP '(?<=nvidia-dkms-)\d+(?=-server)' | sort -rn | head -1)
+if [[ -z "${NVIDIA_VER}" ]]; then
+    echo "ERROR: No versioned nvidia-dkms-*-server package found in apt cache"
+    apt-cache search nvidia-dkms || true
+    exit 1
+fi
+echo "Installing NVIDIA server driver version: ${NVIDIA_VER}"
+apt-get install -y --no-install-recommends \
+    "nvidia-dkms-${NVIDIA_VER}-server" \
+    "nvidia-utils-${NVIDIA_VER}-server"
 
 # Detect the installed NVIDIA DKMS module name + version for explicit build.
 NVIDIA_DKMS_NAME=$(dkms status 2>/dev/null | awk -F'[,/ ]+' '/nvidia/{print $1; exit}')
@@ -91,17 +121,51 @@ apt-get install -y -o Acquire::Retries=3 \
     git curl wget htop tmux \
     ffmpeg libgl1 libglib2.0-0
 
+# ── Docker CE + nvidia-container-toolkit ──────────────────────────────────────
+UBUNTU_CODENAME=$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME}}")
+apt-get install -y --no-install-recommends gnupg ca-certificates
+mkdir -p /usr/share/keyrings /etc/apt/sources.list.d
+
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg \
+    | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${UBUNTU_CODENAME} stable" \
+    > /etc/apt/sources.list.d/docker.list
+
+curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
+    | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
+    | sed 's|deb https://|deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://|g' \
+    > /etc/apt/sources.list.d/nvidia-container-toolkit.list
+
+apt-get update -qq
+apt-get install -y --no-install-recommends \
+    docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \
+    nvidia-container-toolkit
+
+# Wire nvidia runtime into Docker so --gpus all works on first boot.
+nvidia-ctk runtime configure --runtime=docker
+systemctl enable docker
+
+# Add the cloud-init default user (UID 1000) to docker at first boot.
+mkdir -p /etc/cloud/cloud.cfg.d
+cat > /etc/cloud/cloud.cfg.d/99-gpu-groups.cfg <<'EOF'
+runcmd:
+  - usermod -aG docker $(id -un 1000) || true
+EOF
+
 mkdir -p /etc/apt/apt.conf.d
 cat > /etc/apt/apt.conf.d/99-gpu-ami <<'EOF'
 Unattended-Upgrade::Package-Blacklist {
     "linux-";
     "nvidia-";
     "libnvidia-";
+    "docker-";
+    "containerd";
+    "nvidia-container-toolkit";
 };
 EOF
 
 # Rebuild initramfs with nouveau blacklisted and NVIDIA module included.
 update-initramfs -u -k "${KVER}"
 
-
-echo "NVIDIA GPU image setup complete: kernel=${KVER}, NVIDIA driver + DKMS pre-built"
+echo "NVIDIA GPU image setup complete: kernel=${KVER}, NVIDIA driver + DKMS pre-built, Docker + nvidia-container-toolkit installed"

scripts/images/ubuntu-gpu-nvidia.conf

@@ -4,9 +4,12 @@ DISTRO="ubuntu"
 UBUNTU_VERSION="26.04"
 UBUNTU_CODENAME="resolute"
 
-# 12G provides headroom for kernel headers, NVIDIA driver, and DKMS build artefacts
-IMAGE_SIZE="12G"
+# 16G provides headroom for kernel headers, NVIDIA driver, DKMS build artefacts,
+# Docker CE, and nvidia-container-toolkit.
+IMAGE_SIZE="16G"
 
 SETUP_SCRIPT="scripts/images/ubuntu-gpu-nvidia-setup.sh"
 
+AMI_NAME="ubuntu-nvidia-gpu"
+
 SYSTEM_TAG="gpu-vendor=nvidia"

scripts/reset-dev-env.sh

@@ -170,9 +170,10 @@ if [ -e /etc/systemd/network/15-spinifex-veth-wan.netdev ] || \
     sudo networkctl reload 2>/dev/null || true
 fi
 
-# Remove macvlan interfaces created by setup-ovn.sh
+# Legacy: clean up macvlan interfaces left over from pre-veth setup-ovn.sh
+# runs. Safe to remove once no operator system retains these.
 for iface in $(ip -o link show type macvlan 2>/dev/null | awk -F': ' '{print $2}' | grep '^spx-ext-'); do
-    echo "  Deleting macvlan: $iface"
+    echo "  Deleting legacy macvlan: $iface"
     sudo ip link del "$iface" 2>/dev/null || true
 done
 

scripts/setup-ovn.sh

@@ -16,9 +16,7 @@
 # Options:
 #   --management         Also start OVN central services (NB DB, SB DB, ovn-northd)
 #   --wan-bridge=NAME    OVS bridge for WAN traffic (default: auto-detect from default route)
-#   --wan-iface=NAME     Physical NIC to add to the WAN bridge (use with --wan-bridge or --macvlan)
-#   --macvlan            Create macvlan off --wan-iface instead of moving NIC directly.
-#                        SSH-safe for single-NIC hosts where WAN NIC carries SSH.
+#   --wan-iface=NAME     Physical NIC to add to the WAN bridge (use with --wan-bridge)
 #   --dhcp               Obtain gateway IP via DHCP on the WAN bridge interface
 #   --mgmt-bridge=NAME   OVS bridge for system-instance control plane (default: br-mgmt)
 #   --mgmt-cidr=CIDR     IPv4 CIDR to assign on the mgmt bridge (default: 10.15.8.1/24)
@@ -41,9 +39,6 @@
 #   # Dedicated WAN NIC (not your SSH NIC — you take responsibility):
 #   ./scripts/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1
 #
-#   # Single-NIC host (SSH-safe macvlan):
-#   ./scripts/setup-ovn.sh --management --macvlan --wan-iface=eth0
-#
 #   # Compute node joining an existing cluster:
 #   ./scripts/setup-ovn.sh --ovn-remote=tcp:10.0.0.1:6642 --encap-ip=10.0.0.2
 #
@@ -56,7 +51,6 @@ set -e
 MANAGEMENT=false
 WAN_BRIDGE=""
 WAN_IFACE=""
-MACVLAN_MODE=false
 EXTERNAL_DHCP=false
 MGMT_BRIDGE_ENABLED=true
 MGMT_BRIDGE="br-mgmt"
@@ -64,12 +58,19 @@ MGMT_CIDR="10.15.8.1/24"
 MGMT_IFACE=""
 OVN_REMOTE="tcp:127.0.0.1:6642"
 ENCAP_IP=""
+# NODE_NAME is left empty by default. The chassis-id pin block at Step 4
+# only runs when --node-name=NAME is explicitly given. Passing nothing
+# preserves whatever system-id already lives in OVS (gold-image UUID,
+# bootstrap-install.sh's pre-written nodeN, manual ansible value, etc.).
+# Bootstrap callers that want IPsec-cert-identity matching pin the
+# system-id themselves before / after invoking setup-ovn.sh — they
+# don't need this script to do it for them.
+NODE_NAME=""
 
 # Parse arguments
 for arg in "$@"; do
     case "$arg" in
         --management)       MANAGEMENT=true ;;
-        --macvlan)          MACVLAN_MODE=true ;;
         --dhcp)             EXTERNAL_DHCP=true ;;
         --wan-bridge=*)     WAN_BRIDGE="${arg#*=}" ;;
         --wan-iface=*)      WAN_IFACE="${arg#*=}" ;;
@@ -79,6 +80,7 @@ for arg in "$@"; do
         --no-mgmt-bridge)   MGMT_BRIDGE_ENABLED=false ;;
         --ovn-remote=*)     OVN_REMOTE="${arg#*=}" ;;
         --encap-ip=*)       ENCAP_IP="${arg#*=}" ;;
+        --node-name=*)      NODE_NAME="${arg#*=}" ;;
         --help|-h)
             head -50 "$0" | tail -48
             exit 0
@@ -92,15 +94,13 @@ done
 
 # --- WAN bridge auto-detection ---
 # Determine the WAN bridge name and how to set it up.
-WAN_BRIDGE_MODE=""  # "existing", "veth", "direct", "macvlan", or ""
+WAN_BRIDGE_MODE=""  # "existing", "veth", "direct", or ""
 LINUX_BRIDGE=""     # Set when WAN_BRIDGE_MODE="veth" — the Linux bridge behind the veth pair
 
 detect_wan_bridge() {
     # If --wan-bridge was explicitly given, use it
     if [ -n "$WAN_BRIDGE" ]; then
-        if [ "$MACVLAN_MODE" = true ] && [ -n "$WAN_IFACE" ]; then
-            WAN_BRIDGE_MODE="macvlan"
-        elif [ -n "$WAN_IFACE" ]; then
+        if [ -n "$WAN_IFACE" ]; then
             WAN_BRIDGE_MODE="direct"
         elif sudo ovs-vsctl br-exists "$WAN_BRIDGE" 2>/dev/null; then
             WAN_BRIDGE_MODE="existing"
@@ -111,17 +111,6 @@ detect_wan_bridge() {
         return
     fi
 
-    # If --macvlan was given without --wan-bridge, we need --wan-iface
-    if [ "$MACVLAN_MODE" = true ]; then
-        if [ -z "$WAN_IFACE" ]; then
-            echo "ERROR: --macvlan requires --wan-iface=<NIC>"
-            exit 1
-        fi
-        WAN_BRIDGE="br-wan"
-        WAN_BRIDGE_MODE="macvlan"
-        return
-    fi
-
     # Auto-detect: find the default route interface
     local default_dev
     default_dev=$(ip -4 route show default 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="dev") print $(i+1)}' | head -1)
@@ -176,10 +165,7 @@ detect_wan_bridge() {
     echo "  2. Dedicated WAN NIC (NOT your SSH connection):"
     echo "     ./scripts/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=$default_dev"
     echo ""
-    echo "  3. Single-NIC host (SSH-safe macvlan):"
-    echo "     ./scripts/setup-ovn.sh --management --macvlan --wan-iface=$default_dev"
-    echo ""
-    echo "  4. No external networking (overlay-only):"
+    echo "  3. No external networking (overlay-only):"
     echo "     ./scripts/setup-ovn.sh --management --encap-ip=$wan_ip"
     echo "============================================================"
     echo ""
@@ -224,32 +210,31 @@ echo "  OVN Remote (SB):  $OVN_REMOTE"
 echo "  Encap IP:         $ENCAP_IP"
 echo ""
 
-# --- Step 1: Install packages ---
-echo "Step 1: Checking OVN/OVS packages..."
+# Packages (openvswitch-switch, ovn-host, openvswitch-ipsec, strongswan-charon,
+# and ovn-central on management nodes) are baked into the gold image via
+# scripts/tofu-cluster/image-builder/scripts/provision.sh. Runtime apt-get
+# was removed to keep CI off the (flaky) apt-cacher-ng path. If a package is
+# missing, the downstream ovs/ovn commands will fail loudly — the fix is to
+# rebuild the gold image, not to re-add apt-get here.
 
-install_packages() {
-    local missing=()
-    for pkg in openvswitch-switch ovn-host; do
-        if ! dpkg -s "$pkg" >/dev/null 2>&1; then
-            missing+=("$pkg")
-        fi
-    done
-    if [ "$MANAGEMENT" = true ]; then
-        if ! dpkg -s ovn-central >/dev/null 2>&1; then
-            missing+=("ovn-central")
-        fi
-    fi
-
-    if [ ${#missing[@]} -gt 0 ]; then
-        echo "  Installing: ${missing[*]}"
-        sudo apt-get update -qq
-        sudo apt-get install -y -qq "${missing[@]}"
-    else
-        echo "  All packages installed"
-    fi
-}
-
-install_packages
+# strongswan-charon ships an AppArmor profile for /usr/lib/ipsec/charon that
+# only allows reading from /etc/ipsec.*, /etc/strongswan.*, and a few other
+# fixed paths. ovs-monitor-ipsec writes the per-tunnel strongSwan config with
+# absolute paths to our peer cert + key under /etc/spinifex/ipsec/ AND to the
+# cluster CA at /etc/spinifex/ca.pem, so charon hits "Permission denied" on
+# both load paths and surfaces as `no trusted RSA public key found for
+# '<peer>'` (CA can't be loaded → can't validate peer cert chain → no SAs).
+# The profile includes a 'local' override file expressly for site additions
+# — grant reads on /etc/spinifex/** (covers ca.pem AND ipsec/peer.{pem,key})
+# and reload.
+if [ -d /etc/apparmor.d/local ]; then
+    LOCAL_OVERRIDE=/etc/apparmor.d/local/usr.lib.ipsec.charon
+    # Always rewrite — re-runs may have an older, narrower rule (e.g. the
+    # initial /etc/spinifex/ipsec/** only) that misses /etc/spinifex/ca.pem.
+    echo "  Adding AppArmor read grant for /etc/spinifex (cert + CA) to charon profile"
+    echo "/etc/spinifex/** r," | sudo tee "$LOCAL_OVERRIDE" >/dev/null
+    sudo apparmor_parser -r /etc/apparmor.d/usr.lib.ipsec.charon 2>/dev/null || true
+fi
 
 # --- Step 2: Enable services ---
 echo ""
@@ -296,7 +281,6 @@ echo "  br-int: created, fail-mode=secure, up"
 # rule => default management => no carrier => link down). OVS dataplane
 # still forwards, but ovn-controller flow programming and any tooling that
 # probes link state misbehave. Unmanaged=yes keeps OVS in sole control.
-# (mulga-siv-37)
 OVS_INTERNAL_NET=/etc/systemd/network/05-spinifex-ovs-internal.network
 if [ ! -f "$OVS_INTERNAL_NET" ]; then
     sudo tee "$OVS_INTERNAL_NET" >/dev/null <<'NETWORK'
@@ -477,36 +461,6 @@ NETWORK
             echo "  $WAN_BRIDGE: direct bridge on $WAN_IFACE"
             echo "  NOTE: $WAN_IFACE is now an OVS port — no host IP on this NIC"
             ;;
-
-        macvlan)
-            # Create a macvlan sub-interface in bridge mode off the WAN NIC.
-            # The host keeps its IP on the parent NIC — SSH-safe. OVN localnet
-            # traffic flows through the macvlan to the physical wire.
-            if ! ip link show "$WAN_IFACE" >/dev/null 2>&1; then
-                echo "  ERROR: interface $WAN_IFACE does not exist"
-                echo "  Available interfaces:"
-                ip -o link show | awk -F': ' '{print "    " $2}'
-                exit 1
-            fi
-
-            MACVLAN_NAME="spx-ext-${WAN_IFACE}"
-
-            sudo ovs-vsctl --may-exist add-br "$WAN_BRIDGE"
-            sudo ip link set "$WAN_BRIDGE" up
-
-            if ip link show "$MACVLAN_NAME" >/dev/null 2>&1; then
-                echo "  macvlan $MACVLAN_NAME already exists"
-            else
-                sudo ip link add "$MACVLAN_NAME" link "$WAN_IFACE" type macvlan mode bridge
-                echo "  created macvlan: $MACVLAN_NAME (bridge mode) on $WAN_IFACE"
-            fi
-
-            sudo ip link set "$MACVLAN_NAME" up
-            sudo ovs-vsctl --may-exist add-port "$WAN_BRIDGE" "$MACVLAN_NAME"
-            echo "  $WAN_BRIDGE: macvlan port $MACVLAN_NAME on $WAN_IFACE"
-            echo "  NOTE: host keeps its IP on $WAN_IFACE (SSH-safe)"
-            echo "  QUIRK: host cannot reach VMs at their public IPs (macvlan isolation)"
-            ;;
     esac
 
     # --- DHCP: obtain gateway IP for OVN SNAT ---
@@ -514,13 +468,8 @@ NETWORK
         echo ""
         echo "Step 3c: Obtaining external gateway IP via DHCP..."
 
-        # For macvlan mode, DHCP on the macvlan interface (it has L2 access to WAN).
-        # For direct/existing bridge, DHCP on the bridge itself.
-        if [ "$WAN_BRIDGE_MODE" = "macvlan" ]; then
-            DHCP_IFACE="spx-ext-${WAN_IFACE}"
-        else
-            DHCP_IFACE="$WAN_BRIDGE"
-        fi
+        # DHCP on the WAN bridge itself (direct/existing/veth).
+        DHCP_IFACE="$WAN_BRIDGE"
 
         # Run DHCP client to get a lease
         if command -v dhcpcd >/dev/null 2>&1; then
@@ -629,11 +578,33 @@ else
         external_ids:ovn-encap-type="geneve"
 fi
 
-# system-id is owned by the openvswitch-switch package (persisted in
-# /etc/openvswitch/system-id.conf and re-applied on every boot). Read it back
-# rather than overriding — overriding here would drift from the on-disk value
-# and the next reboot would silently flip the chassis identity (mulga-999).
-echo "  system-id:      $(sudo ovs-vsctl get open . external_ids:system-id)"
+# Pin OVS system-id (= OVN chassis-id) to NODE_NAME when --node-name was
+# explicitly given. Reasons to pin:
+#   1. IPsec identity: ovs-monitor-ipsec uses chassis-id as the IKEv2
+#      `@<name>` peer identity. Our per-node IPsec peer cert
+#      (admin.GenerateIPSecPeerCert) carries the cluster node name as CN
+#      + dnsName SAN — see spx admin init/join, which take --node NAME and
+#      bake NAME into the cert. Leaving chassis-id as the package-generated
+#      UUID would cause `received AUTHENTICATION_FAILED` because charon
+#      validates `@<UUID>` against a cert dnsName=<NODE_NAME>.
+#   2. Ops legibility: `ovn-sbctl show` lists chassis by name, not UUID.
+#
+# When --node-name is NOT given (CI bootstrap-install.sh today, dev
+# single-node bring-up, ansible roles that don't yet pass it), leave the
+# system-id alone. A hostname fallback used to live here, but on CI
+# single-node it rewrote system-id from the gold-image UUID to a
+# different hostname while the existing SBDB chassis row kept name=UUID;
+# vpcd's discoverChassis then logged "skipping stale local chassis" and
+# refused to start. Preserving the on-disk value is always safe — any
+# caller that needs IPsec cert identity matching pins it themselves.
+if [ -n "$NODE_NAME" ]; then
+    echo "$NODE_NAME" | sudo tee /etc/openvswitch/system-id.conf >/dev/null
+    sudo ovs-vsctl set Open_vSwitch . external_ids:system-id="$NODE_NAME"
+    echo "  system-id:      $NODE_NAME (pinned via --node-name)"
+else
+    CURRENT_ID=$(sudo ovs-vsctl get Open_vSwitch . external_ids:system-id 2>/dev/null | tr -d '"')
+    echo "  system-id:      ${CURRENT_ID:-<unset>} (preserved; no --node-name given)"
+fi
 echo "  ovn-remote:     $OVN_REMOTE"
 echo "  ovn-encap-ip:   $ENCAP_IP"
 echo "  ovn-encap-type: geneve"
@@ -779,8 +750,20 @@ echo ""
 echo "Step 10: Enabling OVN auto-start on boot..."
 sudo systemctl enable openvswitch-switch 2>/dev/null || true
 sudo systemctl enable ovn-controller 2>/dev/null || true
-echo "  openvswitch-switch: enabled on boot"
-echo "  ovn-controller: enabled on boot"
+# ovs-monitor-ipsec drives strongSwan from OVS DB cert pointers. The daemon's
+# enableOVNIPSec() flips ipsec_encapsulation=true at runtime and silently drops
+# tunnel traffic if this unit isn't already up — enable at provision time so
+# daemon never needs systemd-write capability (only is-active read).
+sudo systemctl enable openvswitch-ipsec.service 2>/dev/null || true
+# When openvswitch-ipsec is installed before strongswan-starter, the deb
+# post-install starts the service against a missing /usr/sbin/ipsec and
+# leaves it in 'failed' state. Clear the failure and restart unconditionally
+# now that strongswan-starter is present.
+sudo systemctl reset-failed openvswitch-ipsec.service 2>/dev/null || true
+sudo systemctl restart openvswitch-ipsec.service 2>/dev/null || true
+echo "  openvswitch-switch:   enabled on boot"
+echo "  ovn-controller:       enabled on boot"
+echo "  openvswitch-ipsec:    enabled on boot"
 
 # --- Step 11: Health check ---
 echo ""
@@ -815,14 +798,6 @@ if [ -n "$WAN_BRIDGE" ]; then
                 echo "  direct bridge:   FAILED ($WAN_IFACE not on $WAN_BRIDGE)"
                 OK=false
             fi
-        elif [ "$WAN_BRIDGE_MODE" = "macvlan" ]; then
-            MACVLAN_NAME="spx-ext-${WAN_IFACE}"
-            if ip link show "$MACVLAN_NAME" >/dev/null 2>&1; then
-                echo "  macvlan:         OK ($MACVLAN_NAME)"
-            else
-                echo "  macvlan:         FAILED ($MACVLAN_NAME not found)"
-                OK=false
-            fi
         fi
     else
         echo "  $WAN_BRIDGE:$(printf '%*s' $((15 - ${#WAN_BRIDGE})) '') FAILED"

scripts/setup.sh

@@ -186,11 +186,13 @@ install_sudoers() {
 spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
 spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
 spinifex-daemon ALL=(root) NOPASSWD: /usr/sbin/dhcpcd
+spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/systemctl is-active openvswitch-ipsec.service
+spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl set NB_Global . ipsec=true
 
 # Spinifex VPC daemon: OVN and OVS read/write, OVN controller status check and DHCP
 spinifex-vpcd ALL=(root) NOPASSWD: /usr/sbin/dhcpcd
 spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
-spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
+spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl, /usr/bin/ovn-appctl
 spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/systemctl is-active --quiet ovn-controller
 spinifex-vpcd ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
 SUDOERS
@@ -214,11 +216,11 @@ install_apt_deps() {
 
         DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq \
             nbdkit \
-            $QEMU_PACKAGES qemu-utils qemu-kvm ovmf qemu-efi-aarch64 less \
+            $QEMU_PACKAGES qemu-utils gdisk qemu-kvm ovmf qemu-efi-aarch64 less \
             libvirt-daemon-system libvirt-clients \
             pciutils \
             jq curl iproute2 netcat-openbsd wget unzip xz-utils file \
-            ovn-central ovn-host openvswitch-switch dhcpcd-base \
+            ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base \
             > /dev/null
 
         info "System dependencies installed"