Merge remote-tracking branch 'origin/feat/reboot-relaunch' into e2e/integration

RebootInstances powered the guest down under -action shutdown=pause and
then ran system_reset + cont in the same QEMU process. On some hosts the
reset left the guest silent in firmware with QMP still answering, while a
fresh QEMU on the same volumes always booted.

A reboot now shuts the guest down over ACPI (forcing it off after 60s, as
before), ends that QEMU, and starts a fresh one on the same mounted volumes,
taps, IMDS datapath and identity, the way nova's soft reboot and Proxmox's
qm reboot do. The instance stays running in Describe, no hooks fire, the
console log is appended rather than truncated, and the existing QMP client
is reconnected in place so the heartbeat carries on.

A relaunched guest that writes nothing to its console within 60s has its
run state, vCPUs, vCPU thread states and registers (sampled twice) logged,
and if it has printed before, its QEMU is replaced once more. A relaunch
that cannot start goes to crash recovery. Stop, terminate and hot-plug
wait for a relaunch in flight so volumes are never sealed under a process
being started. Guest-initiated reboots stay an in-process reset.

The daemon now logs the host kernel, QEMU version and the OVMF package
and checksum at start, and every reboot relaunch carries them.

.github/e2e-cells.json

@@ -8,7 +8,8 @@
     "arch": "x86_64",
     "runner": "ci-single",
     "extra": "",
-    "suite": ""
+    "suite": "",
+    "timeout_minutes": 60
   },
   {
     "cell": 2,
@@ -19,7 +20,8 @@
     "arch": "x86_64",
     "runner": "ci-single",
     "extra": "",
-    "suite": ""
+    "suite": "",
+    "timeout_minutes": 60
   },
   {
     "cell": 3,
@@ -30,7 +32,8 @@
     "arch": "x86_64",
     "runner": "ci-single",
     "extra": "",
-    "suite": ""
+    "suite": "",
+    "timeout_minutes": 60
   },
   {
     "cell": 8,
@@ -200,7 +203,8 @@
     "arch": "x86_64",
     "runner": "ci-single",
     "extra": "",
-    "suite": ""
+    "suite": "",
+    "timeout_minutes": 60
   },
   {
     "cell": 35,

.github/scripts/hypervisor-disk-preflight.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Usage: hypervisor-disk-preflight.sh <env.tfvars>
+#
+# Fails fast when a hypervisor named in the tofu env is short of disk, before
+# `tofu apply` tries to define a domain there. libvirt writes each domain's XML
+# under /etc/libvirt/qemu on the root filesystem and its disks into the pool, and
+# a full one fails the apply minutes in as "Domain Creation Failed".
+#
+# Hosts are reached the way the libvirt provider reaches them (ssh <user>@<name>,
+# resolved through ~/.ssh/config). A host that cannot be reached is reported and
+# skipped: tofu's own error for it is clearer than anything said here.
+#
+# Env: HV_SSH_USER (tf-user), HV_MIN_ROOT_FREE_MB (1024), HV_MIN_POOL_FREE_GB (20).
+set -uo pipefail
+
+tfvars="${1:?usage: $0 <env.tfvars>}"
+user="${HV_SSH_USER:-tf-user}"
+min_root_kb=$(( ${HV_MIN_ROOT_FREE_MB:-1024} * 1024 ))
+min_pool_kb=$(( ${HV_MIN_POOL_FREE_GB:-20} * 1024 * 1024 ))
+
+if [ ! -r "$tfvars" ]; then
+  echo "::error::hypervisor disk preflight: cannot read $tfvars" >&2
+  exit 1
+fi
+
+# One node per line in every env file: { name = "...", ..., storage_pool = "..." }.
+pairs=$(grep -v '^[[:space:]]*#' "$tfvars" |
+  sed -nE 's/.*name[[:space:]]*=[[:space:]]*"([^"]+)".*storage_pool[[:space:]]*=[[:space:]]*"([^"]+)".*/\1 \2/p')
+if [ -z "$pairs" ]; then
+  echo "::warning::hypervisor disk preflight: no nodes with a storage_pool in $tfvars; nothing checked"
+  exit 0
+fi
+
+# Prints "<root-avail-kb> <pool-avail-kb> <pool-path>" for the pool named in $1.
+# shellcheck disable=SC2016 # expanded on the hypervisor, not here
+remote='set -u
+root=$(df -Pk /etc/libvirt/qemu | awk "NR==2{print \$4}")
+path=$(virsh -c qemu:///system pool-dumpxml "$1" 2>/dev/null | sed -n "s:.*<path>\(.*\)</path>.*:\1:p" | head -n1)
+pool=-
+[ -n "$path" ] && pool=$(df -Pk "$path" | awk "NR==2{print \$4}")
+echo "$root $pool ${path:--}"'
+
+fail=0
+while read -r host pool; do
+  [ -n "$host" ] || continue
+  if ! out=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "${user}@${host}" bash -s -- "$pool" <<<"$remote" 2>&1); then
+    echo "::warning::hypervisor disk preflight: cannot reach $host ($out); its disk is unchecked"
+    continue
+  fi
+  read -r root_kb pool_kb pool_path <<<"$(tail -n1 <<<"$out")"
+  if ! [[ "$root_kb" =~ ^[0-9]+$ ]]; then
+    echo "::warning::hypervisor disk preflight: unexpected answer from $host: $out"
+    continue
+  fi
+  echo "$host: root fs $((root_kb / 1024)) MiB free; pool $pool ($pool_path) $([[ "$pool_kb" =~ ^[0-9]+$ ]] && echo "$((pool_kb / 1024 / 1024)) GiB free" || echo "not found")"
+  if [ "$root_kb" -lt "$min_root_kb" ]; then
+    echo "::error::hypervisor $host has $((root_kb / 1024)) MiB free on the filesystem holding /etc/libvirt/qemu (need ${HV_MIN_ROOT_FREE_MB:-1024} MiB): libvirt cannot write a domain definition there. Free space on $host before re-running." >&2
+    fail=1
+  fi
+  if [[ "$pool_kb" =~ ^[0-9]+$ ]] && [ "$pool_kb" -lt "$min_pool_kb" ]; then
+    echo "::error::hypervisor $host has $((pool_kb / 1024 / 1024)) GiB free in storage pool $pool at $pool_path (need ${HV_MIN_POOL_FREE_GB:-20} GiB): the node's disk cannot grow. Free space on $host before re-running." >&2
+    fail=1
+  fi
+done <<<"$pairs"
+
+exit "$fail"

.github/scripts/hypervisor-disk-preflight_test.sh

@@ -0,0 +1,87 @@
+#!/bin/sh
+# hypervisor-disk-preflight_test.sh — drives hypervisor-disk-preflight.sh against
+# a stub ssh that answers for each hypervisor from a canned file, so every
+# verdict is checked without a real host.
+set -eu
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+PREFLIGHT="$SCRIPT_DIR/hypervisor-disk-preflight.sh"
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+# The stub answers "<root-kb> <pool-kb> <path>" from $FAKE/<host>, or exits 255
+# when that host has no file, the way ssh does for a host it cannot reach.
+mkdir -p "$TMP/bin" "$TMP/hosts"
+cat > "$TMP/bin/ssh" <<'STUB'
+#!/bin/sh
+for a in "$@"; do case $a in *@*) host=${a#*@} ;; esac; done
+cat >/dev/null
+if [ -f "$FAKE/$host" ]; then cat "$FAKE/$host"; exit 0; fi
+echo "ssh: Could not resolve hostname $host" >&2
+exit 255
+STUB
+chmod +x "$TMP/bin/ssh"
+PATH="$TMP/bin:$PATH"
+FAKE="$TMP/hosts"
+export PATH FAKE
+
+cat > "$TMP/env.tfvars" <<'TFVARS'
+nodes = [
+  { name = "wattle", wan_ip = "192.0.2.1", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
+  # { name = "retired", wan_ip = "192.0.2.9", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
+  { name = "banksia", wan_ip = "192.0.2.2", storage_pool = "nvme1-pool", base_image = "/x.qcow2" },
+]
+TFVARS
+
+GiB=1048576
+plenty="$((50 * GiB)) $((200 * GiB)) /mnt/drive1/pool"
+
+# run <label> <want-rc> <want-text>
+run() {
+    rc=0
+    out=$(bash "$PREFLIGHT" "$TMP/env.tfvars" 2>&1) || rc=$?
+    if [ "$rc" -ne "$2" ]; then
+        fail "$1: exit $rc, want $2"
+        echo "$out" | sed 's/^/        /'
+        return
+    fi
+    case $out in
+        *"$3"*) pass "$1" ;;
+        *) fail "$1: output lacks '$3'"; echo "$out" | sed 's/^/        /' ;;
+    esac
+}
+
+echo "-- hypervisor-disk-preflight.sh"
+
+echo "$plenty" > "$FAKE/wattle"
+echo "$plenty" > "$FAKE/banksia"
+run "every host has room" 0 "banksia: root fs 51200 MiB free"
+
+echo "0 $((200 * GiB)) /mnt/drive1/pool" > "$FAKE/wattle"
+run "a full root filesystem fails fast" 1 "::error::hypervisor wattle has 0 MiB free on the filesystem holding /etc/libvirt/qemu"
+
+echo "$plenty" > "$FAKE/wattle"
+echo "$((50 * GiB)) $((3 * GiB)) /mnt/nvme1/pool" > "$FAKE/banksia"
+run "a nearly full pool fails fast" 1 "::error::hypervisor banksia has 3 GiB free in storage pool nvme1-pool"
+
+echo "$plenty" > "$FAKE/banksia"
+rm -f "$FAKE/wattle"
+run "an unreachable host is reported, not failed" 0 "::warning::hypervisor disk preflight: cannot reach wattle"
+
+echo "$plenty" > "$FAKE/wattle"
+echo "$((50 * GiB)) - -" > "$FAKE/banksia"
+run "a pool libvirt does not know is reported, not failed" 0 "pool nvme1-pool (-) not found"
+
+echo "0 0 /x" > "$FAKE/retired"
+echo "$plenty" > "$FAKE/banksia"
+run "a commented-out node is never checked" 0 "wattle: root fs"
+
+if [ "$fails" -ne 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi
+echo "  hypervisor-disk-preflight ok"

.github/scripts/suite-sets.sh

@@ -130,6 +130,10 @@ e2e_suite_timeout() {
     # budget on a node stopping and starting: seven units down, a raft election,
     # and the whole service stack back up with every stream caught up.
     kvquorum) echo "40m" ;;
+    # single is the broadest suite and finishes, it does not hang: passing runs
+    # took 24-29m, and an Oracle host ran out at 30m inside the last test's
+    # cleanup. The default left it no headroom at all.
+    single) echo "40m" ;;
     *) echo "30m" ;;
   esac
 }

.github/workflows/e2e-nightly.yml

@@ -184,6 +184,14 @@ jobs:
             echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"
           } >> "${ARTIFACT_DIR}/hypervisor-load.txt"
 
+      # A hypervisor out of disk fails the apply minutes in, as "Domain Creation
+      # Failed" from libvirt writing the domain XML. Say so before starting.
+      - name: Preflight hypervisor disk
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hypervisor-disk-preflight.sh" \
+            "envs/${SPINIFEX_CI_ENV}.tfvars"
+
       - name: Provision VMs
         working-directory: mulga/scripts/tofu-cluster
         env:
@@ -241,6 +249,11 @@ jobs:
           . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
           e2e_suites_assert_subset "e2e-nightly single" "${E2E_SUITES_NIGHTLY_SINGLE}" "${E2E_SUITES_SINGLE}"
 
+          # The select job sized this cell's budget from these same timeouts, so
+          # resolve them here and carry the pairs into the remote loop.
+          SUITE_TMOS=""
+          for s in ${E2E_SUITES_NIGHTLY_SINGLE}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
+
           INVENTORY=$(tofu output -json inventory)
           WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
@@ -250,9 +263,10 @@ jobs:
              for SUITE in ${E2E_SUITES_NIGHTLY_SINGLE}; do \
                echo \"::group::suite \$SUITE\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
                rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
@@ -324,16 +338,20 @@ jobs:
             NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
           done
 
+          SUITE_TMOS=""
+          for s in ${E2E_SUITES_NIGHTLY_MULTI}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
+
           ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
             "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
              for SUITE in ${E2E_SUITES_NIGHTLY_MULTI}; do \
                echo \"::group::suite \$SUITE\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
                  SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
                  SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
                rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \

Makefile

@@ -247,6 +247,12 @@ test-build-scripts:
 		echo "-- $$t"; \
 		sh "$$t" || exit 1; \
 	done
+	@echo -e "\n....Running .github/scripts/*_test.sh...."
+	@for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		sh "$$t" || exit 1; \
+	done
+	shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh .github/scripts/hypervisor-disk-preflight_test.sh
 	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
 	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
 	shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)

architecture/layering/allowlist_test.go

@@ -104,7 +104,6 @@ var allowlist = []allowance{
 
 	{From: "domains/sts/awsapi", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "GetCallerIdentity takes IAMService; needs an IAM capability"},
 
-	{From: "gateway", To: "domains/sts/awsapi", Debt: "gateway (service registration)", Reason: "IAM, STS, EKS and RDS dispatch stays in gateway until the ingress registration seam"},
 	{From: "domains/ecs", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
 	{From: "domains/ecs", To: "domains/ec2/eip", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
 	{From: "domains/ecs", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},

build/systemd/spinifex-firewall.service

@@ -1,11 +1,10 @@
-# spinifex-unit-version: 1
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex host firewall (nftables input policy)
 Documentation=file:/etc/spinifex/firewall/spinifex.nft
 Wants=network-pre.target
 Before=network-pre.target shutdown.target
 DefaultDependencies=no
-ConditionPathExists=/etc/spinifex/firewall/peers.nft
 
 [Service]
 Type=oneshot
@@ -13,8 +12,11 @@ RemainAfterExit=yes
 StandardInput=null
 ProtectSystem=full
 ProtectHome=true
-ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply
-ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply
+# Unconditional: connection tracking loads on every node, so a flow that opens
+# while the policy is not armed survives the arm. The policy itself still loads
+# only where the daemon has written a peer file.
+ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply boot
+ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply boot
 TimeoutStopSec=10s
 
 # No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD

contracts/viperblockd/legacy/v1/README.md

@@ -60,6 +60,10 @@ moved to another node; the local daemon subscribes to stop the guest that
 was using it. Node-addressed, like mount and unmount: the guest is local, and
 no other node's daemon has anything to stop.
 
+The event is sent as a request and repeated until the daemon replies with `VolumeFencedAck`, because the fence often runs while the node's NATS server is restarting and a single publish can reach no subscriber.
+The daemon replies only when the event carries a reply subject, so a plain publish is still accepted.
+Repeats are harmless, since fencing a guest that is already fenced changes nothing.
+
 ## Abandon route
 
 `VolumeAbandonSubject(node)` is `ebs.<node>.abandon`. The daemon uses

contracts/viperblockd/legacy/v1/fence.go

@@ -19,3 +19,11 @@ type VolumeFencedEvent struct {
 	Winner string `json:"winner"`
 	Reason string `json:"reason"`
 }
+
+// VolumeFencedAck is the daemon's answer to a VolumeFencedEvent. Any answer
+// ends the sender's retries; Instance names the guest stopped, if one was, and
+// Error says why the daemon could not act on the event.
+type VolumeFencedAck struct {
+	Instance string `json:"instance,omitempty"`
+	Error    string `json:"error,omitempty"`
+}

contracts/viperblockd/legacy/v1/fence_test.go

@@ -37,3 +37,18 @@ func TestVolumeFencedEvent_JSON(t *testing.T) {
 	require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
 	assert.Equal(t, event, decoded)
 }
+
+// TestVolumeFencedAck_JSON pins the daemon's answer, which is what ends
+// viperblockd's retries of a fence notice.
+func TestVolumeFencedAck_JSON(t *testing.T) {
+	golden := `{"instance":"i-1","error":"bad event"}`
+	ack := viperblocklegacyv1.VolumeFencedAck{Instance: "i-1", Error: "bad event"}
+
+	marshaled, err := json.Marshal(ack)
+	require.NoError(t, err)
+	assert.JSONEq(t, golden, string(marshaled))
+
+	empty, err := json.Marshal(viperblocklegacyv1.VolumeFencedAck{})
+	require.NoError(t, err)
+	assert.JSONEq(t, `{}`, string(empty), "an ack for a volume with no local guest carries nothing")
+}

docs/COMMANDS.md

@@ -26,7 +26,7 @@ Every service is `PartOf=spinifex.target` and ordered by its own `After=`/`Wants
 | `spinifex-qmp-collector.service` | `spx service qmp-collector start` | Reads the `contracts/telemetry/v1` per-VM sidecar, polls telemetry QMP sockets and tap counters, publishes to NATS `metrics.ec2.*` |
 | `spinifex-shutdown.service` | `/bin/true`; the work is its `ExecStop` | Runs `spx admin node drain --local --unless-restarting`. `After=` every service, so it stops first and drains guests while the stack is still up |
 | `spinifex-nats-watchdog.service` / `.timer` | `nats-js-watchdog.sh` | JetStream liveness probe every 30s. `try-restart`s NATS when JetStream stops accepting writes while the process stays alive — what `Restart=on-failure` cannot catch |
-| `spinifex-firewall.service` | `spinifex-firewall-apply` | Oneshot nftables input policy, `WantedBy=sysinit.target` rather than the target. No `ExecStop`: remove it with `nft delete table inet spinifex_filter` |
+| `spinifex-firewall.service` | `spinifex-firewall-apply boot` | Oneshot nftables input policy, `WantedBy=sysinit.target` rather than the target. Loads the `inet spinifex_conntrack` tracking table on every node, and the policy only once the daemon has written a peer file. No `ExecStop`: remove the policy with `spinifex-firewall-apply disable` |
 | `regenerate-ssh-host-keys.service` | `ssh-keygen -A` | Oneshot for AMI-built nodes, conditional on the host key being absent |
 
 #### Operations

docs/PACKAGE_BOUNDARY_MIGRATION.md

@@ -368,6 +368,7 @@ while the branch is in flight; they do not describe the umbrella repository's
 | `spinifex#1354` | `spinifex/gateway/ec2.go` (the EC2 action table, `ec2Handler`, `ec2Actions`, the dispatch pipeline), `ec2_idempotency.go`, `ec2_validation.go`, and the EC2 token-bucket fields of `GatewayConfig` | `spinifex/domains/ec2/awsapi` (`Handler`, `NewHandler`, `Deps`, `Quota`, `Dispatch`, `ActionNames`, `InputHasField`), composed by `spinifex/gateway/ec2.go` | Slice 9a of the EC2 move: the capabilities first, with no behaviour change. The EC2 action table, idempotency and validation move to `domains/ec2/awsapi` behind `Deps{NATS, IAM, Quota, ActiveNodes, ExpectedNodes, NodeIDs, AZ, Endpoint, InstanceStatus}`; the gateway keeps the composition and `EC2_Request`, still on the legacy switch until slice 9b registers EC2 through the ADR-0002 seam. `Quota` is declared in `awsapi` because `domains/admission/quota` imports the EC2 action packages and cannot be imported back, and the gateway wraps `quota.Service` to supply the NATS instance-type resolver its retype gate takes. The instance and spot-instance adapters take the two-method `InstanceProfiles` interface (`ResolveInstanceProfile`, `CanonicalResourceARN`) instead of `iamdomain.IAMService`, which pays off the `spotinstance` to `domains/admission/quota` and `domains/iam` allowlist rows. Allowlist changes are 20 deletions and no additions: the 18 `gateway` to `domains/ec2/awsapi/*` rows and those 2. The pipeline characterisation tests committed first (error envelopes, check order, gateway-local actions, idempotent replay, PassRole, ImportKeyPair padding, launch-template expansion before authorization, RunInstances instance-profile refusal order) pass unchanged. Package-local tests drive every routed action through `Dispatch` against a stub daemon. No change to `dispatch.Actor` or `dispatch.Invocation`. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and MaxResults errors, a no-policy user refused with UnauthorizedOperation (including on the gateway-local DescribeRegions), an assumed-role session allowed and refused per its policy, and one RunInstances to running then terminated. |
 | `spinifex#1355` | `spinifex/gateway/ec2.go` (`EC2_Request` in production, the legacy service switch, its `supportedServices` and JSON tables) and the `Quota` field of `GatewayConfig` | `spinifex/domains/ec2/awsapi` (`NewRegistration`, `OperationInventory`, `ServiceName`), composed by `spinifex/runtime/roles/awsgw` (`ec2.go`: `newEC2Registration`, `advertisedEndpoint`; `ec2quota`) | Slice 9b of the EC2 move: EC2 registers through the ADR-0002 seam as service `ec2` and the legacy gateway switch is empty. The EC2 query-protocol error envelope is kept through `ErrorEnvelopeXML`, and the policy denial match is now `*dispatch.PolicyDenialError`. The characterisation tests committed first also pin the Authorize service name and action, the trace span and audit action, the throttle key (account plus action) and the cluster-unavailable gate, which runs before policy and action lookup for gateway-local and NATS-backed actions alike. They and the 9a tests pass unchanged apart from setup: STS and EC2 test setups now register the services they sign as, and the two tests that claimed a legacy service name are removed because no legacy name remains. `EC2_Request` survives only as a test shim. Allowlist changes are two deletions (the `gateway` to `domains/ec2/awsapi` and `domains/ec2/awsapi/instance` rows) and one addition, the per-service model-coverage row `cmd/aws-model-coverage` to `domains/ec2/awsapi`. `startInstanceStatus` stays in `awsgw.go` because ADR-0007:S6 records the raw instance-state uses by file. No change to `dispatch.Actor` or `dispatch.Invocation`. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and InvalidVolume.NotFound errors, a no-policy user refused with UnauthorizedOperation (including on the gateway-local DescribeRegions), an assumed-role session allowed and refused per its policy, and one RunInstances to running then terminated. The gateway request logs carry `service=ec2` and the per-action label. Not exercised live: a throttled request (the default limits are not reachable from one client; the throttle key is pinned by the characterisation tests) and an agent-signed EC2 call. |
 | `spinifex#1363` | `spinifex/gateway` context keys (`ctxService`, `ctxRegion`, `ctxAction`, the principal and access key id keys, and the unread `ctxQueryArgs`), the duplicate `callerPrincipal` and `principalContext` types, `gateway/caller_arn.go` (`buildCallerARN`), and the direct `policyauth`, `accountadmission` and NATS-connection reads in the auth pipeline | `spinifex/ingress/aws/reqctx` (the request record), `credverify.CallerARN`, and the gateway-consumed capabilities `PolicyEvaluator`, `AccountAdmission` and `Readiness`, composed by `spinifex/runtime/roles/awsgw` (`clusterready`) | Slice 11a of the gateway removal: the request-context record and the ADR-0002:S8 capabilities, with no behaviour change. The record is a copy-on-write value under one context key; the principal is sealed once and a second `WithPrincipal` keeps the first caller. The layering rule `S7_RequestRecordPrivateToIngress` lets only `gateway` and `ingress/aws/*` import `reqctx`, with a fault-injection test, and slice 11c moves the package to `ingress/aws/internal/reqctx` and drops the rule. The caller ARN is built once in `credverify` (root cases kept) and filled on `Principal.CallerARN`. The policy evaluator, account admission (`credential.ErrAccountNotActive`, with `accountadmission.ErrNotActive` an alias) and cluster readiness (`Available() bool`) are interfaces the gateway consumes and fails closed on when nil. The step 1 characterisation tests (throttle against both readiness gates and the policy gate for ECS and IAM, `/admin` skipping `GetService` and the second gate, the nil-connection difference between the two gates, five error writers each minting a request id that differs from the audit record, and the credential-scope region reaching only the span) pass unchanged apart from setup. No allowlist rows change, and there is no change to `dispatch.Actor` or `dispatch.Invocation`. Debt for slice 11c: the test-only `defaultCapabilities` seam (a package variable set by a test `init` so about 60 gateway test files can build a config from `IAMService` alone), the nil-connection case of the early readiness gate, the remaining `NATSConn` readers (`DiscoverActiveNodes` and the field that `awsgw` and the integration harness set), and the `IAMService`, `STSService`, `Config`, `RootCAs` and `InternalSuffix` fields of `GatewayConfig`, which no production code reads. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and InvalidVolume.NotFound errors, a bad access key refused with InvalidClientTokenId on a Query service and InvalidClientTokenIdException on a JSON service, a no-policy user refused with UnauthorizedOperation, and an assumed-role session allowed and refused per its policy. Not exercised live: a suspended-account refusal, a throttled request and the NATS-down 503, which the step 1 tests pin. |
+| `spinifex#1378` | `spinifex/gateway` `oidc_discovery.go` and `registry.go` (the `/oidc/eks` routes and the `/v2` mount), the `DiscoverActiveNodes` method and its cache, and the `GatewayConfig` fields `Config`, `RootCAs`, `InternalSuffix`, `Version`, `Commit`, `IAMService`, `STSService`, `NATSConn` and `ExpectedNodes` | `spinifex/runtime/roles/awsgw` (`PublicRoutes`, `NodeDiscovery.ActiveNodes`, `composeGateway`); the gateway keeps one `PublicRoutes` mount hook and the `Admin` registration name supplied by the role |
 | `spinifex#1334` | `spinifex/gateway/oidc_discovery.go` (`eksAccountKV`, the discovery-document and JWKS read and construction) | `spinifex/domains/eks/cluster` (`OIDCPublisher`, `OIDCDiscoveryDocument`), composed by `spinifex/runtime/roles/awsgw` | The EKS domain now reads its own account bucket and builds the per-cluster discovery document and JWKS responses (ADR-0005:S4), and the gateway only maps the unauthenticated `/oidc/eks/...` routes onto an `OIDCPublisher` interface it is handed. Responses are byte-identical, pinned by `oidc_discovery_current_behaviour_test.go`, whose assertions are unchanged. The gateway no longer imports `domains/eks/cluster` in production code. `STS` keeps its own `ClusterJWKSResolver` (`OIDCJWKSReader`), untouched. Recorded debt, not repaired here: a stored JWKS is served verbatim with 200 even when it is not valid JSON. |
 The EC2 contract row is intentionally different from the directory moves: it creates
 a compatibility boundary. `ec2.cmd.*` retains its deployed one-token NATS

docs/admin/host-firewall/README.md

@@ -155,6 +155,8 @@ Restarting the daemon is what re-arms it: the node rebuilds its peer list from t
 
 `disable` removes Spinifex's own table and the peer file and leaves every other nftables table alone. It is idempotent, and it is the recovery path — it runs even on a node with no policy installed, so it cannot leave you with a node you can neither arm nor disarm.
 
+`disable` leaves `table inet spinifex_conntrack` loaded, and the boot-time unit loads it on every node whether or not the policy is armed. It accepts everything and exists only to keep connection tracking running. Without it, a connection opened while the policy is off, such as an OVN RAFT link formed during that window, is first seen mid-stream when the policy comes back, and its replies are dropped as new connections. Do not delete it.
+
 > [!WARNING]
 > Do not flush the whole nftables ruleset to clear the policy. The VPC daemon writes SNAT and per-elastic-IP forward rules into the `ip nat` and `ip filter` tables and reinstalls them only when it starts, so `nft flush ruleset` breaks elastic IPs silently until the next `systemctl restart spinifex-vpcd`. Use `disable`, which only ever touches `inet spinifex_filter`.
 > On Oracle Linux with IPsec on, a flush also removes `inet spinifex_ipsec_guard`, so Geneve to a peer without an SA goes out in clear until the daemon puts the table back, within 60 seconds.
@@ -171,6 +173,7 @@ Everything lives under `/etc/spinifex/firewall/`. Which files are safe to edit f
 | `open-ports.nft` | The apply helper — the formation window only | No. Reset closed on install and upgrade |
 | `custom.nft` | **You** — `trusted_ssh_peers` and anything else you add | **Yes.** Created once, never rewritten |
 | `mode` | `setup.sh` — whether the install path armed the policy | No |
+| `conntrack.nft` | `setup.sh` — the tracking table, loaded armed or not | No. Edits are overwritten |
 
 The daemon cannot write `/etc` itself, and a blanket `nft` grant would be root-equivalent, so changes go through `/usr/local/lib/spinifex/spinifex-firewall-apply` — a root helper that accepts a fixed set of verbs and re-validates every address handed to it.
 
@@ -190,10 +193,10 @@ Dropped packets are logged, rate-limited to five a minute, so this tells you whe
 sudo journalctl -k | grep 'spinifex-fw drop'
 ```
 
-An empty log with a connection still failing means the policy is not the cause. A node with no table loaded at all is not armed:
+An empty log with a connection still failing means the policy is not the cause. A node with no peer file is not armed, and its boot-time unit loads only the tracking table:
 
 ```bash
-systemctl is-enabled spinifex-firewall.service
+sudo test -e /etc/spinifex/firewall/peers.nft && echo armed || echo not armed
 ```
 
 ## Troubleshooting

docs/security/network-connections/README.md

@@ -237,6 +237,7 @@ Notes that make the difference between this working and locking you out:
 - **Use a dedicated table and never flush the others.** `vpcd` writes MASQUERADE and per-EIP FORWARD rules into the `ip filter` and `ip nat` tables, and only reinstalls them when the service starts. `iptables -F`, `nft flush ruleset`, `ufw enable` and `firewalld` all destroy them silently, and instance networking stays broken until the next restart.
 - **Do not add a `forward` hook.** nftables evaluates every table registered on a hook and any `drop` is final, so a default-deny forward chain here would override `vpcd`'s accepts in the other table and break every routed-NAT instance and every EIP. Filtering forwarded guest traffic is the security group's job, not the host firewall's.
 - **`vpcd` owns a second table named `spinifex_mss`, separate from `inet spinifex_filter`.** It is a `bridge` table on the veth uplink and an `inet` table on the routed uplink, it has a `forward` hook with policy `accept`, and it only rewrites the MSS option of TCP SYN packets on the host hop to the gateway. It never drops or accepts traffic and does not replace the forward-hook advice above. `vpcd` re-installs it every 30 seconds, so a flush is repaired within that interval, and there is none on a physical uplink.
+- **Keep connection tracking running while the policy is unloaded.** The kernel tracks connections only while some rule uses conntrack, so a node with no ruleset tracks nothing. A connection that opens then, such as an OVN RAFT link, is first seen mid-stream when the policy loads, and if the server's end speaks first its packet looks like a new connection to an ephemeral port and is dropped. Spinifex keeps `table inet spinifex_conntrack`, which accepts everything, loaded on every node for this reason.
 - **`output` is deliberately untouched.** The metadata service's reply path egresses through per-instance policy routing; filtering `output` breaks it in ways that are hard to attribute.
 - **On a single-NIC node the cluster and encap sets are the node's public addresses**, because the planes collapse. The rules are still correct, but they are no longer a boundary — an upstream ACL is the only real control there.
 

docs/testing/e2e-coverage.md

@@ -29,6 +29,7 @@ The largest suite, 27 top-level tests. Notable entry points:
 - `TestInstanceMetadata` — IMDS from inside a guest.
 - **`TestIMDSDatapathSurvivesVswitchdRestart`** restarts `ovs-vswitchd` under a guest with an Elastic IP, once through systemd and once with SIGKILL. After each, the guest must answer over its EIP and fetch its instance ID through IMDSv2 within 60 seconds, and `br-imds` must hold all 7 of the guest's own flows. It is sequential because it takes the whole node's datapath down.
 - `TestStopStart`, `TestEarlyRebootLiveness`, `TestGuestChurnDurability`, `TestSnapshotLifecycle`, `TestSnapshotBackedLaunch`, `TestCreateImage`, `TestVolumeLifecycle`, `TestSpotInstanceLifecycle`, `TestLaunchTemplateBoot`.
+- **`TestRebootLoop`** is opt-in and skipped in a whole-suite run. Name it in `-test.run`, or set `SPINIFEX_REBOOT_LOOP_ROUNDS` (default 10), and it reboots one fresh guest through the API that many times and then once from inside, requiring a new boot ID every time.
 - `TestNegativeErrorPaths` — the `8a`–`8t` AWS error-shape cases (malformed AMI ID, invalid instance type, volume in use, detach-root forbidden, unknown or malformed instance IDs on the lifecycle verbs, and so on).
 - **`TestKVReplication`** — the single-node half of the KV replication policy, and it is not the multi-node assertion with a smaller number in it. On one node one replica per bucket is the correct answer rather than the defect, so what it proves is the opposite: that nothing declared a replica count this deployment cannot place, that `spx admin kv replicas` exits 0 and reports `WANT 1`, and that `--repair` run twice against a cluster with nothing to repair changes no bucket's replica count and leaves every bucket serving. The multi-node suites cannot cover any of that, because on three nodes the unplaceable count is the right one. It lives here rather than in its own suite so it runs in every single-node nightly cell, across every host and network permutation.
 
@@ -135,7 +136,7 @@ The create/describe/connect path against a live cluster: a DB VM that actually b
 - **`storagefault`** — what a guest does when its storage backend stops answering, and what stop/start does when the seal failed. **The claim is about corruption, not availability.** SIGSTOPs a cluster-wide service, so it always runs last and in its own cell. Three-node, because the RS(2,1) shard floor and cross-node takeover after a failed seal do not exist on one node.
 - **`partialblock`** — concurrent sub-block writes to the same block surviving a cold reread through NBD from a real guest.
 - **`storagegrowth`** — volume growth and image pull. Requires `external_mode=pool`; skips on a nat env, which has no EIP and therefore no guest SSH.
-- **`reboot`** — a slow guest waited for and keeping what it wrote, and a wedged guest reset once its budget expires. Proves a real reboot by requiring a *new* boot ID.
+- **`reboot`** — a slow guest waited for and keeping what it wrote, and a wedged guest forced off and relaunched once its budget expires. Proves a real reboot by requiring a *new* boot ID.
 - **`diskperf`** — guest disk performance gate, pinned fio profile on a dedicated blank volume. Also requires `external_mode=pool` for guest SSH.
 
 ### Other

internal/testkit/gwfix/deps.go

@@ -0,0 +1,43 @@
+package gwfix
+
+import (
+	"sync"
+
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	"github.com/nats-io/nats.go"
+)
+
+// Deps are the collaborators a test composes a gateway from. The gateway
+// config no longer carries them, so test setup keeps them beside the config.
+type Deps struct {
+	IAM   iamdomain.IAMService
+	STS   stsdomain.STSService
+	NATS  *nats.Conn
+	Nodes int
+}
+
+var (
+	depsMu sync.Mutex
+	depsOf = map[any]*Deps{}
+)
+
+// WithDeps records d for gw and returns gw.
+func WithDeps[P comparable](gw P, d Deps) P {
+	depsMu.Lock()
+	defer depsMu.Unlock()
+	depsOf[gw] = &d
+	return gw
+}
+
+// DepsOf returns the deps recorded for gw, creating an empty set when none.
+func DepsOf[P comparable](gw P) *Deps {
+	depsMu.Lock()
+	defer depsMu.Unlock()
+	d, ok := depsOf[gw]
+	if !ok {
+		d = &Deps{}
+		depsOf[gw] = d
+	}
+	return d
+}

scripts/appliance/firstboot.sh

@@ -90,6 +90,12 @@ else
     log "WARNING: clock not synced, continuing with $(date -uIs)"
 fi
 
+# ── Connection tracking ─────────────────────────────────────────────────────
+# Before OVN opens any link, so arming the firewall later cannot drop one the
+# kernel first saw mid-stream. Non-fatal; the unit reloads it on later boots.
+/usr/local/lib/spinifex/spinifex-firewall-apply track \
+    || log "WARNING: connection tracking not loaded; arming the firewall may drop OVN links"
+
 # ── OVS and OVN ─────────────────────────────────────────────────────────────
 # Start them before setup-ovn.sh so their databases exist when it runs: on
 # physical hardware first-boot DB initialisation outlasts its own timeout.

scripts/setup.sh

@@ -39,6 +39,7 @@ FIREWALL_PEERS="${FIREWALL_DIR}/peers.nft"
 FIREWALL_LOCAL="${FIREWALL_DIR}/local.nft"
 FIREWALL_OPEN="${FIREWALL_DIR}/open-ports.nft"
 FIREWALL_CUSTOM="${FIREWALL_DIR}/custom.nft"
+FIREWALL_TRACK="${FIREWALL_DIR}/conntrack.nft"
 FIREWALL_MODE_FILE="${SPINIFEX_FIREWALL_MODE_FILE:-${FIREWALL_DIR}/mode}"
 FIREWALL_APPLY="/usr/local/lib/spinifex/spinifex-firewall-apply"
 
@@ -749,6 +750,28 @@ RULES
     $SUDO chmod 0644 "$FIREWALL_RULES"
     info "  $FIREWALL_RULES"
 
+    # The kernel only tracks connections while some rule uses conntrack. Without
+    # this table a disarmed node tracks nothing, and a flow opened then (OVN's
+    # RAFT links) is first seen mid-stream at arm time, from whichever end speaks
+    # first. Seen from the server, that is a new flow to an ephemeral port, and
+    # the policy drops it. Kept loaded whether or not the policy is armed.
+    $SUDO tee "$FIREWALL_TRACK" > /dev/null << 'TRACK'
+# Spinifex connection tracking. Managed by scripts/setup.sh — edits are overwritten.
+# Accepts everything and filters nothing; it exists so conntrack keeps running
+# while inet spinifex_filter is not loaded. Never delete it on a running node.
+table inet spinifex_conntrack
+delete table inet spinifex_conntrack
+
+table inet spinifex_conntrack {
+    chain input {
+        type filter hook input priority filter; policy accept;
+        ct state invalid counter
+    }
+}
+TRACK
+    $SUDO chmod 0644 "$FIREWALL_TRACK"
+    info "  $FIREWALL_TRACK"
+
     # Host-local facts the policy needs that the daemon has no view of. Separate
     # from peers.nft because the daemon rewrites that file and would drop this.
     # Refreshed on every setup.sh run, so an sshd port change is picked up by an
@@ -816,6 +839,7 @@ set -eu
 RULES="/etc/spinifex/firewall/spinifex.nft"
 PEERS="/etc/spinifex/firewall/peers.nft"
 OPEN="/etc/spinifex/firewall/open-ports.nft"
+TRACK="/etc/spinifex/firewall/conntrack.nft"
 
 # 0 is a sentinel that keeps the set non-empty, because nft rejects an empty
 # one. No packet can match it: the kernel has no destination port 0.
@@ -837,13 +861,33 @@ write_open_ports() {
 # Deliberately ahead of the policy-file check below: turning the firewall off is
 # the recovery path, and refusing to run it because the policy is missing leaves
 # an operator with a node they can neither arm nor disarm.
+#
+# The tracking table goes in in the same transaction as the delete, so conntrack
+# never stops: a flow that opens while disarmed is then tracked from its SYN,
+# and re-arming does not drop it. The boot unit stays enabled to reload it.
 if [ "${1:-}" = "disable" ]; then
-    nft delete table inet spinifex_filter 2>/dev/null || true
+    if [ -r "$TRACK" ]; then
+        printf 'include "%s"\ntable inet spinifex_filter\ndelete table inet spinifex_filter\n' "$TRACK" | nft -f - \
+            || nft delete table inet spinifex_filter 2>/dev/null || true
+    else
+        nft delete table inet spinifex_filter 2>/dev/null || true
+    fi
     rm -f "$PEERS"
-    systemctl disable --now spinifex-firewall.service >/dev/null 2>&1 || true
     exit 0
 fi
 
+# track loads only the tracking table. boot is the unit's start: tracking on
+# every node, and the policy on top only where the daemon has written peers.
+if [ "${1:-}" = "track" ] || [ "${1:-}" = "boot" ]; then
+    if [ -r "$TRACK" ]; then
+        nft -f "$TRACK"
+    elif [ "$1" = "track" ]; then
+        echo "no tracking table at $TRACK" >&2
+        exit 1
+    fi
+    if [ "$1" = "track" ] || [ ! -r "$PEERS" ]; then exit 0; fi
+fi
+
 # Every remaining verb writes the policy, so it has to exist. Failing here
 # changes nothing, which leaves a node whose daemon has not yet resolved cluster
 # membership reachable rather than cut off from a cluster it cannot name.
@@ -917,10 +961,9 @@ if [ "${1:-}" = "set-peers" ]; then
     fi
     rm -f "$tmp" "$bak"
 
-    # The policy is loaded, so make it survive a reboot too. `disable` turns this
-    # unit off, and forming a cluster means disabling the firewall on every node
-    # first; without this, re-arming would look complete while silently leaving
-    # the node bare after its next boot. Idempotent and quiet in the normal case.
+    # The policy is loaded, so make it survive a reboot too. An older helper's
+    # `disable` turned this unit off; without this, re-arming such a node would
+    # look complete while leaving it bare after its next boot. Idempotent.
     systemctl enable spinifex-firewall.service >/dev/null 2>&1 || true
     exit 0
 fi
@@ -937,6 +980,13 @@ APPLY
     $SUDO chmod 0755 "$FIREWALL_APPLY"
     info "  $FIREWALL_APPLY"
 
+    # Now, not at the next boot: a fresh install forms its cluster before then,
+    # and an upgrade must start tracking before the daemon next arms. A chroot
+    # would load it into the build host's kernel.
+    if [ "${ISO_BUILD:-0}" != "1" ]; then
+        $SUDO "$FIREWALL_APPLY" track || warn "  Could not load $FIREWALL_TRACK: arming later may drop live cluster connections"
+    fi
+
     # Say which state ssh is in either way. A node whose allowlist was never
     # narrowed reads as configured otherwise, and that is the whole risk here.
     _ssh_srcs="$(grep -o '{[^}]*}' "$FIREWALL_CUSTOM" 2>/dev/null | tr -d '{}' | tr -s ' ')"
@@ -1926,9 +1976,9 @@ EOF
     # takes effect once enabled) — without this the JetStream ENOSPC-latch
     # watchdog is inert and a full disk requires a manual restart forever.
     $SUDO systemctl enable --now spinifex-nats-watchdog.timer
-    # Enabled, not started: its ConditionPathExists holds it inert until the
-    # daemon writes a peer file, so this only decides that a node which has one
-    # gets the policy back at boot, before any service opens a socket.
+    # Enabled on every node: at boot it loads connection tracking everywhere,
+    # and the policy only on a node whose daemon has written a peer file, so it
+    # is back before any service opens a socket.
     $SUDO systemctl enable spinifex-firewall.service
     info "Systemd units installed and enabled (per-service users)"
 }

scripts/setup_firewall_conntrack_test.sh

@@ -0,0 +1,214 @@
+#!/bin/sh
+# setup_firewall_conntrack_test.sh — arms the real host firewall under a TCP
+# flow that opened while it was disarmed, in two network namespaces.
+#
+# The policy, the tracking table and spinifex-firewall-apply are cut out of
+# setup.sh with their paths moved under a temp dir, and the helper's verbs are
+# run inside the client namespace. Needs root or passwordless sudo, ip, nft and
+# python3; skips without them. A stub systemctl keeps the helper off the host.
+set -eu
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+SETUP="$SCRIPT_DIR/setup.sh"
+
+echo "-- setup.sh firewall: flows that outlive a disarm"
+
+if [ "$(id -u)" = 0 ]; then SUDO=""
+elif sudo -n true 2>/dev/null; then SUDO="sudo -n"
+else echo "  skip - needs root or passwordless sudo"; exit 0
+fi
+for bin in ip nft python3; do
+    if ! PATH="$PATH:/usr/sbin:/sbin" command -v "$bin" > /dev/null 2>&1; then
+        echo "  skip - $bin not installed"; exit 0
+    fi
+done
+
+TMP=$(mktemp -d)
+S="fwct-s-$$"
+C="fwct-c-$$"
+cleanup() {
+    $SUDO ip netns del "$S" 2> /dev/null || true
+    $SUDO ip netns del "$C" 2> /dev/null || true
+    rm -rf "$TMP"
+}
+trap cleanup EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+# heredoc <tag> — the body of setup.sh's quoted heredoc named <tag>, with the
+# firewall directory moved under $TMP.
+heredoc() {
+    sed -n "/<< '$1'\$/,/^$1\$/p" "$SETUP" | sed '1d;$d' | sed "s#/etc/spinifex/firewall#$TMP/fw#g"
+}
+
+mkdir -p "$TMP/fw" "$TMP/bin"
+heredoc RULES > "$TMP/fw/spinifex.nft"
+heredoc APPLY > "$TMP/apply"
+heredoc TRACK > "$TMP/fw/conntrack.nft"
+printf 'define spinifex_ssh_ports = { 22 }\n' > "$TMP/fw/local.nft"
+printf 'define spinifex_open_ports = { 0 }\n' > "$TMP/fw/open-ports.nft"
+printf 'define trusted_ssh_peers = { 0.0.0.0/0 }\n' > "$TMP/fw/custom.nft"
+printf '#!/bin/sh\nexit 0\n' > "$TMP/bin/systemctl"
+chmod +x "$TMP/bin/systemctl"
+if [ ! -s "$TMP/fw/spinifex.nft" ] || [ ! -s "$TMP/apply" ]; then
+    echo "  FAIL - could not cut the policy or the helper out of setup.sh"; exit 1
+fi
+
+# One process owns both ends, so the order packets are sent in is exact. The
+# server is 10.99.0.1 (a peer) and also holds 10.99.0.9 (not a peer).
+cat > "$TMP/flow.py" << 'PY'
+import os, socket, subprocess, sys
+
+srv_ns, cli_ns, script = sys.argv[1], sys.argv[2], sys.argv[3]
+
+def enter(ns):
+    fd = os.open("/run/netns/" + ns, os.O_RDONLY)
+    os.setns(fd, os.CLONE_NEWNET)
+    os.close(fd)
+
+enter(srv_ns)
+listener = socket.socket()
+listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+listener.bind(("10.99.0.1", 6644))
+listener.listen(1)
+enter(cli_ns)
+
+cli = srv = None
+for step in script.split("@@"):
+    if step == "connect":
+        cli = socket.create_connection(("10.99.0.1", 6644), timeout=3)
+        srv, _ = listener.accept()
+        cli.settimeout(2)
+        srv.settimeout(2)
+    elif step in ("s2c", "c2s"):
+        tx, rx = (srv, cli) if step == "s2c" else (cli, srv)
+        tx.sendall(b"x" * 300)
+        got = b""
+        try:
+            while len(got) < 300:
+                got += rx.recv(300 - len(got))
+        except socket.timeout:
+            print("lost " + step)
+            sys.exit(1)
+    else:
+        subprocess.run(step, shell=True, check=True)
+PY
+
+# dial <client source> <server addr> <port> — a new connection from the server
+# namespace to the client, which listens on the port. 0 when it connects.
+cat > "$TMP/dial.py" << 'PY'
+import os, socket, sys
+
+srv_ns, cli_ns, src, port = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
+fd = os.open("/run/netns/" + cli_ns, os.O_RDONLY)
+os.setns(fd, os.CLONE_NEWNET)
+listener = socket.socket()
+listener.bind(("10.99.0.3", port))
+listener.listen(1)
+fd = os.open("/run/netns/" + srv_ns, os.O_RDONLY)
+os.setns(fd, os.CLONE_NEWNET)
+dialer = socket.socket()
+dialer.bind((src, 0))
+dialer.settimeout(2)
+try:
+    dialer.connect(("10.99.0.3", port))
+except OSError:
+    sys.exit(1)
+PY
+
+# A new pair of namespaces, and no peer file: every case starts disarmed.
+fresh() {
+    $SUDO rm -f "$TMP/fw/peers.nft"
+    $SUDO ip netns del "$S" 2> /dev/null || true
+    $SUDO ip netns del "$C" 2> /dev/null || true
+    $SUDO ip netns add "$S"
+    $SUDO ip netns add "$C"
+    $SUDO ip link add vs netns "$S" type veth peer name vc netns "$C"
+    $SUDO ip -n "$S" addr add 10.99.0.1/24 dev vs
+    $SUDO ip -n "$S" addr add 10.99.0.9/24 dev vs
+    $SUDO ip -n "$C" addr add 10.99.0.3/24 dev vc
+    for l in "$S vs" "$C vc"; do
+        # shellcheck disable=SC2086  # "<netns> <link>"
+        set -- $l
+        $SUDO ip -n "$1" link set lo up
+        $SUDO ip -n "$1" link set "$2" up
+    done
+}
+
+# Helper verbs and raw nft, run as root inside the client namespace.
+in_c="$SUDO env PATH=$TMP/bin:$PATH:/usr/sbin:/sbin ip netns exec $C"
+verb() { echo "$in_c sh $TMP/apply $*"; }
+arm="$(verb set-peers) < $TMP/peers"
+printf '10.99.0.1\n10.99.0.1\n' > "$TMP/peers"
+
+# flow <desc> <steps> — passes when every transfer is delivered.
+flow() {
+    if out=$($SUDO python3 "$TMP/flow.py" "$S" "$C" "$2" 2>&1); then pass "$1"; else fail "$1: $out"; fi
+}
+
+# Control first: with nothing tracking, arming under an open flow must drop the
+# server's next segment, or this kernel cannot show the failure and the cases
+# below prove nothing.
+fresh
+if $SUDO python3 "$TMP/flow.py" "$S" "$C" "connect@@s2c@@c2s@@$arm@@s2c" > /dev/null 2>&1; then
+    fail "control: a flow opened with no conntrack user survived the arm, so this kernel cannot reproduce the drop"
+else
+    pass "control: a flow opened with no conntrack user loses the server's next segment once armed"
+fi
+
+fresh
+flow "a flow opened after disable survives set-peers, server speaking first" \
+    "$(verb disable)@@connect@@s2c@@c2s@@$arm@@s2c@@c2s"
+
+fresh
+flow "a flow opened after boot with no peer file survives set-peers, server speaking first" \
+    "$(verb boot)@@connect@@s2c@@c2s@@$arm@@s2c@@c2s"
+
+fresh
+flow "a flow opened while armed survives disable and a re-arm, server speaking first" \
+    "$arm@@connect@@s2c@@c2s@@$(verb disable)@@s2c@@c2s@@s2c@@$arm@@s2c@@c2s"
+
+fresh
+flow "a flow opened before track survives set-peers once it has carried traffic" \
+    "connect@@s2c@@c2s@@$(verb track)@@s2c@@c2s@@$arm@@s2c@@c2s"
+
+# The tracking table must not open anything the policy drops.
+fresh
+sh -c "$(verb boot) && $arm"
+if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.1 6644; then
+    pass "armed: a peer still reaches a cluster port"
+else
+    fail "armed: a peer cannot reach a cluster port"
+fi
+if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.9 6644; then
+    fail "armed: a non-peer reaches a cluster port"
+else
+    pass "armed: a non-peer is dropped on a cluster port"
+fi
+if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.1 7777; then
+    fail "armed: a peer reaches a port the policy does not open"
+else
+    pass "armed: a peer is dropped on a port the policy does not open"
+fi
+
+# disable leaves only the tracking table, which accepts everything.
+sh -c "$(verb disable)"
+tables=$($in_c nft list tables | tr '\n' ' ')
+case "$tables" in
+    *spinifex_filter*) fail "disable left inet spinifex_filter loaded: $tables" ;;
+    *spinifex_conntrack*) pass "disable removes the policy and keeps the tracking table" ;;
+    *) fail "disable removed the tracking table: '$tables'" ;;
+esac
+if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.9 7777; then
+    pass "disarmed: the tracking table drops nothing"
+else
+    fail "disarmed: the tracking table dropped a connection"
+fi
+
+if [ "$fails" -gt 0 ]; then
+    echo "  $fails check(s) failed"
+    exit 1
+fi
+echo "  setup_firewall_conntrack ok"

scripts/uninstall-spx.sh

@@ -293,18 +293,20 @@ fi
 # file behind it is unreachable after the next reboot and unexplainable to
 # whoever inherits it.
 #
-# Only our own table and our own marked rules, never a global flush: setup.sh
-# creates exactly `table inet spinifex_filter`, and vpcd writes iptables rules
-# carrying the comments below. Anything else on this host belongs to someone
-# else.
+# Only our own tables and our own marked rules, never a global flush: setup.sh
+# creates exactly `table inet spinifex_filter` and `table inet
+# spinifex_conntrack`, and vpcd writes iptables rules carrying the comments
+# below. Anything else on this host belongs to someone else.
 # ---------------------------------------------------------------------------
 log "unloading the host firewall policy"
 if command -v nft >/dev/null 2>&1; then
-    if sudo nft list table inet spinifex_filter >/dev/null 2>&1; then
-        run sudo nft delete table inet spinifex_filter
-    else
-        log "  no inet spinifex_filter table loaded"
-    fi
+    for nft_table in spinifex_filter spinifex_conntrack; do
+        if sudo nft list table inet "$nft_table" >/dev/null 2>&1; then
+            run sudo nft delete table inet "$nft_table"
+        else
+            log "  no inet $nft_table table loaded"
+        fi
+    done
 fi
 if command -v iptables-save >/dev/null 2>&1 && ! $DRY_RUN; then
     for marker in spinifex-nat-egress spinifex-eip-ingress spinifex-imds spinifex-imds-remap; do
@@ -641,9 +643,11 @@ if ! $DRY_RUN; then
     for svc in nats gw daemon storage northstar viperblock vpcd ui; do
         id -u "spinifex-${svc}" >/dev/null 2>&1 && LEFT+=("user spinifex-${svc}")
     done
-    if command -v nft >/dev/null 2>&1 && sudo nft list table inet spinifex_filter >/dev/null 2>&1; then
-        LEFT+=("nft table inet spinifex_filter")
-    fi
+    for nft_table in spinifex_filter spinifex_conntrack; do
+        if command -v nft >/dev/null 2>&1 && sudo nft list table inet "$nft_table" >/dev/null 2>&1; then
+            LEFT+=("nft table inet $nft_table")
+        fi
+    done
     $PURGE_DATA && [ -e "$DATA_DIR" ] && ! mountpoint -q "$DATA_DIR" 2>/dev/null && LEFT+=("$DATA_DIR")
 
     if [ "${#LEFT[@]}" -gt 0 ]; then

scripts/uninstall-spx_test.sh

@@ -158,9 +158,10 @@ fi
 
 # The nft teardown must name our own table. A global flush would take the
 # host's other rules with it.
-grep -q 'nft delete table inet spinifex_filter' "$UNINSTALL" \
-    && pass "nft teardown deletes only inet spinifex_filter" \
-    || fail "nft teardown does not delete inet spinifex_filter by name"
+grep -q 'nft delete table inet "$nft_table"' "$UNINSTALL" \
+    && grep -q 'for nft_table in spinifex_filter spinifex_conntrack; do' "$UNINSTALL" \
+    && pass "nft teardown deletes only inet spinifex_filter and spinifex_conntrack" \
+    || fail "nft teardown does not delete inet spinifex_filter and spinifex_conntrack by name"
 grep -qE 'nft[[:space:]]+(-[a-z][[:space:]]+)*flush ruleset' "$UNINSTALL" \
     && fail "nft teardown flushes the whole ruleset" \
     || pass "nft teardown never flushes the whole ruleset"

spinifex/domains/ec2/awsapi/instance/RebootInstances.go

@@ -34,7 +34,8 @@ func ValidateRebootInstancesInput(input *ec2.RebootInstancesInput) error {
 }
 
 // RebootInstances sends reboot commands to specified instances via NATS.
-// Unlike stop+start, reboot keeps the instance running and sends a QMP system_reset.
+// Unlike stop+start, reboot keeps the instance running, on its node and volumes,
+// while the guest is shut down and started on a fresh QEMU.
 // Returns an empty response on success (AWS returns no state-change data).
 func RebootInstances(ctx context.Context, input *ec2.RebootInstancesInput, natsConn *nats.Conn, accountID string) (*ec2.RebootInstancesOutput, error) {
 	if err := ValidateRebootInstancesInput(input); err != nil {

spinifex/domains/ec2/awsapi/registration.go

@@ -7,14 +7,14 @@ import (
 )
 
 // NewRegistration returns the EC2 AWS Query control-plane registration. Errors
-// render in the EC2 <Response><Errors> body, which the gateway keys on the
-// service name.
+// render in the EC2 <Response><Errors> body.
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
 		ResolveAction: resolveAction,
 		Dispatch:      NewHandler(deps).Dispatch,
 		Errors:        dispatch.ErrorEnvelopeXML,
+		Traits:        dispatch.ServiceTraits{XMLErrorShape: dispatch.XMLErrorShapeEC2},
 		Inventory:     OperationInventory(),
 	}
 }

spinifex/domains/iam/awsapi/registration.go

@@ -376,6 +376,7 @@ func NewRegistration(deps Deps) dispatch.Registration {
 		ResolveAction: resolveAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeXML,
+		Traits:        dispatch.ServiceTraits{XMLThrottleCode: awserrors.ErrorThrottling},
 		Inventory:     OperationInventory(),
 	}
 }

spinifex/domains/network/external/igw.go

@@ -56,10 +56,9 @@ type IGWManager interface {
 	RebindGatewayIP(ctx context.Context, vpcID, newIP string, prefixLen int) error
 }
 
-// RoutedIngressHooks deliver host-side ingress plumbing in routed-NAT mode:
-// Ensure installs the host route sending VPC-CIDR traffic into OVN via the
-// gateway LRP transit IP; Remove tears it down on detach. Both idempotent.
-// Nil hooks (all other modes) are never called.
+// RoutedIngressHooks report routed-NAT ingress changes to the host: Ensure when
+// a VPC's gateway transit IP is set or re-asserted, Remove on detach. Both must
+// be idempotent. Nil hooks (all other modes) are never called.
 type RoutedIngressHooks struct {
 	Ensure func(ctx context.Context, vpcID, vpcCIDR, gwLrpIP string) error
 	Remove func(ctx context.Context, vpcID, vpcCIDR, gwLrpIP string) error
@@ -295,7 +294,7 @@ func (m *igwManager) AttachIGW(ctx context.Context, spec IGWSpec) error {
 // ensureRoutedLeg installs the routed-mode datapath legs for a VPC: the OVN
 // SNAT (VPC CIDR → gateway LRP transit IP) and the host ingress route into
 // it. No-op outside routed mode or without a gateway IP. Idempotent — the
-// reconcile loop replays AttachIGW, re-ensuring host state lost on reboot.
+// reconcile loop replays AttachIGW to re-ensure the SNAT.
 func (m *igwManager) ensureRoutedLeg(ctx context.Context, vpcID, gwLrpIP string) error {
 	if m.natMode != policy.NATModeRouted || gwLrpIP == "" {
 		return nil

spinifex/domains/network/host/routed_ingress.go

@@ -4,9 +4,15 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"net/netip"
 	"strings"
 )
 
+// VPCIngressRouteProto tags every VPC ingress route this package installs, so
+// the routes it may prune are told apart from the transit /24 and the EIP /32s
+// that share the spx-nat veth. Unassigned in iproute2's rt_protos.
+const VPCIngressRouteProto = "201"
+
 // EnsureVPCIngressRoute installs the host route that makes a VPC's private IPs
 // reachable in routed-NAT mode: traffic to the VPC CIDR enters OVN via the
 // gateway LRP's transit IP on the spx-nat veth. Idempotent (route replace).
@@ -18,44 +24,72 @@ func EnsureVPCIngressRoute(ctx context.Context, r Runner, vpcCIDR, gwLrpIP strin
 		return fmt.Errorf("EnsureVPCIngressRoute: gwLrpIP required")
 	}
 	if out, err := r.Run(ctx, "ip", "route", "replace", vpcCIDR,
-		"via", gwLrpIP, "dev", NATTransitHostEnd); err != nil {
+		"via", gwLrpIP, "dev", NATTransitHostEnd, "proto", VPCIngressRouteProto); err != nil {
 		return fmt.Errorf("install VPC ingress route %s via %s: %s: %w", vpcCIDR, gwLrpIP, string(out), err)
 	}
 	slog.Info("VPC ingress route installed", "vpc_cidr", vpcCIDR, "gw_lrp_ip", gwLrpIP)
 	return nil
 }
 
-// VPCIngressRouteVia returns the gateway IP of the existing transit-veth host
-// route for a VPC CIDR, or "" when none exists. Lets callers detect a
-// duplicate-CIDR collision (two VPCs sharing e.g. 172.31.0.0/16) before
-// replacing or deleting a route another VPC installed.
-func VPCIngressRouteVia(ctx context.Context, r Runner, vpcCIDR string) (string, error) {
-	if vpcCIDR == "" {
-		return "", fmt.Errorf("VPCIngressRouteVia: vpcCIDR required")
-	}
-	out, err := r.Run(ctx, "ip", "route", "show", vpcCIDR, "dev", NATTransitHostEnd)
+// ListVPCIngressRoutes returns the VPC ingress routes on this host, VPC CIDR to
+// gateway LRP transit IP. Only routes carrying VPCIngressRouteProto are listed.
+func ListVPCIngressRoutes(ctx context.Context, r Runner) (map[string]string, error) {
+	out, err := r.Run(ctx, "ip", "-o", "route", "show", "dev", NATTransitHostEnd, "proto", VPCIngressRouteProto)
 	if err != nil {
-		return "", fmt.Errorf("show VPC ingress route %s: %s: %w", vpcCIDR, string(out), err)
+		return nil, fmt.Errorf("list VPC ingress routes: %s: %w", string(out), err)
 	}
-	fields := strings.Fields(string(out))
-	for i, f := range fields {
-		if f == "via" && i+1 < len(fields) {
-			return fields[i+1], nil
+	routes := map[string]string{}
+	for line := range strings.SplitSeq(string(out), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) == 0 {
+			continue
 		}
+		dst, err := netip.ParsePrefix(fields[0])
+		if err != nil {
+			slog.Warn("VPC ingress route list: unparseable destination", "line", strings.TrimSpace(line))
+			continue
+		}
+		via := ""
+		for i, f := range fields {
+			if f == "via" && i+1 < len(fields) {
+				via = fields[i+1]
+			}
+		}
+		routes[dst.Masked().String()] = via
 	}
-	return "", nil
+	return routes, nil
 }
 
-// RemoveVPCIngressRoute deletes the host route for a VPC CIDR on IGW detach.
-// A missing route is not an error (idempotent teardown).
+// RemoveVPCIngressRoute deletes the host route for a VPC CIDR. A missing route
+// is not an error (idempotent teardown).
 func RemoveVPCIngressRoute(ctx context.Context, r Runner, vpcCIDR string) error {
 	if vpcCIDR == "" {
 		return fmt.Errorf("RemoveVPCIngressRoute: vpcCIDR required")
 	}
-	if _, err := r.Run(ctx, "ip", "route", "del", vpcCIDR, "dev", NATTransitHostEnd); err != nil {
+	if _, err := r.Run(ctx, "ip", "route", "del", vpcCIDR, "dev", NATTransitHostEnd, "proto", VPCIngressRouteProto); err != nil {
 		slog.Debug("VPC ingress route already absent", "vpc_cidr", vpcCIDR, "err", err)
 		return nil
 	}
 	slog.Info("VPC ingress route removed", "vpc_cidr", vpcCIDR)
 	return nil
 }
+
+// VPCIngressRoutes is this host's VPC ingress route table behind a Runner.
+type VPCIngressRoutes struct {
+	Runner Runner
+}
+
+// List is ListVPCIngressRoutes.
+func (v VPCIngressRoutes) List(ctx context.Context) (map[string]string, error) {
+	return ListVPCIngressRoutes(ctx, v.Runner)
+}
+
+// Ensure is EnsureVPCIngressRoute.
+func (v VPCIngressRoutes) Ensure(ctx context.Context, vpcCIDR, gwLrpIP string) error {
+	return EnsureVPCIngressRoute(ctx, v.Runner, vpcCIDR, gwLrpIP)
+}
+
+// Remove is RemoveVPCIngressRoute.
+func (v VPCIngressRoutes) Remove(ctx context.Context, vpcCIDR string) error {
+	return RemoveVPCIngressRoute(ctx, v.Runner, vpcCIDR)
+}

spinifex/domains/network/host/routed_ingress_test.go

@@ -14,7 +14,7 @@ func TestEnsureVPCIngressRoute_InstallsRoute(t *testing.T) {
 	if err := EnsureVPCIngressRoute(context.Background(), r, "10.0.0.0/16", "100.127.0.10"); err != nil {
 		t.Fatalf("EnsureVPCIngressRoute: %v", err)
 	}
-	want := "ip route replace 10.0.0.0/16 via 100.127.0.10 dev " + NATTransitHostEnd
+	want := "ip route replace 10.0.0.0/16 via 100.127.0.10 dev " + NATTransitHostEnd + " proto " + VPCIngressRouteProto
 	if !r.called(want) {
 		t.Errorf("missing route call:\n  want %q\n  got  %v", want, r.calls)
 	}
@@ -43,43 +43,52 @@ func TestEnsureVPCIngressRoute_RouteFailure(t *testing.T) {
 	}
 }
 
-func TestVPCIngressRouteVia_ParsesHolder(t *testing.T) {
+// The list is what the per-node pass prunes from, so it must carry only the
+// tagged routes and key them exactly as intent renders a VPC CIDR.
+func TestListVPCIngressRoutes_ParsesTaggedRoutes(t *testing.T) {
 	r := newStubRunner()
-	r.expect("ip route show", []byte("172.31.0.0/16 via 100.127.0.239 dev spx-nat-host \n"), nil)
+	r.expect("ip -o route show", []byte("172.31.0.0/16 via 100.127.0.16 \n10.1.0.0/16 via 100.127.0.17 \n"), nil)
 
-	via, err := VPCIngressRouteVia(context.Background(), r, "172.31.0.0/16")
+	routes, err := ListVPCIngressRoutes(context.Background(), r)
 	if err != nil {
-		t.Fatalf("VPCIngressRouteVia: %v", err)
+		t.Fatalf("ListVPCIngressRoutes: %v", err)
 	}
-	if via != "100.127.0.239" {
-		t.Errorf("via = %q, want 100.127.0.239", via)
+	want := map[string]string{"172.31.0.0/16": "100.127.0.16", "10.1.0.0/16": "100.127.0.17"}
+	if len(routes) != len(want) {
+		t.Fatalf("routes = %v, want %v", routes, want)
 	}
-	want := "ip route show 172.31.0.0/16 dev " + NATTransitHostEnd
-	if !r.called(want) {
-		t.Errorf("missing route show call:\n  want %q\n  got  %v", want, r.calls)
+	for cidr, via := range want {
+		if routes[cidr] != via {
+			t.Errorf("routes[%s] = %q, want %q", cidr, routes[cidr], via)
+		}
+	}
+	call := "ip -o route show dev " + NATTransitHostEnd + " proto " + VPCIngressRouteProto
+	if !r.called(call) {
+		t.Errorf("missing list call:\n  want %q\n  got  %v", call, r.calls)
 	}
 }
 
-func TestVPCIngressRouteVia_NoRoute(t *testing.T) {
+func TestListVPCIngressRoutes_Empty(t *testing.T) {
 	r := newStubRunner()
-	r.expect("ip route show", nil, nil)
+	r.expect("ip -o route show", nil, nil)
 
-	via, err := VPCIngressRouteVia(context.Background(), r, "172.31.0.0/16")
+	routes, err := ListVPCIngressRoutes(context.Background(), r)
 	if err != nil {
-		t.Fatalf("VPCIngressRouteVia: %v", err)
+		t.Fatalf("ListVPCIngressRoutes: %v", err)
 	}
-	if via != "" {
-		t.Errorf("via = %q, want empty for absent route", via)
+	if len(routes) != 0 {
+		t.Errorf("routes = %v, want none", routes)
 	}
 }
 
-func TestVPCIngressRouteVia_ValidatesArgs(t *testing.T) {
+// A failed list must surface: read as empty it would prune nothing, but read as
+// a partial table the caller could not tell what it was missing.
+func TestListVPCIngressRoutes_Failure(t *testing.T) {
 	r := newStubRunner()
-	if _, err := VPCIngressRouteVia(context.Background(), r, ""); err == nil {
-		t.Error("empty vpcCIDR: expected error")
-	}
-	if len(r.calls) != 0 {
-		t.Errorf("validation failures must not run commands: %v", r.calls)
+	r.expect("ip -o route show", []byte("Cannot find device"), fmt.Errorf("exit 1"))
+
+	if _, err := ListVPCIngressRoutes(context.Background(), r); err == nil {
+		t.Fatal("expected list failure to be returned")
 	}
 }
 
@@ -90,7 +99,7 @@ func TestRemoveVPCIngressRoute_DeletesRoute(t *testing.T) {
 	if err := RemoveVPCIngressRoute(context.Background(), r, "10.0.0.0/16"); err != nil {
 		t.Fatalf("RemoveVPCIngressRoute: %v", err)
 	}
-	want := "ip route del 10.0.0.0/16 dev " + NATTransitHostEnd
+	want := "ip route del 10.0.0.0/16 dev " + NATTransitHostEnd + " proto " + VPCIngressRouteProto
 	if !r.called(want) {
 		t.Errorf("missing route del call:\n  want %q\n  got  %v", want, r.calls)
 	}

spinifex/domains/network/reconcile/host_ingress.go

@@ -0,0 +1,168 @@
+package reconcile
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/network/ovn"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/policy"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
+	loop "github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Host ingress cadence. The resync is short because a gateway IP rebind after a
+// DHCP lease move writes NB, not KV, so no watch announces it. The retry covers
+// an IGW whose gateway LRP the attaching node has not created yet.
+const (
+	defaultHostIngressResync = 30 * time.Second
+	hostIngressPendingRetry  = 5 * time.Second
+)
+
+// HostIngressRoutes is one host's table of routed-NAT VPC ingress routes, VPC
+// CIDR to the gateway LRP transit IP the traffic enters OVN through.
+type HostIngressRoutes interface {
+	// List returns only the routes Ensure installed, keyed by CIDR.
+	List(ctx context.Context) (map[string]string, error)
+	Ensure(ctx context.Context, cidr, via string) error
+	Remove(ctx context.Context, cidr string) error
+}
+
+// HostIngress converges this host's VPC ingress routes from intent. Every node
+// runs one, without the drift lease: the routes are the host's own, and the
+// attach event that used to install them reaches one vpcd of the queue group.
+type HostIngress struct {
+	// OVN is read only, for each attached VPC's gateway LRP transit IP.
+	OVN    ovn.Client
+	Routes HostIngressRoutes
+	// LoadIntent reads intent fresh each pass. An error abandons the pass, so
+	// an unreadable store never prunes a route.
+	LoadIntent func(ctx context.Context) (IntentState, error)
+	// PreferredVPC wins a CIDR shared by several attached VPCs; see ingressOutranks.
+	PreferredVPC string
+	// Resync is the backstop between passes. Zero means 30s.
+	Resync time.Duration
+}
+
+// hostIngressPlan is the route table a pass converges to. held names CIDRs whose
+// winning VPC has no resolvable gateway IP yet: whatever route they have is kept
+// as it is, since the winner is known but where it enters OVN is not.
+type hostIngressPlan struct {
+	want    map[string]string
+	held    map[string]struct{}
+	pending bool
+}
+
+// ingressOutranks reports whether VPC a takes a shared CIDR from VPC b: the
+// preferred VPC first, then the lower VPC ID. Every node ranks the same intent
+// the same way, so the holder never depends on which node saw which event.
+func ingressOutranks(a, b, preferred string) bool {
+	if a == preferred || b == preferred {
+		return a == preferred && b != preferred
+	}
+	return a < b
+}
+
+// planHostIngress picks one gateway per CIDR across the attached VPCs in intent.
+// gatewayIP is asked only about each CIDR's winner; "" marks it pending.
+func planHostIngress(intent IntentState, preferred string, gatewayIP func(vpcID string) string) hostIngressPlan {
+	winners := map[string]string{}
+	for vpcID := range intent.IGWs {
+		vpc, ok := intent.VPCs[vpcID]
+		if !ok || !vpc.CIDR.IsValid() {
+			continue
+		}
+		cidr := vpc.CIDR.Masked().String()
+		if cur, ok := winners[cidr]; !ok || ingressOutranks(vpcID, cur, preferred) {
+			winners[cidr] = vpcID
+		}
+	}
+	plan := hostIngressPlan{want: map[string]string{}, held: map[string]struct{}{}}
+	for cidr, vpcID := range winners {
+		if gw := gatewayIP(vpcID); gw != "" {
+			plan.want[cidr] = gw
+			continue
+		}
+		plan.held[cidr] = struct{}{}
+		plan.pending = true
+	}
+	return plan
+}
+
+// gatewayTransitIP is the stamped gateway IP of vpcID's gateway LRP, falling
+// back to its network address. "" when the port is absent or unreadable.
+func (h *HostIngress) gatewayTransitIP(ctx context.Context, vpcID string) string {
+	lrp, err := h.OVN.GetLogicalRouterPort(ctx, topology.GatewayRouterPort(vpcID))
+	if err != nil || lrp == nil {
+		return ""
+	}
+	if ip := lrp.ExternalIDs[policy.GatewayIPExtIDKey]; ip != "" {
+		return ip
+	}
+	return gatewayLRPIP(lrp)
+}
+
+// Pass converges the routes once. It asks to be revisited while a winning VPC's
+// gateway is still being created; the loop's resync covers everything else.
+func (h *HostIngress) Pass(ctx context.Context) (time.Duration, error) {
+	intent, err := h.LoadIntent(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("load intent for host ingress routes: %w", err)
+	}
+	have, err := h.Routes.List(ctx)
+	if err != nil {
+		return 0, err
+	}
+	plan := planHostIngress(intent, h.PreferredVPC, func(vpcID string) string {
+		return h.gatewayTransitIP(ctx, vpcID)
+	})
+
+	var errs []error
+	for cidr, via := range plan.want {
+		if have[cidr] == via {
+			continue
+		}
+		if err := h.Routes.Ensure(ctx, cidr, via); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	for cidr, via := range have {
+		_, wanted := plan.want[cidr]
+		_, held := plan.held[cidr]
+		if wanted || held {
+			continue
+		}
+		slog.InfoContext(ctx, "reconcile/host-ingress: pruning route no attached VPC holds",
+			"vpc_cidr", cidr, "gw_lrp_ip", via)
+		if err := h.Routes.Remove(ctx, cidr); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	var revisit time.Duration
+	if plan.pending {
+		revisit = hostIngressPendingRetry
+	}
+	return revisit, errors.Join(errs...)
+}
+
+// RunHostIngress runs h on every intent change, every nudge and every
+// resync until ctx ends. nudge carries the attach and detach events
+// this node handled, which can land before the gateway they announce.
+func RunHostIngress(ctx context.Context, js jetstream.JetStream, h *HostIngress, nudge <-chan struct{}) {
+	resync := h.Resync
+	if resync <= 0 {
+		resync = defaultHostIngressResync
+	}
+	loop.Run(ctx, loop.Config{
+		Name:      "vpcd/host-ingress",
+		Sources:   []loop.Source{intentSource(js)},
+		Reconcile: h.Pass,
+		Trigger:   nudge,
+		Resync:    resync,
+		Debounce:  driftDebounce,
+	})
+}

spinifex/domains/network/reconcile/host_ingress_test.go

@@ -0,0 +1,439 @@
+package reconcile_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"maps"
+	"net/netip"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ec2igw "github.com/mulgadc/spinifex/spinifex/domains/ec2/igw"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/ovn/mock"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/ovn/nbdb"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/policy"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// hostRoutes is one node's host route table.
+type hostRoutes struct {
+	mu     sync.Mutex
+	routes map[string]string
+	lists  int
+}
+
+func newHostRoutes(seed map[string]string) *hostRoutes {
+	r := &hostRoutes{routes: map[string]string{}}
+	maps.Copy(r.routes, seed)
+	return r
+}
+
+func (h *hostRoutes) List(context.Context) (map[string]string, error) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.lists++
+	return maps.Clone(h.routes), nil
+}
+
+func (h *hostRoutes) Ensure(_ context.Context, cidr, via string) error {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.routes[cidr] = via
+	return nil
+}
+
+func (h *hostRoutes) Remove(_ context.Context, cidr string) error {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	delete(h.routes, cidr)
+	return nil
+}
+
+func (h *hostRoutes) listCount() int {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.lists
+}
+
+func (h *hostRoutes) snapshot() map[string]string {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return maps.Clone(h.routes)
+}
+
+// clusterIntent is the control-plane store every node reads.
+type clusterIntent struct {
+	mu   sync.Mutex
+	vpcs map[string]topology.VPCSpec
+	igws map[string]external.IGWSpec
+	err  error
+}
+
+func newClusterIntent() *clusterIntent {
+	return &clusterIntent{vpcs: map[string]topology.VPCSpec{}, igws: map[string]external.IGWSpec{}}
+}
+
+func (c *clusterIntent) addVPC(id, cidr string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.vpcs[id] = topology.VPCSpec{VPCID: id, CIDR: netip.MustParsePrefix(cidr)}
+}
+
+func (c *clusterIntent) attach(vpcID, igwID string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.igws[vpcID] = external.IGWSpec{VPCID: vpcID, InternetGatewayID: igwID}
+}
+
+func (c *clusterIntent) detach(vpcID string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	delete(c.igws, vpcID)
+}
+
+func (c *clusterIntent) load(context.Context) (reconcile.IntentState, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.err != nil {
+		return reconcile.IntentState{}, c.err
+	}
+	return reconcile.IntentState{VPCs: maps.Clone(c.vpcs), IGWs: maps.Clone(c.igws)}, nil
+}
+
+// node is one hypervisor's vpcd as far as host ingress goes.
+type node struct {
+	routes *hostRoutes
+	ingest *reconcile.HostIngress
+}
+
+func newNode(m *mock.Client, intent *clusterIntent, preferred string, seed map[string]string) *node {
+	routes := newHostRoutes(seed)
+	return &node{routes: routes, ingest: &reconcile.HostIngress{
+		OVN: m, Routes: routes, LoadIntent: intent.load, PreferredVPC: preferred,
+	}}
+}
+
+func (n *node) pass(t *testing.T) time.Duration {
+	t.Helper()
+	revisit, err := n.ingest.Pass(t.Context())
+	if err != nil {
+		t.Fatalf("host ingress pass: %v", err)
+	}
+	return revisit
+}
+
+func seedRouter(t *testing.T, m *mock.Client, vpcID, cidr string) {
+	t.Helper()
+	if err := m.CreateLogicalRouter(t.Context(), &nbdb.LogicalRouter{
+		Name:        topology.VPCRouter(vpcID),
+		ExternalIDs: map[string]string{"spinifex:vpc_id": vpcID, "spinifex:cidr": cidr},
+	}); err != nil {
+		t.Fatalf("seed router %s: %v", vpcID, err)
+	}
+}
+
+// seedGateway stands in for the gateway LRP an attach creates in NB.
+func seedGateway(t *testing.T, m *mock.Client, vpcID, gwIP string) {
+	t.Helper()
+	if err := m.CreateLogicalRouterPort(t.Context(), topology.VPCRouter(vpcID), &nbdb.LogicalRouterPort{
+		Name:        topology.GatewayRouterPort(vpcID),
+		Networks:    []string{gwIP + "/24"},
+		ExternalIDs: map[string]string{policy.GatewayIPExtIDKey: gwIP},
+	}); err != nil {
+		t.Fatalf("seed gateway %s: %v", vpcID, err)
+	}
+}
+
+// routedIGW is the IGW manager on the node the queue group hands an event to.
+// Its hooks only nudge, as in vpcd.
+func routedIGW(t *testing.T, m *mock.Client, nudges *int) external.IGWManager {
+	t.Helper()
+	nat, err := policy.NewNATManager(m, policy.NATModeRouted,
+		policy.WithSNATExemptSet(policy.NATExemptSetName, []string{"100.127.0.0/24"}))
+	if err != nil {
+		t.Fatalf("NewNATManager: %v", err)
+	}
+	nudge := func(context.Context, string, string, string) error { *nudges++; return nil }
+	mgr, err := external.NewIGWManager(external.IGWManagerConfig{
+		OVN: m, Routes: policy.NewRouteManager(m), NAT: nat,
+		Pool: &external.ExternalPoolConfig{
+			Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
+			GwLrpRangeStart: "100.127.0.240", GwLrpRangeEnd: "100.127.0.243",
+		},
+		Allocator: external.NewStaticRangeAllocator(m),
+		Chassis:   []string{"chassis-a"}, NATMode: policy.NATModeRouted,
+		RoutedIngress: &external.RoutedIngressHooks{Ensure: nudge, Remove: nudge},
+	})
+	if err != nil {
+		t.Fatalf("NewIGWManager: %v", err)
+	}
+	return mgr
+}
+
+func assertRoutes(t *testing.T, name string, got, want map[string]string) {
+	t.Helper()
+	if !maps.Equal(got, want) {
+		t.Errorf("%s routes = %v, want %v", name, got, want)
+	}
+}
+
+// The attach event reaches one vpcd of the queue group. Every other node must
+// still get the route from intent, or a host that neither handled the event
+// nor won a drift lease has no path into the VPC until a full resync.
+func TestHostIngress_AttachHandledOnOneNodeConvergesEveryNode(t *testing.T) {
+	m := mock.New()
+	intent := newClusterIntent()
+	seedRouter(t, m, "vpc-1", "10.1.0.0/16")
+	intent.addVPC("vpc-1", "10.1.0.0/16")
+	a := newNode(m, intent, "", nil)
+	b := newNode(m, intent, "", nil)
+
+	intent.attach("vpc-1", "igw-1")
+	nudges := 0
+	if err := routedIGW(t, m, &nudges).AttachIGW(t.Context(), external.IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"}); err != nil {
+		t.Fatalf("AttachIGW on node A: %v", err)
+	}
+	if nudges == 0 {
+		t.Fatal("attach on node A never nudged its host ingress loop")
+	}
+
+	a.pass(t)
+	// Node B saw only the intent change, never the event.
+	b.pass(t)
+	want := map[string]string{"10.1.0.0/16": "100.127.0.240"}
+	assertRoutes(t, "node A", a.routes.snapshot(), want)
+	assertRoutes(t, "node B", b.routes.snapshot(), want)
+}
+
+// Detach has the same reach as attach, so the route must leave every node, not
+// only the one that ran DetachIGW.
+func TestHostIngress_DetachPrunesOnEveryNode(t *testing.T) {
+	m := mock.New()
+	intent := newClusterIntent()
+	seedRouter(t, m, "vpc-1", "10.1.0.0/16")
+	intent.addVPC("vpc-1", "10.1.0.0/16")
+	intent.attach("vpc-1", "igw-1")
+	nudges := 0
+	mgr := routedIGW(t, m, &nudges)
+	if err := mgr.AttachIGW(t.Context(), external.IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"}); err != nil {
+		t.Fatalf("AttachIGW: %v", err)
+	}
+	nodes := []*node{newNode(m, intent, "", nil), newNode(m, intent, "", nil), newNode(m, intent, "", nil)}
+	for _, n := range nodes {
+		n.pass(t)
+		assertRoutes(t, "attached", n.routes.snapshot(), map[string]string{"10.1.0.0/16": "100.127.0.240"})
+	}
+
+	intent.detach("vpc-1")
+	if err := mgr.DetachIGW(t.Context(), "vpc-1"); err != nil {
+		t.Fatalf("DetachIGW on node A: %v", err)
+	}
+	for i, n := range nodes {
+		n.pass(t)
+		if got := n.routes.snapshot(); len(got) != 0 {
+			t.Errorf("node %d kept %v after detach, want no ingress route", i, got)
+		}
+	}
+}
+
+// Every account's default VPC is 172.31.0.0/16, so two attached VPCs contend
+// for one host route. Each node must pick the same holder whatever route it
+// had before and whichever order it met the VPCs in.
+func TestHostIngress_SharedCIDRPicksTheSameHolderOnEveryNode(t *testing.T) {
+	cases := []struct {
+		name      string
+		preferred string
+		wantVia   string
+	}{
+		{"preferred VPC wins over a lower ID", "vpc-b", "100.127.0.17"},
+		{"lower VPC ID wins without a preferred VPC", "", "100.127.0.16"},
+		{"a preferred VPC outside the contest changes nothing", "vpc-other", "100.127.0.16"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			m := mock.New()
+			intent := newClusterIntent()
+			for vpcID, gw := range map[string]string{"vpc-a": "100.127.0.16", "vpc-b": "100.127.0.17"} {
+				seedRouter(t, m, vpcID, "172.31.0.0/16")
+				seedGateway(t, m, vpcID, gw)
+				intent.addVPC(vpcID, "172.31.0.0/16")
+				intent.attach(vpcID, "igw-"+vpcID)
+			}
+			want := map[string]string{"172.31.0.0/16": tc.wantVia}
+			// Seeded with each holder, as if the event for either VPC had
+			// landed on that node first; repeated so map order cannot hide a
+			// choice that depends on it.
+			for _, seed := range []string{"", "100.127.0.16", "100.127.0.17"} {
+				for range 20 {
+					var initial map[string]string
+					if seed != "" {
+						initial = map[string]string{"172.31.0.0/16": seed}
+					}
+					n := newNode(m, intent, tc.preferred, initial)
+					n.pass(t)
+					assertRoutes(t, "seed "+seed, n.routes.snapshot(), want)
+				}
+			}
+		})
+	}
+}
+
+// Until the winning VPC's gateway exists the route is left alone: switching it
+// to the loser and back would cut the CIDR over twice for nothing.
+func TestHostIngress_PendingWinnerHoldsTheRouteAndRevisits(t *testing.T) {
+	m := mock.New()
+	intent := newClusterIntent()
+	seedRouter(t, m, "vpc-a", "172.31.0.0/16")
+	seedRouter(t, m, "vpc-b", "172.31.0.0/16")
+	seedGateway(t, m, "vpc-b", "100.127.0.17")
+	intent.addVPC("vpc-a", "172.31.0.0/16")
+	intent.addVPC("vpc-b", "172.31.0.0/16")
+	intent.attach("vpc-a", "igw-a")
+	intent.attach("vpc-b", "igw-b")
+	n := newNode(m, intent, "vpc-a", map[string]string{"172.31.0.0/16": "100.127.0.9"})
+
+	if revisit := n.pass(t); revisit <= 0 {
+		t.Errorf("revisit = %v, want a short retry while vpc-a's gateway is missing", revisit)
+	}
+	assertRoutes(t, "pending", n.routes.snapshot(), map[string]string{"172.31.0.0/16": "100.127.0.9"})
+
+	seedGateway(t, m, "vpc-a", "100.127.0.16")
+	if revisit := n.pass(t); revisit != 0 {
+		t.Errorf("revisit = %v after convergence, want 0", revisit)
+	}
+	assertRoutes(t, "converged", n.routes.snapshot(), map[string]string{"172.31.0.0/16": "100.127.0.16"})
+}
+
+// A restarted vpcd has no memory of the events it handled. Its first pass must
+// rebuild the table from intent: repoint a route left on a stale gateway, add
+// one it never had, and prune one whose VPC detached while it was down.
+func TestHostIngress_RestartConvergesFromIntent(t *testing.T) {
+	m := mock.New()
+	intent := newClusterIntent()
+	for vpcID, v := range map[string][2]string{
+		"vpc-1": {"10.1.0.0/16", "100.127.0.20"},
+		"vpc-2": {"10.2.0.0/16", "100.127.0.21"},
+	} {
+		seedRouter(t, m, vpcID, v[0])
+		seedGateway(t, m, vpcID, v[1])
+		intent.addVPC(vpcID, v[0])
+		intent.attach(vpcID, "igw-"+vpcID)
+	}
+	n := newNode(m, intent, "", map[string]string{
+		"10.1.0.0/16": "100.127.0.99",
+		"10.9.0.0/16": "100.127.0.30",
+	})
+
+	n.pass(t)
+	assertRoutes(t, "after restart", n.routes.snapshot(), map[string]string{
+		"10.1.0.0/16": "100.127.0.20",
+		"10.2.0.0/16": "100.127.0.21",
+	})
+}
+
+// An unreadable store is not an empty one: reading it as empty would strip
+// every VPC's ingress route from every node at once.
+func TestHostIngress_IntentReadFailureNeverPrunes(t *testing.T) {
+	m := mock.New()
+	intent := newClusterIntent()
+	intent.err = errors.New("kv unavailable")
+	seed := map[string]string{"10.1.0.0/16": "100.127.0.20"}
+	n := newNode(m, intent, "", seed)
+
+	if _, err := n.ingest.Pass(t.Context()); err == nil {
+		t.Fatal("expected the intent read failure to be returned")
+	}
+	assertRoutes(t, "after failed read", n.routes.snapshot(), seed)
+}
+
+// The loop has to wake on an intent write by itself, on every node at once,
+// while another node holds the drift lease and no event arrives, well inside
+// its resync.
+func TestRunHostIngress_IntentWriteConvergesEveryNodeWithoutLeaseOrEvent(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	ctx := t.Context()
+	vpcs, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 1)
+	if err != nil {
+		t.Fatalf("create VPC bucket: %v", err)
+	}
+	igws, err := kvutil.GetOrCreateBucket(ctx, js, ec2igw.KVBucketIGW, 1)
+	if err != nil {
+		t.Fatalf("create IGW bucket: %v", err)
+	}
+	put(t, vpcs, "acct/vpc-1", ec2vpc.VPCRecord{VpcId: "vpc-1", CidrBlock: "10.1.0.0/16"})
+	release, elected := reconcile.AcquireLeader(ctx, nc, reconcile.KVBucketVPCDReconcile, "node-elsewhere")
+	if !elected {
+		t.Fatal("could not take the drift lease for another node")
+	}
+	t.Cleanup(release)
+
+	m := mock.New()
+	seedRouter(t, m, "vpc-1", "10.1.0.0/16")
+	seedGateway(t, m, "vpc-1", "100.127.0.20")
+	loopCtx, cancel := context.WithCancel(ctx)
+	var wg sync.WaitGroup
+	t.Cleanup(func() {
+		cancel()
+		wg.Wait()
+	})
+	nodes := make([]*hostRoutes, 3)
+	for i := range nodes {
+		nodes[i] = newHostRoutes(nil)
+		h := &reconcile.HostIngress{
+			OVN: m, Routes: nodes[i], Resync: time.Minute,
+			LoadIntent: func(ctx context.Context) (reconcile.IntentState, error) {
+				return reconcile.LoadIntentFromKV(ctx, js, "us-east-1a")
+			},
+		}
+		wg.Go(func() { reconcile.RunHostIngress(loopCtx, js, h, nil) })
+	}
+	// The startup pass has to be over before the attach is written, or it could
+	// be what converged a node rather than the watch.
+	for _, n := range nodes {
+		for n.listCount() == 0 {
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	want := map[string]string{"10.1.0.0/16": "100.127.0.20"}
+	converged := func() bool {
+		for _, n := range nodes {
+			if !maps.Equal(n.snapshot(), want) {
+				return false
+			}
+		}
+		return true
+	}
+	// Watchers attach asynchronously and see updates only, so the write is
+	// repeated until one lands rather than racing the loop's startup.
+	deadline := time.Now().Add(10 * time.Second)
+	for !converged() {
+		if time.Now().After(deadline) {
+			for i, n := range nodes {
+				t.Errorf("node %d routes = %v 10s after the attach was written, want %v", i, n.snapshot(), want)
+			}
+			t.FailNow()
+		}
+		put(t, igws, "acct/igw-1", ec2igw.IGWRecord{InternetGatewayId: "igw-1", VpcId: "vpc-1", State: "available"})
+		time.Sleep(500 * time.Millisecond)
+	}
+}
+
+func put(t *testing.T, kv jetstream.KeyValue, key string, rec any) {
+	t.Helper()
+	b, err := json.Marshal(rec)
+	if err != nil {
+		t.Fatalf("marshal %s: %v", key, err)
+	}
+	if _, err := kv.Put(t.Context(), key, b); err != nil {
+		t.Fatalf("put %s: %v", key, err)
+	}
+}

spinifex/domains/route53/awsapi/registration.go

@@ -89,6 +89,7 @@ func NewRegistration(deps Deps) dispatch.Registration {
 		ResolveAction: resolveAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeXML,
+		Traits:        dispatch.ServiceTraits{XMLThrottleCode: awserrors.ErrorThrottling, GlobalSigningRegion: "us-east-1"},
 		Inventory:     OperationInventory(),
 	}
 }

spinifex/domains/sts/awsapi/preauth_test.go

@@ -0,0 +1,32 @@
+package awsapi
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestAnonymousArgs_Classification covers the body-peek helper for anonymous action detection.
+func TestAnonymousArgs_Classification(t *testing.T) {
+	t.Run("anonymous action recognised, body restored", func(t *testing.T) {
+		const body = "Action=AssumeRoleWithWebIdentity&WebIdentityToken=tok"
+		req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
+		args, ok := anonymousArgs(req)
+		require.True(t, ok)
+		assert.Equal(t, "AssumeRoleWithWebIdentity", args["Action"])
+
+		restored, _ := io.ReadAll(req.Body)
+		assert.Equal(t, body, string(restored), "body must be restored for downstream read")
+	})
+
+	t.Run("non-anonymous action rejected", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=GetCallerIdentity"))
+		_, ok := anonymousArgs(req)
+		assert.False(t, ok)
+	})
+}

spinifex/domains/sts/awsapi/registration.go

@@ -107,6 +107,7 @@ func NewRegistration(deps Deps) dispatch.Registration {
 		ResolveAction: resolveAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeXML,
+		Traits:        dispatch.ServiceTraits{XMLThrottleCode: awserrors.ErrorThrottling},
 		Inventory:     OperationInventory(),
 	}
 }
@@ -138,6 +139,63 @@ func NewAnonymousAssumeRoleWithWebIdentity(deps Deps) AnonymousFunc {
 	}
 }
 
+// anonymousActions lists the STS actions that carry no SigV4; a web-identity
+// token in the body authenticates them.
+var anonymousActions = map[string]bool{
+	"AssumeRoleWithWebIdentity": true,
+}
+
+// NewPreAuth serves the unsigned web-identity exchange ahead of SigV4. A
+// request with an Authorization header, a body over the payload cap or any
+// other action is left to the signed chain.
+func NewPreAuth(deps Deps) dispatch.PreAuth {
+	run := NewAnonymousAssumeRoleWithWebIdentity(deps)
+	return dispatch.PreAuth{
+		Service: ServiceName,
+		Serve: func(w http.ResponseWriter, r *http.Request) (bool, error) {
+			if r.Method != http.MethodPost || r.Header.Get("Authorization") != "" {
+				return false, nil
+			}
+			args, ok := anonymousArgs(r)
+			if !ok {
+				return false, nil
+			}
+			xmlOutput, err := run(args)
+			if err != nil {
+				return true, err
+			}
+			w.Header().Set("Content-Type", "text/xml")
+			w.WriteHeader(http.StatusOK)
+			if _, err := w.Write(xmlOutput); err != nil {
+				slog.Error("Failed to write STS response", "err", err)
+			}
+			return true, nil
+		},
+	}
+}
+
+// anonymousArgs peeks the request body and reports whether its Action is one
+// served without SigV4. This runs ahead of authentication, so an unauthenticated
+// client controls the allocation: one byte past the cap detects an oversize body,
+// which is handed back unread rather than dispatched.
+func anonymousArgs(r *http.Request) (map[string]string, bool) {
+	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
+	if err != nil {
+		return nil, false
+	}
+	if int64(len(body)) > sigv4.MaxPayloadLen {
+		r.Body = io.NopCloser(io.MultiReader(bytes.NewReader(body), r.Body))
+		return nil, false
+	}
+	r.Body = io.NopCloser(bytes.NewReader(body))
+
+	args, err := query.ParseArgs(string(body))
+	if err != nil || !anonymousActions[args["Action"]] {
+		return nil, false
+	}
+	return args, true
+}
+
 // resolveAction is the one Query Action parser telemetry and dispatch share.
 // It leaves the body readable for the dispatcher and answers "" on any failure.
 func resolveAction(r *http.Request) string {

spinifex/gateway/acm_characterization_test.go

@@ -2,6 +2,7 @@ package gateway
 
 import (
 	"github.com/go-chi/chi/v5"
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
 	acmawsapi "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/stretchr/testify/assert"
@@ -53,7 +54,7 @@ func TestACMPipeline_TraceActionEnricher_SpanNamedServiceAction(t *testing.T) {
 	defer otel.SetTracerProvider(prev)
 
 	gw, _ := acmPipelineApp(t, allowAllACMDocs())
-	subscribeACMCounter(t, gw.NATSConn, "acm.ListCertificates", []byte(`{}`))
+	subscribeACMCounter(t, gwfix.DepsOf(gw).NATS, "acm.ListCertificates", []byte(`{}`))
 
 	r := chi.NewRouter()
 	r.Use(otelsetup.HTTPMiddleware("awsgw"))

spinifex/gateway/actor_projection_characterization_test.go

@@ -95,16 +95,12 @@ func actorProbeGateway(t *testing.T, e *cbEnv) (http.Handler, *[]actorObservatio
 			return nil
 		},
 	}))
-	gw := &GatewayConfig{
+	gw := gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		NATSConn:       e.nc,
 		Region:         gwfix.Region,
-		ExpectedNodes:  1,
-		IAMService:     e.counter,
-		STSService:     e.sts,
-	}
+	}, gwfix.Deps{NATS: e.nc, Nodes: 1, IAM: e.counter, STS: e.sts})
 	withRealAuth(gw)
-	gw.AnonymousSTS = stsawsapi.NewAnonymousAssumeRoleWithWebIdentity(stsawsapi.Deps{STS: gw.STSService, IAM: gw.IAMService})
+	gw.PreAuth = append(gw.PreAuth, stsawsapi.NewPreAuth(stsawsapi.Deps{STS: gwfix.DepsOf(gw).STS, IAM: gwfix.DepsOf(gw).IAM}))
 	require.NoError(t, b.Register(stsRegistration(gw)))
 	gw.Services = b.Build()
 	return gw.SetupRoutes(), &seen
@@ -362,3 +358,9 @@ func TestActorProjection_AgentRolesNotAssumableOverHTTPS(t *testing.T) {
 	assert.Equal(t, e.accountID, o.RDSAccountID)
 	assert.NotEqual(t, global, o.RDSAccountID)
 }
+
+const (
+	testDiscAccount = "000000000001"
+	testDiscCluster = "h1"
+	testDiscIssuer  = "https://10.0.0.1:9999/oidc/eks/ap-southeast-2/000000000001/h1"
+)

spinifex/gateway/admin.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"context"
 	"encoding/json"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/reqctx"
 	"log/slog"
@@ -8,9 +9,16 @@ import (
 	"uuid"
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	operatoradmin "github.com/mulgadc/spinifex/spinifex/operator/admin"
 )
 
+// AdminSurface names the registration the private /admin surface dispatches
+// to and how the request ID reaches it. The composing role supplies both, so
+// the gateway names no service.
+type AdminSurface struct {
+	Service       string
+	WithRequestID func(ctx context.Context, requestID string) context.Context
+}
+
 // Admin_Request serves the private /admin/<Method> surface. The route stays
 // here so it keeps its place ahead of the service selector; the work is the
 // spinifex registration's, dispatched exactly as a Query request is. Only the
@@ -18,9 +26,11 @@ import (
 func (gw *GatewayConfig) Admin_Request(w http.ResponseWriter, r *http.Request) {
 	requestID := uuid.NewV4().String()
 	w.Header().Set("X-Amzn-Requestid", requestID)
-	r = r.WithContext(operatoradmin.WithRequestID(r.Context(), requestID))
+	if gw.Admin.WithRequestID != nil {
+		r = r.WithContext(gw.Admin.WithRequestID(r.Context(), requestID))
+	}
 
-	entry, ok := gw.registered(operatoradmin.ServiceName)
+	entry, ok := gw.registered(gw.Admin.Service)
 	if !ok {
 		slog.Error("Admin: spinifex service is not registered", "requestId", requestID)
 		gw.writeAdminError(w, requestID, awserrors.ErrorInternalError, "")
@@ -29,12 +39,12 @@ func (gw *GatewayConfig) Admin_Request(w http.ResponseWriter, r *http.Request) {
 
 	action := reqctx.Action(r.Context())
 	if err := gw.dispatchRegistered(entry, w, r); err != nil {
-		slog.Log(r.Context(), serviceErrorLevel(operatoradmin.ServiceName, err), "Service request error",
-			"service", operatoradmin.ServiceName, "action", action, "requestId", requestID, "error", err)
+		slog.Log(r.Context(), serviceErrorLevel(gw.Admin.Service, err), "Service request error",
+			"service", gw.Admin.Service, "action", action, "requestId", requestID, "error", err)
 		gw.writeAdminErrorFor(w, requestID, err)
 		return
 	}
-	slog.Info("Service request completed", "service", operatoradmin.ServiceName, "action", action, "requestId", requestID)
+	slog.Info("Service request completed", "service", gw.Admin.Service, "action", action, "requestId", requestID)
 }
 
 // adminErrorBody is the JSON error envelope for /admin. This surface is not an

spinifex/gateway/admin_outcome_test.go

@@ -3,6 +3,7 @@ package gateway
 import (
 	"bytes"
 	"encoding/json"
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
 	"log/slog"
 	"net/http"
 	"strings"
@@ -54,7 +55,7 @@ func TestAdminOutcome_LogFollowsTheRealResult(t *testing.T) {
 	}{
 		{"success", awsidentifiers.DefaultAccountID(), nil, http.StatusOK, ""},
 		{"denied caller", authzAccountID, nil, http.StatusForbidden, "INFO"},
-		{"cluster fault", awsidentifiers.DefaultAccountID(), func(gw *GatewayConfig) { gw.NATSConn = nil }, http.StatusServiceUnavailable, "ERROR"},
+		{"cluster fault", awsidentifiers.DefaultAccountID(), func(gw *GatewayConfig) { gwfix.DepsOf(gw).NATS = nil }, http.StatusServiceUnavailable, "ERROR"},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {

spinifex/gateway/admin_test.go

@@ -62,7 +62,7 @@ func decodeAdminError(t *testing.T, rec *httptest.ResponseRecorder) adminErrorBo
 // An unknown method is rejected before authorization, so it never discloses
 // whether the caller would have been permitted.
 func TestAdminRequestUnknownMethod(t *testing.T) {
-	gw := withOperator(withRealAuth(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}))
+	gw := withOperator(withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()})))
 
 	rec := adminRequest(gw, "PurgeEverything", `{}`)
 
@@ -71,7 +71,7 @@ func TestAdminRequestUnknownMethod(t *testing.T) {
 }
 
 func TestAdminRequestDeniesUnauthorizedCaller(t *testing.T) {
-	gw := withOperator(withRealAuth(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}))
+	gw := withOperator(withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()})))
 
 	rec := adminRequest(gw, "CreateAccount", `{}`)
 
@@ -83,7 +83,7 @@ func TestAdminRequestDeniesUnauthorizedCaller(t *testing.T) {
 // The request ID ties a client failure to the gateway log line, so it must be
 // present on the header and in the body even for a denial.
 func TestAdminRequestAlwaysCarriesRequestID(t *testing.T) {
-	gw := withOperator(withRealAuth(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}))
+	gw := withOperator(withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()})))
 
 	rec := adminRequest(gw, "CreateAccount", `{}`)
 
@@ -106,7 +106,7 @@ func TestWriteAdminErrorFallsBackOnUnknownCode(t *testing.T) {
 // The router serves every verb from one pattern, so the handler is what
 // refuses a GET — a second route would shadow the POST one.
 func TestAdminRequestRejectsNonPost(t *testing.T) {
-	gw := withOperator(withRealAuth(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}))
+	gw := withOperator(withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()})))
 	req := httptest.NewRequest(http.MethodGet, "/admin/CreateAccount", nil)
 	rctx := chi.NewRouteContext()
 	rctx.URLParams.Add("method", "CreateAccount")
@@ -146,7 +146,7 @@ func authorizedAdminRequestForMethod(gw *GatewayConfig, method, body string) *ht
 // An authorized caller must still get a clear answer when the cluster is not
 // reachable, rather than a panic or a silent success.
 func TestAdminRequestReportsClusterUnavailable(t *testing.T) {
-	gw := withOperator(withRealAuth(&GatewayConfig{DisableLogging: true, IAMService: createAccountPolicy()}))
+	gw := withOperator(withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: createAccountPolicy()})))
 
 	rec := authorizedAdminRequest(gw, `{}`)
 
@@ -156,7 +156,7 @@ func TestAdminRequestReportsClusterUnavailable(t *testing.T) {
 // Each admin method is a separate grant. The signup Worker holds
 // spinifex:CreateAccount and must not reach deletion with it.
 func TestCreateAccountGrantDoesNotAllowDeleteAccount(t *testing.T) {
-	gw := withOperator(withRealAuth(&GatewayConfig{DisableLogging: true, IAMService: createAccountPolicy()}))
+	gw := withOperator(withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: createAccountPolicy()})))
 
 	for _, method := range []string{"DeleteAccount", "DescribeAccountDeletion", "ListAccounts"} {
 		t.Run(method, func(t *testing.T) {

spinifex/gateway/auth.go

@@ -148,7 +148,7 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
 					slog.Warn("Auth failure: verification failed",
 						"accessKeyID", sig.Credential.AccessKeyID, "sourceIP", src.logIP,
 						"service", sig.Credential.Service,
-						"action", mismatchAction(r, sig.Credential.Service),
+						"action", gw.mismatchAction(r, sig.Credential.Service),
 						"canonicalRequest", sig.RedactedCanonicalRequest(),
 						"err", sigErr)
 					// Fingerprinted by the signature as well as the key id: each guess at a
@@ -248,9 +248,9 @@ func signingTime(r *http.Request) string {
 
 // mismatchAction recovers the Action for a rejected query-protocol request so the log names
 // the operation. Auth fails before the dispatcher parses it, which otherwise leaves the
-// trace as a bare "POST /". S3 is skipped: its body is object data, not query args.
-func mismatchAction(r *http.Request, service string) string {
-	if service == "s3" {
+// trace as a bare "POST /". A scope whose body is object data is skipped.
+func (gw *GatewayConfig) mismatchAction(r *http.Request, service string) string {
+	if traits, ok := gw.selector().Traits(service); ok && traits.PayloadBody {
 		return ""
 	}
 
@@ -328,7 +328,7 @@ func (gw *GatewayConfig) writeSigV4Error(w http.ResponseWriter, r *http.Request,
 		return
 	}
 
-	xmlError := xmlErrorBody(svc, errorCode, errorMsg.Message, requestID, r.URL.Path)
+	xmlError := gw.xmlErrorBody(svc, errorCode, errorMsg.Message, requestID, r.URL.Path)
 
 	w.Header().Set("Content-Type", envelope.XMLContentType)
 	w.WriteHeader(errorMsg.HTTPCode)
@@ -357,19 +357,13 @@ func signedService(r *http.Request) string {
 	return reqctx.Service(r.Context())
 }
 
-// globalSigningRegions maps a global AWS service to the region its clients sign
-// with when they use the default endpoint. A client given an endpoint override
-// signs with its own configured region instead, so both are accepted.
-var globalSigningRegions = map[string]string{
-	"route53": "us-east-1",
-}
-
 // expectedSigningRegion returns the credential-scope region a request for
 // service must carry: the global service's signing region when the client
-// claimed it, else the gateway's own region.
+// claimed it, else the gateway's own region. A client given an endpoint
+// override signs with its own configured region, so both are accepted.
 func (gw *GatewayConfig) expectedSigningRegion(service, claimed string) string {
-	if global, ok := globalSigningRegions[service]; ok && claimed == global {
-		return global
+	if traits, ok := gw.selector().Traits(service); ok && traits.GlobalSigningRegion != "" && claimed == traits.GlobalSigningRegion {
+		return traits.GlobalSigningRegion
 	}
 	return gw.Region
 }

spinifex/gateway/auth_accountstatus_test.go

@@ -19,7 +19,7 @@ import (
 func statusGateRouter(t *testing.T, svc iamdomain.IAMService) (chi.Router, *http.Request) {
 	t.Helper()
 
-	gw := withEC2(withRealAuth(&GatewayConfig{DisableLogging: true, Region: gwfix.Region, IAMService: svc}))
+	gw := withEC2(withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: gwfix.Region}, gwfix.Deps{IAM: svc})))
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())
 	r.HandleFunc("/*", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })

spinifex/gateway/auth_current_behaviour_test.go

@@ -125,14 +125,10 @@ func newCBEnv(t *testing.T, opts cbEnvOpts) *cbEnv {
 
 // gateway builds a router over the env's services with its own (cold) caches.
 func (e *cbEnv) gateway() http.Handler {
-	gw := &GatewayConfig{
+	gw := gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		NATSConn:       e.nc,
 		Region:         gwfix.Region,
-		ExpectedNodes:  1,
-		IAMService:     e.counter,
-		STSService:     e.sts,
-	}
+	}, gwfix.Deps{NATS: e.nc, Nodes: 1, IAM: e.counter, STS: e.sts})
 	withIAMAndSTS(withRealAuth(gw))
 	return gw.SetupRoutes()
 }

spinifex/gateway/auth_silentmismatch_test.go

@@ -99,14 +99,12 @@ func TestSigV4Auth_FailureLogsLoopbackGatedClientIP(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			rl := authlimit.NewAuthRateLimiter()
 			defer rl.Stop()
-			gw := &GatewayConfig{
+			gw := gwfix.WithDeps(&GatewayConfig{
 				DisableLogging: true,
 				Region:         gwfix.Region,
-				IAMService:     &gwfix.IAM{MasterKey: gwfix.MasterKey},
 				Credentials:    credverify.New(&gwfix.IAM{MasterKey: gwfix.MasterKey}, &gwfix.STS{}),
-				STSService:     &gwfix.STS{},
 				RateLimiter:    rl,
-			}
+			}, gwfix.Deps{IAM: &gwfix.IAM{MasterKey: gwfix.MasterKey}, STS: &gwfix.STS{}})
 			withEC2(gw)
 			handler := gw.SigV4AuthMiddleware()(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
 				t.Fatal("an unknown session credential must not reach the handler")

spinifex/gateway/auth_test.go

@@ -11,6 +11,7 @@ import (
 	ecsawsapi "github.com/mulgadc/spinifex/spinifex/domains/ecs/awsapi"
 	eksawsapi "github.com/mulgadc/spinifex/spinifex/domains/eks/awsapi"
 	ochreawsapi "github.com/mulgadc/spinifex/spinifex/domains/ochre/awsapi"
+	route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
 	taggingawsapi "github.com/mulgadc/spinifex/spinifex/domains/tagging/awsapi"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
@@ -57,11 +58,10 @@ func setupTestApp(accessKey, secretKey string) http.Handler {
 		},
 	}
 
-	gw := withRealAuth(&GatewayConfig{
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
-	})
+	}, gwfix.Deps{IAM: mockSvc}))
 	withServices(gw, awsapi.NewRegistration(awsapi.Deps{}), ec2Registration(gw))
 
 	r := chi.NewRouter()
@@ -90,7 +90,7 @@ func userIDApp(t *testing.T, svc *gwfix.IAM) (http.Handler, *string) {
 		},
 	}
 
-	gw := withEC2(withRealAuth(&GatewayConfig{DisableLogging: true, Region: gwfix.Region, IAMService: svc}))
+	gw := withEC2(withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: gwfix.Region}, gwfix.Deps{IAM: svc})))
 	seen := new(string)
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())
@@ -185,11 +185,10 @@ func TestSigV4Auth_ValidSignatureWithQueryString(t *testing.T) {
 }
 
 func TestSigV4Auth_NilIAMService(t *testing.T) {
-	gw := withEC2(withRealAuth(&GatewayConfig{
+	gw := withEC2(withRealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     nil,
-	}))
+	}, gwfix.Deps{IAM: nil})))
 
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())
@@ -417,11 +416,10 @@ func TestSigV4Auth_ContextPropagation(t *testing.T) {
 		},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
-	})
+	}, gwfix.Deps{IAM: mockSvc}))
 
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())
@@ -499,11 +497,10 @@ func TestSigV4Auth_ContextDoesNotLeakBetweenRequests(t *testing.T) {
 		},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
-	})
+	}, gwfix.Deps{IAM: mockSvc}))
 
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())
@@ -539,11 +536,10 @@ func TestSigV4Auth_ContextDoesNotLeakBetweenRequests(t *testing.T) {
 
 func TestSigV4Auth_LookupAccessKeyUnexpectedError(t *testing.T) {
 	// A non-NoSuchEntity error from LookupAccessKey should yield 500 InternalError.
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     &gwfix.ErrorLookupIAM{Err: errors.New("nats: connection closed")},
-	})
+	}, gwfix.Deps{IAM: &gwfix.ErrorLookupIAM{Err: errors.New("nats: connection closed")}}))
 
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())
@@ -616,11 +612,10 @@ func TestSigV4Auth_SignedContentType(t *testing.T) {
 		},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
-	})
+	}, gwfix.Deps{IAM: mockSvc}))
 
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())
@@ -743,11 +738,10 @@ func TestCheckPolicy_NonRootNoPolicies_Denied(t *testing.T) {
 		},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
-	})
+	}, gwfix.Deps{IAM: mockSvc}))
 
 	handler := setupPolicyTestHandler(gw)
 
@@ -803,11 +797,10 @@ func TestCheckPolicy_NonRootWithAllow_Passes(t *testing.T) {
 		},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
-	})
+	}, gwfix.Deps{IAM: mockSvc}))
 
 	handler := setupPolicyTestHandler(gw)
 
@@ -863,11 +856,10 @@ func TestCheckPolicy_NonRootWithExplicitDeny_Denied(t *testing.T) {
 		},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
-	})
+	}, gwfix.Deps{IAM: mockSvc}))
 
 	handler := setupPolicyTestHandler(gw)
 
@@ -924,11 +916,10 @@ func TestCheckPolicy_RootGlobalAccount_Bypasses(t *testing.T) {
 		},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
-	})
+	}, gwfix.Deps{IAM: mockSvc}))
 
 	handler := setupPolicyTestHandler(gw)
 
@@ -966,11 +957,10 @@ func TestCheckPolicy_MissingAccountID_InternalError(t *testing.T) {
 		},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
-	})
+	}, gwfix.Deps{IAM: mockSvc}))
 
 	handler := setupPolicyTestHandler(gw)
 
@@ -1010,11 +1000,10 @@ func TestCheckPolicy_GetUserPoliciesError_InternalError(t *testing.T) {
 		},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
-	})
+	}, gwfix.Deps{IAM: mockSvc}))
 
 	handler := setupPolicyTestHandler(gw)
 
@@ -1040,12 +1029,10 @@ func TestSigV4Auth_NATSDisconnectedShortCircuit(t *testing.T) {
 	}
 	nc.Close() // disconnected — IsConnected() returns false
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		NATSConn:       nc,
-		IAMService:     nil,
-	})
+	}, gwfix.Deps{NATS: nc, IAM: nil}))
 
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())
@@ -1126,12 +1113,10 @@ func mountSessionGateway(t *testing.T, cred *stsdomain.SessionCredential) (http.
 		},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     iamMock,
-		STSService:     stsMock,
-	})
+	}, gwfix.Deps{IAM: iamMock, STS: stsMock}))
 
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())
@@ -1294,7 +1279,7 @@ func TestSigV4Auth_AKIA_PrincipalTypeUser(t *testing.T) {
 			},
 		},
 	}
-	gw := withEC2RealAuth(&GatewayConfig{DisableLogging: true, Region: gwfix.Region, IAMService: iamMock})
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: gwfix.Region}, gwfix.Deps{IAM: iamMock}))
 
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())
@@ -1324,12 +1309,10 @@ func TestSigV4Auth_Session_STSServiceNil(t *testing.T) {
 		AccessKeys: map[string]*iamdomain.AccessKey{},
 	}
 	_ = encryptedSecret
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     iamMock,
-		STSService:     nil,
-	})
+	}, gwfix.Deps{IAM: iamMock, STS: nil}))
 
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())
@@ -1398,12 +1381,10 @@ func TestCheckPolicy_SessionNameCollision(t *testing.T) {
 		Tokens:   map[string]string{gwfix.SessionAKID: gwfix.SessionToken},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     iamMock,
-		STSService:     stsMock,
-	})
+	}, gwfix.Deps{IAM: iamMock, STS: stsMock}))
 
 	handler := setupPolicyTestHandler(gw)
 
@@ -1442,12 +1423,10 @@ func newAssumedRoleEnforcementGateway(t *testing.T, cred *stsdomain.SessionCrede
 		Sessions: map[string]*stsdomain.SessionCredential{cred.AccessKeyID: cred},
 		Tokens:   map[string]string{cred.AccessKeyID: gwfix.SessionToken},
 	}
-	return withEC2RealAuth(&GatewayConfig{
+	return withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     iamMock,
-		STSService:     stsMock,
-	})
+	}, gwfix.Deps{IAM: iamMock, STS: stsMock}))
 }
 
 // setupPolicyResourceTestHandler is setupPolicyTestHandler for resource-scoped
@@ -1673,7 +1652,8 @@ func TestMismatchAction_RecoversActionAndRewindsBody(t *testing.T) {
 	req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
-	assert.Equal(t, "ImportKeyPair", mismatchAction(req, "ec2"))
+	gw := withServices(&GatewayConfig{})
+	assert.Equal(t, "ImportKeyPair", gw.mismatchAction(req, "ec2"))
 
 	// The error path still writes a response, so the body must survive the peek.
 	rest, err := io.ReadAll(req.Body)
@@ -1684,10 +1664,11 @@ func TestMismatchAction_RecoversActionAndRewindsBody(t *testing.T) {
 func TestMismatchAction_SkipsS3AndPrefersQueryString(t *testing.T) {
 	// An S3 body is object data, so it must never be parsed as query args.
 	s3req := httptest.NewRequest(http.MethodPut, "/bucket/key", bytes.NewReader([]byte("Action=NotReally")))
-	assert.Empty(t, mismatchAction(s3req, "s3"))
+	gw := withServices(&GatewayConfig{})
+	assert.Empty(t, gw.mismatchAction(s3req, "s3"))
 
 	qreq := httptest.NewRequest(http.MethodGet, "/?Action=DescribeInstances", nil)
-	assert.Equal(t, "DescribeInstances", mismatchAction(qreq, "ec2"))
+	assert.Equal(t, "DescribeInstances", gw.mismatchAction(qreq, "ec2"))
 }
 
 func TestRedactedCanonicalRequest_MasksSessionToken(t *testing.T) {
@@ -1829,7 +1810,7 @@ func TestSigV4Auth_ResolvesCtxActionForNonQueryServices(t *testing.T) {
 			},
 		},
 	}
-	gw := withEC2RealAuth(&GatewayConfig{DisableLogging: true, Region: gwfix.Region, IAMService: mockSvc})
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: gwfix.Region}, gwfix.Deps{IAM: mockSvc}))
 	b := dispatch.NewBuilder()
 	require.NoError(t, b.Register(awsapi.NewRegistration(awsapi.Deps{})))
 	require.NoError(t, b.Register(acmawsapi.NewRegistration(acmawsapi.Deps{})))
@@ -1888,7 +1869,7 @@ func TestSigV4Auth_ResolvesCtxActionForNonQueryServices(t *testing.T) {
 // default endpoint and for the configured region against an override. Every
 // other service stays pinned to the gateway's region.
 func TestExpectedSigningRegion(t *testing.T) {
-	gw := &GatewayConfig{Region: "ap-southeast-2"}
+	gw := withServices(&GatewayConfig{Region: "ap-southeast-2"}, route53awsapi.NewRegistration(route53awsapi.Deps{}))
 	for _, tc := range []struct {
 		service, claimed, want string
 	}{
@@ -2022,11 +2003,10 @@ func TestCheckPolicy_RootNonGlobalAccount_StillEvaluated(t *testing.T) {
 		},
 	}
 
-	gw := withEC2RealAuth(&GatewayConfig{
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
-	})
+	}, gwfix.Deps{IAM: mockSvc}))
 
 	handler := setupPolicyTestHandler(gw)
 
@@ -2168,7 +2148,7 @@ func TestSigV4Auth_UnknownAKIDPrefix_NoLookup(t *testing.T) {
 }
 
 func TestWriteSigV4Error_ResponseFormat(t *testing.T) {
-	gw := withEC2RealAuth(&GatewayConfig{DisableLogging: true, Region: gwfix.Region, IAMService: allowAllIAMService()})
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: gwfix.Region}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	testCases := []struct {
 		errorCode      string
@@ -2219,7 +2199,7 @@ func TestWriteSigV4Error_ResponseFormat(t *testing.T) {
 }
 
 func TestWriteSigV4Error_IgnoresClientRequestID(t *testing.T) {
-	gw := withEC2RealAuth(&GatewayConfig{DisableLogging: true, Region: gwfix.Region, IAMService: allowAllIAMService()})
+	gw := withEC2RealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: gwfix.Region}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	w := httptest.NewRecorder()
 	req := httptest.NewRequest(http.MethodGet, "/", nil)

spinifex/gateway/capabilities_testhelpers_test.go

@@ -1,6 +1,8 @@
 package gateway
 
 import (
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
+	operatoradmin "github.com/mulgadc/spinifex/spinifex/operator/admin"
 	"net/http"
 	"time"
 
@@ -29,18 +31,29 @@ func (allowAdmission) Admit(string, string) error { return nil }
 
 type natsReadiness struct{ gw *GatewayConfig }
 
-func (n natsReadiness) Available() bool { return n.gw.NATSConn != nil && n.gw.NATSConn.IsConnected() }
+func (n natsReadiness) Available() bool {
+	return gwfix.DepsOf(n.gw).NATS != nil && gwfix.DepsOf(n.gw).NATS.IsConnected()
+}
 
 // withFakeCapabilities fills whichever capability a test left unset.
 func withFakeCapabilities(gw *GatewayConfig) *GatewayConfig {
+	withTestAdmin(gw)
 	if gw.Policy == nil {
 		gw.Policy = allowPolicy{}
 	}
 	if gw.Admission == nil {
 		gw.Admission = allowAdmission{}
 	}
-	if gw.Readiness == nil && gw.NATSConn != nil {
+	if gw.Readiness == nil && gwfix.DepsOf(gw).NATS != nil {
 		gw.Readiness = natsReadiness{gw}
 	}
 	return gw
 }
+
+// withTestAdmin points the /admin surface at the spinifex registration, as the
+// composing role does.
+func withTestAdmin(gw *GatewayConfig) {
+	if gw.Admin.Service == "" {
+		gw.Admin = AdminSurface{Service: operatoradmin.ServiceName, WithRequestID: operatoradmin.WithRequestID}
+	}
+}

spinifex/gateway/conditionkeys_completeness_test.go

@@ -122,7 +122,7 @@ func emittedKeys(t *testing.T) map[string]string {
 		},
 	}
 
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: &gwfix.IAM{}})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: &gwfix.IAM{}}))
 	union := make(map[string]string)
 	for name, principal := range principals {
 		r := httptest.NewRequest(http.MethodPost, "/?prefix=home/&max-keys=100", nil)
@@ -131,7 +131,7 @@ func emittedKeys(t *testing.T) map[string]string {
 		// Resolved the way the middleware resolves it, so the gate cannot pass on
 		// a value only the test supplies.
 		userID, _, err := stsdomain.ResolveCallerUser(principal.AccountID, principal.PrincipalType,
-			principal.Identity, principal.AssumedRoleID, gw.IAMService)
+			principal.Identity, principal.AssumedRoleID, gwfix.DepsOf(gw).IAM)
 		require.NoError(t, err)
 		principal.UserID = userID
 		for key := range policyauth.RequestConditionKeys(r, principal) {

spinifex/gateway/conditionkeys_test.go

@@ -194,12 +194,11 @@ func conditionedPolicyIAMService(effect string, cond map[string]map[string]iamdo
 // nil keys from checkPolicyResources would leave every other test green while
 // silently killing every conditional Allow on this path.
 func TestCheckPolicyResources_ConditionalAllowHonoursSourceIP(t *testing.T) {
-	gw := withRealAuth(&GatewayConfig{
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		IAMService: conditionedPolicyIAMService("Allow", map[string]map[string]iamdomain.ConditionValue{
-			iampolicy.OpIPAddress: {iampolicy.KeySourceIP: {"10.0.0.0/8"}},
-		}),
-	})
+	}, gwfix.Deps{IAM: conditionedPolicyIAMService("Allow", map[string]map[string]iamdomain.ConditionValue{
+		iampolicy.OpIPAddress: {iampolicy.KeySourceIP: {"10.0.0.0/8"}},
+	})}))
 
 	tests := []struct {
 		name    string
@@ -233,12 +232,11 @@ func TestCheckPolicyResources_ConditionalAllowHonoursSourceIP(t *testing.T) {
 // the one direction where an absent key widens access. Pin it so the omission
 // is a decision rather than an accident.
 func TestCheckPolicyResources_ConditionalDenyHonoursSourceIP(t *testing.T) {
-	gw := withRealAuth(&GatewayConfig{
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		IAMService: conditionedPolicyIAMService("Deny", map[string]map[string]iamdomain.ConditionValue{
-			iampolicy.OpIPAddress: {iampolicy.KeySourceIP: {"203.0.113.0/24"}},
-		}),
-	})
+	}, gwfix.Deps{IAM: conditionedPolicyIAMService("Deny", map[string]map[string]iamdomain.ConditionValue{
+		iampolicy.OpIPAddress: {iampolicy.KeySourceIP: {"203.0.113.0/24"}},
+	})}))
 
 	for _, tt := range []struct {
 		name    string

spinifex/gateway/discover_nodes_test.go

@@ -1,117 +0,0 @@
-package gateway_test
-
-import (
-	"context"
-	"encoding/json"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/gateway"
-	"github.com/nats-io/nats.go"
-)
-
-// respondAsNodes answers the discovery subject as n distinct nodes and reports
-// how many discovery rounds were served.
-func respondAsNodes(t *testing.T, nc *nats.Conn, n int) *atomic.Int64 {
-	t.Helper()
-
-	var rounds atomic.Int64
-	for i := range n {
-		node := string(rune('a' + i))
-		sub, err := nc.Subscribe(clusterv1.NodesDiscoverSubject, func(msg *nats.Msg) {
-			if node == "a" {
-				rounds.Add(1)
-			}
-			payload, err := json.Marshal(clusterv1.NodeDiscoverResponse{Node: node})
-			if err != nil {
-				return
-			}
-			_ = msg.Respond(payload)
-		})
-		if err != nil {
-			t.Fatalf("subscribe: %v", err)
-		}
-		t.Cleanup(func() { _ = sub.Unsubscribe() })
-	}
-	if err := nc.Flush(); err != nil {
-		t.Fatalf("flush: %v", err)
-	}
-	return &rounds
-}
-
-// Discovery fronts eight EC2 handlers, so whatever it costs is added to every
-// one of their calls. The gather counts whoever answers and therefore cannot
-// know when to stop, so it always sits out its whole timeout — that cost
-// has to be paid once in a while, not once per request.
-func TestDiscoverActiveNodesKeepsItsCostOffTheRequestPath(t *testing.T) {
-	const timeout = 50 * time.Millisecond
-	gateway.SetDiscoverActiveNodesTimeoutForTest(t, timeout)
-	_, nc := testutil.StartTestNATS(t)
-	respondAsNodes(t, nc, 4)
-
-	gw := &gateway.GatewayConfig{NATSConn: nc, ExpectedNodes: 4}
-
-	if count := gw.DiscoverActiveNodes(context.Background()); count != 4 {
-		t.Fatalf("DiscoverActiveNodes = %d, want 4", count)
-	}
-
-	start := time.Now()
-	for range 5 {
-		gw.DiscoverActiveNodes(context.Background())
-	}
-	elapsed := time.Since(start)
-
-	// Five uncached calls would cost five timeouts, so two bounds the cached path well clear of that.
-	if elapsed > 2*timeout {
-		t.Errorf("five further discoveries took %v; each was paying the fan-out timeout again", elapsed)
-	}
-}
-
-// Membership does not change per request. Re-deriving it on every call put a
-// fan-out in front of every API call that needs it, which is most of them.
-func TestDiscoverActiveNodesReusesARecentAnswer(t *testing.T) {
-	gateway.SetDiscoverActiveNodesTimeoutForTest(t, 50*time.Millisecond)
-	_, nc := testutil.StartTestNATS(t)
-	rounds := respondAsNodes(t, nc, 3)
-
-	gw := &gateway.GatewayConfig{NATSConn: nc, ExpectedNodes: 3}
-
-	if count := gw.DiscoverActiveNodes(context.Background()); count != 3 {
-		t.Fatalf("first discovery = %d, want 3", count)
-	}
-	first := rounds.Load()
-
-	for range 5 {
-		if count := gw.DiscoverActiveNodes(context.Background()); count != 3 {
-			t.Fatalf("cached discovery = %d, want 3", count)
-		}
-	}
-
-	if got := rounds.Load(); got != first {
-		t.Errorf("discovery ran %d more time(s) within the TTL, want the cached answer reused", got-first)
-	}
-}
-
-// A fallback is not evidence of anything: caching one would make a momentary
-// NATS problem outlive itself for the whole TTL.
-func TestDiscoverActiveNodesDoesNotCacheAFallback(t *testing.T) {
-	gateway.SetDiscoverActiveNodesTimeoutForTest(t, 50*time.Millisecond)
-	_, nc := testutil.StartTestNATS(t)
-
-	gw := &gateway.GatewayConfig{NATSConn: nc, ExpectedNodes: 2}
-
-	if count := gw.DiscoverActiveNodes(context.Background()); count != 2 {
-		t.Fatalf("fallback = %d, want the configured node count", count)
-	}
-
-	rounds := respondAsNodes(t, nc, 2)
-	if count := gw.DiscoverActiveNodes(context.Background()); count != 2 {
-		t.Fatalf("second discovery = %d, want 2", count)
-	}
-	if rounds.Load() == 0 {
-		t.Error("no discovery was published after a fallback, so the fallback had been cached")
-	}
-}

spinifex/gateway/dispatch_seam_test.go

@@ -148,27 +148,25 @@ func dispatchSeamApp(t *testing.T, docs []iamdomain.PolicyDocument, regs ...disp
 		require.NoError(t, b.Register(reg))
 	}
 
-	gw := withRealAuth(&GatewayConfig{
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		NATSConn:       nc,
 		Services:       b.Build(),
-		IAMService: &gwfix.PolicyIAM{
-			IAM: gwfix.IAM{
-				MasterKey: gwfix.MasterKey,
-				AccessKeys: map[string]*iamdomain.AccessKey{
-					gwfix.AccessKey: {
-						AccessKeyID:     gwfix.AccessKey,
-						SecretAccessKey: encryptedSecret,
-						UserName:        "alice",
-						AccountID:       ecrTestAccount,
-						Status:          "Active",
-					},
+	}, gwfix.Deps{NATS: nc, IAM: &gwfix.PolicyIAM{
+		IAM: gwfix.IAM{
+			MasterKey: gwfix.MasterKey,
+			AccessKeys: map[string]*iamdomain.AccessKey{
+				gwfix.AccessKey: {
+					AccessKeyID:     gwfix.AccessKey,
+					SecretAccessKey: encryptedSecret,
+					UserName:        "alice",
+					AccountID:       ecrTestAccount,
+					Status:          "Active",
 				},
 			},
-			GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
 		},
-	})
+		GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
+	}}))
 
 	r := chi.NewRouter()
 	r.Use(audit.Middleware)
@@ -342,7 +340,7 @@ func TestDispatchSeam_NATSDownGetsSame503AsLegacy(t *testing.T) {
 	b := dispatch.NewBuilder()
 	require.NoError(t, b.Register(fakeJSONRegistration(&dispatchCapture{})))
 
-	gw := withRealAuth(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService(), Services: b.Build()})
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Services: b.Build()}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := reqctx.WithService(req.Context(), fakeJSONService)
@@ -369,17 +367,16 @@ func TestDispatchSeam_UnregisteredUnknownServiceStillInvalidAction(t *testing.T)
 
 	encryptedSecret, err := iamdomain.EncryptSecret(gwfix.SecretKey, gwfix.MasterKey)
 	require.NoError(t, err)
-	gw := withRealAuth(&GatewayConfig{
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
 		Services:       b.Build(),
-		IAMService: &gwfix.IAM{
-			MasterKey: gwfix.MasterKey,
-			AccessKeys: map[string]*iamdomain.AccessKey{
-				gwfix.AccessKey: {AccessKeyID: gwfix.AccessKey, SecretAccessKey: encryptedSecret, UserName: "root", Status: "Active"},
-			},
+	}, gwfix.Deps{IAM: &gwfix.IAM{
+		MasterKey: gwfix.MasterKey,
+		AccessKeys: map[string]*iamdomain.AccessKey{
+			gwfix.AccessKey: {AccessKeyID: gwfix.AccessKey, SecretAccessKey: encryptedSecret, UserName: "root", Status: "Active"},
 		},
-	})
+	}}))
 
 	r := chi.NewRouter()
 	r.Use(gw.SigV4AuthMiddleware())

spinifex/gateway/ec2_deletevolume_guard_test.go

@@ -3,6 +3,7 @@ package gateway
 import (
 	"context"
 	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
 	ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
@@ -48,12 +49,10 @@ func TestDeleteVolumeAction_MissingConfiguredNodeIsRetryableAndDeletesNothing(t
 	}
 	require.NoError(t, nc.Flush())
 
-	gw := &GatewayConfig{
+	gw := gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		NATSConn:       nc,
-		IAMService:     allowAllIAMService(),
 		Credentials:    credverify.New(allowAllIAMService(), nil),
-	}
+	}, gwfix.Deps{NATS: nc, IAM: allowAllIAMService()})
 	withEC2(gw, func(d *ec2awsapi.Deps) { d.NodeIDs = []string{"node-1", "node-2"} })
 	req := setupEC2Request("Action=DeleteVolume&VolumeId=vol-0123456789abcdef0", "123456789012")
 	ctx, cancel := context.WithTimeout(req.Context(), 30*time.Second)

spinifex/gateway/ec2_dryrun_test.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
 	ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
@@ -11,7 +12,7 @@ import (
 // The gateway has no NATS connection, so any request that reached its handler
 // would fail ServerInternal; DryRunOperation proves nothing was dispatched.
 func TestEC2Request_DryRunAnswersEveryAction(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: authzRegion}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	actions := ec2awsapi.ActionNames()
 

spinifex/gateway/ec2_validation_test.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
 	"github.com/stretchr/testify/assert"
 	"net/http"
 	"strconv"
@@ -10,7 +11,7 @@ import (
 // Captured from AWS: the combination is refused even when MaxResults is also
 // out of range.
 func TestEC2Request_DescribeInstancesMaxResultsWithInstanceIds(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: authzRegion}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	for _, n := range []string{"2", "5"} {
 		w := serveEC2(gw, "Action=DescribeInstances&InstanceId.1=i-0123456789abcdef0&MaxResults="+n)
@@ -22,12 +23,12 @@ func TestEC2Request_DescribeInstancesMaxResultsWithInstanceIds(t *testing.T) {
 
 // AWS answers DryRun ahead of a MaxResults out of range.
 func TestEC2Request_MaxResultsAfterDryRun(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: authzRegion}, gwfix.Deps{IAM: allowAllIAMService()}))
 	assertDryRun(t, dispatchEC2(t, gw, "Action=DescribeLaunchTemplates&MaxResults=0&DryRun=true"))
 }
 
 func TestEC2Request_MaxResultsInRange(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: authzRegion}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	for _, body := range []string{
 		"Action=DescribeCapacityReservations&MaxResults=1",
@@ -48,7 +49,7 @@ func TestEC2Request_MaxResultsInRange(t *testing.T) {
 
 // The refusals are AWS's own, captured by scripts/aws-compliance/aws-response-diff/scen_e.py.
 func TestEC2Request_MaxResultsOutOfRange(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: authzRegion}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	cases := []struct {
 		action string
@@ -85,7 +86,7 @@ func TestEC2Request_MaxResultsOutOfRange(t *testing.T) {
 // Each of an action's own types reaches the handler, which fails only on the
 // absent NATS connection.
 func TestEC2Request_TagSpecificationAllowedTypes(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: authzRegion}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	for _, rt := range []string{"instance", "volume", "network-interface", "spot-instances-request"} {
 		assertPermitted(t, dispatchEC2(t, gw, tagSpecBody("RunInstances", rt)+"&ImageId=ami-1&MinCount=1&MaxCount=1"))
@@ -94,7 +95,7 @@ func TestEC2Request_TagSpecificationAllowedTypes(t *testing.T) {
 }
 
 func TestEC2Request_TagSpecificationResourceType(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: authzRegion}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	cases := []struct {
 		name, body, code, resourceType string

spinifex/gateway/ecr_characterization_test.go

@@ -59,7 +59,7 @@ func TestECRInventory_ReachableStubsAnswer501(t *testing.T) {
 	}
 	assert.ElementsMatch(t, want, reachableStubs)
 
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	for _, action := range reachableStubs {
 		t.Run(action, func(t *testing.T) {
 			err := gw.serveECR(httptest.NewRecorder(),
@@ -74,7 +74,7 @@ func TestECRInventory_ReachableStubsAnswer501(t *testing.T) {
 // unsupported scanning action answers OperationNotSupportedException (400),
 // not NotImplemented (501).
 func TestECRInventory_ScanningActionsReturnOperationNotSupported(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	for _, action := range awsapi.UnsupportedActionNames() {
 		t.Run(action, func(t *testing.T) {
 			err := gw.serveECR(httptest.NewRecorder(),
@@ -122,7 +122,7 @@ func TestECRPipeline_TraceActionEnricher_SpanNamedServiceAction(t *testing.T) {
 // carrying another service's prefix but naming an action outside awsapi.Actions
 // fails as an unrecognised ECR action, not as a cross-service error.
 func TestECRRequest_ECSPrefixedNonECRActionIsInvalidAction(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	err := gw.serveECR(httptest.NewRecorder(),
 		setupECRRequest("AmazonEC2ContainerServiceV20141113.ListClusters", "{}"))
 	require.Error(t, err)
@@ -132,7 +132,7 @@ func TestECRRequest_ECSPrefixedNonECRActionIsInvalidAction(t *testing.T) {
 // TestECRRequest_TrailingDotIsMissingAction pins that a target ending in "."
 // resolves to an empty action, the same as an absent target.
 func TestECRRequest_TrailingDotIsMissingAction(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	err := gw.serveECR(httptest.NewRecorder(),
 		setupECRRequest(awsapi.TargetPrefix+".", "{}"))
 	require.Error(t, err)
@@ -143,7 +143,7 @@ func TestECRRequest_TrailingDotIsMissingAction(t *testing.T) {
 // any "<prefix>.<Action>" shape (not just the ECR prefix) dispatches on the
 // bare action name today.
 func TestECRRequest_UnrelatedPrefixDispatchesByBareAction(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	err := gw.serveECR(httptest.NewRecorder(), setupECRRequest("a.b.ListRepositories", "{}"))
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorNotImplemented, err.Error())
@@ -154,11 +154,10 @@ func TestECRRequest_UnrelatedPrefixDispatchesByBareAction(t *testing.T) {
 func TestGetAuthorizationToken_IgnoresRequestSuppliedIdentity(t *testing.T) {
 	iss, verify := newECRAuth(t)
 	endpoint := awsapi.RepositoryEndpoint{Region: ecrTestRegion, ServicesDomain: ecrTestSuffix}
-	gw := withECR(withVerifier(&GatewayConfig{
-		Region: ecrTestRegion, InternalSuffix: ecrTestSuffix,
+	gw := withECR(withVerifier(gwfix.WithDeps(&GatewayConfig{
+		Region:         ecrTestRegion,
 		DisableLogging: true,
-		IAMService:     allowAllIAMService(),
-	}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
+	}, gwfix.Deps{IAM: allowAllIAMService()})), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
 
 	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"registryIds":["999999999999"]}`))
 	ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)

spinifex/gateway/ecr_getauthorizationtoken_test.go

@@ -17,11 +17,10 @@ import (
 
 func TestECRRequest_GetAuthorizationTokenDispatched(t *testing.T) {
 	iss, _ := newECRAuth(t)
-	gw := withECR(withVerifier(&GatewayConfig{
-		Region: ecrTestRegion, InternalSuffix: ecrTestSuffix,
+	gw := withECR(withVerifier(gwfix.WithDeps(&GatewayConfig{
+		Region:         ecrTestRegion,
 		DisableLogging: true,
-		IAMService:     allowAllIAMService(),
-	}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, awsapi.RepositoryEndpoint{
+	}, gwfix.Deps{IAM: allowAllIAMService()})), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, awsapi.RepositoryEndpoint{
 		Region: ecrTestRegion, ServicesDomain: ecrTestSuffix,
 	})})
 	w := httptest.NewRecorder()
@@ -33,11 +32,10 @@ func TestECRRequest_GetAuthorizationTokenDispatched(t *testing.T) {
 func TestGetAuthorizationToken_MintsUsableToken(t *testing.T) {
 	iss, verify := newECRAuth(t)
 	endpoint := awsapi.RepositoryEndpoint{Region: ecrTestRegion, ServicesDomain: ecrTestSuffix}
-	gw := withECR(withVerifier(&GatewayConfig{
-		Region: ecrTestRegion, InternalSuffix: ecrTestSuffix,
+	gw := withECR(withVerifier(gwfix.WithDeps(&GatewayConfig{
+		Region:         ecrTestRegion,
 		DisableLogging: true,
-		IAMService:     allowAllIAMService(),
-	}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
+	}, gwfix.Deps{IAM: allowAllIAMService()})), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
 
 	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("{}"))
 	ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
@@ -78,7 +76,7 @@ func TestGetAuthorizationToken_MintsUsableToken(t *testing.T) {
 }
 
 func TestGetAuthorizationToken_MissingComposition(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	req := setupECRRequest(awsapi.TargetPrefix+".GetAuthorizationToken", "{}")
 	err := gw.serveECR(httptest.NewRecorder(), req)
 	require.Error(t, err)
@@ -86,10 +84,9 @@ func TestGetAuthorizationToken_MissingComposition(t *testing.T) {
 }
 
 func TestGetAuthorizationToken_NoIssuerNotImplemented(t *testing.T) {
-	gw := withECR(withVerifier(&GatewayConfig{
+	gw := withECR(withVerifier(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		IAMService:     allowAllIAMService(),
-	}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(nil, awsapi.RepositoryEndpoint{})})
+	}, gwfix.Deps{IAM: allowAllIAMService()})), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(nil, awsapi.RepositoryEndpoint{})})
 	req := setupECRRequest(awsapi.TargetPrefix+".GetAuthorizationToken", "{}")
 	ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
 	err := gw.serveECR(httptest.NewRecorder(), req.WithContext(ctx))
@@ -107,11 +104,10 @@ func TestGetAuthorizationToken_PathedUserRoundTrips(t *testing.T) {
 	iamSvc.userPolicies[ecrPrincipalTestAccount+"|dev"] = allow
 	iss, verify := newECRAuth(t)
 	endpoint := awsapi.RepositoryEndpoint{Region: ecrTestRegion, ServicesDomain: ecrTestSuffix}
-	gw := withECR(withVerifier(&GatewayConfig{
-		Region: ecrTestRegion, InternalSuffix: ecrTestSuffix,
+	gw := withECR(withVerifier(gwfix.WithDeps(&GatewayConfig{
+		Region:         ecrTestRegion,
 		DisableLogging: true,
-		IAMService:     iamSvc,
-	}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
+	}, gwfix.Deps{IAM: iamSvc})), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
 
 	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("{}"))
 	ctx := withPrincipalField(req.Context(), fieldAccountID, ecrPrincipalTestAccount)
@@ -138,7 +134,7 @@ func TestGetAuthorizationToken_PathedUserRoundTrips(t *testing.T) {
 	claims, err := verify.Verify(jwtStr)
 	require.NoError(t, err)
 	assert.Equal(t, ecrPathedUserARN, claims.Subject)
-	got, failure := credverify.New(gw.IAMService, gw.STSService).Rehydrate(claims.AccessKeyID, claims.AccountID)
+	got, failure := credverify.New(gwfix.DepsOf(gw).IAM, gwfix.DepsOf(gw).STS).Rehydrate(claims.AccessKeyID, claims.AccountID)
 	require.Nil(t, failure)
 	assert.Equal(t, ecrPathedUserARN, got.CallerARN)
 }
@@ -146,11 +142,10 @@ func TestGetAuthorizationToken_PathedUserRoundTrips(t *testing.T) {
 func TestGetAuthorizationToken_ProxyEndpointCarriesPort(t *testing.T) {
 	iss, _ := newECRAuth(t)
 	endpoint := awsapi.RepositoryEndpoint{Region: ecrTestRegion, ServicesDomain: ecrTestSuffix, RegistryPort: "9999"}
-	gw := withECR(withVerifier(&GatewayConfig{
-		Region: ecrTestRegion, InternalSuffix: ecrTestSuffix,
+	gw := withECR(withVerifier(gwfix.WithDeps(&GatewayConfig{
+		Region:         ecrTestRegion,
 		DisableLogging: true,
-		IAMService:     allowAllIAMService(),
-	}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
+	}, gwfix.Deps{IAM: allowAllIAMService()})), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
 
 	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("{}"))
 	ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)

spinifex/gateway/ecr_test.go

@@ -1,17 +1,11 @@
 package gateway
 
 import (
-	"encoding/json"
 	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/reqctx"
 	"net/http"
 	"net/http/httptest"
 	"strings"
-	"testing"
-
-	"github.com/go-chi/chi/v5"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 // setupECRRequest builds an ECR control-plane request context. It carries a
@@ -31,23 +25,3 @@ func setupECRRequest(target, body string) *http.Request {
 	ctx = withPrincipalField(ctx, fieldAccessKey, "AKIAECRTESTCONTROLPL1")
 	return req.WithContext(ctx)
 }
-
-func TestOCIRegistry_StubReturns501(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
-	r := chi.NewRouter()
-	gw.mountRegistry(r)
-
-	w := httptest.NewRecorder()
-	r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v2/team/app/tags/list", nil))
-
-	assert.Equal(t, http.StatusNotImplemented, w.Code)
-
-	var body struct {
-		Errors []struct {
-			Code string `json:"code"`
-		} `json:"errors"`
-	}
-	require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body))
-	require.Len(t, body.Errors, 1)
-	assert.Equal(t, "UNSUPPORTED", body.Errors[0].Code)
-}

spinifex/gateway/ecs_passrole_test.go

@@ -19,7 +19,7 @@ func TestECSRegisterTaskDefinition_DenialRendersAccessDenied403(t *testing.T) {
 			return gwfix.AllowPolicyResource("ecs:RegisterTaskDefinition", "*"), nil
 		},
 	}
-	gw := withRealAuth(&GatewayConfig{DisableLogging: true, IAMService: mock})
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: mock}))
 	req := setupECSUserRequest("AmazonEC2ContainerServiceV20141113.RegisterTaskDefinition",
 		registerTaskDefinitionBody(t, testECSPassRoleTargetARN))
 

spinifex/gateway/ecs_test.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/reqctx"
 	"github.com/stretchr/testify/assert"
@@ -13,7 +14,7 @@ import (
 
 // An action whose real handler has not yet landed still resolves to the 501 stub.
 func TestECSRequest_KnownActionNotImplemented(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	err := gw.serveECS(httptest.NewRecorder(),
 		setupECSRequest("AmazonEC2ContainerServiceV20141113.UpdateCluster", "{}"))
 	require.Error(t, err)
@@ -27,7 +28,7 @@ func TestECSRequest_MissingAccountID(t *testing.T) {
 	req.Header.Set("X-Amz-Target", "AmazonEC2ContainerServiceV20141113.ListClusters")
 	req = withTestIdentity(req.WithContext(reqctx.WithService(req.Context(), "ecs")))
 
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	// Account-less requests are rejected by the policy gate, which runs ahead of
 	// the handler's own account guard.
 	err := gw.serveECS(httptest.NewRecorder(), req)
@@ -36,14 +37,14 @@ func TestECSRequest_MissingAccountID(t *testing.T) {
 }
 
 func TestECSRequest_MissingTarget(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	err := gw.serveECS(httptest.NewRecorder(), setupECSRequest("", ""))
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorMissingAction, err.Error())
 }
 
 func TestECSRequest_UnknownAction(t *testing.T) {
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	err := gw.serveECS(httptest.NewRecorder(),
 		setupECSRequest("AmazonEC2ContainerServiceV20141113.MadeUpAction", "{}"))
 	require.Error(t, err)

spinifex/gateway/error_envelope_test.go

@@ -237,7 +237,7 @@ func TestErrorEnvelope_ClusterUnavailable(t *testing.T) {
 	b := dispatch.NewBuilder()
 	require.NoError(t, b.Register(envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil)))
 	require.NoError(t, b.Register(envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c, nil)))
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService(), Services: b.Build()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Services: b.Build()}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	call := func(svc string) *http.Response {
 		req := httptest.NewRequest(http.MethodPost, "/", nil)

spinifex/gateway/gateway.go

@@ -3,29 +3,20 @@ package gateway
 
 import (
 	"context"
-	"crypto/x509"
-	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/pipeline"
 	authlimit "github.com/mulgadc/spinifex/spinifex/ingress/aws/ratelimit"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/reqctx"
 	"log/slog"
 	"net/http"
 	"strconv"
-	"sync"
-	"time"
 	"uuid"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/mulgadc/bluebottle/pkg/iampolicy"
 	bbotel "github.com/mulgadc/bluebottle/pkg/otelsetup"
 	"github.com/mulgadc/bluebottle/pkg/ratelimit"
-	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
-	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
-	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
-	stsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sts/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/audit"
@@ -34,7 +25,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
-	"github.com/nats-io/nats.go"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 )
@@ -43,29 +33,12 @@ import (
 // facts, auth and throttling collaborators, and lazily built caches. Its methods are the
 // per-service request handlers.
 type GatewayConfig struct {
-	Debug          bool       `json:"debug"`
-	DisableLogging bool       `json:"disable_logging"`
-	NATSConn       *nats.Conn // Shared NATS connection for service communication
-	Config         string     // Shared AWS Gateway config for S3 auth
-	// RootCAs is the cluster CA pool, used to verify predastore meta nodes
-	// dialed directly for GetStorageStatus. Nil disables that verification,
-	// so admin storage status reports every meta node unreachable rather than
-	// dialing with no trust root.
-	RootCAs       *x509.CertPool
-	ExpectedNodes int    // Number of expected spinifex nodes for multi-node operations
-	Region        string // Region this gateway is running in
-	// The last discovered node count and when it was discovered, so the
-	// discovery fan-out runs once per activeNodesTTL rather than once per
-	// request that needs to know how many nodes to wait for.
-	activeNodesMu    sync.RWMutex
-	activeNodesCount int
-	activeNodesAt    time.Time
-	InternalSuffix   string // Internal DNS suffix for AWS-parity endpoints (e.g. spinifex.internal)
-	IAMService       iamdomain.IAMService
-	STSService       stsdomain.STSService
-	// AnonymousSTS serves the unsigned STS actions that authenticate by a token
-	// in the body. Nil answers them with InternalError.
-	AnonymousSTS stsawsapi.AnonymousFunc
+	Debug          bool   `json:"debug"`
+	DisableLogging bool   `json:"disable_logging"`
+	Region         string // Region this gateway is running in
+	// PreAuth serves requests that authenticate by their own payload, ahead of
+	// SigV4 and throttling, in order.
+	PreAuth []dispatch.PreAuth
 
 	// Credentials verifies SigV4 credentials. Nil fails every request closed.
 	Credentials credential.Verifier
@@ -81,15 +54,13 @@ type GatewayConfig struct {
 	// early refusal in the auth middleware and makes the service gate answer
 	// cluster-unavailable.
 	Readiness pipeline.Readiness
-	Version   string // Build-time version string (set from cmd.Version)
-	Commit    string // Build-time commit hash (set from cmd.Commit)
-	// OIDC serves the unauthenticated EKS OIDC discovery and JWKS endpoints. Nil
-	// answers them with a 500 (e.g. in unit tests of unrelated routes).
-	OIDC OIDCPublisher
-	// RegistryHandler serves the registry host's /v2 subtree. It sits outside the
-	// SigV4 group and the AWS seam, which ADR-0002:S5 keeps off the registry. Nil
-	// answers 501.
-	RegistryHandler http.Handler
+	// PublicRoutes mounts the routes served ahead of SigV4 and the throttle.
+	// They authenticate themselves or need no authentication. Nil mounts none.
+	PublicRoutes func(chi.Router)
+
+	// Admin configures the private /admin surface. Without a Service, every
+	// /admin request is answered InternalError.
+	Admin AdminSurface
 
 	// Services holds the signed AWS services dispatched through the
 	// registration seam. Nil means nothing is registered, so every signed
@@ -159,19 +130,15 @@ func (gw *GatewayConfig) SetupRoutes() http.Handler {
 		r.Use(audit.RequestLogger)
 	}
 
-	// Anonymous STS (AssumeRoleWithWebIdentity) is dispatched ahead of SigV4 —
-	// these calls carry a web-identity JWT, not AWS credentials.
-	r.Use(gw.anonymousSTSInterceptor)
+	// Pre-authentication handlers run ahead of SigV4: their requests carry a
+	// payload token, not AWS credentials.
+	r.Use(gw.preAuthInterceptor)
 
-	// Unauthenticated OIDC discovery endpoints (IRSA) bypass auth and throttle.
-	r.Group(func(pub chi.Router) {
-		pub.Get("/oidc/eks/{region}/{accountID}/{clusterName}/.well-known/openid-configuration", gw.OIDCDiscoveryDocument)
-		pub.Get("/oidc/eks/{region}/{accountID}/{clusterName}/keys", gw.OIDCJWKS)
-	})
-
-	// OCI Distribution registry (/v2/*). Token/host-authenticated rather than
-	// SigV4-credential-scoped, so it mounts outside the SigV4 group.
-	gw.mountRegistry(r)
+	// Public routes: unauthenticated or self-authenticating, so outside the
+	// SigV4 group.
+	if gw.PublicRoutes != nil {
+		gw.PublicRoutes(r)
+	}
 
 	// Authenticated AWS API surface.
 	r.Group(func(auth chi.Router) {
@@ -202,6 +169,25 @@ func (gw *GatewayConfig) SetupRoutes() http.Handler {
 	return r
 }
 
+// preAuthInterceptor offers each request to the pre-authentication handlers in
+// order; one that takes it ends the chain, and its errors render under the
+// service it declares.
+func (gw *GatewayConfig) preAuthInterceptor(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		for _, pre := range gw.PreAuth {
+			handled, err := pre.Serve(w, r)
+			if !handled {
+				continue
+			}
+			if err != nil {
+				gw.ErrorHandler(w, r.WithContext(reqctx.WithService(r.Context(), pre.Service)), err)
+			}
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
 // throttleKeyFuncs returns the KeyFunc slice for the API throttle middleware,
 // keyed by account-id and action from the SigV4 auth context.
 func (gw *GatewayConfig) throttleKeyFuncs() []ratelimit.KeyFunc {
@@ -261,7 +247,7 @@ func (gw *GatewayConfig) writeClusterUnavailable(w http.ResponseWriter, r *http.
 		return
 	}
 
-	xmlBody := xmlErrorBody(svc, awserrors.ErrorServiceUnavailable, clusterUnavailableMsg, requestID, r.URL.Path)
+	xmlBody := gw.xmlErrorBody(svc, awserrors.ErrorServiceUnavailable, clusterUnavailableMsg, requestID, r.URL.Path)
 
 	w.Header().Set("Content-Type", envelope.XMLContentType)
 	w.WriteHeader(http.StatusServiceUnavailable)
@@ -276,8 +262,8 @@ func (gw *GatewayConfig) writeThrottleError(w http.ResponseWriter, r *http.Reque
 	svc := reqctx.Service(r.Context())
 
 	errorCode := awserrors.ErrorRequestLimitExceeded
-	if svc == "iam" || svc == "sts" || svc == "eks" || svc == "route53" {
-		errorCode = awserrors.ErrorThrottling
+	if traits, ok := gw.selector().Traits(svc); ok && traits.XMLThrottleCode != "" {
+		errorCode = traits.XMLThrottleCode
 	}
 	errorMsg := awserrors.LookupErrorMessage(svc, errorCode)
 
@@ -296,7 +282,7 @@ func (gw *GatewayConfig) writeThrottleError(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	xmlErr := xmlErrorBody(svc, errorCode, errorMsg.Message, requestID, r.URL.Path)
+	xmlErr := gw.xmlErrorBody(svc, errorCode, errorMsg.Message, requestID, r.URL.Path)
 
 	w.Header().Set("Content-Type", envelope.XMLContentType)
 	w.WriteHeader(errorMsg.HTTPCode)
@@ -454,7 +440,7 @@ func (gw *GatewayConfig) ErrorHandler(w http.ResponseWriter, r *http.Request, er
 		return
 	}
 
-	xmlError := xmlErrorBody(svc, code, errorMsg.Message, requestId, r.URL.Path)
+	xmlError := gw.xmlErrorBody(svc, code, errorMsg.Message, requestId, r.URL.Path)
 
 	slog.Debug("Generated error response", "error", err, "code", code, "xml", string(xmlError), "requestId", requestId)
 
@@ -471,24 +457,26 @@ func ParseAWSQueryArgs(body string) (map[string]string, error) {
 	return query.ParseArgs(body)
 }
 
-// xmlErrorBody renders an error in the XML envelope svc's clients expect. The
-// one place the service-to-envelope mapping lives, so a service cannot be
-// added to some emitters and missed by others. JSON services never reach
-// here — every caller checks jsonErrorService first.
-func xmlErrorBody(svc, code, message, requestID, resource string) []byte {
-	switch svc {
-	case "s3":
+// xmlErrorBody renders an error in the XML envelope svc's clients expect, from
+// the shape its registration (or the composition, for a scope it does not
+// serve) declares. JSON services never reach here — every caller checks
+// errorProtocol first. No scope yet means the signature never parsed; that
+// keeps EC2's shape rather than changing a path this is not about.
+func (gw *GatewayConfig) xmlErrorBody(svc, code, message, requestID, resource string) []byte {
+	if svc == "" {
+		return envelope.EC2Body(code, message, requestID)
+	}
+	traits, _ := gw.selector().Traits(svc)
+	switch traits.XMLErrorShape {
+	case dispatch.XMLErrorShapeS3:
 		return envelope.S3Body(code, message, requestID, resource)
-	case "ec2", "spinifex", "":
-		// Both speak the Action-parameter query protocol, and their clients
-		// parse the EC2 <Response><Errors> shape, not the generic ErrorResponse.
-		// Empty means the signature never parsed, so no service is known yet;
-		// keep EC2's shape there rather than changing a path this is not about.
+	case dispatch.XMLErrorShapeEC2:
+		// The Action-parameter query clients parse the EC2 <Response><Errors>
+		// shape, not the generic ErrorResponse.
 		return envelope.EC2Body(code, message, requestID)
 	default:
-		// iam, sts, elasticloadbalancing, rds, and any unenumerated or
-		// unserved scope: the generic REST-XML ErrorResponse envelope, never
-		// EC2's shape, which a REST-XML client cannot deserialize.
+		// Any other or unserved scope: the generic REST-XML ErrorResponse,
+		// never EC2's shape, which a REST-XML client cannot deserialize.
 		return envelope.IAMBody(code, message, requestID)
 	}
 }
@@ -496,78 +484,6 @@ func xmlErrorBody(svc, code, message, requestID, resource string) []byte {
 // How long a discovered node count is reused before it is gathered again.
 // Membership does not change per request, and re-deriving it on every call put
 // the discovery fan-out's whole timeout in front of every API call it fronts.
-const activeNodesTTL = 5 * time.Second
-
-// discoverActiveNodesTimeout is how long the discovery fan-out collects replies.
-// A var only so tests can shorten it.
-var discoverActiveNodesTimeout = 500 * time.Millisecond
-
-// DiscoverActiveNodes discovers the number of active spinifex daemon nodes in the
-// cluster by publishing a discovery request and counting unique responses. It
-// carries the request context so the discovery fan-out joins the caller's trace.
-// Returns the number of active nodes (minimum 1 if fallback is needed).
-//
-// The result is cached for activeNodesTTL. The gather itself stays unbounded:
-// it counts whoever answers, which may legitimately exceed the configured node
-// count during a grow, and bounding it by that count would stop early and drop
-// a live node's resources from every fan-out that follows.
-func (gw *GatewayConfig) DiscoverActiveNodes(ctx context.Context) int {
-	if gw.NATSConn == nil {
-		slog.WarnContext(ctx, "DiscoverActiveNodes: NATS connection not available, using ExpectedNodes fallback", "fallback", gw.ExpectedNodes)
-		return gw.ExpectedNodes
-	}
-
-	if count, ok := gw.cachedActiveNodes(); ok {
-		return count
-	}
-
-	frames, _, err := natsmsg.Gather(ctx, gw.NATSConn, clusterv1.NodesDiscoverSubject, []byte("{}"),
-		natsmsg.GatherOpts{Timeout: discoverActiveNodesTimeout})
-	if err != nil {
-		slog.ErrorContext(ctx, "DiscoverActiveNodes: fan-out failed, using ExpectedNodes fallback", "err", err, "fallback", gw.ExpectedNodes)
-		return gw.ExpectedNodes
-	}
-
-	nodesSeen := make(map[string]bool)
-	for _, frame := range frames {
-		var response clusterv1.NodeDiscoverResponse
-		if err := json.Unmarshal(frame.Data, &response); err != nil {
-			slog.DebugContext(ctx, "DiscoverActiveNodes: Failed to unmarshal response", "err", err)
-			continue
-		}
-		nodesSeen[response.Node] = true
-	}
-
-	activeNodes := len(nodesSeen)
-	if activeNodes == 0 {
-		slog.WarnContext(ctx, "DiscoverActiveNodes: No nodes responded, using ExpectedNodes fallback", "fallback", gw.ExpectedNodes)
-		return gw.ExpectedNodes
-	}
-
-	gw.rememberActiveNodes(activeNodes)
-	slog.DebugContext(ctx, "DiscoverActiveNodes: Discovered active nodes", "count", activeNodes)
-	return activeNodes
-}
-
-// cachedActiveNodes returns a count discovered within the TTL. Only a real
-// discovery is cached: a fallback is not evidence of anything and caching one
-// would make a momentary NATS problem outlive itself.
-func (gw *GatewayConfig) cachedActiveNodes() (int, bool) {
-	gw.activeNodesMu.RLock()
-	defer gw.activeNodesMu.RUnlock()
-	if gw.activeNodesCount == 0 || time.Since(gw.activeNodesAt) > activeNodesTTL {
-		return 0, false
-	}
-	return gw.activeNodesCount, true
-}
-
-func (gw *GatewayConfig) rememberActiveNodes(count int) {
-	gw.activeNodesMu.Lock()
-	defer gw.activeNodesMu.Unlock()
-	gw.activeNodesCount = count
-	gw.activeNodesAt = time.Now()
-}
-
 // recordResolvedAction renames the current span to service.action and tags
 // aws.service/aws.action. Called from both SigV4 auth and checkPolicyResource,
 // so it must be idempotent — the last call before the response wins.

spinifex/gateway/gateway_test.go

@@ -1,7 +1,6 @@
 package gateway
 
 import (
-	"context"
 	"encoding/json"
 	"encoding/xml"
 	"errors"
@@ -10,10 +9,13 @@ import (
 	acmawsapi "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi"
 	ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
 	awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
+	elbv2awsapi "github.com/mulgadc/spinifex/spinifex/domains/elbv2/awsapi"
+	rdsawsapi "github.com/mulgadc/spinifex/spinifex/domains/rds/awsapi"
 	route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
 	taggingawsapi "github.com/mulgadc/spinifex/spinifex/domains/tagging/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/reqctx"
+	operatoradmin "github.com/mulgadc/spinifex/spinifex/operator/admin"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -24,7 +26,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	awsec2 "github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/bluebottle/pkg/ratelimit"
-	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ochreawsapi "github.com/mulgadc/spinifex/spinifex/domains/ochre/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -74,9 +75,15 @@ func TestXMLErrorBody_EnvelopePerService(t *testing.T) {
 		{"route53", "ErrorResponse"},
 	}
 
+	gw := withServices(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}),
+		ec2Registration(&GatewayConfig{}), operatoradmin.NewRegistration(operatoradmin.Deps{}),
+		iamRegistration(&GatewayConfig{}), stsRegistration(&GatewayConfig{}),
+		elbv2awsapi.NewRegistration(elbv2awsapi.Deps{}), rdsawsapi.NewRegistration(rdsawsapi.Deps{}),
+		route53awsapi.NewRegistration(route53awsapi.Deps{}))
+
 	for _, tc := range tests {
 		t.Run(tc.svc, func(t *testing.T) {
-			body := xmlErrorBody(tc.svc, "AccessDenied", "denied", "req-1", "/path")
+			body := gw.xmlErrorBody(tc.svc, "AccessDenied", "denied", "req-1", "/path")
 
 			assert.Equal(t, tc.root, xmlRootName(t, body))
 			assert.Contains(t, string(body), "<Code>AccessDenied</Code>")
@@ -85,7 +92,7 @@ func TestXMLErrorBody_EnvelopePerService(t *testing.T) {
 }
 
 func TestErrorHandler_IAMService(t *testing.T) {
-	gw := withIAM(withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}))
+	gw := withIAM(withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()})))
 
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "iam")
@@ -107,7 +114,7 @@ func TestErrorHandler_IAMService(t *testing.T) {
 }
 
 func TestErrorHandler_UnknownError(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "ec2")
@@ -128,7 +135,7 @@ func TestErrorHandler_UnknownError(t *testing.T) {
 }
 
 func TestErrorHandler_WrappedErrorCode(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "ec2")
@@ -152,7 +159,7 @@ func TestErrorHandler_WrappedErrorCode(t *testing.T) {
 // fidelity gap: a call site that names the blocking resource via awserrors.Errorf
 // must have that wording reach the client instead of the generic ErrorLookup text.
 func TestErrorHandler_PrefersCallSiteMessage(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "ec2")
@@ -177,7 +184,7 @@ func TestErrorHandler_PrefersCallSiteMessage(t *testing.T) {
 // ResourceInUseException collision check: ACM's DeleteCertificate must not
 // surface EKS's "cluster already exists" wording for the shared wire code.
 func TestErrorHandler_ACMResourceInUse_UsesACMWording(t *testing.T) {
-	gw := withACM(withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}), acmawsapi.Deps{})
+	gw := withACM(withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()})), acmawsapi.Deps{})
 	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := reqctx.WithService(req.Context(), "acm")
 	req = req.WithContext(ctx)
@@ -198,7 +205,7 @@ func TestErrorHandler_ACMResourceInUse_UsesACMWording(t *testing.T) {
 // own ResourceInUseException must keep its existing wording, unaffected by the
 // ACM-specific override.
 func TestErrorHandler_EKSResourceInUse_UsesEKSWording(t *testing.T) {
-	gw := withContainers(withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}))
+	gw := withContainers(withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()})))
 	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := reqctx.WithService(req.Context(), "eks")
 	req = req.WithContext(ctx)
@@ -218,7 +225,7 @@ func TestErrorHandler_EKSResourceInUse_UsesEKSWording(t *testing.T) {
 // case: an error path that supplies no message (the vast majority of call
 // sites) must keep rendering exactly today's ErrorLookup text, unchanged.
 func TestErrorHandler_NoMessageSupplied_MatchesErrorLookup(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "ec2")
@@ -240,7 +247,7 @@ func TestErrorHandler_NoMessageSupplied_MatchesErrorLookup(t *testing.T) {
 // validation code outside EC2: the default must not be EC2's Reserved Instance
 // or resource-ID wording, whichever envelope the service uses.
 func TestErrorHandler_NonEC2ValidationDefaults_DropEC2Wording(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	b := dispatch.NewBuilder()
 	require.NoError(t, b.Register(awsapi.NewRegistration(awsapi.Deps{})))
 	require.NoError(t, b.Register(acmawsapi.NewRegistration(acmawsapi.Deps{})))
@@ -267,7 +274,7 @@ func TestErrorHandler_NonEC2ValidationDefaults_DropEC2Wording(t *testing.T) {
 }
 
 func TestErrorHandler_ELBv2Service(t *testing.T) {
-	gw := withELBv2(withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}))
+	gw := withELBv2(withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()})))
 
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "elasticloadbalancing")
@@ -289,7 +296,7 @@ func TestErrorHandler_ELBv2Service(t *testing.T) {
 }
 
 func TestErrorHandler_EC2Service(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "ec2")
@@ -309,7 +316,7 @@ func TestErrorHandler_EC2Service(t *testing.T) {
 }
 
 func TestErrorHandler_IgnoresClientRequestID(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "ec2")
@@ -327,7 +334,7 @@ func TestErrorHandler_IgnoresClientRequestID(t *testing.T) {
 }
 
 func TestErrorHandler_ContentTypeXML(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "ec2")
@@ -346,95 +353,6 @@ func startTestNATS(t *testing.T) *nats.Conn {
 	return nc
 }
 
-func TestDiscoverActiveNodes_NilNATS(t *testing.T) {
-	gw := &GatewayConfig{
-		ExpectedNodes: 3,
-		NATSConn:      nil,
-	}
-
-	result := gw.DiscoverActiveNodes(context.Background())
-	assert.Equal(t, 3, result)
-}
-
-func TestDiscoverActiveNodes_NoResponders(t *testing.T) {
-	SetDiscoverActiveNodesTimeoutForTest(t, 50*time.Millisecond)
-	nc := startTestNATS(t)
-
-	gw := &GatewayConfig{
-		ExpectedNodes: 5,
-		NATSConn:      nc,
-	}
-
-	result := gw.DiscoverActiveNodes(context.Background())
-	assert.Equal(t, 5, result)
-}
-
-func TestDiscoverActiveNodes_WithResponders(t *testing.T) {
-	SetDiscoverActiveNodesTimeoutForTest(t, 50*time.Millisecond)
-	nc := startTestNATS(t)
-
-	for _, nodeName := range []string{"node-1", "node-2"} {
-		name := nodeName
-		_, err := nc.Subscribe(clusterv1.NodesDiscoverSubject, func(msg *nats.Msg) {
-			resp := clusterv1.NodeDiscoverResponse{Node: name}
-			data, _ := json.Marshal(resp)
-			msg.Respond(data)
-		})
-		require.NoError(t, err)
-	}
-	require.NoError(t, nc.Flush())
-
-	gw := &GatewayConfig{
-		ExpectedNodes: 1,
-		NATSConn:      nc,
-	}
-
-	result := gw.DiscoverActiveNodes(context.Background())
-	assert.Equal(t, 2, result)
-}
-
-func TestDiscoverActiveNodes_InvalidJSON(t *testing.T) {
-	SetDiscoverActiveNodesTimeoutForTest(t, 50*time.Millisecond)
-	nc := startTestNATS(t)
-
-	_, err := nc.Subscribe(clusterv1.NodesDiscoverSubject, func(msg *nats.Msg) {
-		msg.Respond([]byte("not json"))
-	})
-	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
-
-	gw := &GatewayConfig{
-		ExpectedNodes: 4,
-		NATSConn:      nc,
-	}
-
-	result := gw.DiscoverActiveNodes(context.Background())
-	assert.Equal(t, 4, result)
-}
-
-func TestDiscoverActiveNodes_DuplicateNodes(t *testing.T) {
-	SetDiscoverActiveNodesTimeoutForTest(t, 50*time.Millisecond)
-	nc := startTestNATS(t)
-
-	for range 2 {
-		_, err := nc.Subscribe(clusterv1.NodesDiscoverSubject, func(msg *nats.Msg) {
-			resp := clusterv1.NodeDiscoverResponse{Node: "same-node"}
-			data, _ := json.Marshal(resp)
-			msg.Respond(data)
-		})
-		require.NoError(t, err)
-	}
-	require.NoError(t, nc.Flush())
-
-	gw := &GatewayConfig{
-		ExpectedNodes: 5,
-		NATSConn:      nc,
-	}
-
-	result := gw.DiscoverActiveNodes(context.Background())
-	assert.Equal(t, 1, result)
-}
-
 func TestParseAWSQueryArgs(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -656,7 +574,7 @@ func TestParseAWSQueryArgs_CapturedClientBodies(t *testing.T) {
 }
 
 func TestRequest_UnsupportedService(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := reqctx.WithService(req.Context(), "s3")
 	req = req.WithContext(ctx)
@@ -684,7 +602,7 @@ func TestRequest_MalformedQueryString_EndToEnd(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			// A live NATS connection bypasses the cluster-unavailable gate.
-			gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, NATSConn: connectedNATS(t), IAMService: allowAllIAMService()})
+			gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{NATS: connectedNATS(t), IAM: allowAllIAMService()}))
 			if tc.service == "elasticloadbalancing" {
 				withELBv2(gw)
 			}
@@ -711,7 +629,7 @@ func TestRequest_MalformedQueryString_EndToEnd(t *testing.T) {
 }
 
 func TestRequest_EC2MissingAction(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, NATSConn: connectedNATS(t), IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{NATS: connectedNATS(t), IAM: allowAllIAMService()}))
 	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(""))
 	ctx := reqctx.WithService(req.Context(), "ec2")
 	ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
@@ -728,7 +646,7 @@ func TestRequest_EC2MissingAction(t *testing.T) {
 }
 
 func TestRequest_IAMNilService(t *testing.T) {
-	gw := withIAM(withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: nil, NATSConn: connectedNATS(t)}))
+	gw := withIAM(withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: nil, NATS: connectedNATS(t)})))
 	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=CreateUser&UserName=test"))
 	ctx := reqctx.WithService(req.Context(), "iam")
 	ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
@@ -766,7 +684,7 @@ func setupEC2Request(body string, accountID string) *http.Request {
 }
 
 func TestEC2Request_MissingAction(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	req := setupEC2Request("", "123456789012")
 	w := httptest.NewRecorder()
 
@@ -776,7 +694,7 @@ func TestEC2Request_MissingAction(t *testing.T) {
 }
 
 func TestEC2Request_UnknownAction(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	req := setupEC2Request("Action=FakeAction", "123456789012")
 	w := httptest.NewRecorder()
 
@@ -786,7 +704,7 @@ func TestEC2Request_UnknownAction(t *testing.T) {
 }
 
 func TestEC2Request_NilNATSNonLocalAction(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, NATSConn: nil, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{NATS: nil, IAM: allowAllIAMService()}))
 	req := setupEC2Request("Action=DescribeInstances", "123456789012")
 	w := httptest.NewRecorder()
 
@@ -796,12 +714,10 @@ func TestEC2Request_NilNATSNonLocalAction(t *testing.T) {
 }
 
 func TestEC2Request_NilNATSLocalAction(t *testing.T) {
-	gw := withEC2(withEC2Verifier(&GatewayConfig{
+	gw := withEC2(withEC2Verifier(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		NATSConn:       nil,
 		Region:         "us-east-1",
-		IAMService:     allowAllIAMService(),
-	}), func(d *ec2awsapi.Deps) { d.AZ = "us-east-1a" })
+	}, gwfix.Deps{NATS: nil, IAM: allowAllIAMService()})), func(d *ec2awsapi.Deps) { d.AZ = "us-east-1a" })
 	req := setupEC2Request("Action=DescribeRegions", "123456789012")
 	w := httptest.NewRecorder()
 
@@ -817,7 +733,7 @@ func TestEC2Request_NilNATSLocalAction(t *testing.T) {
 }
 
 func TestEC2Request_MissingAccountID(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, NATSConn: nil, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{NATS: nil, IAM: allowAllIAMService()}))
 	// Use a local action so we don't fail on nil NATS first
 	req := setupEC2Request("Action=DescribeRegions", "")
 	w := httptest.NewRecorder()
@@ -830,7 +746,7 @@ func TestEC2Request_MissingAccountID(t *testing.T) {
 
 // Static attributes are served without NATS; only default-vpc needs a lookup.
 func TestEC2Request_DescribeAccountAttributes(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, NATSConn: nil, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{NATS: nil, IAM: allowAllIAMService()}))
 	req := setupEC2Request("Action=DescribeAccountAttributes&AttributeName.1=max-instances", "123456789012")
 	w := httptest.NewRecorder()
 
@@ -862,7 +778,7 @@ func TestEC2Request_DescribeAccountAttributes_DefaultVPC(t *testing.T) {
 	require.NoError(t, err)
 	t.Cleanup(func() { _ = sub.Unsubscribe() })
 
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{NATS: nc, IAM: allowAllIAMService()}))
 	req := setupEC2Request("Action=DescribeAccountAttributes&AttributeName.1=default-vpc", "123456789012")
 	w := httptest.NewRecorder()
 
@@ -872,12 +788,10 @@ func TestEC2Request_DescribeAccountAttributes_DefaultVPC(t *testing.T) {
 }
 
 func TestEC2Request_DescribeAvailabilityZones(t *testing.T) {
-	gw := withEC2(withEC2Verifier(&GatewayConfig{
+	gw := withEC2(withEC2Verifier(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		NATSConn:       nil,
 		Region:         "us-east-1",
-		IAMService:     allowAllIAMService(),
-	}), func(d *ec2awsapi.Deps) { d.AZ = "us-east-1a" })
+	}, gwfix.Deps{NATS: nil, IAM: allowAllIAMService()})), func(d *ec2awsapi.Deps) { d.AZ = "us-east-1a" })
 	req := setupEC2Request("Action=DescribeAvailabilityZones", "123456789012")
 	w := httptest.NewRecorder()
 
@@ -904,7 +818,7 @@ func shortenPolicyResolveRetry(t *testing.T) {
 // --- Throttle middleware integration tests ---
 
 func TestThrottleKeyFuncs_ExtractsAccountAndAction(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	keyFuncs := gw.throttleKeyFuncs()
 	require.Len(t, keyFuncs, 2)
 
@@ -923,7 +837,7 @@ func TestThrottleKeyFuncs_ExtractsAccountAndAction(t *testing.T) {
 }
 
 func TestThrottleKeyFuncs_UnknownAction(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	keyFuncs := gw.throttleKeyFuncs()
 
 	// No ctxAction in context — should return "unknown".
@@ -937,7 +851,7 @@ func TestThrottleKeyFuncs_UnknownAction(t *testing.T) {
 }
 
 func TestThrottleKeyFuncs_MissingAccountID(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 	keyFuncs := gw.throttleKeyFuncs()
 
 	// No ctxAccountID in context — should return an error.
@@ -949,7 +863,7 @@ func TestThrottleKeyFuncs_MissingAccountID(t *testing.T) {
 }
 
 func TestWriteThrottleError_EC2(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := reqctx.WithService(req.Context(), "ec2")
@@ -967,7 +881,7 @@ func TestWriteThrottleError_EC2(t *testing.T) {
 }
 
 func TestWriteThrottleError_IAM(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withIAM(withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()})))
 
 	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := reqctx.WithService(req.Context(), "iam")
@@ -987,7 +901,7 @@ func TestWriteThrottleError_IAM(t *testing.T) {
 // Route 53 throttles with a 400 carrying the Throttling code, which SDK
 // retryers classify by code rather than status.
 func TestWriteThrottleError_Route53(t *testing.T) {
-	gw := &GatewayConfig{DisableLogging: true}
+	gw := withServices(&GatewayConfig{DisableLogging: true}, route53awsapi.NewRegistration(route53awsapi.Deps{}))
 	req := httptest.NewRequest(http.MethodGet, "/2013-04-01/hostedzone", nil)
 	req = req.WithContext(reqctx.WithService(req.Context(), "route53"))
 	w := httptest.NewRecorder()
@@ -1028,11 +942,10 @@ func TestThrottleMiddleware_Integration(t *testing.T) {
 	throttler := ratelimit.New(cfg)
 	defer throttler.Stop()
 
-	gw := withEC2Verifier(&GatewayConfig{
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Throttler:      throttler,
-		IAMService:     allowAllIAMService(),
-	})
+	}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusOK)
@@ -1069,7 +982,7 @@ func TestThrottleMiddleware_Integration(t *testing.T) {
 
 func TestThrottleMiddleware_DisabledConfig(t *testing.T) {
 	// When Throttler is nil, SetupRoutes skips middleware — no panic.
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, Throttler: nil, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Throttler: nil}, gwfix.Deps{IAM: allowAllIAMService()}))
 	handler := gw.SetupRoutes()
 
 	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=DescribeInstances"))
@@ -1093,7 +1006,7 @@ func TestThrottleMiddleware_PerActionIsolation(t *testing.T) {
 	throttler := ratelimit.New(cfg)
 	defer throttler.Stop()
 
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusOK)
@@ -1124,7 +1037,7 @@ func TestThrottleMiddleware_PerActionIsolation(t *testing.T) {
 }
 
 func TestRequest_ClusterUnavailableNilConn_EC2(t *testing.T) {
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()}))
 
 	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := reqctx.WithService(req.Context(), "ec2")
@@ -1144,7 +1057,7 @@ func TestRequest_ClusterUnavailableNilConn_EC2(t *testing.T) {
 }
 
 func TestRequest_ClusterUnavailableNilConn_IAM(t *testing.T) {
-	gw := withIAM(withEC2Verifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}))
+	gw := withIAM(withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{IAM: allowAllIAMService()})))
 
 	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := reqctx.WithService(req.Context(), "iam")
@@ -1167,7 +1080,7 @@ func TestRequest_ClusterUnavailableClosedConn(t *testing.T) {
 	require.NoError(t, err)
 	nc.Close()
 
-	gw := withEC2Verifier(&GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()})
+	gw := withEC2Verifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{NATS: nc, IAM: allowAllIAMService()}))
 
 	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := reqctx.WithService(req.Context(), "ec2")

spinifex/gateway/gwhelpers_test.go

@@ -68,7 +68,7 @@ import (
 // role composes it. An unset deps.NATS takes gw.NATSConn.
 func withACM(gw *GatewayConfig, deps acmawsapi.Deps) *GatewayConfig {
 	if deps.NATS == nil {
-		deps.NATS = gw.NATSConn
+		deps.NATS = gwfix.DepsOf(gw).NATS
 	}
 	b := dispatch.NewBuilder()
 	if err := b.Register(acmawsapi.NewRegistration(deps)); err != nil {
@@ -87,26 +87,24 @@ func acmPipelineApp(t *testing.T, docs []iamdomain.PolicyDocument) (*GatewayConf
 	require.NoError(t, err)
 	_, nc := testutil.StartTestNATS(t)
 
-	gw := withVerifier(&GatewayConfig{
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		NATSConn:       nc,
-		IAMService: &gwfix.PolicyIAM{
-			IAM: gwfix.IAM{
-				MasterKey: gwfix.MasterKey,
-				AccessKeys: map[string]*iamdomain.AccessKey{
-					gwfix.AccessKey: {
-						AccessKeyID:     gwfix.AccessKey,
-						SecretAccessKey: encryptedSecret,
-						UserName:        "alice",
-						AccountID:       authzAccountID,
-						Status:          "Active",
-					},
+	}, gwfix.Deps{NATS: nc, IAM: &gwfix.PolicyIAM{
+		IAM: gwfix.IAM{
+			MasterKey: gwfix.MasterKey,
+			AccessKeys: map[string]*iamdomain.AccessKey{
+				gwfix.AccessKey: {
+					AccessKeyID:     gwfix.AccessKey,
+					SecretAccessKey: encryptedSecret,
+					UserName:        "alice",
+					AccountID:       authzAccountID,
+					Status:          "Active",
 				},
 			},
-			GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
 		},
-	})
+		GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
+	}}))
 
 	withACM(gw, acmawsapi.Deps{})
 
@@ -215,20 +213,19 @@ func newRDSGatewayWithResolver(
 	}
 	// Both branches are wired: an assumed-role principal resolves by role, and a
 	// test that only stubbed users would miss it entirely.
-	return withRealAuth(&GatewayConfig{
+	return withRealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         "ap-southeast-2",
-		IAMService: &gwfix.PolicyIAM{
-			GetUserPoliciesFn: counted,
-			GetRolePoliciesFn: counted,
-		},
-	}), probe
+	}, gwfix.Deps{IAM: &gwfix.PolicyIAM{
+		GetUserPoliciesFn: counted,
+		GetRolePoliciesFn: counted,
+	}})), probe
 }
 
 // withRDS registers RDS on gw with the NATS connection and node count it holds,
 // as the awsgw role composes it. Set both on gw before calling.
 func withRDS(gw *GatewayConfig) *GatewayConfig {
-	deps := rdsawsapi.Deps{NATS: gw.NATSConn, ExpectedNodes: gw.ExpectedNodes}
+	deps := rdsawsapi.Deps{NATS: gwfix.DepsOf(gw).NATS, ExpectedNodes: gwfix.DepsOf(gw).Nodes}
 	if quotaFor(gw) != nil {
 		deps.Quota = quotaFor(gw)
 	}
@@ -244,7 +241,7 @@ func withRDS(gw *GatewayConfig) *GatewayConfig {
 // as the awsgw role composes it. Set both on gw before calling.
 func withELBv2(gw *GatewayConfig) *GatewayConfig {
 	b := dispatch.NewBuilder()
-	reg := elbv2awsapi.NewRegistration(elbv2awsapi.Deps{NATS: gw.NATSConn, Quota: quotaFor(gw)})
+	reg := elbv2awsapi.NewRegistration(elbv2awsapi.Deps{NATS: gwfix.DepsOf(gw).NATS, Quota: quotaFor(gw)})
 	if err := b.Register(reg); err != nil {
 		panic(err)
 	}
@@ -257,8 +254,8 @@ func withELBv2(gw *GatewayConfig) *GatewayConfig {
 func serveAccountLimits(t *testing.T, gw *GatewayConfig, accountID, body string) ([]byte, error) {
 	t.Helper()
 	_, nc := testutil.StartTestNATS(t)
-	gw.NATSConn = nc
-	gw.IAMService = allowAllIAMService()
+	gwfix.DepsOf(gw).NATS = nc
+	gwfix.DepsOf(gw).IAM = allowAllIAMService()
 	withELBv2(gw)
 
 	w := httptest.NewRecorder()
@@ -305,26 +302,24 @@ func elbv2PipelineApp(t *testing.T, docs []iamdomain.PolicyDocument, quota *admi
 	require.NoError(t, err)
 	_, nc := testutil.StartTestNATS(t)
 
-	gw := withVerifier(&GatewayConfig{
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		NATSConn:       nc,
-		IAMService: &gwfix.PolicyIAM{
-			IAM: gwfix.IAM{
-				MasterKey: gwfix.MasterKey,
-				AccessKeys: map[string]*iamdomain.AccessKey{
-					gwfix.AccessKey: {
-						AccessKeyID:     gwfix.AccessKey,
-						SecretAccessKey: encryptedSecret,
-						UserName:        "alice",
-						AccountID:       authzAccountID,
-						Status:          "Active",
-					},
+	}, gwfix.Deps{NATS: nc, IAM: &gwfix.PolicyIAM{
+		IAM: gwfix.IAM{
+			MasterKey: gwfix.MasterKey,
+			AccessKeys: map[string]*iamdomain.AccessKey{
+				gwfix.AccessKey: {
+					AccessKeyID:     gwfix.AccessKey,
+					SecretAccessKey: encryptedSecret,
+					UserName:        "alice",
+					AccountID:       authzAccountID,
+					Status:          "Active",
 				},
 			},
-			GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
 		},
-	})
+		GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
+	}}))
 	withELBv2(withQuota(gw, quota))
 
 	r := chi.NewRouter()
@@ -380,6 +375,10 @@ func withServices(gw *GatewayConfig, regs ...dispatch.Registration) *GatewayConf
 			panic(err)
 		}
 	}
+	// The composition declares the scopes it knows but does not serve.
+	if err := b.DeclareScope("s3", dispatch.ServiceTraits{XMLErrorShape: dispatch.XMLErrorShapeS3, PayloadBody: true}); err != nil {
+		panic(err)
+	}
 	gw.Services = b.Build()
 	withFakeCapabilities(gw)
 	return gw
@@ -394,9 +393,9 @@ func withIAM(gw *GatewayConfig) *GatewayConfig {
 // and node count gw holds, as the awsgw role composes it. Set them on gw first.
 func iamRegistration(gw *GatewayConfig) dispatch.Registration {
 	return iamawsapi.NewRegistration(iamawsapi.Deps{
-		IAM: gw.IAMService,
+		IAM: gwfix.DepsOf(gw).IAM,
 		CountInstanceProfileAssociations: func(ctx context.Context, accountID, profileARN string) (int, error) {
-			return ec2instanceapi.CountInstanceProfileAssociations(ctx, gw.NATSConn, gw.DiscoverActiveNodes(ctx), accountID, profileARN)
+			return ec2instanceapi.CountInstanceProfileAssociations(ctx, gwfix.DepsOf(gw).NATS, activeNodes(gw)(ctx), accountID, profileARN)
 		},
 	})
 }
@@ -788,11 +787,10 @@ func (svc *echoIAMService) UpdateAssumeRolePolicy(accountID string, input *iam.U
 
 // setupIAMRequestHandler wires an http.Handler for IAM_Request with injected SigV4 context values.
 func setupIAMRequestHandler(svc iamdomain.IAMService) http.Handler {
-	gw := &GatewayConfig{
+	gw := gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		IAMService:     svc,
 		Credentials:    credverify.New(svc, nil),
-	}
+	}, gwfix.Deps{IAM: svc})
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := r.Context()
 		ctx = reqctx.WithService(ctx, "iam")
@@ -885,20 +883,20 @@ type iamExample struct {
 // stsRegistration is the STS registration over the IAMService and STSService
 // gw holds, as the awsgw role composes it. Set them on gw first.
 func stsRegistration(gw *GatewayConfig) dispatch.Registration {
-	return stsawsapi.NewRegistration(stsawsapi.Deps{STS: gw.STSService, IAM: gw.IAMService})
+	return stsawsapi.NewRegistration(stsawsapi.Deps{STS: gwfix.DepsOf(gw).STS, IAM: gwfix.DepsOf(gw).IAM})
 }
 
 // withIAMAndSTS registers IAM and STS together on gw, with the unsigned
 // web-identity capability, as awsgw composes them.
 func withIAMAndSTS(gw *GatewayConfig) *GatewayConfig {
-	gw.AnonymousSTS = stsawsapi.NewAnonymousAssumeRoleWithWebIdentity(stsawsapi.Deps{STS: gw.STSService, IAM: gw.IAMService})
+	gw.PreAuth = append(gw.PreAuth, stsawsapi.NewPreAuth(stsawsapi.Deps{STS: gwfix.DepsOf(gw).STS, IAM: gwfix.DepsOf(gw).IAM}))
 	return withServices(gw, iamRegistration(gw), stsRegistration(gw))
 }
 
 // withSTS registers STS alone on gw and composes its unsigned web-identity
 // capability, as awsgw does.
 func withSTS(gw *GatewayConfig) *GatewayConfig {
-	gw.AnonymousSTS = stsawsapi.NewAnonymousAssumeRoleWithWebIdentity(stsawsapi.Deps{STS: gw.STSService, IAM: gw.IAMService})
+	gw.PreAuth = append(gw.PreAuth, stsawsapi.NewPreAuth(stsawsapi.Deps{STS: gwfix.DepsOf(gw).STS, IAM: gwfix.DepsOf(gw).IAM}))
 	return withServices(gw, stsRegistration(gw))
 }
 
@@ -1071,11 +1069,9 @@ func setupSTSRequestHandler(p stsRequestParams) http.Handler {
 	if p.iamSvc == nil {
 		p.iamSvc = &flexMockIAMService{}
 	}
-	gw := withRealAuth(&GatewayConfig{
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		STSService:     p.stsSvc,
-		IAMService:     p.iamSvc,
-	})
+	}, gwfix.Deps{STS: p.stsSvc, IAM: p.iamSvc}))
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := r.Context()
 		ctx = reqctx.WithService(ctx, "sts")
@@ -1162,26 +1158,24 @@ func taggingPipelineApp(t *testing.T, docs []iamdomain.PolicyDocument, agg *tagg
 	require.NoError(t, err)
 	_, nc := testutil.StartTestNATS(t)
 
-	gw := withVerifier(&GatewayConfig{
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		NATSConn:       nc,
-		IAMService: &gwfix.PolicyIAM{
-			IAM: gwfix.IAM{
-				MasterKey: gwfix.MasterKey,
-				AccessKeys: map[string]*iamdomain.AccessKey{
-					gwfix.AccessKey: {
-						AccessKeyID:     gwfix.AccessKey,
-						SecretAccessKey: encryptedSecret,
-						UserName:        "alice",
-						AccountID:       authzAccountID,
-						Status:          "Active",
-					},
+	}, gwfix.Deps{NATS: nc, IAM: &gwfix.PolicyIAM{
+		IAM: gwfix.IAM{
+			MasterKey: gwfix.MasterKey,
+			AccessKeys: map[string]*iamdomain.AccessKey{
+				gwfix.AccessKey: {
+					AccessKeyID:     gwfix.AccessKey,
+					SecretAccessKey: encryptedSecret,
+					UserName:        "alice",
+					AccountID:       authzAccountID,
+					Status:          "Active",
 				},
 			},
-			GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
 		},
-	})
+		GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
+	}}))
 	withTagging(gw, agg)
 
 	r := chi.NewRouter()
@@ -1223,7 +1217,7 @@ func newTaggingCBGateway(t *testing.T) (*GatewayConfig, *ec2tags.TagsServiceImpl
 		})
 	})
 
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService(), NATSConn: nc})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true, Region: authzRegion}, gwfix.Deps{IAM: allowAllIAMService(), NATS: nc}))
 	return gw, svc
 }
 
@@ -1267,29 +1261,25 @@ func operatorPipelineApp(t *testing.T, accountID string, docs []iamdomain.Policy
 	require.NoError(t, err)
 	_, nc := testutil.StartTestNATS(t)
 
-	gw := &GatewayConfig{
+	gw := gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		NATSConn:       nc,
-		Version:        "v9.9.9-test",
-		Commit:         "abc1234",
-		IAMService: &gwfix.PolicyIAM{
-			IAM: gwfix.IAM{
-				MasterKey: gwfix.MasterKey,
-				AccessKeys: map[string]*iamdomain.AccessKey{
-					gwfix.AccessKey: {
-						AccessKeyID:     gwfix.AccessKey,
-						SecretAccessKey: encryptedSecret,
-						UserName:        "admin",
-						AccountID:       accountID,
-						Status:          "Active",
-					},
+	}, gwfix.Deps{NATS: nc, IAM: &gwfix.PolicyIAM{
+		IAM: gwfix.IAM{
+			MasterKey: gwfix.MasterKey,
+			AccessKeys: map[string]*iamdomain.AccessKey{
+				gwfix.AccessKey: {
+					AccessKeyID:     gwfix.AccessKey,
+					SecretAccessKey: encryptedSecret,
+					UserName:        "admin",
+					AccountID:       accountID,
+					Status:          "Active",
 				},
 			},
-			GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
 		},
-	}
-	gw.IAMService = operatorIAM{gw.IAMService.(*gwfix.PolicyIAM)}
+		GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
+	}})
+	gwfix.DepsOf(gw).IAM = operatorIAM{gwfix.DepsOf(gw).IAM.(*gwfix.PolicyIAM)}
 	if mutate != nil {
 		mutate(gw)
 	}
@@ -1374,15 +1364,14 @@ var spinifexAdminActions = func() map[string]bool {
 func withOperator(gw *GatewayConfig) *GatewayConfig {
 	b := dispatch.NewBuilder()
 	deps := operatoradmin.Deps{
-		Version:     gw.Version,
-		Commit:      gw.Commit,
-		NATS:        gw.NATSConn,
-		RootCAs:     gw.RootCAs,
-		ActiveNodes: gw.DiscoverActiveNodes,
+		Version:     "v9.9.9-test",
+		Commit:      "abc1234",
+		NATS:        gwfix.DepsOf(gw).NATS,
+		ActiveNodes: activeNodes(gw),
 
-		IAM:           gw.IAMService,
+		IAM:           gwfix.DepsOf(gw).IAM,
 		Quota:         quotaFor(gw),
-		ExpectedNodes: gw.ExpectedNodes,
+		ExpectedNodes: gwfix.DepsOf(gw).Nodes,
 	}
 	reg := operatoradmin.NewRegistration(deps)
 	if err := b.Register(reg); err != nil {
@@ -1531,7 +1520,7 @@ type jsonErrorEnvelope struct {
 // role composes it. An unset deps.NATS takes gw.NATSConn.
 func withECR(gw *GatewayConfig, deps awsapi.Deps) *GatewayConfig {
 	if deps.NATS == nil {
-		deps.NATS = gw.NATSConn
+		deps.NATS = gwfix.DepsOf(gw).NATS
 	}
 	b := dispatch.NewBuilder()
 	if err := b.Register(awsapi.NewRegistration(deps)); err != nil {
@@ -1550,27 +1539,24 @@ func ecrPipelineApp(t *testing.T, docs []iamdomain.PolicyDocument) (*GatewayConf
 	require.NoError(t, err)
 	_, nc := testutil.StartTestNATS(t)
 
-	gw := withVerifier(&GatewayConfig{
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		InternalSuffix: ecrTestSuffix,
-		NATSConn:       nc,
-		IAMService: &gwfix.PolicyIAM{
-			IAM: gwfix.IAM{
-				MasterKey: gwfix.MasterKey,
-				AccessKeys: map[string]*iamdomain.AccessKey{
-					gwfix.AccessKey: {
-						AccessKeyID:     gwfix.AccessKey,
-						SecretAccessKey: encryptedSecret,
-						UserName:        "alice",
-						AccountID:       ecrTestAccount,
-						Status:          "Active",
-					},
+	}, gwfix.Deps{NATS: nc, IAM: &gwfix.PolicyIAM{
+		IAM: gwfix.IAM{
+			MasterKey: gwfix.MasterKey,
+			AccessKeys: map[string]*iamdomain.AccessKey{
+				gwfix.AccessKey: {
+					AccessKeyID:     gwfix.AccessKey,
+					SecretAccessKey: encryptedSecret,
+					UserName:        "alice",
+					AccountID:       ecrTestAccount,
+					Status:          "Active",
 				},
 			},
-			GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
 		},
-	})
+		GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
+	}}))
 
 	withECR(gw, awsapi.Deps{})
 
@@ -1749,11 +1735,11 @@ func (gw *GatewayConfig) serveECS(w http.ResponseWriter, r *http.Request) error
 // mock IAM service it holds, as the awsgw role composes them.
 func withContainers(gw *GatewayConfig) *GatewayConfig {
 	b := dispatch.NewBuilder()
-	if err := b.Register(ecsawsapi.NewRegistration(ecsawsapi.Deps{NATS: gw.NATSConn})); err != nil {
+	if err := b.Register(ecsawsapi.NewRegistration(ecsawsapi.Deps{NATS: gwfix.DepsOf(gw).NATS})); err != nil {
 		panic(err)
 	}
-	roles, _ := gw.IAMService.(IAMServiceLike)
-	if err := b.Register(eksawsapi.NewRegistration(eksawsapi.Deps{NATS: gw.NATSConn, Roles: mockRoleResolver{iam: roles}})); err != nil {
+	roles, _ := gwfix.DepsOf(gw).IAM.(IAMServiceLike)
+	if err := b.Register(eksawsapi.NewRegistration(eksawsapi.Deps{NATS: gwfix.DepsOf(gw).NATS, Roles: mockRoleResolver{iam: roles}})); err != nil {
 		panic(err)
 	}
 	gw.Services = b.Build()
@@ -1860,11 +1846,11 @@ func serveEC2(gw *GatewayConfig, body string) *httptest.ResponseRecorder {
 // rest.
 func ec2Registration(gw *GatewayConfig, mutate ...func(*ec2awsapi.Deps)) dispatch.Registration {
 	deps := ec2awsapi.Deps{
-		NATS:          gw.NATSConn,
-		IAM:           gw.IAMService,
+		NATS:          gwfix.DepsOf(gw).NATS,
+		IAM:           gwfix.DepsOf(gw).IAM,
 		Quota:         ec2quota.Adapter{Service: quotaFor(gw)},
-		ActiveNodes:   gw.DiscoverActiveNodes,
-		ExpectedNodes: gw.ExpectedNodes,
+		ActiveNodes:   activeNodes(gw),
+		ExpectedNodes: gwfix.DepsOf(gw).Nodes,
 	}
 	for _, m := range mutate {
 		m(&deps)
@@ -1992,15 +1978,12 @@ func ec2Pipeline(t *testing.T, mutate func(*ec2PipelineEnv), statements ...iamdo
 	_, nc, _ := testutil.StartTestJetStream(t)
 	daemon := newEC2Daemon(t, nc)
 
-	cfg := &GatewayConfig{
+	cfg := gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		NATSConn:       nc,
-		IAMService:     svc,
-		ExpectedNodes:  1,
-	}
+	}, gwfix.Deps{NATS: nc, IAM: svc, Nodes: 1})
 	if env.NoNATS {
-		cfg.NATSConn = nil
+		gwfix.DepsOf(cfg).NATS = nil
 	}
 	gw := withEC2(withRealAuth(cfg), func(d *ec2awsapi.Deps) {
 		d.AZ = gwfix.Region + "a"
@@ -2159,3 +2142,9 @@ func (d *ec2Daemon) reply(subject, body string) {
 	defer d.mu.Unlock()
 	d.replies[subject] = body
 }
+
+// activeNodes answers the configured node count, which is what discovery falls
+// back to when no daemon answers; no gateway test runs a responder.
+func activeNodes(gw *GatewayConfig) func(context.Context) int {
+	return func(context.Context) int { return gwfix.DepsOf(gw).Nodes }
+}

spinifex/gateway/iam_request_test.go

@@ -164,11 +164,10 @@ func TestIAMRequest_ListUsers_Success(t *testing.T) {
 }
 
 func TestIAMRequest_MissingAccountID(t *testing.T) {
-	gw := &GatewayConfig{
+	gw := gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		IAMService:     &flexMockIAMService{},
 		Credentials:    credverify.New(&flexMockIAMService{}, nil),
-	}
+	}, gwfix.Deps{IAM: &flexMockIAMService{}})
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "iam")
 		r = withTestIdentity(r.WithContext(ctx))
@@ -188,11 +187,10 @@ func TestIAMRequest_MissingAccountID(t *testing.T) {
 }
 
 func TestIAMRequest_NilService(t *testing.T) {
-	gw := &GatewayConfig{
+	gw := gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		IAMService:     nil,
 		Credentials:    credverify.New(nil, nil),
-	}
+	}, gwfix.Deps{IAM: nil})
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := r.Context()
 		ctx = reqctx.WithService(ctx, "iam")

spinifex/gateway/mount_pins_test.go

@@ -0,0 +1,70 @@
+package gateway
+
+//test:in-package — pins the route order of the unauthenticated mounts against
+// the SigV4 group and the throttle, through the real router.
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/go-chi/chi/v5"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+func mountsApp(t *testing.T) http.Handler {
+	t.Helper()
+	gw := reqctxGateway(t, reqctxGatewayEnv{NATS: natsUp, Throttler: newThrottler(t)})
+	gw.PublicRoutes = func(r chi.Router) {
+		ok := func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }
+		r.Get("/oidc/eks/{region}/{accountID}/{clusterName}/.well-known/openid-configuration", ok)
+		r.Get("/oidc/eks/{region}/{accountID}/{clusterName}/keys", ok)
+		r.Mount("/v2", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+			w.WriteHeader(http.StatusTeapot)
+		}))
+	}
+	return gw.SetupRoutes()
+}
+
+// OIDC discovery and the registry sit ahead of SigV4 and the throttle: an
+// unsigned caller is served, however many times, with a bucket of one.
+func TestMounts_PublicRoutesPrecedeSigV4AndThrottle(t *testing.T) {
+	app := mountsApp(t)
+	for _, tc := range []struct {
+		path string
+		want int
+	}{
+		{"/oidc/eks/ap-southeast-2/123456789012/c/.well-known/openid-configuration", http.StatusOK},
+		{"/oidc/eks/ap-southeast-2/123456789012/c/keys", http.StatusOK},
+		{"/v2/", http.StatusTeapot},
+		{"/v2/team/app/tags/list", http.StatusTeapot},
+	} {
+		for range 4 {
+			resp := doRequest(app, httptest.NewRequest(http.MethodGet, tc.path, nil))
+			assert.Equal(t, tc.want, resp.StatusCode, tc.path)
+		}
+	}
+}
+
+// Only the exact /v2 subtree is the registry; a sibling path stays behind SigV4.
+func TestMounts_RegistryPrefixIsExact(t *testing.T) {
+	resp := doRequest(mountsApp(t), httptest.NewRequest(http.MethodGet, "/v20/x", nil))
+	assert.NotEqual(t, http.StatusTeapot, resp.StatusCode)
+	assert.GreaterOrEqual(t, resp.StatusCode, http.StatusBadRequest)
+}
+
+// Unsigned /admin is refused by SigV4, so the mount cannot be reached without it.
+func TestMounts_AdminSitsBehindSigV4(t *testing.T) {
+	resp := doRequest(mountsApp(t), httptest.NewRequest(http.MethodPost, "/admin/CreateAccount", nil))
+	assert.GreaterOrEqual(t, resp.StatusCode, http.StatusBadRequest)
+}
+
+// Without an admin registration name the surface refuses, rather than
+// dispatching to whatever service the empty name selects.
+func TestAdmin_UnconfiguredSurfaceAnswersInternalError(t *testing.T) {
+	gw := withServices(&GatewayConfig{DisableLogging: true})
+	rec := adminRequest(gw, "CreateAccount", `{}`)
+	assert.Equal(t, http.StatusInternalServerError, rec.Code)
+	assert.Equal(t, awserrors.ErrorInternalError, decodeAdminError(t, rec).Error.Code)
+}

spinifex/gateway/oidc_discovery.go

@@ -1,37 +0,0 @@
-package gateway
-
-import (
-	"log/slog"
-	"net/http"
-
-	"github.com/go-chi/chi/v5"
-)
-
-// OIDCPublisher serves the unauthenticated per-cluster OIDC discovery document
-// and JWKS (IRSA). The EKS domain owns the records they are built from; the
-// gateway only maps the route onto the account and cluster.
-type OIDCPublisher interface {
-	ServeDiscoveryDocument(w http.ResponseWriter, r *http.Request, accountID, clusterName string)
-	ServeJWKS(w http.ResponseWriter, r *http.Request, accountID, clusterName string)
-}
-
-// OIDCDiscoveryDocument serves the cluster's OpenID discovery document.
-func (gw *GatewayConfig) OIDCDiscoveryDocument(w http.ResponseWriter, r *http.Request) {
-	if gw.OIDC == nil {
-		slog.Error("OIDC discovery: publisher not configured")
-		http.Error(w, "internal error", http.StatusInternalServerError)
-		return
-	}
-	gw.OIDC.ServeDiscoveryDocument(w, r, chi.URLParam(r, "accountID"), chi.URLParam(r, "clusterName"))
-}
-
-// OIDCJWKS serves the cluster's JWKS document as referenced by the discovery
-// document's jwks_uri.
-func (gw *GatewayConfig) OIDCJWKS(w http.ResponseWriter, r *http.Request) {
-	if gw.OIDC == nil {
-		slog.Error("OIDC JWKS: publisher not configured")
-		http.Error(w, "internal error", http.StatusInternalServerError)
-		return
-	}
-	gw.OIDC.ServeJWKS(w, r, chi.URLParam(r, "accountID"), chi.URLParam(r, "clusterName"))
-}

spinifex/gateway/operator_characterization_test.go

@@ -3,6 +3,7 @@ package gateway
 import (
 	"encoding/json"
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
 	admissionquota "github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
@@ -101,7 +102,7 @@ func TestOperatorAdmin_ListAccounts_JSONSuccess(t *testing.T) {
 // Unlike the Query surface, /admin authorizes first and only then checks the
 // cluster, so NATS down is visible only to an authorized caller.
 func TestOperatorAdmin_NATSDownAnswers503OnlyAfterAuthorization(t *testing.T) {
-	down := func(gw *GatewayConfig) { gw.NATSConn = nil }
+	down := func(gw *GatewayConfig) { gwfix.DepsOf(gw).NATS = nil }
 
 	_, app := operatorPipelineApp(t, awsidentifiers.DefaultAccountID(), allowAllOperatorDocs(), down)
 	resp := adminCall(t, app, http.MethodPost, "ListAccounts", "spinifex", `{}`)
@@ -227,7 +228,7 @@ func TestOperatorQuery_ModelAccessWithoutStoreIsServerInternal(t *testing.T) {
 // The cluster-unavailable check is in the shared Request path, ahead of
 // authorization, so even a caller with no grant sees 503 when NATS is down.
 func TestOperatorQuery_NATSDownAnswers503BeforeAuthorization(t *testing.T) {
-	_, app := operatorPipelineApp(t, authzAccountID, nil, func(gw *GatewayConfig) { gw.NATSConn = nil })
+	_, app := operatorPipelineApp(t, authzAccountID, nil, func(gw *GatewayConfig) { gwfix.DepsOf(gw).NATS = nil })
 	resp := doRequest(app, spinifexQuery(t, "GetVersion", nil))
 
 	require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)

spinifex/gateway/operator_instance_metrics_test.go

@@ -18,7 +18,7 @@ import (
 func TestOperatorQuery_GetInstanceMetrics_PolicyOnAnotherInstanceIsDenied(t *testing.T) {
 	otherARN := "arn:aws:ec2:" + gwfix.Region + ":" + streamAccount + ":instance/i-0fffffffffffffff0"
 	gw, app := operatorPipelineApp(t, streamAccount, instanceMetricsDocs(otherARN), nil)
-	answerWatches(t, gw.NATSConn)
+	answerWatches(t, gwfix.DepsOf(gw).NATS)
 	srv := httptest.NewServer(app)
 	t.Cleanup(srv.Close)
 
@@ -32,7 +32,7 @@ func TestOperatorQuery_GetInstanceMetrics_PolicyOnAnotherInstanceIsDenied(t *tes
 func TestOperatorQuery_GetInstanceMetrics_StreamsThroughThePipeline(t *testing.T) {
 	instanceARN := "arn:aws:ec2:" + gwfix.Region + ":" + streamAccount + ":instance/" + streamInstance
 	gw, app := operatorPipelineApp(t, streamAccount, instanceMetricsDocs(instanceARN), nil)
-	answerWatches(t, gw.NATSConn)
+	answerWatches(t, gwfix.DepsOf(gw).NATS)
 	srv := httptest.NewServer(app)
 	t.Cleanup(srv.Close)
 

spinifex/gateway/quota_helpers_test.go

@@ -65,10 +65,9 @@ func quotaTestGateway(t *testing.T) (*GatewayConfig, string) {
 	quota.SetOverrides(kv)
 
 	svc := &quotaIAMService{known: account}
-	return withQuota(withVerifier(&GatewayConfig{
+	return withQuota(withVerifier(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		IAMService:     svc,
-	}), quota), account
+	}, gwfix.Deps{IAM: svc})), quota), account
 }
 
 // quotaStubChecker answers AccountTerminating from a fixed set, so a test can

spinifex/gateway/ratelimit_test.go

@@ -49,14 +49,12 @@ func setupTestAppWithRateLimiter(t *testing.T, accessKey, secretKey string, rl *
 	require.NoError(t, err)
 	t.Cleanup(nc.Close)
 
-	gw := &GatewayConfig{
+	gw := gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     mockSvc,
 		Credentials:    credverify.New(mockSvc, nil),
 		RateLimiter:    rl,
-		NATSConn:       nc,
-	}
+	}, gwfix.Deps{IAM: mockSvc, NATS: nc})
 	withEC2(gw)
 
 	r := chi.NewRouter()

spinifex/gateway/rds_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"github.com/aws/aws-sdk-go/service/rds"
 	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
 	rdsinstance "github.com/mulgadc/spinifex/spinifex/domains/rds/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
@@ -30,7 +31,7 @@ func TestRDSRequest_CreateDBInstance_EmptyMasterUserPasswordReturns400(t *testin
 	require.NoError(t, err)
 	t.Cleanup(func() { _ = sub.Unsubscribe() })
 
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{NATS: nc, IAM: allowAllIAMService()}))
 	req := setupRDSRequest("Action=CreateDBInstance&DBInstanceIdentifier=orders-db&Engine=postgres&" +
 		"DBInstanceClass=db.t3.medium&AllocatedStorage=20&MasterUsername=appuser&DBName=orders")
 
@@ -45,7 +46,7 @@ func TestRDSRequest_CreateDBInstance_EmptyMasterUserPasswordReturns400(t *testin
 
 func TestRDSRequest_MalformedScalarReturnsHTTP400(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
-	gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()})
+	gw := withVerifier(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{NATS: nc, IAM: allowAllIAMService()}))
 	for _, body := range []string{
 		"Action=CreateDBInstance&AllocatedStorage=abc",
 		"Action=CreateDBInstance&MultiAZ=yes",

spinifex/gateway/real_auth_exception_test.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/accountadmission"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/clusterready"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
@@ -12,10 +13,11 @@ import (
 // internals (the middleware stash, the admin method gate, the account gate)
 // use it; every other gateway test runs on the fake verifier.
 func withRealAuth(gw *GatewayConfig) *GatewayConfig {
-	gw.Credentials = credverify.New(gw.IAMService, gw.STSService)
-	gw.Policy = policyauth.New(gw.IAMService, policyauth.WithRetryBase(policyRetryBase))
-	gw.Admission = accountadmission.New(gw.IAMService)
-	if gw.NATSConn != nil {
+	withTestAdmin(gw)
+	gw.Credentials = credverify.New(gwfix.DepsOf(gw).IAM, gwfix.DepsOf(gw).STS)
+	gw.Policy = policyauth.New(gwfix.DepsOf(gw).IAM, policyauth.WithRetryBase(policyRetryBase))
+	gw.Admission = accountadmission.New(gwfix.DepsOf(gw).IAM)
+	if gwfix.DepsOf(gw).NATS != nil {
 		gw.Readiness = realReadiness{gw}
 	}
 	return gw
@@ -31,5 +33,5 @@ func withEC2RealAuth(gw *GatewayConfig) *GatewayConfig {
 type realReadiness struct{ gw *GatewayConfig }
 
 func (r realReadiness) Available() bool {
-	return r.gw.NATSConn != nil && clusterready.New(r.gw.NATSConn).Available()
+	return gwfix.DepsOf(r.gw).NATS != nil && clusterready.New(gwfix.DepsOf(r.gw).NATS).Available()
 }

spinifex/gateway/registry.go

@@ -1,24 +0,0 @@
-package gateway
-
-import (
-	"net/http"
-
-	"github.com/go-chi/chi/v5"
-)
-
-// mountRegistry mounts the registry handler at /v2, ahead of the SigV4 group. The
-// registry authenticates by its own token rather than SigV4, so it is not part of the
-// AWS seam. Only the exact /v2 subtree is the registry; /v20 and the like stay SigV4.
-func (gw *GatewayConfig) mountRegistry(r chi.Router) {
-	h := gw.RegistryHandler
-	if h == nil {
-		h = http.HandlerFunc(registryNotConfigured)
-	}
-	r.Mount("/v2", h)
-}
-
-func registryNotConfigured(w http.ResponseWriter, _ *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(http.StatusNotImplemented)
-	_, _ = w.Write([]byte(`{"errors":[{"code":"UNSUPPORTED","message":"registry not configured"}]}`))
-}

spinifex/gateway/reqctx_pipeline_characterization_test.go

@@ -97,13 +97,11 @@ func reqctxGateway(t *testing.T, env reqctxGatewayEnv, statements ...iamdomain.S
 			nc.Close()
 		}
 	}
-	gw := withRealAuth(&GatewayConfig{
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         reqctxRegion,
-		NATSConn:       nc,
-		IAMService:     svc,
 		Throttler:      env.Throttler,
-	})
+	}, gwfix.Deps{NATS: nc, IAM: svc}))
 	regs := []dispatch.Registration{
 		ecsawsapi.NewRegistration(ecsawsapi.Deps{NATS: nc}),
 		iamRegistration(gw),
@@ -115,6 +113,7 @@ func reqctxGateway(t *testing.T, env reqctxGatewayEnv, statements ...iamdomain.S
 			ResolveAction: func(*http.Request) string { return "ProbeAction" },
 			Dispatch:      env.Probe,
 			Errors:        dispatch.ErrorEnvelopeXML,
+			Traits:        dispatch.ServiceTraits{GlobalSigningRegion: "us-east-1"},
 			Inventory:     dispatch.Inventory{Registered: []string{"ProbeAction"}},
 		})
 	}

spinifex/gateway/requestaudit_test.go

@@ -29,13 +29,12 @@ func auditRouter(t *testing.T, keys map[string]*iamdomain.AccessKey) (http.Handl
 	rl := authlimit.NewAuthRateLimiter()
 	t.Cleanup(rl.Stop)
 
-	gw := &GatewayConfig{
+	gw := gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		IAMService:     &gwfix.IAM{MasterKey: gwfix.MasterKey, AccessKeys: keys},
 		Credentials:    credverify.New(&gwfix.IAM{MasterKey: gwfix.MasterKey, AccessKeys: keys}, nil),
 		RateLimiter:    rl,
-	}
+	}, gwfix.Deps{IAM: &gwfix.IAM{MasterKey: gwfix.MasterKey, AccessKeys: keys}})
 	withEC2(gw)
 
 	var got *audit.Record

spinifex/gateway/service_metadata_pins_test.go

@@ -0,0 +1,152 @@
+package gateway
+
+import (
+	"bytes"
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+	stsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sts/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/reqctx"
+)
+
+// The throttle rejection per service, through the registrations production
+// composes where there is one. EKS is a JSON registration, so it never takes
+// the XML arm.
+func TestWriteThrottleError_ByService(t *testing.T) {
+	for _, tc := range []struct {
+		svc, code, contains string
+		status              int
+		contentType         string
+		with                func(*GatewayConfig) *GatewayConfig
+	}{
+		{"iam", awserrors.ErrorThrottling, "<ErrorResponse>", http.StatusServiceUnavailable, "application/xml", withIAM},
+		{"sts", awserrors.ErrorThrottling, "<ErrorResponse>", http.StatusServiceUnavailable, "application/xml", withSTS},
+		{"eks", "", `"__type"`, 0, envelope.JSONContentType, withContainers},
+		{"s3", awserrors.ErrorRequestLimitExceeded, "<Error>", http.StatusServiceUnavailable, "application/xml", func(gw *GatewayConfig) *GatewayConfig { return withServices(gw) }},
+		{"unserved-service", awserrors.ErrorRequestLimitExceeded, "<ErrorResponse>", http.StatusServiceUnavailable, "application/xml", nil},
+		{"", awserrors.ErrorRequestLimitExceeded, "<Response>", http.StatusServiceUnavailable, "application/xml", nil},
+	} {
+		t.Run(tc.svc, func(t *testing.T) {
+			gw := &GatewayConfig{DisableLogging: true}
+			if tc.with != nil {
+				gw = tc.with(gw)
+			}
+			req := httptest.NewRequest(http.MethodPost, "/", nil)
+			req = req.WithContext(reqctx.WithService(req.Context(), tc.svc))
+			w := httptest.NewRecorder()
+
+			gw.writeThrottleError(w, req)
+
+			body, _ := io.ReadAll(w.Result().Body)
+			if tc.status != 0 {
+				assert.Equal(t, tc.status, w.Result().StatusCode)
+			}
+			assert.Equal(t, tc.contentType, w.Result().Header.Get("Content-Type"))
+			if tc.code != "" {
+				assert.Contains(t, string(body), "<Code>"+tc.code+"</Code>")
+			}
+			assert.Contains(t, string(body), tc.contains)
+		})
+	}
+}
+
+// An unparsed signature leaves no service, and the rejection keeps EC2's shape.
+func TestXMLErrorBody_EmptyServiceKeepsEC2Shape(t *testing.T) {
+	body := (&GatewayConfig{}).xmlErrorBody("", "AccessDenied", "denied", "req-1", "/")
+
+	assert.Equal(t, "Response", xmlRootName(t, body))
+	assert.Contains(t, string(body), "<Errors>")
+}
+
+// A global service accepts its own signing region and the gateway's, and
+// nothing else; a regional service accepts only the gateway's.
+func TestReqctxPipeline_GlobalSigningRegionRejectsOthers(t *testing.T) {
+	probe := func(w http.ResponseWriter, _ dispatch.Invocation) error {
+		w.WriteHeader(http.StatusOK)
+		return nil
+	}
+	app := reqctxGateway(t, reqctxGatewayEnv{NATS: natsUp, Probe: probe}, statement("Allow", "*", "*")).SetupRoutes()
+
+	body := []byte("Action=ProbeAction")
+	for _, tc := range []struct {
+		region string
+		want   int
+	}{
+		{"us-east-1", http.StatusOK},
+		{reqctxRegion, http.StatusOK},
+		{"eu-west-1", http.StatusForbidden},
+	} {
+		t.Run(tc.region, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body))
+			req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
+			got := doRequest(app, signReqctx(t, req, body, "route53", tc.region))
+			assert.Equal(t, tc.want, got.StatusCode)
+		})
+	}
+}
+
+// The anonymous exchange runs ahead of authentication, so an oversize body is
+// handed back unread to the signed chain instead of being dispatched.
+func TestAnonymousSTS_OversizeBodyPassesThroughUnread(t *testing.T) {
+	gw := &GatewayConfig{DisableLogging: true, PreAuth: []dispatch.PreAuth{stsawsapi.NewPreAuth(stsawsapi.Deps{})}}
+	big := "Action=AssumeRoleWithWebIdentity&WebIdentityToken=" + strings.Repeat("a", int(sigv4.MaxPayloadLen)+1)
+
+	var seen []byte
+	next := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { seen, _ = io.ReadAll(r.Body) })
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(big))
+	rec := httptest.NewRecorder()
+
+	gw.preAuthInterceptor(next).ServeHTTP(rec, req)
+
+	assert.Equal(t, big, string(seen), "the whole body must reach the next handler")
+}
+
+// With no anonymous capability composed, the action fails closed.
+func TestAnonymousSTS_NilCapabilityIsAnInternalError(t *testing.T) {
+	gw := &GatewayConfig{DisableLogging: true, PreAuth: []dispatch.PreAuth{stsawsapi.NewPreAuth(stsawsapi.Deps{})}}
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=AssumeRoleWithWebIdentity&WebIdentityToken=x"))
+	rec := httptest.NewRecorder()
+
+	gw.preAuthInterceptor(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+		t.Fatal("must not fall through to the signed chain")
+	})).ServeHTTP(rec, req)
+
+	require.Equal(t, http.StatusInternalServerError, rec.Code)
+	assert.Contains(t, rec.Body.String(), "<Code>"+awserrors.ErrorInternalError+"</Code>")
+}
+
+// A pre-authentication request is served before SigV4 and before the
+// throttle, so an exhausted bucket never reaches it.
+func TestPreAuth_RunsAheadOfSigV4AndThrottle(t *testing.T) {
+	served := 0
+	gw := withIAM(withVerifier(gwfix.WithDeps(&GatewayConfig{
+		DisableLogging: true,
+		Throttler:      newThrottler(t),
+		PreAuth: []dispatch.PreAuth{{
+			Service: "svc",
+			Serve: func(w http.ResponseWriter, _ *http.Request) (bool, error) {
+				served++
+				w.WriteHeader(http.StatusOK)
+				return true, nil
+			},
+		}},
+	}, gwfix.Deps{IAM: allowAllIAMService()})))
+	app := gw.SetupRoutes()
+
+	for range 5 {
+		req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=Anything"))
+		assert.Equal(t, http.StatusOK, doRequest(app, req).StatusCode)
+	}
+	assert.Equal(t, 5, served)
+}

spinifex/gateway/sts_anonymous.go

@@ -1,80 +0,0 @@
-package gateway
-
-import (
-	"bytes"
-	"errors"
-	"github.com/mulgadc/spinifex/spinifex/ingress/aws/reqctx"
-	"io"
-	"log/slog"
-	"net/http"
-
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-)
-
-// anonymousSTSInterceptor routes unsigned IRSA bootstrap actions
-// (AssumeRoleWithWebIdentity) to the STS dispatcher before the SigV4 middleware.
-// Callers carry no AWS credentials — authentication is via the ServiceAccount JWT
-// in the body. Signed requests (Authorization header present) pass through unchanged.
-func (gw *GatewayConfig) anonymousSTSInterceptor(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.Method == http.MethodPost && r.Header.Get("Authorization") == "" {
-			if args, ok := gw.anonymousSTSArgs(r); ok {
-				r = r.WithContext(reqctx.WithService(r.Context(), "sts"))
-				if err := gw.serveAnonymousSTS(w, args); err != nil {
-					gw.ErrorHandler(w, r, err)
-				}
-				return
-			}
-		}
-		next.ServeHTTP(w, r)
-	})
-}
-
-// anonymousSTSArgs peeks the request body, parses query args, and reports
-// whether the Action is permitted without SigV4. Non-anonymous requests pass through unchanged.
-func (gw *GatewayConfig) anonymousSTSArgs(r *http.Request) (map[string]string, bool) {
-	// This runs ahead of SigV4, so an unauthenticated client controls the
-	// allocation. Read one byte past the cap to detect an oversized body and
-	// hand it back to the normal chain unread rather than dispatching it.
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, false
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		r.Body = io.NopCloser(io.MultiReader(bytes.NewReader(body), r.Body))
-		return nil, false
-	}
-	r.Body = io.NopCloser(bytes.NewReader(body))
-
-	args, err := ParseAWSQueryArgs(string(body))
-	if err != nil || !anonymousSTSActions[args["Action"]] {
-		return nil, false
-	}
-	return args, true
-}
-
-// anonymousSTSActions lists STS actions that carry no SigV4; authenticated by
-// a web-identity JWT instead. anonymousSTSInterceptor routes these before auth.
-var anonymousSTSActions = map[string]bool{
-	"AssumeRoleWithWebIdentity": true,
-}
-
-// serveAnonymousSTS runs an unsigned STS action through the composed
-// capability and writes its XML response.
-func (gw *GatewayConfig) serveAnonymousSTS(w http.ResponseWriter, args map[string]string) error {
-	if gw.AnonymousSTS == nil {
-		slog.Error("STS: anonymous service not initialized")
-		return errors.New(awserrors.ErrorInternalError)
-	}
-	xmlOutput, err := gw.AnonymousSTS(args)
-	if err != nil {
-		return err
-	}
-	w.Header().Set("Content-Type", "text/xml")
-	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(xmlOutput); err != nil {
-		slog.Error("Failed to write STS response", "err", err)
-	}
-	return nil
-}

spinifex/gateway/sts_anonymous_test.go

@@ -2,50 +2,29 @@ package gateway
 
 import (
 	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
+	stsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sts/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"io"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
 )
 
-// TestAnonymousSTSArgs_Classification covers the body-peek helper for anonymous action detection.
-func TestAnonymousSTSArgs_Classification(t *testing.T) {
-	gw := &GatewayConfig{DisableLogging: true}
-
-	t.Run("anonymous action recognised, body restored", func(t *testing.T) {
-		const body = "Action=AssumeRoleWithWebIdentity&WebIdentityToken=tok"
-		req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
-		args, ok := gw.anonymousSTSArgs(req)
-		require.True(t, ok)
-		assert.Equal(t, "AssumeRoleWithWebIdentity", args["Action"])
-
-		restored, _ := io.ReadAll(req.Body)
-		assert.Equal(t, body, string(restored), "body must be restored for downstream read")
-	})
-
-	t.Run("non-anonymous action rejected", func(t *testing.T) {
-		req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=GetCallerIdentity"))
-		_, ok := gw.anonymousSTSArgs(req)
-		assert.False(t, ok)
-	})
-}
-
 // TestAnonymousSTS_SignedRequestFallsThrough confirms a request with an Authorization
 // header is not intercepted — it passes through to the signed surface.
 func TestAnonymousSTS_SignedRequestFallsThrough(t *testing.T) {
-	gw := &GatewayConfig{
+	gw := gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
-		STSService: &flexMockSTSService{
-			assumeWebIdentityFn: func(*sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error) {
-				t.Fatal("signed request must not reach the anonymous STS dispatcher")
-				return nil, nil
-			},
+	}, gwfix.Deps{STS: &flexMockSTSService{
+		assumeWebIdentityFn: func(*sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error) {
+			t.Fatal("signed request must not reach the anonymous STS dispatcher")
+			return nil, nil
 		},
-	}
+	}})
 
+	gw.PreAuth = []dispatch.PreAuth{stsawsapi.NewPreAuth(stsawsapi.Deps{STS: gwfix.DepsOf(gw).STS})}
 	nextCalled := false
 	next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { nextCalled = true })
 
@@ -54,6 +33,6 @@ func TestAnonymousSTS_SignedRequestFallsThrough(t *testing.T) {
 	req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=...")
 
 	rec := httptest.NewRecorder()
-	gw.anonymousSTSInterceptor(next).ServeHTTP(rec, req)
+	gw.preAuthInterceptor(next).ServeHTTP(rec, req)
 	assert.True(t, nextCalled, "signed request must pass through to next handler")
 }

spinifex/gateway/sts_request_test.go

@@ -58,7 +58,7 @@ func TestSTSRequest_AssumeRole_NilIAMService_InternalError(t *testing.T) {
 			return nil, nil
 		},
 	}
-	gw := withRealAuth(&GatewayConfig{DisableLogging: true, STSService: svc, IAMService: nil})
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{STS: svc, IAM: nil}))
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "sts")
 		ctx = withPrincipalField(ctx, fieldAccountID, awsidentifiers.GlobalAccountID)
@@ -186,23 +186,21 @@ func TestSTSRequest_GetCallerIdentity_PathedUserThroughMiddleware(t *testing.T)
 	const pathed = "arn:aws:iam::123456789012:user/eng/alice"
 	encryptedSecret, err := iamdomain.EncryptSecret(gwfix.SecretKey, gwfix.MasterKey)
 	require.NoError(t, err)
-	gw := withRealAuth(&GatewayConfig{
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{
 		DisableLogging: true,
 		Region:         gwfix.Region,
-		STSService:     &flexMockSTSService{},
-		IAMService: &gwfix.IAM{
-			MasterKey: gwfix.MasterKey,
-			AccessKeys: map[string]*iamdomain.AccessKey{gwfix.AccessKey: {
-				AccessKeyID: gwfix.AccessKey, SecretAccessKey: encryptedSecret,
-				UserName: "alice", AccountID: "123456789012", Status: "Active",
-			}},
-			GetUserFn: func(string, *iam.GetUserInput) (*iam.GetUserOutput, error) {
-				return &iam.GetUserOutput{User: &iam.User{
-					UserName: aws.String("alice"), UserId: aws.String("AIDAALICE"), Arn: aws.String(pathed),
-				}}, nil
-			},
+	}, gwfix.Deps{STS: &flexMockSTSService{}, IAM: &gwfix.IAM{
+		MasterKey: gwfix.MasterKey,
+		AccessKeys: map[string]*iamdomain.AccessKey{gwfix.AccessKey: {
+			AccessKeyID: gwfix.AccessKey, SecretAccessKey: encryptedSecret,
+			UserName: "alice", AccountID: "123456789012", Status: "Active",
+		}},
+		GetUserFn: func(string, *iam.GetUserInput) (*iam.GetUserOutput, error) {
+			return &iam.GetUserOutput{User: &iam.User{
+				UserName: aws.String("alice"), UserId: aws.String("AIDAALICE"), Arn: aws.String(pathed),
+			}}, nil
 		},
-	})
+	}}))
 	withServices(gw, stsRegistration(gw), ec2Registration(gw))
 	handler := gw.SigV4AuthMiddleware()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if err := gw.STS_Request(w, r); err != nil {
@@ -343,7 +341,7 @@ func TestSTSRequest_MissingAction(t *testing.T) {
 }
 
 func TestSTSRequest_NilService_InternalError(t *testing.T) {
-	gw := withRealAuth(&GatewayConfig{DisableLogging: true, STSService: nil, IAMService: &flexMockIAMService{}})
+	gw := withRealAuth(gwfix.WithDeps(&GatewayConfig{DisableLogging: true}, gwfix.Deps{STS: nil, IAM: &flexMockIAMService{}}))
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := reqctx.WithService(r.Context(), "sts")
 		ctx = withPrincipalField(ctx, fieldAccountID, awsidentifiers.GlobalAccountID)

spinifex/gateway/verifier_helper_test.go

@@ -5,6 +5,6 @@ import "github.com/mulgadc/spinifex/internal/testkit/gwfix"
 // withVerifier authenticates through the IAM service the config holds, so a
 // test that changes it changes authentication too.
 func withVerifier(gw *GatewayConfig) *GatewayConfig {
-	gw.Credentials = &gwfix.Verifier{IAM: gw.IAMService}
+	gw.Credentials = &gwfix.Verifier{IAM: gwfix.DepsOf(gw).IAM}
 	return withFakeCapabilities(gw)
 }

spinifex/ingress/aws/dispatch/preauth.go

@@ -0,0 +1,13 @@
+package dispatch
+
+import "net/http"
+
+// PreAuth serves a request that carries no credentials ahead of
+// authentication and throttling, because its own payload authenticates it.
+// Serve reports handled=false, having written nothing and left the body
+// readable, for a request that is not its own. An error it returns for a
+// handled request renders under Service.
+type PreAuth struct {
+	Service string
+	Serve   func(w http.ResponseWriter, r *http.Request) (handled bool, err error)
+}

spinifex/ingress/aws/dispatch/registry.go

@@ -160,6 +160,8 @@ type Registration struct {
 	// Unset, a REST-JSON or JSON 1.0 registration gets the protocol's own
 	// throttle error and any other keeps the gateway default. JSON only.
 	Throttle *ErrorSpec
+	// Traits are the per-service behaviours the pipeline reads by scope.
+	Traits ServiceTraits
 	// QueryErrorCodes maps an error code to the legacy query code sent in
 	// x-amzn-query-error; a code absent from it is sent as itself. JSON 1.0
 	// only.
@@ -194,6 +196,7 @@ var serviceNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
 // concurrent use; Build returns the immutable registry the server reads.
 type Builder struct {
 	entries   map[string]*Entry
+	scopes    map[string]ServiceTraits
 	inventory map[string]Inventory
 	sealed    bool
 }
@@ -202,6 +205,7 @@ type Builder struct {
 func NewBuilder() *Builder {
 	return &Builder{
 		entries:   make(map[string]*Entry),
+		scopes:    make(map[string]ServiceTraits),
 		inventory: make(map[string]Inventory),
 	}
 }
@@ -235,6 +239,7 @@ func (b *Builder) Register(reg Registration) error {
 		handle:     reg.Dispatch,
 		errors:     reg.Errors,
 		throttle:   reg.Throttle,
+		traits:     reg.Traits,
 		queryCodes: maps.Clone(reg.QueryErrorCodes),
 		internal:   reg.InternalErrorCode,
 		namespace:  reg.ErrorNamespace,
@@ -247,11 +252,32 @@ func (b *Builder) Register(reg Registration) error {
 	return nil
 }
 
+// DeclareScope declares the traits of a credential scope this gateway knows
+// but does not serve, so a rejection for it still renders as its clients
+// expect. It fails on an invalid name or a scope a registration serves.
+func (b *Builder) DeclareScope(scope string, traits ServiceTraits) error {
+	if b.sealed {
+		return ErrSealed
+	}
+	if !serviceNamePattern.MatchString(scope) {
+		return fmt.Errorf("dispatch: invalid scope name %q", scope)
+	}
+	if _, ok := b.entries[scope]; ok {
+		return fmt.Errorf("dispatch: scope %q is served by a registration", scope)
+	}
+	if _, ok := b.scopes[scope]; ok {
+		return fmt.Errorf("dispatch: scope %q is already declared", scope)
+	}
+	b.scopes[scope] = traits
+	return nil
+}
+
 // Build seals the builder and returns the registry. Further Register calls fail.
 func (b *Builder) Build() *Registry {
 	b.sealed = true
 	return &Registry{
 		entries:   maps.Clone(b.entries),
+		scopes:    maps.Clone(b.scopes),
 		inventory: maps.Clone(b.inventory),
 	}
 }
@@ -289,6 +315,9 @@ func validate(reg Registration) ([]string, error) {
 }
 
 func validateErrorSpecs(reg Registration) error {
+	if reg.Traits.XMLThrottleCode != "" && reg.Errors != ErrorEnvelopeXML {
+		return errors.New("XML throttle code needs the XML error envelope")
+	}
 	if t := reg.Throttle; t != nil {
 		if !reg.Errors.IsJSON() {
 			return errors.New("throttle error needs a JSON error envelope")
@@ -363,6 +392,7 @@ func sortedClone(s []string) []string {
 // for concurrent use because nothing mutates it after Build.
 type Registry struct {
 	entries   map[string]*Entry
+	scopes    map[string]ServiceTraits
 	inventory map[string]Inventory
 }
 
@@ -372,6 +402,16 @@ func (r *Registry) Lookup(service string) (*Entry, bool) {
 	return e, ok
 }
 
+// Traits returns the traits of a scope: a registration's own, else those
+// declared for a scope the gateway does not serve.
+func (r *Registry) Traits(scope string) (ServiceTraits, bool) {
+	if e, ok := r.entries[scope]; ok {
+		return e.traits, true
+	}
+	t, ok := r.scopes[scope]
+	return t, ok
+}
+
 // Inventory returns a fresh copy of every registration's operation inventory,
 // keyed by inventory name with each list sorted.
 func (r *Registry) Inventory() map[string]Inventory {
@@ -394,12 +434,16 @@ type Entry struct {
 	errors  ErrorEnvelope
 
 	throttle   *ErrorSpec
+	traits     ServiceTraits
 	queryCodes map[string]string
 	internal   string
 	namespace  string
 	namespaces map[string]string
 }
 
+// Traits returns the registration's declared service traits.
+func (e *Entry) Traits() ServiceTraits { return e.traits }
+
 // Service returns the registration's primary credential-scope name.
 func (e *Entry) Service() string { return e.service }
 

spinifex/ingress/aws/dispatch/registry_test.go

@@ -364,3 +364,38 @@ func TestEntryErrorNamespacePerCodeOverride(t *testing.T) {
 	assert.Equal(t, "com.amazonaws.sqs", e.ErrorNamespace("QueueDoesNotExist"))
 	assert.Equal(t, "com.amazon.coral.service", e.ErrorNamespace("SerializationException"))
 }
+
+func TestRegisterXMLThrottleCodeNeedsXMLEnvelope(t *testing.T) {
+	r := validReg("svc")
+	r.Errors = ErrorEnvelopeXML
+	r.Traits = ServiceTraits{XMLThrottleCode: "Throttling"}
+	b := NewBuilder()
+	require.NoError(t, b.Register(r))
+	e, _ := b.Build().Lookup("svc")
+	assert.Equal(t, "Throttling", e.Traits().XMLThrottleCode)
+
+	r = validReg("other")
+	r.Errors = ErrorEnvelopeJSON
+	r.Traits = ServiceTraits{XMLThrottleCode: "Throttling"}
+	assert.Error(t, NewBuilder().Register(r))
+}
+
+func TestDeclareScope(t *testing.T) {
+	b := NewBuilder()
+	require.NoError(t, b.Register(validReg("svc")))
+	require.NoError(t, b.DeclareScope("other", ServiceTraits{XMLErrorShape: XMLErrorShapeS3}))
+
+	assert.Error(t, b.DeclareScope("svc", ServiceTraits{}), "a served service cannot be declared")
+	assert.Error(t, b.DeclareScope("other", ServiceTraits{}), "a scope is declared once")
+	assert.Error(t, b.DeclareScope("Bad Name", ServiceTraits{}))
+
+	reg := b.Build()
+	traits, ok := reg.Traits("other")
+	require.True(t, ok)
+	assert.Equal(t, XMLErrorShapeS3, traits.XMLErrorShape)
+	_, served := reg.Lookup("other")
+	assert.False(t, served)
+	_, ok = reg.Traits("unknown")
+	assert.False(t, ok)
+	assert.ErrorIs(t, b.DeclareScope("late", ServiceTraits{}), ErrSealed)
+}

spinifex/ingress/aws/dispatch/selector.go

@@ -44,6 +44,14 @@ func (s *Selector) Lookup(service string) (*Entry, bool) {
 	return s.registry.Lookup(service)
 }
 
+// Traits returns the traits declared for a credential scope, served or not.
+func (s *Selector) Traits(service string) (ServiceTraits, bool) {
+	if s.registry == nil {
+		return ServiceTraits{}, false
+	}
+	return s.registry.Traits(service)
+}
+
 // Served reports whether either dispatch path serves service.
 func (s *Selector) Served(service string) bool {
 	if _, ok := s.Lookup(service); ok {

spinifex/ingress/aws/dispatch/traits.go

@@ -0,0 +1,32 @@
+package dispatch
+
+// XMLErrorShape selects the XML body an XML error renders in. The zero value
+// is the generic REST-XML ErrorResponse.
+type XMLErrorShape int
+
+const (
+	// XMLErrorShapeGeneric is the <ErrorResponse> envelope.
+	XMLErrorShapeGeneric XMLErrorShape = iota
+	// XMLErrorShapeEC2 is the EC2 <Response><Errors> envelope.
+	XMLErrorShapeEC2
+	// XMLErrorShapeS3 is the S3 <Error> envelope.
+	XMLErrorShapeS3
+)
+
+// ServiceTraits are the behaviours the pipeline applies by credential scope
+// without naming a service: the owning registration declares them, and the
+// composition declares them for a scope it knows but does not serve.
+type ServiceTraits struct {
+	// XMLThrottleCode is the code of the XML throttle rejection. Unset, the
+	// gateway default RequestLimitExceeded applies. XML envelopes only.
+	XMLThrottleCode string
+	// XMLErrorShape is the XML body the scope's errors render in.
+	XMLErrorShape XMLErrorShape
+	// GlobalSigningRegion is the credential-scope region clients of a global
+	// service sign with on its default endpoint. Unset, only the gateway's own
+	// region is accepted.
+	GlobalSigningRegion string
+	// PayloadBody marks a scope whose request body is object data, never
+	// query arguments, so nothing reads an Action out of it.
+	PayloadBody bool
+}

spinifex/operator/admin/registration.go

@@ -93,6 +93,7 @@ func NewRegistration(deps Deps) dispatch.Registration {
 		ResolveAction: resolveAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeXML,
+		Traits:        dispatch.ServiceTraits{XMLErrorShape: dispatch.XMLErrorShapeEC2},
 		Inventory:     dispatch.Inventory{Registered: ActionNames()},
 	}
 }

spinifex/operator/host/systemd/units_gen.go

@@ -9,7 +9,7 @@ var Units = map[string]string{
 	"regenerate-ssh-host-keys.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Regenerate SSH host keys\nConditionPathExists=!/etc/ssh/ssh_host_ed25519_key\nBefore=ssh.service\n\n[Service]\nType=oneshot\nExecStart=/usr/bin/ssh-keygen -A\nRemainAfterExit=yes\n# No ExecStop: the keygen process has already exited by the time this unit\n# is stopped, so there is nothing to signal.\nTimeoutStopSec=5\n\n[Install]\nWantedBy=multi-user.target\n",
 	"spinifex-awsgw.service":           "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex AWS Gateway\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-gw\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service awsgw start\nRestart=on-failure\nRestartSec=5\n# No SIGTERM handler in this process: it exits immediately on signal, so this\n# only bounds an unexpectedly wedged process before SIGKILL.\nTimeoutStopSec=10\n\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/\nEnvironment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/awsgw /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
 	"spinifex-daemon.service":          "# spinifex-unit-version: 5\n[Unit]\nDescription=Spinifex Daemon (VM Orchestration)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service\n# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet\nWants=openvswitch-ipsec.service spinifex-hostnet.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see\nEnvironmentFile=-/etc/spinifex/host.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-daemon\nGroup=spinifex\nExecStartPre=/usr/local/bin/spx admin hostnet check\nExecStart=/usr/local/bin/spx service spinifex start\nRestart=on-failure\nRestartSec=5\n# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)\nKillMode=process\nTimeoutStopSec=15\n# RG-4: infra tier — reaped only after every guest\nOOMScoreAdjust=-500\n# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead\nMemoryHigh=75%\nMemoryMax=85%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\nEnvironment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/\n# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,\n# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.\nAmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE\nProtectSystem=full\nPrivateDevices=no\nDevicePolicy=closed\nDeviceAllow=/dev/kvm rw\nDeviceAllow=/dev/net/tun rw\nDeviceAllow=/dev/vhost-net rw\nDeviceAllow=char-vfio rw\nDeviceAllow=/dev/vfio/vfio rw\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\n# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS\n# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.\nReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n# LimitMEMLOCK=infinity: VFIO pins all guest RAM\nLimitMEMLOCK=infinity\n\n[Install]\nWantedBy=spinifex.target\n",
-	"spinifex-firewall.service":        "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex host firewall (nftables input policy)\nDocumentation=file:/etc/spinifex/firewall/spinifex.nft\nWants=network-pre.target\nBefore=network-pre.target shutdown.target\nDefaultDependencies=no\nConditionPathExists=/etc/spinifex/firewall/peers.nft\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nStandardInput=null\nProtectSystem=full\nProtectHome=true\nExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply\nExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply\nTimeoutStopSec=10s\n\n# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD\n# rules with it. Remove the policy deliberately with\n# `nft delete table inet spinifex_filter`.\n\n[Install]\nWantedBy=sysinit.target\n",
+	"spinifex-firewall.service":        "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex host firewall (nftables input policy)\nDocumentation=file:/etc/spinifex/firewall/spinifex.nft\nWants=network-pre.target\nBefore=network-pre.target shutdown.target\nDefaultDependencies=no\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nStandardInput=null\nProtectSystem=full\nProtectHome=true\n# Unconditional: connection tracking loads on every node, so a flow that opens\n# while the policy is not armed survives the arm. The policy itself still loads\n# only where the daemon has written a peer file.\nExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply boot\nExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply boot\nTimeoutStopSec=10s\n\n# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD\n# rules with it. Remove the policy deliberately with\n# `nft delete table inet spinifex_filter`.\n\n[Install]\nWantedBy=sysinit.target\n",
 	"spinifex-guests.slice":            "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex guest VM tier\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-guests -> spinifex), sibling of\n# spinifex-system. Parent for the per-instance spinifex-instance-<id>.slice\n# leaves (0002 RG-7). Inherits the spinifex.slice ceiling; no extra limit here.\n",
 	"spinifex-hostnet.service":         "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex host network wiring (veth pairs, mgmt address, encap route)\n# PartOf/WantedBy ovs-vswitchd: re-applied whenever OVS starts; daemon and vpcd only Want it\nRequires=ovs-vswitchd.service\nAfter=ovs-vswitchd.service NetworkManager-wait-online.service\nPartOf=ovs-vswitchd.service\n# Stopping OVS stops ovn-controller through its Requires= chain and starting OVS\n# does not bring it back, so the start that re-applies hostnet restarts it too\nWants=ovn-controller.service\nBefore=spinifex.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/usr/local/bin/spx admin hostnet apply\n# Covers the 120s wait behind setup-ovn.sh plus the 60s wait for links\nTimeoutStartSec=240\nTimeoutStopSec=10\nProtectSystem=full\nProtectHome=yes\nPrivateTmp=yes\n\n[Install]\nWantedBy=spinifex.target ovs-vswitchd.service\n",
 	"spinifex-nats-watchdog.service":   "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex NATS JetStream liveness watchdog\nPartOf=spinifex.target\nAfter=spinifex-nats.service\n\n[Service]\nType=oneshot\n# Runs as root: the probes are a local curl to the NATS monitoring port and a\n# canary KV round-trip via spx, and the remedy is `systemctl try-restart\n# spinifex-nats` when JetStream stops accepting writes while the process stays\n# alive (which Restart=on-failure cannot catch).\nExecStart=/var/lib/spinifex/nats-js-watchdog.sh\n# Sized for the probe loop's own worst case (3 samples x (5s curl + 5s canary)\n# + 2 gaps x 2s = 34s): a stop mid-cycle lets the script finish rather than\n# cutting it off. Only the failing path runs that long; a healthy node exits on\n# the first sample.\nTimeoutStopSec=45\n# Same 34s worst case bounds the start side: without this the unit inherits\n# oneshot's infinite default, so a genuinely hung run would sit in\n# activating(start) forever and could not be retriggered by its timer.\nTimeoutStartSec=120\n\n# Hardening (kept compatible with talking to PID1 over /run/systemd).\nNoNewPrivileges=yes\nProtectHome=yes\nPrivateTmp=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectControlGroups=yes\nRestrictRealtime=yes\nLockPersonality=yes\nRestrictSUIDSGID=yes\n",

spinifex/operator/host/systemd/units_invariants_test.go

@@ -426,8 +426,11 @@ var unitBodyHashes = map[string]map[int]string{
 		4: "f66e1240cc5655f6a13f3b56ca282c73e2fe289f2e51d0ce624d14b90f12f838",
 		5: "d3d25d7eb0051ccf97f70838ddff763bade38bef934b8f85733e048151ac9319",
 	},
-	"spinifex-firewall.service": {1: "01d0a7a79f47eedca02aaab3ff97a0b6462d61b7834e16c7336a7b96034dc392"},
-	"spinifex-guests.slice":     {1: "22dac772e81a9a716db98415a4cf590885c9cf19d4290cc21e645e5fe15bc793"},
+	"spinifex-firewall.service": {
+		1: "01d0a7a79f47eedca02aaab3ff97a0b6462d61b7834e16c7336a7b96034dc392",
+		2: "0786c7312ccf8099910a8545416e859b40116e574a740c0d561b6527e4b38f84",
+	},
+	"spinifex-guests.slice": {1: "22dac772e81a9a716db98415a4cf590885c9cf19d4290cc21e645e5fe15bc793"},
 	"spinifex-hostnet.service": {
 		1: "4a110daac63b652aef49e358f22ac8a27a7797261b231d5b1184e171cccb8f87",
 		2: "13fd4231bb850198616ba31f25dfc2ec0aa9edb2c3f170320aa6f7cc6142ba77",

spinifex/operator/installer/firstboot/firstboot.go

@@ -140,6 +140,12 @@ fi
 # Set hostname
 hostnamectl set-hostname %s
 
+# Connection tracking before OVN opens its RAFT links: a link the kernel first
+# sees mid-stream is dropped when the firewall arms. Non-fatal, as the policy
+# that needs it is optional; the unit enabled below reloads it on later boots.
+/usr/local/lib/spinifex/spinifex-firewall-apply track \
+    || echo "[firstboot] WARNING: connection tracking not loaded; arming the firewall may drop OVN links"
+
 %s
 
 # Create default nameservers if DHCP server returns blank
@@ -225,8 +231,8 @@ systemctl enable spinifex.target spinifex-banner.service
 # setup.sh enables this on the package install path, but the ISO chroot returns
 # before that line (no systemd to talk to), so the ISO needs it here. An nft
 # ruleset is runtime state and does not survive a reboot; without this the unit
-# never reapplies the policy and the node comes back up open. Enabled, not
-# started: ConditionPathExists holds it inert until the daemon writes a peer file.
+# never reapplies the policy and the node comes back up open. It loads the
+# policy only once the daemon has written a peer file; until then it only tracks.
 systemctl enable spinifex-firewall.service
 # WantedBy=timers.target only self-activates once enabled — without this the
 # JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart.

spinifex/operator/installer/firstboot/firstboot_test.go

@@ -133,6 +133,30 @@ func TestWriteScriptEnablesNatsWatchdogTimer(t *testing.T) {
 	}
 }
 
+// A RAFT link that opens before conntrack runs is first seen mid-stream when
+// the firewall arms, and its server-side packets are dropped as new flows. A
+// provisioning controller forms the cluster after firstboot, so both modes.
+func TestWriteScriptLoadsConntrackBeforeOVN(t *testing.T) {
+	for _, skip := range []bool{false, true} {
+		root := t.TempDir()
+		makeRootDirs(t, root)
+		if err := Write(root, Config{Hostname: "test-node", SkipFormation: skip}); err != nil {
+			t.Fatalf("Write: %v", err)
+		}
+		content := readScript(t, root)
+
+		track := strings.Index(content, "/usr/local/lib/spinifex/spinifex-firewall-apply track")
+		if track < 0 {
+			t.Fatalf("SkipFormation=%v: firstboot never loads connection tracking", skip)
+		}
+		for _, later := range []string{"systemctl start ovn-central", "/usr/local/bin/setup-ovn.sh", "spx admin init --", "spx admin join"} {
+			if i := strings.Index(content, later); i >= 0 && i < track {
+				t.Errorf("SkipFormation=%v: %q runs before connection tracking is loaded", skip, later)
+			}
+		}
+	}
+}
+
 func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)

spinifex/providers/ebs/viperblock/viperblock.go

@@ -214,6 +214,10 @@ type Config struct {
 	fenceWatchEvery time.Duration
 	processAlive    func(pid int) bool
 
+	// fenceNoticeEvery overrides the wait between fence notice attempts. Zero
+	// means defaultFenceNoticeRetry.
+	fenceNoticeEvery time.Duration
+
 	// leases excludes a second viperblock engine on a volume this node has
 	// open. Nil means exclusion cannot be established, and every engine open
 	// refuses rather than proceeding blind.

spinifex/providers/ebs/viperblock/volume_fence.go

@@ -42,20 +42,28 @@ const fenceKillTimeout = 30 * time.Second
 // nodes writing one volume. Reclaiming correctly needs a state transition that
 // atomically replaces the lost in-memory lease and starts a new renewal loop,
 // and that is an optimisation, not a safety property.
-//
-// The holder is read only to say who won, which is what the operator needs and
-// not something the decision turns on.
 func (cfg *Config) onVolumeLeaseLost(ctx context.Context, volumeName string, kind leaseLossKind) {
 	if cfg.leases == nil {
 		return
 	}
+	cfg.fenceVolume(ctx, volumeName, kind)
+}
 
-	// Read only to name the winner for the operator. No holder, or one this node
-	// cannot read, still fences: it cannot show it is the only writer, and
-	// continuing to write is the one option that can corrupt.
+// fenceHolderReadTimeout bounds the read that names the winner. It runs after
+// the kill, so it cannot delay the fence, but the daemon's notice waits on it.
+const fenceHolderReadTimeout = 2 * time.Second
+
+// fenceWinner names who holds volumeName now, for the operator. The decision to
+// fence never turns on it: no holder, or one this node cannot read, still
+// fences, because this node cannot show it is the only writer.
+func (cfg *Config) fenceWinner(ctx context.Context, volumeName string, kind leaseLossKind) (string, otelsetup.FenceOutcome) {
 	winner := "unknown"
-	if owner, held := cfg.leases.currentOwner(ctx, volumeName); held && owner != cfg.leases.owner {
-		winner = owner
+	if cfg.leases != nil {
+		readCtx, cancel := context.WithTimeout(ctx, fenceHolderReadTimeout)
+		if owner, held := cfg.leases.currentOwner(readCtx, volumeName); held && owner != cfg.leases.owner {
+			winner = owner
+		}
+		cancel()
 	}
 
 	outcome := otelsetup.FenceOutcomeExpired
@@ -65,17 +73,21 @@ func (cfg *Config) onVolumeLeaseLost(ctx context.Context, volumeName string, kin
 	case winner != "unknown":
 		outcome = otelsetup.FenceOutcomeTaken
 	}
-	cfg.fenceVolume(ctx, volumeName, winner, outcome)
+	return winner, outcome
 }
 
 // fenceVolume tears this node's export down because the volume belongs to
 // somebody else now. The guest loses its disk immediately, which is the point:
 // an export left up is a second writer.
 //
+// The kill precedes anything that waits on JetStream: validity lapses only
+// volumeLeaseAcquireMargin before the server can re-grant the entry, and a read
+// of the store this node just failed to reach would spend it with nbdkit alive.
+//
 // Deliberately does not seal. This node's copy is the stale one, and sealing it
 // would publish an older state over the winner's. For the same reason the dirty
 // marker is left alone — the winner has taken it over and it now names them.
-func (cfg *Config) fenceVolume(ctx context.Context, volumeName, winner string, outcome otelsetup.FenceOutcome) {
+func (cfg *Config) fenceVolume(ctx context.Context, volumeName string, kind leaseLossKind) {
 	matched, found := cfg.lookupMountedVolume(volumeName)
 	if !found {
 		// The export went away on its own between losing the lease and getting
@@ -85,12 +97,14 @@ func (cfg *Config) fenceVolume(ctx context.Context, volumeName, winner string, o
 	}
 
 	slog.ErrorContext(ctx, "fencing volume: the lease moved while this node had it open, so the guest is losing its disk",
-		"volume", volumeName, "previous_owner", cfg.leases.owner, "winner", winner,
-		"pid", matched.PID)
+		"volume", volumeName, "previous_owner", cfg.leaseOwner(), "pid", matched.PID)
 
-	if err := cfg.tearDownExport(ctx, matched, "fence"); err != nil {
+	killErr := cfg.tearDownExport(ctx, matched, "fence")
+	winner, outcome := cfg.fenceWinner(ctx, volumeName, kind)
+
+	if killErr != nil {
 		slog.ErrorContext(ctx, "fence FAILED: nbdkit did not exit, so this node is still a writer on a volume it does not own",
-			"volume", volumeName, "pid", matched.PID, "winner", winner, "err", err)
+			"volume", volumeName, "pid", matched.PID, "winner", winner, "err", killErr)
 		otelsetup.RecordVolumeFence(ctx, otelsetup.FenceOutcomeKillFailed)
 		if !cfg.writerAlive(matched.PID) {
 			// The kill was refused rather than ignored, so nothing here says what
@@ -104,6 +118,8 @@ func (cfg *Config) fenceVolume(ctx context.Context, volumeName, winner string, o
 		return
 	}
 
+	slog.WarnContext(ctx, "volume fenced: the export is gone and this node is no longer writing it",
+		"volume", volumeName, "pid", matched.PID, "winner", winner)
 	otelsetup.RecordVolumeFence(ctx, outcome)
 	cfg.publishVolumeFenced(ctx, volumeName, winner)
 }
@@ -285,9 +301,26 @@ func (cfg *Config) finishTeardown(ctx context.Context, matched MountedVolume, wh
 	cfg.releaseVolumeLease(ctx, matched.Lease)
 }
 
-// publishVolumeFenced tells the local daemon a guest is now running against a
-// disk that is gone. Best effort: the fence has already happened, and a guest
-// left running with dead I/O is a worse outcome to report than to cause.
+// Fence notice delivery. The notice is the only way the daemon learns its guest
+// lost its disk, and the fence often runs while this node's NATS is restarting,
+// so it is retried until the daemon answers rather than published once.
+const (
+	fenceNoticeTimeout       = 2 * time.Second
+	defaultFenceNoticeRetry  = 2 * time.Second
+	fenceNoticeLogEveryTries = 15
+)
+
+// fenceNoticeRetry is how long a failed notice waits before the next attempt.
+func (cfg *Config) fenceNoticeRetry() time.Duration {
+	if cfg.fenceNoticeEvery > 0 {
+		return cfg.fenceNoticeEvery
+	}
+	return defaultFenceNoticeRetry
+}
+
+// publishVolumeFenced tells the local daemon a guest is running against a disk
+// that is gone, until the daemon acknowledges. It gives up on a closed
+// connection, or once the volume is exported here again by a newer mount.
 func (cfg *Config) publishVolumeFenced(ctx context.Context, volumeName, winner string) {
 	if cfg.nc == nil {
 		return
@@ -304,9 +337,46 @@ func (cfg *Config) publishVolumeFenced(ctx context.Context, volumeName, winner s
 		return
 	}
 	subject := viperblocklegacyv1.VolumeFencedSubject(cfg.NodeName)
-	if err := cfg.nc.Publish(subject, payload); err != nil {
-		slog.ErrorContext(ctx, "fence: could not announce the fenced volume, the guest will be left with dead I/O",
-			"volume", volumeName, "subject", subject, "err", err)
+	began := time.Now()
+
+	for attempt := 1; ; attempt++ {
+		reqCtx, cancel := context.WithTimeout(ctx, fenceNoticeTimeout)
+		reply, err := cfg.nc.RequestWithContext(reqCtx, subject, payload)
+		cancel()
+		if err == nil {
+			var ack viperblocklegacyv1.VolumeFencedAck
+			if jerr := json.Unmarshal(reply.Data, &ack); jerr == nil && ack.Error != "" {
+				slog.ErrorContext(ctx, "fence: the daemon could not act on the fenced volume",
+					"volume", volumeName, "subject", subject, "err", ack.Error)
+			} else if attempt > 1 {
+				slog.WarnContext(ctx, "fence: the daemon acknowledged the fenced volume after retrying",
+					"volume", volumeName, "instance", ack.Instance, "attempts", attempt,
+					"unacknowledged_ms", otelsetup.Millis(time.Since(began)))
+			}
+			return
+		}
+
+		if cfg.nc.IsClosed() {
+			slog.ErrorContext(ctx, "fence: connection closed before the daemon acknowledged the fenced volume, the guest may be left with dead I/O",
+				"volume", volumeName, "subject", subject, "attempts", attempt)
+			return
+		}
+		if _, again := cfg.lookupMountedVolume(volumeName); again {
+			slog.WarnContext(ctx, "fence: volume exported here again before the daemon acknowledged the fence, dropping the stale notice",
+				"volume", volumeName, "attempts", attempt)
+			return
+		}
+		if attempt == 1 || attempt%fenceNoticeLogEveryTries == 0 {
+			slog.ErrorContext(ctx, "fence: the daemon has not acknowledged the fenced volume, retrying; until it does the guest keeps running with dead I/O",
+				"volume", volumeName, "subject", subject, "attempts", attempt,
+				"unacknowledged_ms", otelsetup.Millis(time.Since(began)), "err", err)
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(cfg.fenceNoticeRetry()):
+		}
 	}
 }
 

spinifex/providers/ebs/viperblock/volume_fence_invariants_test.go

@@ -0,0 +1,254 @@
+package viperblock
+
+//test:in-package — the fence, the lease store and the notice it sends are all
+//unexported, and the ordering they guarantee has no exported surface.
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// hangingHolderKV stands in for a lease store this node has just failed to
+// reach: once armed, reading the holder blocks until released or cancelled.
+type hangingHolderKV struct {
+	jetstream.KeyValue
+
+	armed   atomic.Bool
+	entered chan struct{}
+	release chan struct{}
+}
+
+func (k *hangingHolderKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
+	if !k.armed.Load() {
+		return k.KeyValue.Get(ctx, key)
+	}
+	select {
+	case k.entered <- struct{}{}:
+	default:
+	}
+	select {
+	case <-k.release:
+		return k.KeyValue.Get(ctx, key)
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+// hangingFenceConfig is fencedConfig with a lease store whose holder read hangs
+// once armed. The wrapper goes in before the lease exists, so no renewal
+// goroutine ever sees the store change under it.
+func hangingFenceConfig(t *testing.T, natsURL, volumeName string) (*Config, *hangingHolderKV) {
+	t.Helper()
+
+	leases := newTestLeases(t, natsURL, "node-a")
+	kv := &hangingHolderKV{KeyValue: leases.kv, entered: make(chan struct{}, 1), release: make(chan struct{})}
+	leases.kv = kv
+
+	cfg := &Config{NodeName: "node-a", leases: leases, dirty: newTestDirty(t, natsURL, "node-a")}
+	cfg.leases = cfg.bindLeaseFence(cfg.leases)
+
+	lease, err := cfg.acquireVolumeLease(t.Context(), volumeName)
+	require.NoError(t, err)
+	cfg.MountedVolumes = []MountedVolume{{Name: volumeName, Lease: lease, PID: stubExport(t)}}
+	t.Cleanup(func() { cfg.releaseVolumeLease(context.Background(), lease) })
+
+	kv.armed.Store(true)
+	return cfg, kv
+}
+
+// TestS4_FenceStopsTheWriterBeforeReadingTheHolder enforces ADR-0007:S4: a
+// fence must be "effective fencing that prevents the former node from continuing
+// as a conflicting writer". Local validity lapses only volumeLeaseAcquireMargin
+// before the server can re-grant the entry, so nbdkit has to be dead inside that
+// margin even when the read naming the winner never comes back.
+func TestS4_FenceStopsTheWriterBeforeReadingTheHolder(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencekillfirst"
+	cfg, kv := hangingFenceConfig(t, natsURL, volumeName)
+	pid := cfg.MountedVolumes[0].PID
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.onVolumeLeaseLost(context.Background(), volumeName, leaseLostStalled) }()
+	t.Cleanup(func() { close(kv.release); <-done })
+
+	require.Eventuallyf(t, func() bool { return !hostprocess.ProcessAlive(pid) },
+		volumeLeaseAcquireMargin, 10*time.Millisecond,
+		"ADR-0007:S4: nbdkit (pid %d) was still alive %s after the lease was lost, with the holder read hanging; "+
+			"the server re-grants the entry after that margin, so this node would still be writing when another takes over. "+
+			"Fix: kill the export in fenceVolume (volume_fence.go) before anything that reads JetStream",
+		pid, volumeLeaseAcquireMargin)
+
+	select {
+	case <-kv.entered:
+	case <-time.After(volumeLeaseAcquireMargin):
+		t.Fatal("the fence never read the holder after the kill")
+	}
+	cfg.mu.Lock()
+	mounted := len(cfg.MountedVolumes)
+	cfg.mu.Unlock()
+	assert.Zero(t, mounted, "ADR-0007:S4: the export must be gone before the fence waits on the store")
+}
+
+// TestS4_FenceHolderReadIsBounded enforces ADR-0007:S4 for the part of the fence
+// after the kill. The holder read only names the winner, and the daemon's notice
+// waits behind it, so a store that never answers must not hold the guest up.
+func TestS4_FenceHolderReadIsBounded(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencereadbounded"
+	cfg, kv := hangingFenceConfig(t, natsURL, volumeName)
+	t.Cleanup(func() { close(kv.release) })
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.onVolumeLeaseLost(context.Background(), volumeName, leaseLostStalled) }()
+
+	select {
+	case <-kv.entered:
+	case <-time.After(volumeLeaseAcquireMargin):
+		t.Fatal("ADR-0007:S4: the fence never read the holder, so this test proved nothing about bounding it")
+	}
+
+	limit := fenceHolderReadTimeout + 2*time.Second
+	select {
+	case <-done:
+	case <-time.After(limit):
+		t.Fatalf("ADR-0007:S4: the fence was still waiting on an unanswered holder read after %s; "+
+			"the read is diagnostics and must not delay the notice that stops the guest. "+
+			"Fix: bound currentOwner's context in fenceWinner (volume_fence.go)", limit)
+	}
+}
+
+// fenceResponder answers fence notices as the daemon does and records them.
+type fenceResponder struct {
+	mu     sync.Mutex
+	events []viperblocklegacyv1.VolumeFencedEvent
+}
+
+func (r *fenceResponder) handle(msg *nats.Msg) {
+	var event viperblocklegacyv1.VolumeFencedEvent
+	_ = json.Unmarshal(msg.Data, &event)
+	r.mu.Lock()
+	r.events = append(r.events, event)
+	r.mu.Unlock()
+	payload, _ := json.Marshal(viperblocklegacyv1.VolumeFencedAck{Instance: "i-fenced"})
+	_ = msg.Respond(payload)
+}
+
+func (r *fenceResponder) count() int {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return len(r.events)
+}
+
+// noticeConfig is fencedConfig with a connection to announce on and a short
+// retry interval.
+func noticeConfig(t *testing.T, natsURL, volumeName string) (*Config, *nats.Conn) {
+	t.Helper()
+	cfg := fencedConfig(t, natsURL, "node-a", volumeName)
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	t.Cleanup(nc.Close)
+	cfg.nc = nc
+	cfg.fenceNoticeEvery = 20 * time.Millisecond
+	return cfg, nc
+}
+
+// TestS4_FenceNoticeSurvivesAMissedDelivery enforces ADR-0007:S4 on the daemon's
+// half of the fence. The notice is the only way the daemon learns its guest lost
+// its disk, and a node's NATS restarting mid-fence leaves nobody subscribed for
+// the first send. It has to arrive once the daemon is back, however late.
+func TestS4_FenceNoticeSurvivesAMissedDelivery(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticelate"
+	cfg, _ := noticeConfig(t, natsURL, volumeName)
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.onVolumeLeaseLost(context.Background(), volumeName, leaseLostStalled) }()
+
+	// Nobody is listening yet: the daemon is still reconnecting.
+	time.Sleep(10 * cfg.fenceNoticeEvery)
+
+	daemonConn, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	t.Cleanup(daemonConn.Close)
+	responder := &fenceResponder{}
+	_, err = daemonConn.Subscribe(viperblocklegacyv1.VolumeFencedSubject("node-a"), responder.handle)
+	require.NoError(t, err)
+	require.NoError(t, daemonConn.Flush())
+
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("ADR-0007:S4: the fence never finished announcing to a daemon that came back")
+	}
+	require.GreaterOrEqualf(t, responder.count(), 1,
+		"ADR-0007:S4: a daemon that subscribed after the first notice never heard the fence, so its guest runs on with no disk. "+
+			"Fix: retry publishVolumeFenced (volume_fence.go) until the daemon acknowledges")
+	assert.Equal(t, volumeName, responder.events[0].Volume)
+
+	seen := responder.count()
+	time.Sleep(5 * cfg.fenceNoticeEvery)
+	assert.Equal(t, seen, responder.count(), "an acknowledged notice must not be sent again")
+}
+
+// TestS4_StaleFenceNoticeIsDropped is the limit on that retry. Once the volume is
+// exported here again it belongs to a newer mount, and a notice that finally
+// lands then would stop the guest that mount is serving.
+func TestS4_StaleFenceNoticeIsDropped(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticestale"
+	cfg, _ := noticeConfig(t, natsURL, volumeName)
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.onVolumeLeaseLost(context.Background(), volumeName, leaseLostStalled) }()
+
+	require.Eventually(t, func() bool {
+		cfg.mu.Lock()
+		defer cfg.mu.Unlock()
+		return len(cfg.MountedVolumes) == 0
+	}, volumeLeaseAcquireMargin, 10*time.Millisecond)
+	cfg.mu.Lock()
+	cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{Name: volumeName})
+	cfg.mu.Unlock()
+
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("ADR-0007:S4: a fence notice kept retrying after its volume was exported here again")
+	}
+}
+
+// TestS4_FenceNoticeStopsWithTheConnection keeps a shutting-down viperblockd from
+// leaving a notice goroutine spinning on a connection that cannot carry it.
+func TestS4_FenceNoticeStopsWithTheConnection(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticeclosed"
+	cfg, nc := noticeConfig(t, natsURL, volumeName)
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.onVolumeLeaseLost(context.Background(), volumeName, leaseLostStalled) }()
+
+	time.Sleep(5 * cfg.fenceNoticeEvery)
+	nc.Close()
+
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("a fence notice kept retrying on a closed connection")
+	}
+}

spinifex/providers/ebs/viperblock/volume_fence_test.go

@@ -135,7 +135,7 @@ func TestVolumeFence_KillFailureLeavesTheVolumeMounted(t *testing.T) {
 	// rather than left to the test's own cleanup.
 	ctx, cancel := context.WithCancel(t.Context())
 	watched := make(chan struct{})
-	go func() { defer close(watched); cfg.fenceVolume(ctx, volumeName, "node-b", "taken") }()
+	go func() { defer close(watched); cfg.fenceVolume(ctx, volumeName, leaseLostToPeer) }()
 	defer func() { cancel(); <-watched }()
 
 	assert.Never(t, func() bool {
@@ -168,7 +168,7 @@ func TestVolumeFence_CompletesOnceTheWriterFinallyExits(t *testing.T) {
 	cfg.processAlive = func(int) bool { return probes.Add(1) <= 2 }
 	cfg.fenceWatchEvery = 10 * time.Millisecond
 
-	cfg.fenceVolume(t.Context(), volumeName, "node-b", "taken")
+	cfg.fenceVolume(t.Context(), volumeName, leaseLostToPeer)
 
 	cfg.mu.Lock()
 	mounted := len(cfg.MountedVolumes)
@@ -192,7 +192,7 @@ func TestVolumeFence_DoesNotCompleteForAWriterItCannotSee(t *testing.T) {
 	cfg.processAlive = func(int) bool { return false }
 	cfg.fenceWatchEvery = 10 * time.Millisecond
 
-	cfg.fenceVolume(t.Context(), volumeName, "node-b", "taken")
+	cfg.fenceVolume(t.Context(), volumeName, leaseLostToPeer)
 
 	cfg.mu.Lock()
 	mounted := len(cfg.MountedVolumes)
@@ -216,7 +216,7 @@ func TestVolumeFence_DoesNotSealOnTheWayOut(t *testing.T) {
 	cfg := fencedConfig(t, natsURL, "node-a", volumeName)
 	cfg.sealVolume = func(context.Context, string) error { sealed = true; return nil }
 
-	cfg.fenceVolume(t.Context(), volumeName, "node-b", "taken")
+	cfg.fenceVolume(t.Context(), volumeName, leaseLostToPeer)
 
 	assert.False(t, sealed, "fencing must never seal: this node's copy is behind the winner's")
 }
@@ -231,7 +231,7 @@ func TestVolumeFence_LeavesTheDirtyMarkerToTheWinner(t *testing.T) {
 
 	cfg := fencedConfig(t, natsURL, "node-a", volumeName)
 
-	cfg.fenceVolume(t.Context(), volumeName, "node-b", "taken")
+	cfg.fenceVolume(t.Context(), volumeName, leaseLostToPeer)
 
 	record, ok := cfg.dirty.holder(t.Context(), volumeName)
 	require.True(t, ok, "fencing must leave the marker behind for the winner to take over")
@@ -249,7 +249,7 @@ func TestVolumeFence_AlreadyUnmountedIsNotAnError(t *testing.T) {
 		dirty:    newTestDirty(t, natsURL, "node-a"),
 	}
 
-	cfg.fenceVolume(t.Context(), "vol-fencenotmounted", "node-b", "taken")
+	cfg.fenceVolume(t.Context(), "vol-fencenotmounted", leaseLostToPeer)
 }
 
 // TestUnmountVolume_RefusesToSealUnderALiveWriter is the unmount side of the

spinifex/runtime/compute/vm/fence.go

@@ -3,9 +3,12 @@ package vm
 import (
 	"context"
 	"log/slog"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 )
 
 // fenceStateReasonCode is the StateReason a fenced instance carries. Distinct
@@ -78,7 +81,7 @@ func (m *Manager) FenceVolume(ctx context.Context, volumeID, reason string) stri
 		"instanceId", instance.ID, "volumeId", volumeID, "reason", reason)
 
 	m.goroutineWg.Go(func() {
-		m.shutdownQEMU(instance)
+		m.killFencedQEMU(ctx, instance)
 		m.cleanupTapDevices(instance)
 		m.Inspect(instance, func(v *VM) { v.LastNode = m.deps.NodeID })
 		if err := m.writeRunningState(); err != nil {
@@ -89,6 +92,37 @@ func (m *Manager) FenceVolume(ctx context.Context, volumeID, reason string) stri
 	return instance.ID
 }
 
+// fencedQEMUKillTimeout bounds the wait for a SIGKILLed fenced guest to exit.
+const fencedQEMUKillTimeout = 10 * time.Second
+
+// killFencedQEMU ends a fenced guest without a powerdown. Its disk is gone, so a
+// graceful shutdown has nothing to flush to, and while it waits the guest's taps
+// and process outlive its ownership as a survivor starts it again.
+func (m *Manager) killFencedQEMU(ctx context.Context, instance *VM) {
+	pid, err := hostprocess.ReadPidFile(instance.ID)
+	if err != nil || pid <= 0 {
+		// No PID to kill, so ask QEMU itself; quit skips the guest's shutdown.
+		if instance.QMPClient != nil {
+			if _, qerr := sendQMPCommandWithTimeout(ctx, instance.QMPClient,
+				qmp.QMPCommand{Execute: "quit"}, instance.ID, qmpCommandTimeout); qerr != nil {
+				slog.WarnContext(ctx, "FenceVolume: QMP quit failed (VM may already be stopped)",
+					"instanceId", instance.ID, "err", qerr)
+			}
+		}
+		return
+	}
+	if !hostprocess.ProcessAlive(pid) {
+		_ = hostprocess.RemovePidFile(instance.ID)
+		return
+	}
+	if err := hostprocess.ForceKillProcess(pid, fencedQEMUKillTimeout); err != nil {
+		slog.ErrorContext(ctx, "FenceVolume: failed to kill the fenced guest",
+			"instanceId", instance.ID, "pid", pid, "err", err)
+		return
+	}
+	_ = hostprocess.RemovePidFile(instance.ID)
+}
+
 // instanceUsingVolume returns the local instance holding volumeID, or nil. The
 // map is this node's own running set, so a hit means the guest is here.
 func (m *Manager) instanceUsingVolume(volumeID string) *VM {

spinifex/runtime/compute/vm/fence_invariants_test.go

@@ -0,0 +1,132 @@
+package vm
+
+//test:in-package — the fence teardown, the I/O-error resume and the QMP client
+//they drive are unexported manager behaviour with no exported surface.
+
+import (
+	"os/exec"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fencedGuestTeardownBound is how long a fenced guest may take to go. Well
+// under pidFileRemovalTimeout, so a teardown that tries a graceful shutdown first
+// cannot meet it.
+const fencedGuestTeardownBound = 5 * time.Second
+
+// fencedGuest is a running instance with a live stand-in QEMU behind its PID
+// file and a QMP monitor that answers, using volumeID.
+func fencedGuest(t *testing.T, volumeID string) (*Manager, *VM, int, func(string) int, *recordingInstanceCleaner) {
+	t.Helper()
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	hostprocess.ShortenProcessExitPollForTest(t, 5*time.Millisecond)
+
+	cmd := exec.Command("sleep", "300")
+	require.NoError(t, cmd.Start())
+	var reaped sync.WaitGroup
+	reaped.Go(func() { _ = cmd.Wait() })
+	t.Cleanup(func() { _ = cmd.Process.Kill(); reaped.Wait() })
+
+	const id = "i-fencedguest"
+	require.NoError(t, hostprocess.WritePidFile(id, cmd.Process.Pid))
+
+	sockPath, count := startRecordingQMPListener(t)
+	client, err := qmp.NewQMPClient(sockPath)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = client.Conn.Close() })
+
+	cleaner := &recordingInstanceCleaner{}
+	m := NewManagerWithDeps(Deps{NodeID: "node-a", InstanceCleaner: cleaner})
+	instance := &VM{
+		ID:          id,
+		Status:      StateRunning,
+		Instance:    &ec2.Instance{},
+		QMPClient:   client,
+		EBSRequests: EBSRequests{Requests: []EBSRequest{{Name: volumeID}}},
+	}
+	m.Insert(instance)
+	return m, instance, cmd.Process.Pid, count, cleaner
+}
+
+// waitFenceTeardown waits for FenceVolume's teardown goroutine, failing past
+// fencedGuestTeardownBound.
+func waitFenceTeardown(t *testing.T, m *Manager) time.Duration {
+	t.Helper()
+	began := time.Now()
+	finished := make(chan struct{})
+	go func() { m.goroutineWg.Wait(); close(finished) }()
+	select {
+	case <-finished:
+	case <-time.After(fencedGuestTeardownBound):
+		t.Fatalf("ADR-0007:S4: the fenced guest's teardown was still running after %s; "+
+			"a fenced guest has lost its disk, and while it lingers a survivor can start the instance again. "+
+			"Fix: kill it in FenceVolume (fence.go) instead of shutting it down gracefully", fencedGuestTeardownBound)
+	}
+	return time.Since(began)
+}
+
+// TestS4_FencedGuestIsKilledWithoutAGracefulShutdown enforces ADR-0007:S4, "a
+// fence that the former node's realization path respects". The guest's export is
+// already gone, so an ACPI powerdown can flush nothing, and the seconds it spends
+// trying are seconds the guest runs on this node while a survivor starts it.
+func TestS4_FencedGuestIsKilledWithoutAGracefulShutdown(t *testing.T) {
+	m, instance, pid, qmpCalls, _ := fencedGuest(t, "vol-fencedkill")
+
+	m.FenceVolume(t.Context(), "vol-fencedkill", "lease moved to node-b")
+	took := waitFenceTeardown(t, m)
+
+	assert.Falsef(t, hostprocess.ProcessAlive(pid),
+		"ADR-0007:S4: the fenced guest's QEMU (pid %d) survived its teardown", pid)
+	assert.Zero(t, qmpCalls("system_powerdown"),
+		"ADR-0007:S4: a fenced guest has no disk to flush, so it must not be asked to power down")
+	_, perr := hostprocess.ReadPidFile(instance.ID)
+	assert.Error(t, perr, "a SIGKILLed QEMU never removes its own PID file, so the fence must")
+	t.Logf("fenced guest torn down in %s", took)
+}
+
+// TestS7_FenceStopsButNeverDestroysTheInstance enforces ADR-0003:S7 on the fast
+// teardown: the fence is an automatic path, so the instance is left stopped with
+// its volumes, never terminated or stripped of them.
+func TestS7_FenceStopsButNeverDestroysTheInstance(t *testing.T) {
+	m, instance, _, _, cleaner := fencedGuest(t, "vol-fencedkeep")
+
+	m.FenceVolume(t.Context(), "vol-fencedkeep", "lease moved to node-b")
+	waitFenceTeardown(t, m)
+
+	assert.Equal(t, StateError, m.Status(instance),
+		"ADR-0003:S7: a fenced instance must be left stopped, not terminated")
+	cleaner.mu.Lock()
+	defer cleaner.mu.Unlock()
+	assert.Empty(t, cleaner.deleteVolumes, "ADR-0003:S7: a fence must never delete the instance's volumes")
+	assert.Empty(t, cleaner.detachAndDeleteENI, "ADR-0003:S7: a fence must never delete the instance's interfaces")
+	assert.Len(t, instance.EBSRequests.Requests, 1, "ADR-0003:S7: the fenced instance keeps its volume attachments")
+}
+
+// TestS4_FencedGuestIsNeverResumed enforces ADR-0007:S4 on the I/O-error path. A
+// fenced guest is paused on the write its dead export refused, and the heartbeat
+// may have read it running a moment before the fence landed.
+func TestS4_FencedGuestIsNeverResumed(t *testing.T) {
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	m, instance, qmpCalls := pausedGuest(t)
+	m.Inspect(instance, func(v *VM) {
+		v.Instance = &ec2.Instance{}
+		v.EBSRequests = EBSRequests{Requests: []EBSRequest{{Name: "vol-fencedpaused"}}}
+	})
+
+	m.FenceVolume(t.Context(), "vol-fencedpaused", "lease moved to node-b")
+	m.goroutineWg.Wait()
+
+	for range 3 {
+		m.resumeAfterIOError(t.Context(), instance)
+	}
+	assert.Zero(t, qmpCalls("cont"),
+		"ADR-0007:S4: a guest whose volume is fenced must never be resumed. "+
+			"Fix: check the fence in resumeAfterIOError (lifecycle.go) before sending cont")
+}

spinifex/runtime/compute/vm/hoststack.go

@@ -0,0 +1,129 @@
+package vm
+
+import (
+	"bufio"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"log/slog"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+)
+
+// HostVirtStack names the host software a guest boots on: the kernel that runs
+// KVM, the QEMU binary, and the UEFI firmware. A firmware or vCPU hang cannot be
+// reproduced without these, and nothing else records them.
+type HostVirtStack struct {
+	Kernel          string
+	QEMU            string
+	QEMUVersion     string
+	Firmware        string
+	FirmwarePackage string
+	FirmwareSHA256  string
+}
+
+// hostStackCommandTimeout bounds each version probe so a wedged package
+// database cannot hold up daemon start.
+const hostStackCommandTimeout = 5 * time.Second
+
+var (
+	hostStackOnce   sync.Once
+	hostStackCached HostVirtStack
+)
+
+// CurrentHostVirtStack returns the host stack, probed once per process.
+func CurrentHostVirtStack() HostVirtStack {
+	hostStackOnce.Do(func() { hostStackCached = detectHostVirtStack(qemuArch(runtime.GOARCH)) })
+	return hostStackCached
+}
+
+// LogHostVirtStack records the host stack at daemon start.
+func LogHostVirtStack() {
+	slog.Info("Host virtualization stack", CurrentHostVirtStack().logAttrs()...)
+}
+
+func (s HostVirtStack) logAttrs() []any {
+	return []any{
+		"host_kernel", s.Kernel,
+		"qemu_binary", s.QEMU,
+		"qemu_version", s.QEMUVersion,
+		"firmware_code", s.Firmware,
+		"firmware_package", s.FirmwarePackage,
+		"firmware_sha256", s.FirmwareSHA256,
+	}
+}
+
+// qemuArch maps a Go architecture onto the names the firmware and QEMU binary
+// tables are keyed by.
+func qemuArch(goarch string) string {
+	if goarch == "amd64" {
+		return "x86_64"
+	}
+	return goarch
+}
+
+func detectHostVirtStack(arch string) HostVirtStack {
+	var s HostVirtStack
+	if b, err := os.ReadFile("/proc/sys/kernel/osrelease"); err == nil {
+		s.Kernel = strings.TrimSpace(string(b))
+	}
+	if bin, err := resolveQEMUBinary(arch); err == nil {
+		s.QEMU = bin
+		s.QEMUVersion = firstLine(runHostProbe(bin, "--version"))
+	}
+	if code, _, _, err := FirmwarePaths(arch); err == nil {
+		s.Firmware = code
+		s.FirmwarePackage = packageOwning(code)
+		s.FirmwareSHA256 = fileSHA256(code)
+	}
+	return s
+}
+
+// packageOwning names the distro package and version that installed path, on
+// either packaging system. Empty when neither knows it.
+func packageOwning(path string) string {
+	if out := firstLine(runHostProbe("dpkg-query", "-S", path)); out != "" {
+		pkg, _, _ := strings.Cut(out, ":")
+		if ver := firstLine(runHostProbe("dpkg-query", "-W", "-f=${Version}", pkg)); ver != "" {
+			return pkg + " " + ver
+		}
+		return pkg
+	}
+	return firstLine(runHostProbe("rpm", "-qf", path))
+}
+
+func runHostProbe(name string, args ...string) string {
+	ctx, cancel := context.WithTimeout(context.Background(), hostStackCommandTimeout)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, name, args...).Output()
+	if err != nil {
+		return ""
+	}
+	return string(out)
+}
+
+func firstLine(s string) string {
+	sc := bufio.NewScanner(strings.NewReader(s))
+	if sc.Scan() {
+		return strings.TrimSpace(sc.Text())
+	}
+	return ""
+}
+
+func fileSHA256(path string) string {
+	f, err := os.Open(path)
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return ""
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}

spinifex/runtime/compute/vm/hoststack_test.go

@@ -0,0 +1,51 @@
+package vm
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestDetectHostVirtStack names the binary, its version line and the exact
+// firmware build, which is what a boot hang report has to carry to be
+// reproducible on another host.
+func TestDetectHostVirtStack(t *testing.T) {
+	dir := t.TempDir()
+	qemu := filepath.Join(dir, "qemu-system-x86_64")
+	require.NoError(t, os.WriteFile(qemu,
+		[]byte("#!/bin/sh\necho 'QEMU emulator version 10.0.13 (Debian 1:10.0.13+ds-0+deb13u1)'\necho 'Copyright'\n"), 0o755))
+	withQEMUResolver(t, func(string) (string, error) { return qemu, nil })
+
+	code := filepath.Join(dir, "OVMF_CODE.fd")
+	vars := filepath.Join(dir, "OVMF_VARS.fd")
+	require.NoError(t, os.WriteFile(code, []byte("firmware"), 0o600))
+	require.NoError(t, os.WriteFile(vars, []byte("vars"), 0o600))
+	prev := FirmwarePathCandidates
+	FirmwarePathCandidates = map[string][]FirmwareCandidate{"x86_64": {{Code: code, VarsTemplate: vars}}}
+	t.Cleanup(func() { FirmwarePathCandidates = prev })
+
+	s := detectHostVirtStack("x86_64")
+
+	sum := sha256.Sum256([]byte("firmware"))
+	assert.Equal(t, qemu, s.QEMU)
+	assert.Equal(t, "QEMU emulator version 10.0.13 (Debian 1:10.0.13+ds-0+deb13u1)", s.QEMUVersion)
+	assert.Equal(t, code, s.Firmware)
+	assert.Equal(t, hex.EncodeToString(sum[:]), s.FirmwareSHA256)
+	assert.Empty(t, s.FirmwarePackage, "a file no package installed has no owner to name")
+	assert.NotEmpty(t, s.Kernel)
+}
+
+// TestPackageOwning_DistroFirmware resolves the package of a file the host's
+// package manager does own, when there is one to ask.
+func TestPackageOwning_DistroFirmware(t *testing.T) {
+	const path = "/bin/sh"
+	if runHostProbe("dpkg-query", "-S", path) == "" && runHostProbe("rpm", "-qf", path) == "" {
+		t.Skip("no package manager owns /bin/sh on this host")
+	}
+	assert.NotEmpty(t, packageOwning(path))
+}

spinifex/runtime/compute/vm/lifecycle.go

@@ -20,7 +20,6 @@ import (
 
 	telemetryv1 "github.com/mulgadc/spinifex/contracts/telemetry/v1"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instancetypes"
-	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
@@ -103,142 +102,6 @@ func (m *Manager) Start(ctx context.Context, id string) error {
 	return m.launch(ctx, instance)
 }
 
-const rebootRunningTimeout = 4 * time.Second
-
-// rebootStatusPollInterval is how often Reboot samples the run state after
-// system_reset. A var so tests need not wait out the real interval.
-var rebootStatusPollInterval = 100 * time.Millisecond
-
-// rebootPowerdownTimeout is how long the guest gets to shut itself down before
-// it is reset under. Nova's shutdown_timeout, and the same shape as the hard
-// reboot AWS documents once a guest will not go quietly.
-var rebootPowerdownTimeout = 60 * time.Second
-
-// Reboot admits a reboot and runs it in the background, the way EC2 does: the
-// answer says the request was accepted, not that the guest is back. Asking a
-// guest to shut down takes as long as the guest takes, which is far longer than
-// any caller waits, so only the two admission failures can be reported at all.
-func (m *Manager) Reboot(ctx context.Context, id string) error {
-	instance, ok := m.Get(id)
-	if !ok {
-		return ErrInstanceNotFound
-	}
-	if status := m.Status(instance); status != StateRunning {
-		return fmt.Errorf("%w: cannot reboot instance %s in state %s",
-			ErrInvalidTransition, id, status)
-	}
-
-	// Tell the heartbeat this guest is meant to be stopped for a moment: it
-	// quits a guest it finds in the shutdown run state, which is the state a
-	// reboot passes through. Claiming the flag also serialises reboots, so a
-	// repeated request joins the one in flight instead of racing it.
-	if !instance.rebooting.CompareAndSwap(false, true) {
-		slog.InfoContext(ctx, "Reboot already in flight, ignoring the repeat", "instanceId", id)
-		return nil
-	}
-
-	// Detached rather than derived: the caller is answered immediately, so its
-	// context is cancelled long before the guest has finished shutting down.
-	rebootCtx := context.WithoutCancel(ctx)
-	m.goroutineWg.Go(func() {
-		defer instance.rebooting.Store(false)
-		if err := m.rebootNow(rebootCtx, instance); err != nil {
-			// Nobody is left to return this to. The guest is either back or it
-			// is not, and the heartbeat is what notices the second case.
-			slog.ErrorContext(rebootCtx, "Reboot failed", "instanceId", id, "err", err)
-		}
-	})
-	return nil
-}
-
-// rebootNow asks the guest to shut itself down, then resets and resumes it in
-// the same QEMU process. The VM remains in StateRunning throughout. A guest
-// that will not power down inside its budget is hard-reset, and that is logged.
-func (m *Manager) rebootNow(ctx context.Context, instance *VM) error {
-	// A reset alone never reaches the guest kernel, so it syncs nothing and
-	// every dirty page is lost. Ask first; reset only once asking has failed.
-	// A guest whose QEMU would exit rather than pause is reset without asking,
-	// because a powerdown there leaves nothing to reset and the guest down.
-	switch {
-	case !qemuPausesOnShutdown(instance):
-		slog.InfoContext(ctx, "Guest predates the paused-shutdown flag, hard-resetting",
-			"instanceId", instance.ID)
-	default:
-		if err := m.gracefulPowerdown(ctx, instance, rebootPowerdownTimeout); err != nil {
-			slog.WarnContext(ctx, "Guest did not power down for reboot, hard-resetting",
-				"instanceId", instance.ID, "budget_ms", otelsetup.Millis(rebootPowerdownTimeout), "err", err)
-		}
-	}
-
-	rebootCtx, cancel := context.WithTimeout(ctx, rebootRunningTimeout)
-	defer cancel()
-
-	if _, err := sendQMPCommandWithTimeout(rebootCtx, instance.QMPClient,
-		qmp.QMPCommand{Execute: "system_reset"}, instance.ID, contextTimeRemaining(rebootCtx)); err != nil {
-		return fmt.Errorf("QMP system_reset: %w", err)
-	}
-	return m.waitForQMPRunning(rebootCtx, instance, rebootStatusPollInterval)
-}
-
-// waitForQMPRunning polls until QEMU reports running or ctx expires. Paused,
-// prelaunch and shutdown guests receive cont so a reset cannot leave their
-// vCPUs parked — a reset from a powered-down guest lands in one of the three.
-func (m *Manager) waitForQMPRunning(ctx context.Context, instance *VM, pollInterval time.Duration) error {
-	var lastStatus qmp.Status
-	for {
-		if err := ctx.Err(); err != nil {
-			return fmt.Errorf("wait for QMP running (last status %q): %w", lastStatus.Status, err)
-		}
-		if status := m.Status(instance); status != StateRunning {
-			return fmt.Errorf("%w: reboot interrupted for instance %s in state %s",
-				ErrInvalidTransition, instance.ID, status)
-		}
-
-		qmpStatus, err := queryQMPStatus(ctx, instance, contextTimeRemaining(ctx))
-		if err != nil {
-			return fmt.Errorf("verify QMP running: %w", err)
-		}
-		lastStatus = qmpStatus
-		if qmpStatus.Running {
-			return nil
-		}
-
-		if qmpStatus.Status == "paused" || qmpStatus.Status == "prelaunch" ||
-			qmpStatus.Status == qmpStatusShutdown {
-			if status := m.Status(instance); status != StateRunning {
-				return fmt.Errorf("%w: cannot resume instance %s in state %s",
-					ErrInvalidTransition, instance.ID, status)
-			}
-			if _, err := sendQMPCommandWithTimeout(ctx, instance.QMPClient,
-				qmp.QMPCommand{Execute: "cont"}, instance.ID, contextTimeRemaining(ctx)); err != nil {
-				return fmt.Errorf("QMP cont from status %q: %w", qmpStatus.Status, err)
-			}
-		}
-
-		timer := time.NewTimer(pollInterval)
-		select {
-		case <-ctx.Done():
-			timer.Stop()
-			return fmt.Errorf("wait for QMP running (last status %q): %w", lastStatus.Status, ctx.Err())
-		case <-timer.C:
-		}
-	}
-}
-
-// contextTimeRemaining returns the active context deadline budget. Reboot
-// always installs a deadline before issuing QMP commands.
-func contextTimeRemaining(ctx context.Context) time.Duration {
-	deadline, ok := ctx.Deadline()
-	if !ok {
-		return qmpCommandTimeout
-	}
-	remaining := time.Until(deadline)
-	if remaining <= 0 {
-		return time.Nanosecond
-	}
-	return remaining
-}
-
 // launchStillValid returns true when status is still pending/stopped/provisioning.
 // Returns false if a concurrent terminate took ownership; launch must bail.
 func (m *Manager) launchStillValid(instance *VM) bool {
@@ -313,7 +176,7 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
 	}
 
 	_, qemuSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.start_qemu")
-	qemuErr := m.startQEMU(instance)
+	qemuErr := m.startQEMU(instance, false)
 	endSpanWithError(qemuSpan, qemuErr)
 	if qemuErr != nil {
 		slog.ErrorContext(ctx, "Failed to launch instance", "err", qemuErr)
@@ -408,8 +271,6 @@ const (
 // waitForNBDReady is a test seam over nbd.WaitForNBDReady.
 var waitForNBDReady = nbd.WaitForNBDReady
 
-// startQEMU launches the QEMU process for instance and waits for startup to
-// confirm.
 // qemuStderrLevel reports the level a QEMU stderr line belongs at. QEMU
 // announces its own exit on stderr, so the notice for the signal a stop sends
 // it is the shutdown working rather than a fault.
@@ -420,7 +281,10 @@ func qemuStderrLevel(line string) slog.Level {
 	return slog.LevelError
 }
 
-func (m *Manager) startQEMU(instance *VM) error {
+// startQEMU launches the QEMU process for instance and waits for startup to
+// confirm. appendConsole keeps the console log of the process it replaces, so a
+// reboot does not erase what the guest printed before it.
+func (m *Manager) startQEMU(instance *VM, appendConsole bool) error {
 	pidFile, err := hostprocess.GeneratePidFile(instance.ID)
 	if err != nil {
 		slog.Error("Failed to generate PID file", "err", err)
@@ -640,6 +504,7 @@ func (m *Manager) startQEMU(instance *VM) error {
 		}
 		launchCfg = &composed
 	}
+	launchCfg.ConsoleLogAppend = appendConsole
 
 	// An NBD export is waited on until it accepts connections; a block export
 	// is local to the host and only has to exist. An invalid request has no
@@ -697,6 +562,7 @@ func (m *Manager) startQEMU(instance *VM) error {
 		}
 
 		slog.Info("VM started successfully", "pid", cmd.Process.Pid)
+		instance.qemuPID.Store(int64(cmd.Process.Pid))
 
 		oomScore := guestOOMScore(instance.ManagedBy)
 		if err := hostprocess.SetOOMScore(cmd.Process.Pid, oomScore); err != nil {
@@ -735,6 +601,12 @@ func (m *Manager) startQEMU(instance *VM) error {
 			return
 		}
 
+		// A reboot ends this process on purpose and starts its replacement, so
+		// the exit belongs to the reboot rather than to crash recovery.
+		if instance.qemuPID.Load() != int64(cmd.Process.Pid) {
+			slog.Info("Replaced VM process exited", "instance", instance.ID, "pid", cmd.Process.Pid, "err", waitErr)
+			return
+		}
 		if waitErr != nil {
 			if m.deps.CrashHandler != nil {
 				m.deps.CrashHandler(instance, waitErr)
@@ -1072,6 +944,12 @@ func (m *Manager) qmpHeartbeat(instance *VM) {
 // qmpHeartbeatPoll performs one status check and returns false after crash
 // recovery takes ownership.
 func (m *Manager) qmpHeartbeatPoll(instance *VM) bool {
+	// A reboot ends and replaces this QEMU, so the socket going away and the
+	// guest sitting powered off are both expected. Reboot bounds its own window.
+	if instance.rebooting.Load() {
+		return true
+	}
+
 	slog.Debug("QMP heartbeat", "instance", instance.ID)
 	ctx := context.WithValue(context.Background(), noTraceKey{}, true)
 	qmpStatus, err := queryQMPStatus(ctx, instance, qmpCommandTimeout)
@@ -1084,11 +962,11 @@ func (m *Manager) qmpHeartbeatPoll(instance *VM) bool {
 		return true
 	}
 
-	// A guest that powered itself off leaves QEMU paused rather than gone,
-	// because that window is what a reboot needs. Nobody is rebooting this one,
-	// so close it: quit lets the process exit and recovery proceed as before.
+	// A guest that powered itself off leaves QEMU paused rather than gone.
+	// Nobody is rebooting this one, so close it: quit lets the process exit and
+	// recovery proceed as before.
 	if err == nil && qmpStatus.Status == qmpStatusShutdown &&
-		m.Status(instance) == StateRunning && !instance.rebooting.Load() {
+		m.Status(instance) == StateRunning {
 		slog.InfoContext(ctx, "Guest powered itself off, ending QEMU", "instance", instance.ID)
 		if _, quitErr := sendQMPCommand(ctx, instance.QMPClient,
 			qmp.QMPCommand{Execute: "quit"}, instance.ID); quitErr != nil {
@@ -1098,11 +976,7 @@ func (m *Manager) qmpHeartbeatPoll(instance *VM) bool {
 		return true
 	}
 
-	// A reboot in flight is meant to be stopped for a moment, so counting it as
-	// unresponsive would mark a healthy guest impaired for doing what it was
-	// asked. Reboot bounds its own window; this only declines to judge it.
-	if err == nil && !qmpStatus.Running && m.Status(instance) == StateRunning &&
-		!instance.rebooting.Load() {
+	if err == nil && !qmpStatus.Running && m.Status(instance) == StateRunning {
 		err = fmt.Errorf("QMP reported non-running status %q", qmpStatus.Status)
 	}
 	if err != nil {
@@ -1138,6 +1012,15 @@ func (m *Manager) qmpHeartbeatPoll(instance *VM) bool {
 // retrying a backend that is still down is nearly free, while a guest still
 // paused minutes after its backend returned is an outage of our own making.
 func (m *Manager) resumeAfterIOError(ctx context.Context, instance *VM) {
+	// A fenced guest's disk belongs to another node, and the heartbeat may have
+	// read Running just before the fence landed. Resuming it serves nobody.
+	var stopped bool
+	m.Inspect(instance, func(v *VM) { stopped = v.Status != StateRunning || VolumeFenced(v.Instance) })
+	if stopped {
+		slog.InfoContext(ctx, "Not resuming a guest that is fenced or no longer running", "instance", instance.ID)
+		return
+	}
+
 	var attempt int
 	m.UpdateState(instance.ID, func(v *VM) {
 		v.Health.IOErrorResumes++

spinifex/runtime/compute/vm/lifecycle_test.go

@@ -1001,7 +1001,7 @@ func TestStartQEMU_DirectBoot_PrimaryTapError(t *testing.T) {
 		ENIMac:       "02:aa:bb:cc:dd:01",
 	}
 
-	err := m.startQEMU(instance)
+	err := m.startQEMU(instance, false)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "setup tap device")
@@ -1029,7 +1029,7 @@ func TestStartQEMU_DirectBoot_BridgeEnsureError(t *testing.T) {
 		ENIMac:       "02:aa:bb:cc:dd:04",
 	}
 
-	err := m.startQEMU(instance)
+	err := m.startQEMU(instance, false)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "ensure IMDS bridge")
@@ -1064,7 +1064,7 @@ func TestStartQEMU_DirectBoot_ExtraENITapError(t *testing.T) {
 		},
 	}
 
-	err := m.startQEMU(instance)
+	err := m.startQEMU(instance, false)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "extra ENI")
@@ -1085,7 +1085,7 @@ func TestStartQEMU_DirectBoot_MgmtTapError(t *testing.T) {
 		MgmtMAC:      "02:aa:bb:cc:dd:ff",
 	}
 
-	err := m.startQEMU(instance)
+	err := m.startQEMU(instance, false)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "mgmt tap")
@@ -1110,7 +1110,7 @@ func TestStartQEMU_DiskBoot_MgmtTapCreated(t *testing.T) {
 		MgmtMAC:      "02:aa:bb:cc:dd:ee",
 	}
 
-	err := m.startQEMU(instance)
+	err := m.startQEMU(instance, false)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "mgmt tap")
@@ -1132,7 +1132,7 @@ func TestStartQEMU_DirectBoot_InstanceTypeNotFound(t *testing.T) {
 		DirectBoot:   true,
 	}
 
-	err := m.startQEMU(instance)
+	err := m.startQEMU(instance, false)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "x9.enormous")
@@ -1148,7 +1148,7 @@ func TestStartQEMU_BlockDeviceMissing(t *testing.T) {
 	instance := directBootInstance("i-blk-missing",
 		EBSRequest{Name: "vol-blk", BlockDevice: missing, BlockSizeBytes: 1 << 20})
 
-	err := m.startQEMU(instance)
+	err := m.startQEMU(instance, false)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "block device not present for vol-blk")
@@ -2047,12 +2047,12 @@ func TestStartQEMU_DirectBoot_LaunchesMountedDataVolumeWithoutSavingIt(t *testin
 	second := nbdListener(t)
 
 	instance := directBootInstance("i-db-vol", EBSRequest{Name: "vol-data-a", NBDURI: first})
-	require.NoError(t, m.startQEMU(instance))
+	require.NoError(t, m.startQEMU(instance, false))
 
 	// A later launch from the same saved base, after the volume was remounted
 	// on a different endpoint.
 	instance.EBSRequests.Requests[0].NBDURI = second
-	require.NoError(t, m.startQEMU(instance))
+	require.NoError(t, m.startQEMU(instance, false))
 
 	got := launches()
 	require.Len(t, got, 2)
@@ -2094,7 +2094,7 @@ func TestStartQEMU_DirectBoot_LaunchesTheEndpointItProbed(t *testing.T) {
 		instance.EBSRequests.Mu.Unlock()
 		return orig(uri, timeout)
 	}
-	require.NoError(t, m.startQEMU(instance))
+	require.NoError(t, m.startQEMU(instance, false))
 
 	assert.Equal(t, []string{probedEP}, probed)
 	got := launches()
@@ -2109,7 +2109,7 @@ func TestStartQEMU_DirectBoot_NoVolumesLaunchesTheBase(t *testing.T) {
 	launches := withFakeQEMU(t)
 
 	instance := directBootInstance("i-db-novol")
-	require.NoError(t, m.startQEMU(instance))
+	require.NoError(t, m.startQEMU(instance, false))
 
 	got := launches()
 	require.Len(t, got, 1)
@@ -2134,7 +2134,7 @@ func TestStartQEMU_DirectBoot_InvalidDataRequestIsNotLaunched(t *testing.T) {
 			launches := withFakeQEMU(t)
 
 			instance := directBootInstance("i-db-bad", tt.req)
-			err := m.startQEMU(instance)
+			err := m.startQEMU(instance, false)
 
 			require.Error(t, err)
 			assert.Contains(t, err.Error(), tt.wantErr)
@@ -2155,7 +2155,7 @@ func TestStartQEMU_DiskBoot_DataVolumeStaysOnThePCIPath(t *testing.T) {
 		{Name: "vol-data-a", NBDURI: ep},
 	}
 	require.NoError(t, os.WriteFile(strings.TrimPrefix(instance.EBSRequests.Requests[0].NBDURI, "nbd:unix:"), nil, 0o600))
-	require.NoError(t, m.startQEMU(instance))
+	require.NoError(t, m.startQEMU(instance, false))
 
 	got := launches()
 	require.Len(t, got, 1)

spinifex/runtime/compute/vm/reboot.go

@@ -0,0 +1,359 @@
+package vm
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
+)
+
+// A reboot is a clean guest shutdown followed by a fresh QEMU process on the
+// same volumes, taps and identity, the way nova's soft reboot and Proxmox's
+// qm reboot do it. Resetting the paused process in place instead left some
+// guests silent in firmware, while a new process on the same disks always boots.
+
+// rebootPowerdownTimeout is how long the guest gets to shut itself down before
+// its QEMU is ended without asking. Nova's shutdown_timeout, and the same shape
+// as the hard reboot AWS documents once a guest will not go quietly.
+var rebootPowerdownTimeout = 60 * time.Second
+
+// rebootBootProgressTimeout is how long a relaunched guest has to write its
+// first console byte. Firmware prints within seconds even on a nested host, so
+// silence past this is a hang, not a slow boot.
+var rebootBootProgressTimeout = 60 * time.Second
+
+// rebootBootProgressPoll is how often the console log is sampled while waiting.
+var rebootBootProgressPoll = time.Second
+
+// bootDiagSampleGap separates the two register samples in a hang report, so a
+// frozen RIP can be told from a vCPU spinning in a loop.
+var bootDiagSampleGap = 2 * time.Second
+
+const (
+	// relaunchQuitTimeout bounds the quit sent to the process being replaced.
+	relaunchQuitTimeout = 5 * time.Second
+	// relaunchExitTimeout is how long a quit QEMU gets to exit before SIGKILL.
+	// QEMU flushes its drives on quit, so this is not cut short.
+	relaunchExitTimeout = 20 * time.Second
+	// relaunchKillTimeout bounds the wait after SIGKILL.
+	relaunchKillTimeout = 10 * time.Second
+	// bootDiagCommandTimeout bounds each QMP query in a hang report.
+	bootDiagCommandTimeout = 5 * time.Second
+)
+
+// startRelaunchedQEMU starts the replacement process. A var so tests can stand
+// in for QEMU.
+var startRelaunchedQEMU = func(m *Manager, instance *VM) error {
+	return m.startQEMU(instance, true)
+}
+
+// errRelaunchFailed marks a reboot that ended the old QEMU and could not start
+// its replacement. The guest is down, so crash recovery takes over.
+var errRelaunchFailed = errors.New("relaunch after reboot failed")
+
+// Reboot admits a reboot and runs it in the background, the way EC2 does: the
+// answer says the request was accepted, not that the guest is back. Asking a
+// guest to shut down takes as long as the guest takes, which is far longer than
+// any caller waits, so only the two admission failures can be reported at all.
+func (m *Manager) Reboot(ctx context.Context, id string) error {
+	instance, ok := m.Get(id)
+	if !ok {
+		return ErrInstanceNotFound
+	}
+	if status := m.Status(instance); status != StateRunning {
+		return fmt.Errorf("%w: cannot reboot instance %s in state %s",
+			ErrInvalidTransition, id, status)
+	}
+
+	// Claiming the flag tells the heartbeat to leave the guest alone while it
+	// is down, and serialises reboots, so a repeated request joins the one in
+	// flight instead of racing it.
+	if !instance.rebooting.CompareAndSwap(false, true) {
+		slog.InfoContext(ctx, "Reboot already in flight, ignoring the repeat", "instanceId", id)
+		return nil
+	}
+
+	// Detached rather than derived: the caller is answered immediately, so its
+	// context is cancelled long before the guest has finished shutting down.
+	rebootCtx := context.WithoutCancel(ctx)
+	m.goroutineWg.Go(func() {
+		err := m.rebootNow(rebootCtx, instance)
+		instance.rebooting.Store(false)
+		if err == nil {
+			return
+		}
+		// A stop or terminate that lands mid-reboot owns the guest now.
+		if errors.Is(err, ErrInvalidTransition) {
+			slog.InfoContext(rebootCtx, "Reboot gave way to a lifecycle change", "instanceId", id, "reason", err)
+			return
+		}
+		slog.ErrorContext(rebootCtx, "Reboot failed", "instanceId", id, "err", err)
+		// The old process is gone and no new one runs, which is a crash in
+		// all but name: recovery relaunches it under the restart policy.
+		if errors.Is(err, errRelaunchFailed) {
+			m.HandleCrash(instance, err)
+		}
+	})
+	return nil
+}
+
+// rebootNow shuts the guest down, replaces its QEMU, and watches the new one
+// reach the console. The instance stays in StateRunning throughout.
+func (m *Manager) rebootNow(ctx context.Context, instance *VM) error {
+	// Ending QEMU on a running guest syncs nothing and every dirty page is
+	// lost. Ask first; force it only once asking has failed.
+	if err := m.gracefulPowerdown(ctx, instance, rebootPowerdownTimeout); err != nil {
+		slog.WarnContext(ctx, "Guest did not power down for reboot, forcing it off",
+			"instanceId", instance.ID, "budget_ms", otelsetup.Millis(rebootPowerdownTimeout), "err", err)
+	}
+
+	baseline, err := m.relaunchGuest(ctx, instance)
+	if err != nil {
+		return err
+	}
+	if m.bootProgressed(ctx, instance, baseline) {
+		slog.InfoContext(ctx, "Guest rebooted into a fresh QEMU", "instanceId", instance.ID)
+		return nil
+	}
+	m.captureBootDiagnostics(ctx, instance, "no console output from the relaunched guest")
+
+	// Silence is only evidence for a guest that has printed before. One that
+	// never writes to its serial console is left to boot in its own time.
+	if baseline == 0 {
+		slog.WarnContext(ctx, "Relaunched guest has written nothing to its console, and never has",
+			"instanceId", instance.ID)
+		return nil
+	}
+
+	slog.WarnContext(ctx, "Relaunched guest is silent, replacing its QEMU once more", "instanceId", instance.ID)
+	baseline, err = m.relaunchGuest(ctx, instance)
+	if err != nil {
+		return err
+	}
+	if m.bootProgressed(ctx, instance, baseline) {
+		slog.InfoContext(ctx, "Guest rebooted on the second fresh QEMU", "instanceId", instance.ID)
+		return nil
+	}
+	m.captureBootDiagnostics(ctx, instance, "no console output after a second relaunch")
+	return fmt.Errorf("guest %s wrote nothing to its console within %s of two fresh launches",
+		instance.ID, rebootBootProgressTimeout)
+}
+
+// relaunchGuest ends the guest's QEMU and starts a fresh one on the same
+// mounted volumes, rebuilding taps and the command line as a start does. It
+// returns the console log size before the new process ran.
+func (m *Manager) relaunchGuest(ctx context.Context, instance *VM) (int64, error) {
+	// Hot-plug and stop both act on the process, so neither may land on one
+	// that is being replaced.
+	instance.attachMu.Lock()
+	defer instance.attachMu.Unlock()
+	instance.relaunchMu.Lock()
+	defer instance.relaunchMu.Unlock()
+
+	if status := m.Status(instance); status != StateRunning {
+		return 0, fmt.Errorf("%w: reboot of %s interrupted in state %s", ErrInvalidTransition, instance.ID, status)
+	}
+	if err := m.endQEMUForRelaunch(ctx, instance); err != nil {
+		return 0, fmt.Errorf("end QEMU for reboot: %w", err)
+	}
+	// A stop that arrived while the old process was ending owns the instance
+	// now and finds no process to stop, which is what it wanted.
+	if status := m.Status(instance); status != StateRunning {
+		return 0, fmt.Errorf("%w: reboot of %s interrupted in state %s", ErrInvalidTransition, instance.ID, status)
+	}
+
+	baseline := consoleLogSize(instance)
+	if err := startRelaunchedQEMU(m, instance); err != nil {
+		return 0, fmt.Errorf("%w: start QEMU: %w", errRelaunchFailed, err)
+	}
+	fresh, err := newQMPClientWithHandshake(ctx, instance)
+	if err != nil {
+		if pid, pidErr := hostprocess.ReadPidFile(instance.ID); pidErr == nil && pid > 0 {
+			_ = hostprocess.ForceKillProcess(pid, relaunchKillTimeout)
+		}
+		return 0, fmt.Errorf("%w: connect QMP: %w", errRelaunchFailed, err)
+	}
+	m.adoptQMPConnection(instance, fresh)
+
+	if err := m.writeRunningState(); err != nil {
+		slog.WarnContext(ctx, "Failed to persist state after reboot relaunch", "instanceId", instance.ID, "err", err)
+	}
+	slog.InfoContext(ctx, "Reboot relaunched QEMU", append([]any{"instanceId", instance.ID,
+		"pid", instance.qemuPID.Load()}, CurrentHostVirtStack().logAttrs()...)...)
+	return baseline, nil
+}
+
+// endQEMUForRelaunch ends the guest's process and waits until it has exited,
+// so its NBD connections are closed before the replacement opens them.
+func (m *Manager) endQEMUForRelaunch(ctx context.Context, instance *VM) error {
+	pid, _ := hostprocess.ReadPidFile(instance.ID)
+	// Disown it first: its exit is the reboot's doing, not a crash.
+	instance.qemuPID.Store(0)
+
+	if q := instance.QMPClient; q != nil && q.Conn != nil {
+		if _, err := sendQMPCommandWithTimeout(ctx, q, qmp.QMPCommand{Execute: "quit"},
+			instance.ID, relaunchQuitTimeout); err != nil {
+			// QEMU may close the socket before its reply; the exit is the answer.
+			slog.DebugContext(ctx, "QMP quit for reboot", "instanceId", instance.ID, "err", err)
+		}
+	}
+	if pid > 0 && hostprocess.WaitForProcessExit(pid, relaunchExitTimeout) != nil {
+		slog.WarnContext(ctx, "QEMU did not exit after quit, killing it for reboot",
+			"instanceId", instance.ID, "pid", pid)
+		if err := hostprocess.ForceKillProcess(pid, relaunchKillTimeout); err != nil {
+			return fmt.Errorf("kill QEMU pid %d: %w", pid, err)
+		}
+	}
+
+	if q := instance.QMPClient; q != nil {
+		q.Mu.Lock()
+		if q.Conn != nil {
+			_ = q.Conn.Close()
+		}
+		q.Dead = true
+		q.Mu.Unlock()
+	}
+	return nil
+}
+
+// adoptQMPConnection moves a fresh monitor connection into the instance's
+// existing client, so the heartbeat and every holder of the pointer carry on
+// against the new process.
+func (m *Manager) adoptQMPConnection(instance *VM, fresh *qmp.QMPClient) {
+	q := instance.QMPClient
+	if q == nil {
+		instance.QMPClient = fresh
+		go m.qmpHeartbeat(instance)
+		return
+	}
+	q.Mu.Lock()
+	defer q.Mu.Unlock()
+	if q.Conn != nil {
+		_ = q.Conn.Close()
+	}
+	q.Conn, q.Decoder, q.Encoder, q.Path = fresh.Conn, fresh.Decoder, fresh.Encoder, fresh.Path
+	q.Dead = false
+}
+
+// consoleLogSize is the console log's length, or zero when there is none.
+func consoleLogSize(instance *VM) int64 {
+	if instance.Config.ConsoleLogPath == "" {
+		return 0
+	}
+	info, err := os.Stat(instance.Config.ConsoleLogPath)
+	if err != nil {
+		return 0
+	}
+	return info.Size()
+}
+
+// bootProgressed waits for the console log to grow past baseline. It also
+// reports true once the instance leaves running, because whoever moved it owns
+// it now and there is nothing left to judge. A guest with no log passes.
+func (m *Manager) bootProgressed(ctx context.Context, instance *VM, baseline int64) bool {
+	if instance.Config.ConsoleLogPath == "" {
+		return true
+	}
+	deadline := time.Now().Add(rebootBootProgressTimeout)
+	for {
+		if consoleLogSize(instance) > baseline || m.Status(instance) != StateRunning {
+			return true
+		}
+		if !time.Now().Before(deadline) {
+			return false
+		}
+		timer := time.NewTimer(rebootBootProgressPoll)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return true
+		case <-timer.C:
+		}
+	}
+}
+
+// captureBootDiagnostics logs what QEMU and the host say about a guest that
+// has gone silent, before anything is done about it: run state, vCPUs, and
+// every vCPU's registers sampled twice.
+func (m *Manager) captureBootDiagnostics(ctx context.Context, instance *VM, reason string) {
+	query := func(cmd qmp.QMPCommand) string {
+		resp, err := sendQMPCommandWithTimeout(ctx, instance.QMPClient, cmd, instance.ID, bootDiagCommandTimeout)
+		if err != nil {
+			return "error: " + err.Error()
+		}
+		return string(resp.Return)
+	}
+	registers := qmp.QMPCommand{Execute: "human-monitor-command",
+		Arguments: map[string]any{"command-line": "info registers -a"}}
+
+	pid := instance.qemuPID.Load()
+	if pid == 0 {
+		// A process adopted across a daemon restart has no waiter, only a pid file.
+		if p, err := hostprocess.ReadPidFile(instance.ID); err == nil {
+			pid = int64(p)
+		}
+	}
+	attrs := []any{"instanceId", instance.ID, "reason", reason, "pid", pid,
+		"console_bytes", consoleLogSize(instance)}
+	attrs = append(attrs, "query_status", query(qmp.QMPCommand{Execute: "query-status"}))
+	cpus := query(qmp.QMPCommand{Execute: "query-cpus-fast"})
+	attrs = append(attrs, "query_cpus_fast", cpus, "vcpu_threads", vcpuThreadStates(pid, cpus))
+	attrs = append(attrs, "registers_first", query(registers))
+
+	timer := time.NewTimer(bootDiagSampleGap)
+	select {
+	case <-ctx.Done():
+		timer.Stop()
+	case <-timer.C:
+	}
+	attrs = append(attrs, "registers_second", query(registers), "vcpu_threads_second", vcpuThreadStates(pid, cpus))
+	attrs = append(attrs, CurrentHostVirtStack().logAttrs()...)
+	slog.ErrorContext(ctx, "Guest is silent after reboot; QEMU state follows", attrs...)
+}
+
+// vcpuThreadStates reads each vCPU thread's scheduler state and CPU time from
+// /proc, which tells a thread spinning in the guest from one asleep in KVM_RUN.
+func vcpuThreadStates(pid int64, queryCPUsFast string) string {
+	if pid <= 0 {
+		return ""
+	}
+	var cpus []qmp.CPUInfoFast
+	if err := json.Unmarshal([]byte(queryCPUsFast), &cpus); err != nil {
+		return ""
+	}
+	parts := make([]string, 0, len(cpus))
+	for _, cpu := range cpus {
+		task := fmt.Sprintf("/proc/%d/task/%d", pid, cpu.ThreadID)
+		stat, err := os.ReadFile(task + "/stat")
+		if err != nil {
+			continue
+		}
+		wchan, _ := os.ReadFile(task + "/wchan")
+		parts = append(parts, fmt.Sprintf("cpu%d tid=%d %s wchan=%s",
+			cpu.CPUIndex, cpu.ThreadID, threadStatFields(string(stat)), strings.TrimSpace(string(wchan))))
+	}
+	return strings.Join(parts, "; ")
+}
+
+// threadStatFields picks state, utime and stime out of a /proc stat line. The
+// command name may hold spaces, so fields are counted from its closing paren.
+func threadStatFields(stat string) string {
+	_, rest, ok := strings.CutLast(stat, ")")
+	if !ok {
+		return ""
+	}
+	fields := strings.Fields(rest)
+	// After the paren: state is field 3 of the line, utime 14, stime 15.
+	if len(fields) < 13 {
+		return ""
+	}
+	return fmt.Sprintf("state=%s utime=%s stime=%s", fields[0], fields[11], fields[12])
+}

spinifex/runtime/compute/vm/reboot_test.go

@@ -0,0 +1,497 @@
+package vm
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeQEMU stands in for successive QEMU processes of one guest. Each start is
+// a new generation: a real child process behind the pid file, and a monitor on
+// the instance's QMP socket. quit ends that generation the way QEMU does.
+type fakeQEMU struct {
+	t          *testing.T
+	instanceID string
+	socket     string
+
+	mu       sync.Mutex
+	gen      int
+	commands []string // "<gen>:<execute>"
+	ln       net.Listener
+	conns    []net.Conn
+	child    *exec.Cmd
+	// poweredOff is per generation: whether the guest acknowledged the button.
+	poweredOff bool
+
+	// ignorePowerButton keeps the guest running through system_powerdown.
+	ignorePowerButton bool
+	// onCommand runs for each command before the reply, for a test to act
+	// mid-reboot.
+	onCommand func(gen int, cmd qmp.QMPCommand)
+}
+
+func newFakeQEMU(t *testing.T, instanceID string) *fakeQEMU {
+	t.Helper()
+	dir := t.TempDir()
+	t.Setenv("XDG_RUNTIME_DIR", dir)
+	hostprocess.ShortenProcessExitPollForTest(t, 5*time.Millisecond)
+	f := &fakeQEMU{t: t, instanceID: instanceID, socket: filepath.Join(dir, "qmp.sock")}
+	t.Cleanup(f.stop)
+	return f
+}
+
+// start brings up the next generation.
+func (f *fakeQEMU) start() error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	child := exec.Command("sleep", "300")
+	if err := child.Start(); err != nil {
+		return err
+	}
+	// Reaped as soon as it dies, so a liveness probe does not see a zombie.
+	go func() { _ = child.Wait() }()
+	if err := hostprocess.WritePidFile(f.instanceID, child.Process.Pid); err != nil {
+		return err
+	}
+	_ = os.Remove(f.socket)
+	ln, err := net.Listen("unix", f.socket)
+	if err != nil {
+		return err
+	}
+	f.gen++
+	f.ln, f.child, f.poweredOff = ln, child, false
+	gen := f.gen
+	go f.accept(ln, gen)
+	return nil
+}
+
+func (f *fakeQEMU) accept(ln net.Listener, gen int) {
+	for {
+		conn, err := ln.Accept()
+		if err != nil {
+			return
+		}
+		f.mu.Lock()
+		f.conns = append(f.conns, conn)
+		f.mu.Unlock()
+		go f.serve(conn, gen)
+	}
+}
+
+func (f *fakeQEMU) serve(conn net.Conn, gen int) {
+	enc := json.NewEncoder(conn)
+	dec := json.NewDecoder(conn)
+	if err := enc.Encode(map[string]any{"QMP": map[string]any{"capabilities": []string{}}}); err != nil {
+		return
+	}
+	for {
+		var cmd qmp.QMPCommand
+		if err := dec.Decode(&cmd); err != nil {
+			return
+		}
+		f.mu.Lock()
+		f.commands = append(f.commands, fmt.Sprintf("%d:%s", gen, cmd.Execute))
+		hook := f.onCommand
+		f.mu.Unlock()
+		if hook != nil {
+			hook(gen, cmd)
+		}
+
+		reply := map[string]any{"return": map[string]any{}}
+		f.mu.Lock()
+		switch cmd.Execute {
+		case "system_powerdown":
+			if !f.ignorePowerButton {
+				f.poweredOff = true
+			}
+		case "query-status":
+			if f.poweredOff {
+				reply = qmpStatusResponse(qmpStatusShutdown, false)
+			} else {
+				reply = qmpStatusResponse("running", true)
+			}
+		case "query-cpus-fast":
+			reply = map[string]any{"return": []map[string]any{{"cpu-index": 0, "thread-id": 1}}}
+		case "human-monitor-command":
+			reply = map[string]any{"return": "RIP=000000000000fff0"}
+		}
+		f.mu.Unlock()
+		if err := enc.Encode(reply); err != nil {
+			return
+		}
+		if cmd.Execute == "quit" {
+			f.exit(gen)
+			return
+		}
+	}
+}
+
+// exit ends generation gen: the process dies, its pid file and socket go.
+func (f *fakeQEMU) exit(gen int) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if gen != f.gen || f.child == nil {
+		return
+	}
+	_ = f.child.Process.Kill()
+	_ = hostprocess.WaitForProcessExit(f.child.Process.Pid, 5*time.Second)
+	_ = hostprocess.RemovePidFile(f.instanceID)
+	_ = f.ln.Close()
+	for _, c := range f.conns {
+		_ = c.Close()
+	}
+	f.conns, f.child = nil, nil
+}
+
+func (f *fakeQEMU) stop() {
+	f.mu.Lock()
+	gen := f.gen
+	f.mu.Unlock()
+	f.exit(gen)
+}
+
+func (f *fakeQEMU) generation() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.gen
+}
+
+func (f *fakeQEMU) log() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return slices.Clone(f.commands)
+}
+
+// rebootRig is a running guest on a fakeQEMU, with the reboot's timings cut
+// down and the relaunch seam pointed at the fake. writes says, per relaunch,
+// whether the new guest prints to its console.
+type rebootRig struct {
+	m       *Manager
+	vm      *VM
+	qemu    *fakeQEMU
+	console string
+	starts  int
+}
+
+func newRebootRig(t *testing.T, deps Deps, consoleSoFar string, writes ...bool) *rebootRig {
+	t.Helper()
+	setFast := func(p *time.Duration, d time.Duration) {
+		prev := *p
+		*p = d
+		t.Cleanup(func() { *p = prev })
+	}
+	setFast(&rebootPowerdownTimeout, 200*time.Millisecond)
+	setFast(&powerdownPollInterval, 5*time.Millisecond)
+	setFast(&rebootBootProgressTimeout, 150*time.Millisecond)
+	setFast(&rebootBootProgressPoll, 5*time.Millisecond)
+	setFast(&bootDiagSampleGap, time.Millisecond)
+
+	r := &rebootRig{qemu: newFakeQEMU(t, "i-1")}
+	r.console = filepath.Join(filepath.Dir(r.qemu.socket), "console.log")
+	require.NoError(t, os.WriteFile(r.console, []byte(consoleSoFar), 0o600))
+	require.NoError(t, r.qemu.start())
+
+	r.vm = &VM{ID: "i-1", Status: StateRunning,
+		Config: Config{QMPSocket: r.qemu.socket, ConsoleLogPath: r.console}}
+	client, err := newQMPClientWithHandshake(t.Context(), r.vm)
+	require.NoError(t, err)
+	r.vm.QMPClient = client
+
+	previous := startRelaunchedQEMU
+	startRelaunchedQEMU = func(*Manager, *VM) error {
+		r.starts++
+		if err := r.qemu.start(); err != nil {
+			return err
+		}
+		if r.starts <= len(writes) && writes[r.starts-1] {
+			fh, err := os.OpenFile(r.console, os.O_APPEND|os.O_WRONLY, 0)
+			if err != nil {
+				return err
+			}
+			defer fh.Close()
+			_, err = fmt.Fprintf(fh, "BdsDxe: boot %d\n", r.starts)
+			return err
+		}
+		return nil
+	}
+	t.Cleanup(func() { startRelaunchedQEMU = previous })
+
+	r.m = NewManagerWithDeps(deps)
+	r.m.Insert(r.vm)
+	return r
+}
+
+func (r *rebootRig) reboot(t *testing.T) {
+	t.Helper()
+	require.NoError(t, r.m.Reboot(t.Context(), "i-1"))
+	r.m.WaitForBackgroundWork()
+}
+
+// TestReboot_ReplacesQEMUAfterACleanShutdown is the whole reboot: the guest is
+// asked down, its QEMU quit once it has, and a fresh one started. Nothing is
+// reset in place, nothing the instance owns moves, and Describe sees running.
+func TestReboot_ReplacesQEMUAfterACleanShutdown(t *testing.T) {
+	var upCalls, downCalls int
+	mounter := &fakeVolumeMounter{}
+	r := newRebootRig(t, Deps{
+		VolumeMounter: mounter,
+		Hooks: ManagerHooks{
+			OnInstanceUp:   func(*VM) error { upCalls++; return nil },
+			OnInstanceDown: func(string) { downCalls++ },
+		},
+	}, "first boot\n", true)
+	client := r.vm.QMPClient
+
+	r.reboot(t)
+
+	assert.Equal(t, 2, r.qemu.generation(), "the reboot must run on a fresh QEMU")
+	log := r.qemu.log()
+	assert.NotContains(t, log, "1:system_reset", "a reboot must not reset the old process in place")
+	assert.Less(t, slices.Index(log, "1:system_powerdown"), slices.Index(log, "1:quit"),
+		"the guest is asked down before its QEMU is ended")
+	assert.Equal(t, StateRunning, r.m.Status(r.vm))
+	assert.Zero(t, upCalls+downCalls, "a reboot is not a stop and start; no hooks fire")
+	assert.Empty(t, mounter.mounted, "volumes stay mounted across the relaunch")
+	assert.Empty(t, mounter.unmounted, "volumes stay mounted across the relaunch")
+
+	// The same client now talks to the new process, so the heartbeat and every
+	// other holder of it carry on.
+	assert.Same(t, client, r.vm.QMPClient)
+	_, err := queryQMPStatus(t.Context(), r.vm, time.Second)
+	require.NoError(t, err)
+	assert.Contains(t, r.qemu.log(), "2:query-status")
+
+	console, err := os.ReadFile(r.console)
+	require.NoError(t, err)
+	assert.Equal(t, "first boot\nBdsDxe: boot 1\n", string(console),
+		"the console log carries on across the reboot")
+}
+
+// TestReboot_WedgedGuestIsForcedOffAndRelaunched is the AWS contract: a guest
+// that will not shut down cleanly is rebooted anyway once its budget is spent.
+func TestReboot_WedgedGuestIsForcedOffAndRelaunched(t *testing.T) {
+	r := newRebootRig(t, Deps{}, "x", true)
+	r.qemu.ignorePowerButton = true
+	start := time.Now()
+
+	r.reboot(t)
+
+	assert.GreaterOrEqual(t, time.Since(start), rebootPowerdownTimeout,
+		"the guest is given its whole budget before it is forced off")
+	assert.Equal(t, 2, r.qemu.generation())
+	assert.Contains(t, r.qemu.log(), "1:quit")
+	assert.Equal(t, StateRunning, r.m.Status(r.vm))
+}
+
+// TestReboot_SilentGuestIsDiagnosedThenRelaunchedOnce covers the hang this
+// design exists for. A relaunched guest that has printed before and now prints
+// nothing gets its QEMU state captured first, then one more fresh process.
+func TestReboot_SilentGuestIsDiagnosedThenRelaunchedOnce(t *testing.T) {
+	r := newRebootRig(t, Deps{}, "first boot\n", false, true)
+
+	r.reboot(t)
+
+	require.Equal(t, 3, r.qemu.generation(), "a silent relaunch is replaced exactly once")
+	log := r.qemu.log()
+	for _, cmd := range []string{"2:query-status", "2:query-cpus-fast", "2:human-monitor-command", "2:quit"} {
+		assert.Contains(t, log, cmd)
+	}
+	assert.Less(t, slices.Index(log, "2:human-monitor-command"), slices.Index(log, "2:quit"),
+		"the silent process is captured before it is ended")
+	assert.Equal(t, StateRunning, r.m.Status(r.vm))
+}
+
+// TestReboot_GuestThatNeverPrintedIsNotRelaunchedAgain: silence from a guest
+// that never writes to its serial console is not evidence of a hang.
+func TestReboot_GuestThatNeverPrintedIsNotRelaunchedAgain(t *testing.T) {
+	r := newRebootRig(t, Deps{}, "", false)
+
+	r.reboot(t)
+
+	assert.Equal(t, 2, r.qemu.generation())
+	assert.Contains(t, r.qemu.log(), "2:human-monitor-command", "the silence is still recorded")
+	assert.NotContains(t, r.qemu.log(), "2:quit")
+}
+
+// TestReboot_FailedRelaunchGoesToCrashRecovery: once the old process is gone,
+// a replacement that will not start is a down guest, and recovery owns it.
+func TestReboot_FailedRelaunchGoesToCrashRecovery(t *testing.T) {
+	r := newRebootRig(t, Deps{}, "x")
+	startRelaunchedQEMU = func(*Manager, *VM) error { return errors.New("no qemu") }
+
+	r.reboot(t)
+
+	var crashes int
+	r.m.Inspect(r.vm, func(v *VM) { crashes = v.Health.CrashCount })
+	assert.Equal(t, 1, crashes, "the failed relaunch is handed to crash recovery")
+}
+
+// TestReboot_StopWhileOldQEMUEndsSkipsTheRelaunch: a stop that lands while the
+// old process is ending owns the instance, so no new process is started.
+func TestReboot_StopWhileOldQEMUEndsSkipsTheRelaunch(t *testing.T) {
+	r := newRebootRig(t, Deps{}, "x", true)
+	r.qemu.onCommand = func(_ int, cmd qmp.QMPCommand) {
+		if cmd.Execute == "quit" {
+			r.m.UpdateState("i-1", func(v *VM) { v.Status = StateStopping })
+		}
+	}
+
+	r.reboot(t)
+
+	assert.Equal(t, 1, r.qemu.generation(), "no QEMU is started for an instance being stopped")
+}
+
+// TestReboot_StopWaitsForTheRelaunchBeforeUnmounting is the data-safety
+// invariant: a stop never seals volumes under a QEMU the reboot is starting.
+func TestReboot_StopWaitsForTheRelaunchBeforeUnmounting(t *testing.T) {
+	mounter := &fakeVolumeMounter{}
+	r := newRebootRig(t, Deps{VolumeMounter: mounter}, "x", true)
+
+	stopDone := make(chan struct{})
+	inner := startRelaunchedQEMU
+	startRelaunchedQEMU = func(m *Manager, instance *VM) error {
+		go func() {
+			defer close(stopDone)
+			_ = m.shutdownAndUnmount(instance)
+		}()
+		select {
+		case <-stopDone:
+			t.Error("the stop finished while the relaunch still owned the process")
+		case <-time.After(100 * time.Millisecond):
+		}
+		mounter.mu.Lock()
+		assert.Empty(t, mounter.unmounted, "volumes unmounted under a QEMU being started")
+		mounter.mu.Unlock()
+		return inner(m, instance)
+	}
+
+	r.reboot(t)
+	<-stopDone
+	assert.Equal(t, []string{"i-1"}, mounter.unmounted, "the stop runs once the relaunch is done")
+	assert.Contains(t, r.qemu.log(), "2:quit", "the stop ends the process the reboot started")
+}
+
+// TestReboot_RepeatJoinsTheOneInFlight asserts two reboots of one guest do not
+// race each other through the shutdown and relaunch.
+func TestReboot_RepeatJoinsTheOneInFlight(t *testing.T) {
+	r := newRebootRig(t, Deps{}, "x", true, true)
+	release := make(chan struct{})
+	r.qemu.onCommand = func(gen int, cmd qmp.QMPCommand) {
+		if gen == 1 && cmd.Execute == "system_powerdown" {
+			<-release
+		}
+	}
+
+	require.NoError(t, r.m.Reboot(t.Context(), "i-1"))
+	require.NoError(t, r.m.Reboot(t.Context(), "i-1"))
+	close(release)
+	r.m.WaitForBackgroundWork()
+
+	assert.Equal(t, 2, r.qemu.generation(), "the repeat must join the reboot in flight")
+}
+
+// TestReboot_AnswersBeforeTheGuestIsBack is the contract EC2 documents: the
+// reply says the request was accepted, long before the guest is down.
+func TestReboot_AnswersBeforeTheGuestIsBack(t *testing.T) {
+	r := newRebootRig(t, Deps{}, "x", true)
+	release := make(chan struct{})
+	r.qemu.onCommand = func(gen int, cmd qmp.QMPCommand) {
+		if gen == 1 && cmd.Execute == "system_powerdown" {
+			<-release
+		}
+	}
+
+	done := make(chan error, 1)
+	go func() { done <- r.m.Reboot(t.Context(), "i-1") }()
+	select {
+	case err := <-done:
+		require.NoError(t, err)
+	case <-time.After(5 * time.Second):
+		t.Error("Reboot blocked on the guest shutting down; callers time out well inside that")
+	}
+	close(release)
+	r.m.WaitForBackgroundWork()
+}
+
+func TestReboot_InstanceNotFound(t *testing.T) {
+	err := NewManager().Reboot(t.Context(), "i-missing")
+	assert.ErrorIs(t, err, ErrInstanceNotFound)
+}
+
+func TestReboot_NotRunning(t *testing.T) {
+	m := NewManager()
+	m.Insert(&VM{ID: "i-1", Status: StateStopped})
+	assert.ErrorIs(t, m.Reboot(t.Context(), "i-1"), ErrInvalidTransition)
+}
+
+// TestQMPHeartbeatPoll_LeavesARebootingGuestAlone: a reboot powers the guest
+// off and replaces its process, so the heartbeat must neither quit it nor
+// count the missing socket against it.
+func TestQMPHeartbeatPoll_LeavesARebootingGuestAlone(t *testing.T) {
+	recorder := &qmpRecorder{}
+	qmpClient, cancel := newMockQMPClient(t, func(cmd qmp.QMPCommand) map[string]any {
+		recorder.record(cmd)
+		return qmpStatusResponse(qmpStatusShutdown, false)
+	})
+	defer cancel()
+
+	m := NewManager()
+	instance := &VM{ID: "i-1", Status: StateRunning, QMPClient: qmpClient}
+	instance.rebooting.Store(true)
+	m.Insert(instance)
+
+	assert.True(t, m.qmpHeartbeatPoll(instance))
+	assert.Empty(t, recorder.executes(), "a rebooting guest must not be quit")
+	assert.Zero(t, instance.Health.QMPConsecutiveFailures, "a reboot in flight is not a failed heartbeat")
+}
+
+func TestThreadStatFields(t *testing.T) {
+	// The comm field holds a space and a paren, which a plain split misreads.
+	stat := "4242 (CPU 0/K)VM) R 1 2 3 4 5 6 7 8 9 10 777 888 0 0"
+	assert.Equal(t, "state=R utime=777 stime=888", threadStatFields(stat))
+	assert.Empty(t, threadStatFields("garbage"))
+}
+
+func TestExecute_ConsoleLogAppendOnlyWhenAsked(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		cfg    Config
+		want   string
+		absent string
+	}{
+		{"socket fresh", Config{SerialSocket: "/s", ConsoleLogPath: "/c"}, "logfile=/c", "logappend"},
+		{"socket reboot", Config{SerialSocket: "/s", ConsoleLogPath: "/c", ConsoleLogAppend: true}, "logfile=/c,logappend=on", ""},
+		{"file reboot", Config{MachineType: "microvm", ConsoleLogPath: "/c", ConsoleLogAppend: true}, "file,id=console0,path=/c,append=on", ""},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := minimalConfig()
+			cfg.MachineType, cfg.SerialSocket = tc.cfg.MachineType, tc.cfg.SerialSocket
+			cfg.ConsoleLogPath, cfg.ConsoleLogAppend = tc.cfg.ConsoleLogPath, tc.cfg.ConsoleLogAppend
+			cmd, err := cfg.Execute()
+			require.NoError(t, err)
+			var chardev string
+			for i, a := range cmd.Args {
+				if a == "-chardev" && i+1 < len(cmd.Args) {
+					chardev = cmd.Args[i+1]
+				}
+			}
+			assert.Contains(t, chardev, tc.want)
+			if tc.absent != "" {
+				assert.NotContains(t, chardev, tc.absent)
+			}
+		})
+	}
+}

spinifex/runtime/compute/vm/shutdown.go

@@ -6,8 +6,6 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
-	"slices"
-	"strings"
 	"sync"
 	"time"
 
@@ -36,31 +34,9 @@ const (
 var powerdownPollInterval = 500 * time.Millisecond
 
 // errPowerdownTimedOut reports a guest still running at the end of its budget.
-// Callers escalate: Stop to SIGKILL, Reboot to a hard reset.
+// Callers escalate: Stop to SIGKILL, Reboot to ending QEMU without asking.
 var errPowerdownTimedOut = errors.New("guest did not power down within its budget")
 
-// qemuPausesOnShutdown reports whether this guest's QEMU was launched with the
-// shutdown action that leaves it paused. A guest started before that flag
-// exits on powerdown instead, and a reboot would have nothing left to reset.
-// Indirected so a test can describe a guest without one running behind it.
-var qemuPausesOnShutdown = func(instance *VM) bool {
-	pid, err := hostprocess.ReadPidFile(instance.ID)
-	if err != nil || pid <= 0 {
-		return false
-	}
-	cmdline, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid))
-	if err != nil {
-		return false
-	}
-	return cmdlineHasPauseAction(cmdline)
-}
-
-// cmdlineHasPauseAction looks for the shutdown action in a NUL-separated
-// /proc cmdline. Split out from the /proc read so the matching is testable.
-func cmdlineHasPauseAction(cmdline []byte) bool {
-	return slices.Contains(strings.Split(string(cmdline), "\x00"), "shutdown=pause")
-}
-
 // gracefulPowerdown presses the guest's power button until it shuts itself
 // down, so it syncs and unmounts rather than losing its dirty pages. QEMU is
 // left paused in the shutdown run state, not exited — the caller decides.
@@ -146,7 +122,8 @@ func (m *Manager) Stop(id string) error {
 }
 
 // stopOne runs the stop sequence shared by Stop and StopAll:
-// Stopping → stopCleanup → Stopped → migrate to "stopped" KV → OnInstanceDown.
+// Stopping → stopCleanup → release auto-assigned IP → Stopped → migrate to
+// "stopped" KV → OnInstanceDown.
 // The bool reports whether migration removed the instance (caller must persist).
 //
 // A failed volume seal (ErrVolumeSealFailed) does not abort the sequence —
@@ -162,6 +139,12 @@ func (m *Manager) stopOne(instance *VM) (bool, error) {
 
 	m.UpdateState(instance.ID, func(v *VM) { v.LastNode = m.deps.NodeID })
 
+	// Released before the stopped state is published: DescribeInstances reads
+	// the record from that moment, and must never show a borrowed address on it.
+	if instance.DesiredState == DesiredStopped {
+		m.releaseAutoAssignedPublicIP(instance)
+	}
+
 	if err := m.transitionWithPrecheck(instance, StateStopped); err != nil {
 		slog.Error("Failed to transition to stopped", "instanceId", instance.ID, "err", err)
 	}
@@ -178,8 +161,6 @@ func (m *Manager) stopOne(instance *VM) (bool, error) {
 		return false, sealErr
 	}
 
-	m.releaseAutoAssignedPublicIP(instance)
-
 	if !m.MigrateStoppedToSharedKV(instance) {
 		// Either StateStore unavailable / write failed (instance stays in
 		// local map; restoreInstances retries on next boot) OR a concurrent
@@ -704,11 +685,16 @@ func (m *Manager) ForgetSuperseded(instance *VM) {
 // unmounting: a fenced node's volumes belong to another node now, and an
 // unmount would seal this node's stale copy over theirs.
 func (m *Manager) shutdownQEMU(instance *VM) {
+	// Wait out a reboot swapping the process, then stop whichever QEMU is
+	// current. Without this a stop could seal volumes under a fresh process.
+	instance.relaunchMu.Lock()
+	defer instance.relaunchMu.Unlock()
+
 	if instance.QMPClient != nil {
 		ctx := context.Background()
 		// The whole graceful budget is the pid-file wait it replaces, so a stop
-		// takes no longer than it used to. quit closes the paused-on-shutdown
-		// window a reboot needs and a stop does not.
+		// takes no longer than it used to. quit ends the QEMU the guest left
+		// paused when it powered off.
 		if err := m.gracefulPowerdown(ctx, instance, pidFileRemovalTimeout); err != nil {
 			slog.Warn("Guest did not power down gracefully", "id", instance.ID, "err", err)
 		} else if _, err := sendQMPCommand(ctx, instance.QMPClient,