fix(rds): grow and delete a DB instance whose VM is stopped with its volume attached (#1513 )

* fix(rds): grow and delete a DB instance whose VM is stopped with its volume attached * test(rds): run the stopped-terminate commander test in parallel * fix(rds): hold a delete behind an active modify and detach on terminate only the terminated instance's volume * fix(rds): repeat the guarded detach when a stopped terminate left the data volume attached * fix(rds): take the data volume out of a stopped DB VM's record before releasing it and terminating the VM * fix(rds): tear down from the re-read record and keep an accepted delete when a modify fails * fix(rds): release a stopped DB VM's data volume the way a stopped-instance detach does
fix(ec2): relaunch a direct-boot system guest without demanding a root volume (#1512 )
2026-11-04 18:23:51 +00:00 · 2026-11-04 10:08:36 -06:00 · 2026-11-04 09:07:39 -06:00 · 2026-11-04 22:51:49 +10:00 · 2026-11-04 22:19:23 +10:00 · 2026-11-04 21:30:07 +10:00
278 changed files with 21380 additions and 746 deletions
@@ -1,15 +0,0 @@
-# CODEOWNERS — GitHub PR review enforcement
-#
-# Format: <path pattern>  <owner> [<owner> …]
-# Order matters — last matching pattern wins.
-#
-# Set the matching branch protection rule on `dev` and `main` to
-# "Require review from Code Owners" or this file is advisory only.
-
-# Network layer is governed by its invariant tests.
-# Changes here must respect those clauses
-# see spinifex/domains/network/invariants/
-# mulga docs/development/proposals/region-local-architecture/0006-spinifex-network-layer-contract.md (proposal, not accepted)
-/spinifex/domains/network/          @joshsiv-mulga
-/spinifex/runtime/roles/vpcd/ @joshsiv-mulga
-/spinifex/domains/network/invariants/ @joshsiv-mulga
@@ -6,20 +6,6 @@

 <!-- The notable changes, as bullets. -->

-## Account-scoped lifecycle impact
-
-<!-- Complete this section only when the PR changes account-scoped resource lifecycle, account termination, resource cleanup, or a lifecycle boundary. Otherwise write: N/A. -->
-
- Lifecycle owner and resource family:
- Owner-local ordering, recovery, retained-state, or force impact:
- Cross-domain dependency or ownership impact:
- Evidence added or rechecked:
- ADR-0008 lifecycle register / ADR-0001 migration-record / bead updates:
-
-## Mulga design
-
-<!-- One line, so a reviewer knows whether the design records in the private mulga repository need to change. Write either "Updated: <mulga PR or bead id>" or "Not needed: <reason>". This is a reminder, not a gate. -->
-
 ## Community Contribution License

 - [ ] All community contributions in this pull request are licensed to Mulga under the terms of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). By creating this pull request I represent that I have the right to license these contributions to Mulga under the Apache License 2.0.
@@ -1,87 +0,0 @@
-#!/bin/sh
-# hypervisor-disk-preflight_test.sh — drives hypervisor-disk-preflight.sh against
-# a stub ssh that answers for each hypervisor from a canned file, so every
-# verdict is checked without a real host.
-set -eu
-
-SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-PREFLIGHT="$SCRIPT_DIR/hypervisor-disk-preflight.sh"
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-# The stub answers "<root-kb> <pool-kb> <path>" from $FAKE/<host>, or exits 255
-# when that host has no file, the way ssh does for a host it cannot reach.
-mkdir -p "$TMP/bin" "$TMP/hosts"
-cat > "$TMP/bin/ssh" <<'STUB'
-#!/bin/sh
-for a in "$@"; do case $a in *@*) host=${a#*@} ;; esac; done
-cat >/dev/null
-if [ -f "$FAKE/$host" ]; then cat "$FAKE/$host"; exit 0; fi
-echo "ssh: Could not resolve hostname $host" >&2
-exit 255
-STUB
-chmod +x "$TMP/bin/ssh"
-PATH="$TMP/bin:$PATH"
-FAKE="$TMP/hosts"
-export PATH FAKE
-
-cat > "$TMP/env.tfvars" <<'TFVARS'
-nodes = [
-  { name = "wattle", wan_ip = "192.0.2.1", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
-  # { name = "retired", wan_ip = "192.0.2.9", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
-  { name = "banksia", wan_ip = "192.0.2.2", storage_pool = "nvme1-pool", base_image = "/x.qcow2" },
-]
-TFVARS
-
-GiB=1048576
-plenty="$((50 * GiB)) $((200 * GiB)) /mnt/drive1/pool"
-
-# run <label> <want-rc> <want-text>
-run() {
-    rc=0
-    out=$(bash "$PREFLIGHT" "$TMP/env.tfvars" 2>&1) || rc=$?
-    if [ "$rc" -ne "$2" ]; then
-        fail "$1: exit $rc, want $2"
-        echo "$out" | sed 's/^/        /'
-        return
-    fi
-    case $out in
-        *"$3"*) pass "$1" ;;
-        *) fail "$1: output lacks '$3'"; echo "$out" | sed 's/^/        /' ;;
-    esac
-}
-
-echo "-- hypervisor-disk-preflight.sh"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$plenty" > "$FAKE/banksia"
-run "every host has room" 0 "banksia: root fs 51200 MiB free"
-
-echo "0 $((200 * GiB)) /mnt/drive1/pool" > "$FAKE/wattle"
-run "a full root filesystem fails fast" 1 "::error::hypervisor wattle has 0 MiB free on the filesystem holding /etc/libvirt/qemu"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$((50 * GiB)) $((3 * GiB)) /mnt/nvme1/pool" > "$FAKE/banksia"
-run "a nearly full pool fails fast" 1 "::error::hypervisor banksia has 3 GiB free in storage pool nvme1-pool"
-
-echo "$plenty" > "$FAKE/banksia"
-rm -f "$FAKE/wattle"
-run "an unreachable host is reported, not failed" 0 "::warning::hypervisor disk preflight: cannot reach wattle"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$((50 * GiB)) - -" > "$FAKE/banksia"
-run "a pool libvirt does not know is reported, not failed" 0 "pool nvme1-pool (-) not found"
-
-echo "0 0 /x" > "$FAKE/retired"
-echo "$plenty" > "$FAKE/banksia"
-run "a commented-out node is never checked" 0 "wattle: root fs"
-
-if [ "$fails" -ne 0 ]; then
-    echo "$fails failure(s)"
-    exit 1
-fi
-echo "  hypervisor-disk-preflight ok"
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY and
+# never fails the job: the check is a prompt, not a gate.
+set -euo pipefail
+shopt -s nocasematch
+
+section_re='^#{1,6}[[:space:]]*Mulga design[[:space:]]*$'
+line_re='^[[:space:]]*(Updated|Not needed)[[:space:]]*:[[:space:]]*(.*)$'
+placeholder_re='<[^<>]*>'
+
+filled() {
+  local body=$1 line value kept inside=0 pre rest
+  body=${body//$'\r'/}
+  while [[ $body == *'<!--'*'-->'* ]]; do
+    pre=${body%%'<!--'*}
+    rest=${body#*'<!--'}
+    body=$pre${rest#*'-->'}
+  done
+  while IFS= read -r line; do
+    if [[ $line =~ $section_re ]]; then
+      inside=1
+    elif ((inside)) && [[ ${line#"${line%%[![:space:]]*}"} == '#'* ]]; then
+      return 1
+    elif ((inside)) && [[ $line =~ $line_re ]]; then
+      value=${BASH_REMATCH[2]} kept=''
+      while [[ $value =~ $placeholder_re ]]; do
+        kept+=${value%%"${BASH_REMATCH[0]}"*}
+        value=${value#*"${BASH_REMATCH[0]}"}
+      done
+      [[ $kept$value == *[![:space:]]* ]] && return 0
+    fi
+  done <<<"$body"
+  return 1
+}
+
+if filled "${PR_BODY:-}"; then
+  echo "The Mulga design line is filled in."
+else
+  echo '::warning title=Mulga design::Add a "## Mulga design" section with one line: "Updated: <mulga PR>" or "Not needed: <reason>".'
+fi
@@ -1,36 +0,0 @@
-#!/usr/bin/env python3
-"""Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY; never fails the job."""
-
-import os
-import re
-
-COMMENT_RE = re.compile(r"<!--.*?-->", re.S)
-SECTION_RE = re.compile(r"^#{1,6}\s*Mulga design\s*$", re.I)
-LINE_RE = re.compile(r"^\s*(Updated|Not needed)\s*:\s*(.*?)\s*$", re.I)
-PLACEHOLDER_RE = re.compile(r"<[^<>]*>")
-
-
-def filled(body):
-    inside = False
-    for line in COMMENT_RE.sub("", body or "").splitlines():
-        if SECTION_RE.match(line):
-            inside = True
-        elif inside and line.lstrip().startswith("#"):
-            return False
-        elif inside:
-            m = LINE_RE.match(line)
-            if m and PLACEHOLDER_RE.sub("", m.group(2)).strip():
-                return True
-    return False
-
-
-def main():
-    if filled(os.environ.get("PR_BODY", "")):
-        print("The Mulga design line is filled in.")
-    else:
-        print('::warning title=Mulga design::Add a line under "## Mulga design": "Updated: <mulga PR or bead id>" or "Not needed: <reason>".')
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())
@@ -1,23 +0,0 @@
-import unittest
-
-from mulga_design_check import filled
-
-
-class FilledTest(unittest.TestCase):
-    def test_accepts_either_form(self):
-        self.assertTrue(filled("## Mulga design\nUpdated: mulga#275"))
-        self.assertTrue(filled("## Mulga design\n\nnot needed: internal refactor\n## Next"))
-
-    def test_rejects_unanswered(self):
-        self.assertFalse(filled(""))
-        self.assertFalse(filled("## Mulga design\n<!-- Updated: x -->"))
-        self.assertFalse(filled("## Mulga design\nUpdated: <mulga PR or bead id>"))
-        self.assertFalse(filled("## Mulga design\nUpdated:"))
-
-    def test_scoped_to_the_section(self):
-        self.assertFalse(filled("Updated: mulga#1"))
-        self.assertFalse(filled("## Mulga design\n## Other\nUpdated: mulga#1"))
-
-
-if __name__ == "__main__":
-    unittest.main()
@@ -1,47 +0,0 @@
-#!/bin/sh
-# wait-for-memory_test.sh — drives wait-for-memory.sh against a fake meminfo
-# that a background writer changes mid-wait, so each verdict is checked without
-# a loaded host.
-set -eu
-
-SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-WAIT="$SCRIPT_DIR/wait-for-memory.sh"
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-meminfo() { printf 'MemTotal:       65536000 kB\nMemAvailable:   %s kB\n' "$(( $1 * 1024 ))" > "$TMP/meminfo"; }
-MEMINFO="$TMP/meminfo"
-MEM_WAIT_POLL_S=1
-export MEMINFO MEM_WAIT_POLL_S
-
-meminfo 30000
-if MEM_WAIT_TIMEOUT_S=5 bash "$WAIT" 26624 > "$TMP/out" 2>&1; then
-  pass "enough memory returns at once"
-else
-  fail "enough memory returns at once: $(cat "$TMP/out")"
-fi
-
-meminfo 21577
-( sleep 2; meminfo 30000 ) &
-if MEM_WAIT_TIMEOUT_S=20 bash "$WAIT" 26624 > "$TMP/out" 2>&1 && grep -q waiting "$TMP/out"; then
-  pass "memory freed mid-wait is admitted"
-else
-  fail "memory freed mid-wait is admitted: $(cat "$TMP/out")"
-fi
-wait
-
-meminfo 21577
-if MEM_WAIT_TIMEOUT_S=2 bash "$WAIT" 26624 > "$TMP/out" 2>&1; then
-  fail "memory that never frees fails: exited 0"
-elif grep -q '::error::need 26624 MiB available, have 21577 MiB' "$TMP/out" && grep -q 'RSS' "$TMP/out"; then
-  pass "memory that never frees fails and names the biggest processes"
-else
-  fail "memory that never frees fails and names the biggest processes: $(cat "$TMP/out")"
-fi
-
-[ "$fails" -eq 0 ] || { echo "wait-for-memory: $fails failed"; exit 1; }
-echo "wait-for-memory: all passed"
@@ -1017,7 +1017,7 @@ jobs:
        id: siblings
        env:
          BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
        working-directory: mulga/spinifex

      - uses: actions/checkout@v7.0.1
@@ -1275,7 +1275,7 @@ jobs:
        id: siblings
        env:
          BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
        working-directory: mulga/spinifex

      - uses: actions/checkout@v7.0.1
@@ -1423,7 +1423,7 @@ jobs:
        id: siblings
        env:
          BRANCH: ${{ github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
        working-directory: spinifex

      - name: Checkout Predastore
@@ -1547,7 +1547,7 @@ jobs:
        id: siblings
        env:
          BRANCH: ${{ github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
        working-directory: spinifex

      - name: Checkout Predastore
@@ -2,7 +2,8 @@ name: Mulga design

 # Reminds the author to say whether the Mulga design records need to change.
 # It reads only the pull request body, uses no secrets, and warns without ever
-# failing: the check is a prompt, not a gate.
+# failing: the check is a prompt, not a gate. Fork PRs are skipped because their
+# authors cannot see the design records.

 on:
  pull_request:
@@ -16,7 +17,7 @@ permissions:
 jobs:
  mulga_design:
    name: Mulga design line
-    if: ${{ !endsWith(github.actor, '[bot]') }}
+    if: ${{ !endsWith(github.actor, '[bot]') && github.event.pull_request.head.repo.full_name == github.repository }}
    runs-on: ubuntu-26.04
    steps:
      - name: Checkout Code
@@ -28,6 +29,4 @@ jobs:
        env:
          PR_BODY: ${{ github.event.pull_request.body }}
        working-directory: .github/scripts
-        run: |
-          python3 -m unittest -q test_mulga_design_check
-          python3 mulga_design_check.py
+        run: bash mulga-design-check.sh
@@ -113,7 +113,7 @@ jobs:
        id: siblings
        env:
          BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
        working-directory: spinifex

      - uses: actions/checkout@v7.0.1
@@ -84,7 +84,7 @@ jobs:

      - name: Resolve sibling release tags from go.mod
        id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod

      - name: Checkout Viperblock
        uses: actions/checkout@v7.0.1
@@ -18,7 +18,7 @@ jobs:

      - name: Resolve sibling release tags from go.mod
        id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod

      - name: Checkout Predastore
        uses: actions/checkout@v7.0.1
@@ -85,7 +85,7 @@ jobs:

      - name: Resolve sibling release tags from go.mod
        id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod

      - name: Checkout Predastore
        uses: actions/checkout@v7.0.1
@@ -335,7 +335,7 @@ jobs:

      - name: Resolve sibling release tags from go.mod
        id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod

      - name: Checkout Predastore
        uses: actions/checkout@v7.0.1
@@ -34,7 +34,7 @@ jobs:
        id: siblings
        env:
          BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
        working-directory: spinifex

      - name: Checkout Viperblock
@@ -36,7 +36,7 @@ jobs:
        id: siblings
        env:
          BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
        working-directory: spinifex

      - name: Checkout Viperblock
@@ -166,7 +166,7 @@ jobs:
        id: siblings
        env:
          BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
        working-directory: spinifex

      - name: Checkout Viperblock
@@ -268,7 +268,7 @@ jobs:
        id: siblings
        env:
          BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
        working-directory: spinifex

      - name: Checkout Viperblock
@@ -27,6 +27,7 @@ build/microvm/sqs-worker/.stage.*/
 build/microvm/lambda-runtime/vmlinuz
 build/microvm/lambda-runtime/initramfs.cpio.gz
 build/microvm/lambda-runtime/.stage.*/
+build/microvm/lambda-testlink-runtime/
 build/qboot/
 !spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/
@@ -42,11 +42,15 @@ ifdef QUIET
  _COVQ  = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' | grep -v 'coverage: 0\.0%' || true; }; exit $${PIPESTATUS[0]}
  _RACEQ = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' || true; }; exit $${PIPESTATUS[0]}
  _SECQ  = >
+  _HDR   = :
+  _RUNSH = sh "$$t" >"$$log" 2>&1 || { echo "-- $$t"; cat "$$log"; exit 1; }
 else
  _Q     =
  _COVQ  =
  _RACEQ =
  _SECQ  = 2>&1 | tee
+  _HDR   = echo -e
+  _RUNSH = echo "-- $$t"; sh "$$t" || exit 1
 endif

 build: go_build build-installer build-lb-agent generate-aws-model-coverage
@@ -78,6 +82,35 @@ go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
 	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go

+# Controlled-test only: builds the tagged daemon and its hard-kill initializer
+# together, pinning the initializer's digest into that daemon build. The daemon
+# keeps its symbols so the pin can be read back from the binary.
+# LAMBDA_IMAGE_DIGEST is the test-link runtime image digest that
+# build-lambda-testlink-image prints; it is read raw and validated before use.
+LAMBDA_TESTLINK_DIR := ./bin/lambda-testlink
+LAMBDA_TESTLINK_PKG := github.com/mulgadc/spinifex/spinifex/runtime/daemon
+LAMBDA_TESTLINK_PIN := $(LAMBDA_TESTLINK_PKG).lambdaHardKillSHA256
+LAMBDA_DIGEST_RE := ^[0-9a-f]{64}$$
+build-lambda-testlink: override export LAMBDA_IMAGE_DIGEST := $(value LAMBDA_IMAGE_DIGEST)
+build-lambda-testlink:
+	@[[ "$$LAMBDA_IMAGE_DIGEST" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"; exit 1; }
+	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
+	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
+	@rm -f $(LAMBDA_TESTLINK_DIR)/hardkill.so $(LAMBDA_TESTLINK_DIR)/$(GO_PROJECT_NAME)
+	@mkdir -p $(LAMBDA_TESTLINK_DIR).tmp
+	@rm -f $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME)
+	cc -std=c17 -O2 -Wall -Wextra -Werror -shared -fPIC -o $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so spinifex/runtime/compute/lambda/testlink/hardkill/hardkill.c -lrt
+	scripts/check-glibc-ceiling.sh 2.34 $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so
+	@digest=$$(sha256sum $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so | cut -d' ' -f1); \
+	test $${#digest} -eq 64 || { echo "initializer digest missing"; exit 1; }; \
+	echo "initializer sha256 $$digest"; \
+	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
+	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ && \
+	go build $(GO_BUILD_MOD) -tags lambdatestlink -ldflags "$(subst -s -w ,,$(LDFLAGS)) $$pin" -o $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	@mkdir -p $(LAMBDA_TESTLINK_DIR)
+	mv $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) $(LAMBDA_TESTLINK_DIR)/
+	@rmdir $(LAMBDA_TESTLINK_DIR).tmp
+
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
 	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
@@ -193,6 +226,24 @@ bin/lambda-agent: go.mod go.sum $$(call go_local_srcs,./cmd/lambda-agent)
 	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/lambda-agent

 build-lambda-runtime-microvm-image: $(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) ## Build the Lambda runtime microVM kernel and initramfs (not shipped)
+	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz absent
+
+# Controlled-test only: a separately built guest whose agent has the
+# lambdatestlink tag, in its own outputs, never touching bin/lambda-agent or the
+# default image. It prints the digest to pass as LAMBDA_IMAGE_DIGEST.
+LAMBDA_TESTLINK_IMAGE_DIR := $(MICROVM_OUT_DIR)/lambda-testlink-runtime
+build-lambda-testlink-image: ## Build the test-link Lambda guest image and print its digest (not shipped)
+	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
+	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
+	$(MAKE) --no-print-directory $(MICROVM_OUT_DIR)/qboot.rom
+	@mkdir -p bin $(LAMBDA_TESTLINK_IMAGE_DIR)
+	GOWORK=off GOFLAGS= CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -tags lambdatestlink -ldflags '-s -w' -o bin/lambda-agent-testlink ./cmd/lambda-agent
+	./scripts/build-lambda-runtime-microvm-image.sh bin/lambda-agent-testlink $(LAMBDA_TESTLINK_IMAGE_DIR)
+	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_TESTLINK_IMAGE_DIR)/initramfs.cpio.gz present
+	install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom $(LAMBDA_TESTLINK_IMAGE_DIR)/qboot.rom
+	@digest=$$(GOWORK=off go run -tags lambdatestlink ./cmd/lambda-image-digest $(LAMBDA_TESTLINK_IMAGE_DIR)) && \
+	[[ "$$digest" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "image digest not computed"; exit 1; }; \
+	echo "LAMBDA_IMAGE_DIGEST=$$digest"

 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
@@ -240,27 +291,29 @@ preflight:
 NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh scripts/lib/stage-sqs-store-microvm.sh

 test-build-scripts:
-	@echo -e "\n....Running build/scripts/**/*_test.sh...."
-	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running build/scripts/**/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find build/scripts -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
 	done
-	@echo -e "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
-	@for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
+		$(_RUNSH); \
 	done
-	@echo -e "\n....Running .github/scripts/*_test.sh...."
-	@for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running scripts/terraform/latitude-spx/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find scripts/terraform/latitude-spx -maxdepth 1 -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
 	done
-	shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh .github/scripts/hypervisor-disk-preflight_test.sh \
-		.github/scripts/wait-for-memory.sh .github/scripts/wait-for-memory_test.sh
-	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
-	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
-	shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
-	@echo "  test-build-scripts ok"
+	@$(_HDR) "\n....Running .github/scripts/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
+	done
+	$(_Q)shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh \
+		.github/scripts/wait-for-memory.sh \
+		.github/scripts/mulga-design-check.sh
+	@$(_HDR) "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
+	$(_Q)shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
+	$(_Q)shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
+	@$(_HDR) "  test-build-scripts ok"

 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
@@ -327,7 +380,10 @@ test-cover:
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
 	$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ ./spinifex/runtime/compute/lambda/testlink/ ./spinifex/domains/network/host/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink(Executor|Reply)_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink,lambdatestlinkseam -timeout 120s -run '^TestLambdaTestLinkSeam_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink' ./spinifex/runtime/daemon/ $(_RACEQ)

 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -538,7 +594,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/

-.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-lambda-testlink build-lambda-testlink-image build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \
@@ -70,8 +70,8 @@ The platform serves **405 AWS API operations** across the services below. See ex
 | **RDS** | Databases | PostgreSQL and MariaDB, with automated backups | Available |
 | **Route 53** | DNS | [Northstar](https://github.com/mulgadc/northstar): authoritative DNS with hosted zones | Available |
 | **Bedrock** | AI deployment | vLLM for text generation and Hugging Face TEI for embeddings and reranking | Q4 2026 |
-| **Lambda** | Serverless functions | | Q1 2027 |
-| **SQS** | Message queues | | Q1 2027 |
+| **Lambda** | Serverless functions | QEMU `microvm` execution environments | Q1 2027 |
+| **SQS** | Message queues | A dedicated NATS JetStream store in a per-Region QEMU `microvm` guest | Q1 2027 |

 Roadmap services ship under the same AWS API surface, so code written for AWS today works the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"

+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 )

@@ -29,20 +30,20 @@ func defaultStartup() startup {

 // connect loads the identity, seals fw_cfg whether or not the load succeeded,
 // then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
-func (s startup) connect(ctx context.Context) (*tls.Conn, int, error) {
+func (s startup) connect(ctx context.Context) (*tls.Conn, lambdav1.Scope, int, error) {
 	id, loadErr := hostlink.LoadIdentity(s.fwCfgDir)
 	if err := hostlink.SealFwCfg(s.fwCfgRoot, s.unload); err != nil {
-		return nil, exitRefused, errors.Join(loadErr, err)
+		return nil, "", exitRefused, errors.Join(loadErr, err)
 	}
 	if loadErr != nil {
-		return nil, exitRefused, loadErr
+		return nil, "", exitRefused, loadErr
 	}
 	conn, err := hostlink.Dial(ctx, id)
 	if err != nil {
 		if ctx.Err() != nil {
 			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
 		}
-		return nil, exitLink, err
+		return nil, "", exitLink, err
 	}
-	return conn, 0, nil
+	return conn, id.Scope, 0, nil
 }
@@ -0,0 +1,66 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+)
+
+const entryGrant = "test-grant"
+
+// testGrant is the grant the host launched this VM under.
+type testGrant struct {
+	grantID, attemptID, environmentID string
+	generation                        uint64
+}
+
+// readGrant reads the test-grant fw_cfg entry: grant ID, attempt ID,
+// environment ID and generation, one per line.
+func readGrant(fwCfgDir string) (testGrant, error) {
+	b, err := os.ReadFile(filepath.Join(fwCfgDir, entryGrant, "raw"))
+	if err != nil {
+		return testGrant{}, fmt.Errorf("read %s: %w", entryGrant, err)
+	}
+	lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n")
+	if len(lines) != 4 {
+		return testGrant{}, errors.New(entryGrant + " entry malformed")
+	}
+	g := testGrant{grantID: lines[0], attemptID: lines[1], environmentID: lines[2]}
+	g.generation, err = strconv.ParseUint(lines[3], 10, 64)
+	if err != nil || g.generation == 0 || g.grantID == "" || g.attemptID == "" || g.environmentID == "" {
+		return testGrant{}, errors.New(entryGrant + " entry malformed")
+	}
+	return g, nil
+}
+
+// checkBinding refuses a run unless the grant names the authenticated environment
+// generation, the request carries that grant and its deadline is still ahead. The
+// clock check is a local guard only: it cannot detect skew or a superseded grant.
+func checkBinding(scope lambdav1.Scope, g testGrant, req hostlink.Request, now time.Time) error {
+	parts := strings.Split(string(scope), "/")
+	if len(parts) != 5 {
+		return errors.New("authenticated scope malformed")
+	}
+	if parts[3] != g.environmentID || parts[4] != strconv.FormatUint(g.generation, 10) {
+		return errors.New("grant is not for the authenticated environment generation")
+	}
+	if req.GrantID != g.grantID || req.AttemptID != g.attemptID {
+		return errors.New("request does not carry this VM's grant")
+	}
+	if req.DeadlineMs <= 0 {
+		return errors.New("request has no deadline")
+	}
+	if !now.Before(time.UnixMilli(req.DeadlineMs)) {
+		return errors.New("request deadline already passed")
+	}
+	return nil
+}
@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 )

-const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
+const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/1"

 // guest is a fake fw_cfg tree; unload removes it as unloading the module would.
 type guest struct {
@@ -18,6 +18,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode"

 	"golang.org/x/sys/unix"
 )
@@ -28,6 +29,7 @@ import (
 const (
 	entryPackageSize   = "test-package-size"
 	entryPackageSHA256 = "test-package-sha256"
+	entryHandler       = "test-handler"

 	devMaxZipBytes    = 16 << 20
 	devMaxUnzipBytes  = 64 << 20
@@ -37,8 +39,9 @@ const (
 )

 type packageSpec struct {
-	size   int64
-	sha256 [sha256.Size]byte
+	size    int64
+	sha256  [sha256.Size]byte
+	handler string
 }

 type unpackLimits struct {
@@ -76,6 +79,13 @@ func readPackageSpec(fwCfgDir string) (packageSpec, error) {
 		return spec, errors.New("package sha256 is not 64 hex digits")
 	}
 	copy(spec.sha256[:], b)
+	if spec.handler, err = read(entryHandler); err != nil {
+		return spec, err
+	}
+	if spec.handler == "" || len(spec.handler) > 128 || strings.IndexFunc(spec.handler, unicode.IsSpace) >= 0 ||
+		strings.ContainsRune(spec.handler, 0) {
+		return spec, errors.New("handler must be 1..128 characters with no whitespace")
+	}
 	return spec, nil
 }

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"syscall"
 	"testing"

@@ -165,19 +166,23 @@ func TestUnpackChecksSizeAndHash(t *testing.T) {
 func TestReadPackageSpecBounds(t *testing.T) {
 	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 	cases := map[string]struct {
-		size, hash string
-		want       string
+		size, hash, handler string
+		want                string
 	}{
-		"valid":     {"10", sum, ""},
-		"zero":      {"0", sum, "outside"},
-		"too large": {"16777217", sum, "outside"},
-		"not hex":   {"10", "zz", "64 hex digits"},
-		"short":     {"10", sum[:62], "64 hex digits"},
+		"valid":            {"10", sum, "bootstrap.h", ""},
+		"zero":             {"0", sum, "bootstrap.h", "outside"},
+		"too large":        {"16777217", sum, "bootstrap.h", "outside"},
+		"not hex":          {"10", "zz", "bootstrap.h", "64 hex digits"},
+		"short":            {"10", sum[:62], "bootstrap.h", "64 hex digits"},
+		"empty handler":    {"10", sum, "", "handler"},
+		"handler space":    {"10", sum, "a b", "handler"},
+		"128-char handler": {"10", sum, strings.Repeat("h", 128), ""},
+		"129-char handler": {"10", sum, strings.Repeat("h", 129), "handler"},
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
 			dir := t.TempDir()
-			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash, entryHandler: tc.handler} {
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
 				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
 			}
@@ -185,6 +190,7 @@ func TestReadPackageSpecBounds(t *testing.T) {
 			if tc.want == "" {
 				require.NoError(t, err)
 				assert.Equal(t, int64(10), spec.size)
+				assert.Equal(t, tc.handler, spec.handler)
 				return
 			}
 			assert.ErrorContains(t, err, tc.want)
@@ -25,7 +25,7 @@ func start(ctx context.Context) int {

 // run authenticates to the host and holds the connection until it closes.
 func (s startup) run(ctx context.Context) (int, error) {
-	conn, code, err := s.connect(ctx)
+	conn, _, code, err := s.connect(ctx)
 	if err != nil {
 		return code, err
 	}
@@ -5,7 +5,6 @@ package main
 import (
 	"context"
 	"errors"
-	"flag"
 	"fmt"
 	"log/slog"
 	"net"
@@ -24,8 +23,6 @@ const (
 	taskRoot    = "/var/task"
 )

-var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler passed to the bootstrap")
-
 // testlink relays one invocation over the test carrier on the authenticated
 // connection; the carrier is not the guest protocol.
 type testlink struct {
@@ -54,13 +51,14 @@ func start(ctx context.Context) int {
 func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
-	conn, code, err := t.connect(parent)
+	grant, grantErr := readGrant(t.fwCfgDir)
+	conn, scope, code, err := t.connect(parent)
 	if err != nil {
-		return code, errors.Join(err, specErr)
+		return code, errors.Join(err, specErr, grantErr)
 	}
 	defer conn.Close()
-	if specErr != nil {
-		return exitRefused, specErr
+	if err := errors.Join(specErr, grantErr); err != nil {
+		return exitRefused, err
 	}
 	link := hostlink.NewTestStream(conn)

@@ -76,6 +74,10 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
+	// Nothing of the tenant's is installed or run for an unbound request.
+	if err := checkBinding(scope, grant, first, time.Now()); err != nil {
+		return exitRefused, err
+	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -93,7 +95,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	defer cancel(nil)
 	cmd, err := t.startBootstrap(ctx, []string{
 		"AWS_LAMBDA_RUNTIME_API=" + ln.Addr().String(),
-		"_HANDLER=" + *handler,
+		"_HANDLER=" + spec.handler,
 		"LAMBDA_TASK_ROOT=" + taskRoot,
 		"PATH=/bin:/sbin:/usr/bin",
 	})
@@ -25,12 +25,18 @@ import (

 const helperEnv = "LAMBDA_AGENT_TEST_BOOTSTRAP"

+// helperHandlerOut names the file the helper bootstrap writes its _HANDLER to.
+const helperHandlerOut = "LAMBDA_AGENT_TEST_HANDLER_OUT"
+
 // TestHelperBootstrap is the fake function bootstrap, run as a subprocess: it
 // echoes one invocation through the Runtime API, then waits for the next.
 func TestHelperBootstrap(t *testing.T) {
 	if os.Getenv(helperEnv) != "1" {
 		return
 	}
+	if out := os.Getenv(helperHandlerOut); out != "" {
+		require.NoError(t, os.WriteFile(out, []byte(os.Getenv("_HANDLER")), 0o600))
+	}
 	base := "http://" + os.Getenv("AWS_LAMBDA_RUNTIME_API") + "/2018-06-01/runtime/invocation/"
 	resp, err := http.Get(base + "next")
 	require.NoError(t, err)
@@ -44,11 +50,19 @@ func TestHelperBootstrap(t *testing.T) {
 	_, _ = http.Get(base + "next")
 }

+// testGrantEntry grants g-1 for attempt a-1 on the fixture scope's env-1/1.
+const testGrantEntry = "g-1\na-1\nenv-1\n1\n"
+
+// testHandler is the handler the fixture's host configures.
+const testHandler = "bootstrap.fixture"
+
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
 	task    string
 	sealed  bool
+	// handlerOut receives the _HANDLER the bootstrap process itself saw.
+	handlerOut string
 }

 // testlink builds a guest whose fw_cfg also carries a controlled package,
@@ -61,9 +75,16 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	sum := sha256.Sum256(zipBytes)
 	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
 	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
+	if _, ok := entries[entryHandler]; !ok {
+		entries[entryHandler] = []byte(testHandler)
+	}
+	if _, ok := entries[entryGrant]; !ok {
+		entries[entryGrant] = []byte(testGrantEntry)
+	}
 	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
+	b.handlerOut = filepath.Join(t.TempDir(), "handler")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -77,7 +98,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			b.n++
 			b.started <- struct{}{}
 			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestHelperBootstrap$")
-			cmd.Env = append(env, helperEnv+"=1")
+			cmd.Env = append(env, helperEnv+"=1", helperHandlerOut+"="+b.handlerOut)
 			return cmd, cmd.Start()
 		},
 	}
@@ -132,6 +153,9 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
+	seen, err := os.ReadFile(b.handlerOut)
+	require.NoError(t, err, "bootstrap process did not report its handler")
+	assert.Equal(t, testHandler, string(seen), "bootstrap process not given the configured handler")
 	assert.True(t, b.sealed, "task dir not made read-only")
 	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
 }
@@ -218,7 +242,8 @@ func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {

 	conn := h.guest(t)
 	defer conn.Close()
-	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1",
+		DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli()}))
 	res := wait(t, done, "agent to stop")
 	assert.Equal(t, exitRefused, res.code)
 	assert.ErrorContains(t, res.err, "install package")
@@ -244,3 +269,87 @@ func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
 	_, err := os.Stat(tl.fwCfgRoot)
 	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
 }
+
+// A request that does not carry this VM's grant, or a grant for another
+// environment generation than the one authenticated, installs and runs
+// nothing of the tenant's.
+func TestTestlinkUnboundRequestNeverStartsBootstrap(t *testing.T) {
+	cases := map[string]struct {
+		grant            string
+		grantID, attempt string
+		deadlineMs       int64
+		want             string
+	}{
+		"frame with no deadline":      {testGrantEntry, "g-1", "a-1", -1, "no deadline"},
+		"frame with expired deadline": {testGrantEntry, "g-1", "a-1", time.Now().Add(-time.Second).UnixMilli(), "deadline already passed"},
+		"frame for another grant":     {testGrantEntry, "g-2", "a-1", 0, "does not carry this VM's grant"},
+		"frame for another attempt":   {testGrantEntry, "g-1", "a-2", 0, "does not carry this VM's grant"},
+		"frame with no grant":         {testGrantEntry, "", "", 0, "does not carry this VM's grant"},
+		"grant for another env":       {"g-1\na-1\nenv-2\n1\n", "g-1", "a-1", 0, "not for the authenticated environment"},
+		"grant for another gen":       {"g-1\na-1\nenv-1\n2\n", "g-1", "a-1", 0, "not for the authenticated environment"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			h := newHost(t)
+			h.accept(ctx)
+			var b bootstrapStarts
+			entries := h.entries()
+			entries[entryGrant] = []byte(tc.grant)
+			done := runTestlinkAsync(ctx, b.testlink(t, entries))
+			conn := h.guest(t)
+			defer conn.Close()
+			deadline := time.Now().Add(5 * time.Second).UnixMilli()
+			switch {
+			case tc.deadlineMs < 0:
+				deadline = 0
+			case tc.deadlineMs > 0:
+				deadline = tc.deadlineMs
+			}
+			req := hostlink.Request{GrantID: tc.grantID, AttemptID: tc.attempt, RequestID: "r-1",
+				DeadlineMs: deadline, Payload: []byte(`{}`)}
+			require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
+
+			res := wait(t, done, "agent to stop")
+			assert.Equal(t, exitRefused, res.code)
+			assert.ErrorContains(t, res.err, tc.want)
+			assert.Zero(t, b.n)
+			assert.False(t, b.sealed)
+			assert.NoFileExists(t, filepath.Join(b.task, "bootstrap"), "package installed")
+		})
+	}
+}
+
+// A missing or malformed grant entry refuses before any frame is awaited.
+func TestTestlinkBadGrantEntryRefuses(t *testing.T) {
+	cases := map[string]*string{
+		"missing":       nil,
+		"too few lines": ptr("g-1\na-1\nenv-1\n"),
+		"generation 0":  ptr("g-1\na-1\nenv-1\n0\n"),
+		"empty grant":   ptr("\na-1\nenv-1\n1\n"),
+	}
+	for name, grant := range cases {
+		t.Run(name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			h := newHost(t)
+			h.accept(ctx)
+			var b bootstrapStarts
+			entries := h.entries()
+			if grant != nil {
+				entries[entryGrant] = []byte(*grant)
+			}
+			tl := b.testlink(t, entries)
+			if grant == nil {
+				require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryGrant)))
+			}
+			res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
+			assert.Equal(t, exitRefused, res.code)
+			assert.ErrorContains(t, res.err, entryGrant)
+			assert.Zero(t, b.n)
+		})
+	}
+}
+
+func ptr(s string) *string { return &s }
@@ -0,0 +1,221 @@
+// Command lambda-image-agent-tag checks whether the lambda-agent archived in a
+// built Lambda guest initramfs carries the lambdatestlink build tag. Build
+// information is recorded by the build; it is not proof of how the agent behaves.
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"debug/buildinfo"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+const (
+	agentPath    = "usr/local/bin/lambda-agent"
+	maxArchive   = 1 << 30
+	maxAgent     = 256 << 20
+	maxName      = 4096
+	newcHeader   = 110
+	trailerName  = "TRAILER!!!"
+	modeTypeMask = 0o170000
+	modeDir      = 0o040000
+	modeRegular  = 0o100000
+)
+
+var agentParents = []string{"usr", "usr/local", "usr/local/bin"}
+
+func main() {
+	if len(os.Args) != 3 || (os.Args[2] != "present" && os.Args[2] != "absent") {
+		fmt.Fprintln(os.Stderr, "usage: lambda-image-agent-tag <initramfs.cpio.gz> present|absent")
+		os.Exit(2)
+	}
+	if err := run(os.Args[1], os.Args[2] == "present"); err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[1], err)
+		os.Exit(1)
+	}
+}
+
+func run(path string, want bool) error {
+	fi, err := os.Lstat(path)
+	if err != nil {
+		return err
+	}
+	if !fi.Mode().IsRegular() {
+		return errors.New("not a regular file")
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	agent, err := archivedAgent(f)
+	if err != nil {
+		return err
+	}
+	return checkTag(agent, want)
+}
+
+// archivedAgent returns the single agent entry after checking listed names,
+// entry types, the agent's link count and zero-only padding after the trailer.
+// It is not a full kernel-equivalent parser; 070702 CRCs, for one, are unchecked.
+func archivedAgent(r io.Reader) ([]byte, error) {
+	zr, err := gzip.NewReader(bufio.NewReader(r))
+	if err != nil {
+		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
+	}
+	raw, err := io.ReadAll(io.LimitReader(zr, maxArchive+1))
+	if err != nil {
+		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
+	}
+	if len(raw) > maxArchive {
+		return nil, fmt.Errorf("archive larger than %d bytes", maxArchive)
+	}
+
+	seen := map[string]bool{}
+	var agent []byte
+	off := 0
+	for {
+		e, next, err := parseEntry(raw, off)
+		if err != nil {
+			return nil, fmt.Errorf("entry at offset %d: %w", off, err)
+		}
+		off = next
+		if e.name == trailerName {
+			break
+		}
+		if !canonical(e.name) {
+			return nil, fmt.Errorf("entry name %q is not canonical", e.name)
+		}
+		if seen[e.name] {
+			return nil, fmt.Errorf("entry %q appears more than once", e.name)
+		}
+		seen[e.name] = true
+		for _, p := range agentParents {
+			if e.name == p && e.mode&modeTypeMask != modeDir {
+				return nil, fmt.Errorf("%s is not a directory", p)
+			}
+		}
+		if e.name == agentPath {
+			for _, p := range agentParents {
+				if !seen[p] {
+					return nil, fmt.Errorf("%s has no %s directory entry before it", agentPath, p)
+				}
+			}
+			if e.mode&modeTypeMask != modeRegular {
+				return nil, fmt.Errorf("%s is not a regular file", agentPath)
+			}
+			if e.nlink != 1 {
+				return nil, fmt.Errorf("%s has %d links, want 1", agentPath, e.nlink)
+			}
+			if len(e.data) > maxAgent {
+				return nil, fmt.Errorf("%s is larger than %d bytes", agentPath, maxAgent)
+			}
+			agent = e.data
+		}
+	}
+	if i := bytes.IndexFunc(raw[off:], func(r rune) bool { return r != 0 }); i >= 0 {
+		return nil, fmt.Errorf("data after the archive trailer at offset %d", off+i)
+	}
+	if agent == nil {
+		return nil, fmt.Errorf("no %s entry", agentPath)
+	}
+	return agent, nil
+}
+
+type entry struct {
+	name  string
+	mode  uint64
+	nlink uint64
+	data  []byte
+}
+
+func parseEntry(raw []byte, off int) (entry, int, error) {
+	if len(raw)-off < newcHeader {
+		return entry{}, 0, errors.New("truncated header")
+	}
+	h := raw[off : off+newcHeader]
+	if m := string(h[:6]); m != "070701" && m != "070702" {
+		return entry{}, 0, fmt.Errorf("magic %q is not newc", m)
+	}
+	field := func(i int) (uint64, error) {
+		return strconv.ParseUint(string(h[6+8*i:14+8*i]), 16, 32)
+	}
+	mode, err1 := field(1)
+	nlink, err2 := field(4)
+	size, err3 := field(6)
+	nameSize, err4 := field(11)
+	if err := errors.Join(err1, err2, err3, err4); err != nil {
+		return entry{}, 0, fmt.Errorf("bad header field: %w", err)
+	}
+	if nameSize < 2 || nameSize > maxName {
+		return entry{}, 0, fmt.Errorf("name size %d out of range", nameSize)
+	}
+	nameEnd := off + newcHeader + int(nameSize)
+	dataOff := align4(nameEnd)
+	dataEnd := dataOff + int(size)
+	if dataEnd > len(raw) || align4(dataEnd) > len(raw) {
+		return entry{}, 0, errors.New("truncated entry")
+	}
+	name := raw[off+newcHeader : nameEnd]
+	if name[len(name)-1] != 0 || bytes.IndexByte(name[:len(name)-1], 0) >= 0 {
+		return entry{}, 0, errors.New("name is not one NUL-terminated string")
+	}
+	return entry{name: string(name[:len(name)-1]), mode: mode, nlink: nlink, data: raw[dataOff:dataEnd]}, align4(dataEnd), nil
+}
+
+func align4(n int) int { return (n + 3) &^ 3 }
+
+// canonical accepts "." and relative names with no empty, "." or ".." component.
+func canonical(name string) bool {
+	if name == "." {
+		return true
+	}
+	if name == "" || strings.HasPrefix(name, "/") {
+		return false
+	}
+	for c := range strings.SplitSeq(name, "/") {
+		if c == "" || c == "." || c == ".." {
+			return false
+		}
+	}
+	return true
+}
+
+func checkTag(agent []byte, want bool) error {
+	info, err := buildinfo.Read(bytes.NewReader(agent))
+	if err != nil {
+		return fmt.Errorf("no Go build information in %s: %w", agentPath, err)
+	}
+	return decide(info, want)
+}
+
+// decide refuses build information with no recorded settings, since their
+// absence does not show the tag was absent.
+func decide(info *buildinfo.BuildInfo, want bool) error {
+	if len(info.Settings) == 0 {
+		return fmt.Errorf("%s records no build settings", agentPath)
+	}
+	if hasTag(info, "lambdatestlink") != want {
+		return fmt.Errorf("%s: lambdatestlink build tag present=%v, want %v", agentPath, !want, want)
+	}
+	return nil
+}
+
+func hasTag(info *buildinfo.BuildInfo, tag string) bool {
+	for _, s := range info.Settings {
+		if s.Key != "-tags" {
+			continue
+		}
+		if slices.Contains(strings.Split(s.Value, ","), tag) {
+			return true
+		}
+	}
+	return false
+}
@@ -0,0 +1,154 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"debug/buildinfo"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type fileEntry struct {
+	name  string
+	mode  uint32
+	nlink uint32
+	data  string
+}
+
+func dir(name string) fileEntry { return fileEntry{name: name, mode: modeDir | 0o755, nlink: 2} }
+func file(name, data string) fileEntry {
+	return fileEntry{name: name, mode: modeRegular | 0o755, nlink: 1, data: data}
+}
+func symlink(name, target string) fileEntry {
+	return fileEntry{name: name, mode: 0o120777, nlink: 1, data: target}
+}
+
+// newc writes entries and a trailer in the format the kernel unpacks.
+func newc(entries ...fileEntry) []byte {
+	var b bytes.Buffer
+	for i, e := range append(entries, fileEntry{name: trailerName, nlink: 1}) {
+		fmt.Fprintf(&b, "070701%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X",
+			i+1, e.mode, 0, 0, e.nlink, 0, len(e.data), 0, 0, 0, 0, len(e.name)+1, 0)
+		b.WriteString(e.name)
+		b.WriteByte(0)
+		pad(&b)
+		b.WriteString(e.data)
+		pad(&b)
+	}
+	return b.Bytes()
+}
+
+func pad(b *bytes.Buffer) {
+	for b.Len()%4 != 0 {
+		b.WriteByte(0)
+	}
+}
+
+func gz(parts ...[]byte) []byte {
+	var b bytes.Buffer
+	for _, p := range parts {
+		w := gzip.NewWriter(&b)
+		_, _ = w.Write(p)
+		_ = w.Close()
+	}
+	return b.Bytes()
+}
+
+func withAgent(agent fileEntry) []byte {
+	return newc(dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent, file("init", "#!/bin/sh"))
+}
+
+func TestArchivedAgent_ReturnsTheOneArchivedAgent(t *testing.T) {
+	got, err := archivedAgent(bytes.NewReader(gz(withAgent(file(agentPath, "agent-bytes")))))
+	require.NoError(t, err)
+	require.Equal(t, "agent-bytes", string(got))
+
+	padded := append(withAgent(file(agentPath, "agent-bytes")), make([]byte, 1024)...)
+	got, err = archivedAgent(bytes.NewReader(gz(padded)))
+	require.NoError(t, err, "zero padding after the trailer")
+	require.Equal(t, "agent-bytes", string(got))
+}
+
+func TestArchivedAgent_Refuses(t *testing.T) {
+	parents := []fileEntry{dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin")}
+	agent := file(agentPath, "untagged")
+	small := newc(append(parents, agent)...)
+	second := newc(append(parents[1:], file(agentPath, "tagged"))...)
+	// The second archive starts inside what would be the first one's final block.
+	sameBlock := append(append([]byte{}, small...), second...)
+	for len(sameBlock)%512 != 0 {
+		sameBlock = append(sameBlock, 0)
+	}
+
+	for name, archive := range map[string][]byte{
+		"aliased duplicate":            gz(newc(append(parents, agent, file("././"+agentPath, "tagged"))...)),
+		"leading slash alias":          gz(newc(append(parents, agent, file("/"+agentPath, "tagged"))...)),
+		"dot-dot component":            gz(newc(append(parents, agent, file("usr/local/../local/bin/lambda-agent", "x"))...)),
+		"empty component":              gz(newc(append(parents, agent, file("usr//local/bin/lambda-agent", "x"))...)),
+		"exact duplicate":              gz(newc(append(parents, agent, file(agentPath, "tagged"))...)),
+		"duplicate directory":          gz(newc(dir("usr"), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent)),
+		"missing directory entry":      gz(newc(dir("usr"), dir("usr/local/bin"), agent)),
+		"directory after the agent":    gz(newc(dir("usr"), dir("usr/local"), agent, dir("usr/local/bin"))),
+		"parent is a symlink":          gz(newc(dir("usr"), dir("usr/local"), symlink("usr/local/bin", "elsewhere"), agent)),
+		"agent is a symlink":           gz(newc(append(parents, symlink(agentPath, "/bin/sh"))...)),
+		"hard-linked agent":            gz(newc(append(parents, fileEntry{name: agentPath, mode: modeRegular | 0o755, nlink: 2, data: "x"})...)),
+		"no agent":                     gz(newc(parents...)),
+		"second archive appended":      gz(small, second),
+		"second archive in same block": gz(sameBlock),
+		"non-zero byte after trailer":  gz(append(append([]byte{}, small...), 0, 0, 1)),
+		"truncated archive":            gz(small[:len(small)-20]),
+		"not newc":                     gz([]byte("not cpio at all, but long enough to fill one newc header: " + string(make([]byte, 120)))),
+		"not gzip":                     small,
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, err := archivedAgent(bytes.NewReader(archive))
+			require.Error(t, err)
+		})
+	}
+}
+
+func TestRun_RefusesASymlinkedImagePath(t *testing.T) {
+	d := t.TempDir()
+	img := filepath.Join(d, "initramfs.cpio.gz")
+	require.NoError(t, os.WriteFile(img, gz(withAgent(file(agentPath, "x"))), 0o644))
+	require.NoError(t, os.Symlink(img, filepath.Join(d, "link")))
+	require.ErrorContains(t, run(filepath.Join(d, "link"), false), "not a regular file")
+}
+
+func TestHasTag_MatchesWholeTagsOnly(t *testing.T) {
+	info := func(tags string) *buildinfo.BuildInfo {
+		return &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}, {Key: "-tags", Value: tags}}}
+	}
+	require.True(t, hasTag(info("lambdatestlink,fips140v1.0"), "lambdatestlink"))
+	require.False(t, hasTag(info("fips140v1.0"), "lambdatestlink"))
+	require.False(t, hasTag(info("lambdatestlinkx,xlambdatestlink"), "lambdatestlink"))
+	require.False(t, hasTag(&buildinfo.BuildInfo{}, "lambdatestlink"))
+}
+
+// The test binary carries real build information, so checkTag reads it and
+// refuses the opposite answer; bytes with no build information are refused.
+func TestCheckTag_ReadsRealBuildInformation(t *testing.T) {
+	exe, err := os.Executable()
+	require.NoError(t, err)
+	raw, err := os.ReadFile(exe)
+	require.NoError(t, err)
+	info, err := buildinfo.Read(bytes.NewReader(raw))
+	require.NoError(t, err)
+	tagged := hasTag(info, "lambdatestlink")
+	require.NoError(t, checkTag(raw, tagged))
+	require.Error(t, checkTag(raw, !tagged))
+	require.Error(t, checkTag([]byte("not a Go binary"), false))
+}
+
+func TestDecide_RefusesMissingBuildSettings(t *testing.T) {
+	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, false), "no build settings")
+	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, true), "no build settings")
+	untagged := &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}}}
+	require.NoError(t, decide(untagged, false))
+	require.Error(t, decide(untagged, true))
+}
@@ -0,0 +1,26 @@
+//go:build lambdatestlink
+
+// Command lambda-image-digest prints the test-link runtime image digest of one
+// directory, selecting the boot files the same way the daemon does. The digest
+// identifies the bytes; it does not authenticate where they came from.
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Fprintln(os.Stderr, "usage: lambda-image-digest <image-dir>")
+		os.Exit(2)
+	}
+	digest, err := testlink.DirDigest(os.Args[1])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	fmt.Println(digest)
+}
@@ -110,7 +110,7 @@ Three entry points, in the order they run:

 ### `iam` — everything IAM and STS

-Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
+Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, a user rename needing `iam:UpdateUser` on both the old and the new name, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.

 ### `cert` — the cluster CA and TLS identity

@@ -3,24 +3,28 @@
 # + initramfs: a guest running the Lambda guest agent, with virtio-net for its
 # host control connection. It is not shipped.
 #
+# Usage: build-lambda-runtime-microvm-image.sh [agent-binary [output-dir]]
+#
 # Inputs:
 #   $REPO_ROOT/build/microvm/lambda-runtime/{init.sh,inittab,run-agent}
-#   $REPO_ROOT/bin/lambda-agent  (make bin/lambda-agent)
+#   agent binary, default $REPO_ROOT/bin/lambda-agent (make bin/lambda-agent)
 #
-# Outputs:
-#   $REPO_ROOT/build/microvm/lambda-runtime/vmlinuz
-#   $REPO_ROOT/build/microvm/lambda-runtime/initramfs.cpio.gz
+# Outputs, in output-dir (default $REPO_ROOT/build/microvm/lambda-runtime):
+#   vmlinuz
+#   initramfs.cpio.gz
 #
 # Rootfs strategy and module stripping: scripts/lib/microvm-image.sh.
 # shellcheck disable=SC2034 # variables are consumed by the sourced library
 set -euo pipefail

 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-BUILD_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
-INIT_SH="$BUILD_DIR/init.sh"
-INITTAB="$BUILD_DIR/inittab"
-RUN_AGENT="$BUILD_DIR/run-agent"
-LAMBDA_AGENT_BIN="$REPO_ROOT/bin/lambda-agent"
+INPUT_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
+INIT_SH="$INPUT_DIR/init.sh"
+INITTAB="$INPUT_DIR/inittab"
+RUN_AGENT="$INPUT_DIR/run-agent"
+LAMBDA_AGENT_BIN="${1:-$REPO_ROOT/bin/lambda-agent}"
+# The library stages beside BUILD_DIR, so it names the output, not the inputs.
+BUILD_DIR="${2:-$INPUT_DIR}"
 TAG="[build-lambda-runtime-microvm-image]"

 ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
@@ -38,7 +42,8 @@ MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
 source "$REPO_ROOT/scripts/lib/microvm-image.sh"

 echo "$TAG repo root: $REPO_ROOT"
-echo "$TAG build dir: $BUILD_DIR"
+echo "$TAG input dir: $INPUT_DIR"
+echo "$TAG output dir: $BUILD_DIR"

 microvm_check_tools

@@ -57,6 +62,9 @@ if ! file -b "$LAMBDA_AGENT_BIN" | grep -q "statically linked"; then
    exit 1
 fi

+# Absolute, because packing the initramfs runs from inside the chroot.
+mkdir -p "$BUILD_DIR"
+BUILD_DIR=$(cd "$BUILD_DIR" && pwd)
 microvm_init_dirs
 microvm_make_rootfs
 microvm_empty_dev
@@ -0,0 +1,81 @@
+#!/bin/sh
+# build-lambda-runtime-microvm-image_test.sh — a build into a chosen output
+# directory stages and publishes there, and leaves the default input directory
+# untouched.
+#
+# The script runs in a fixture repo against the real library, with only the
+# rootfs, kernel and packing steps replaced so no image is downloaded.
+set -eu
+
+SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- build-lambda-runtime-microvm-image.sh output directory"
+
+REPO="$TMP/repo"
+IN="$REPO/build/microvm/lambda-runtime"
+mkdir -p "$REPO/scripts/lib" "$IN" "$REPO/bin" "$TMP/bin"
+cp "$SCRIPTS_DIR/build-lambda-runtime-microvm-image.sh" "$REPO/scripts/"
+for f in init.sh inittab run-agent; do printf '%s\n' "$f" > "$IN/$f"; done
+printf 'default image\n' > "$IN/vmlinuz"
+printf '#!/bin/sh\n' > "$REPO/bin/agent-testlink"
+chmod +x "$REPO/bin/agent-testlink"
+printf '#!/bin/sh\necho "ELF 64-bit, statically linked"\n' > "$TMP/bin/file"
+chmod +x "$TMP/bin/file"
+BEFORE=$(cd "$IN" && ls -A | cksum)
+
+# The real library, with the steps that need a rootfs or kernel replaced.
+# Packing runs from inside the chroot as the real step does, and records where
+# the stage is while it still exists.
+cat > "$REPO/scripts/lib/microvm-image.sh" <<EOF
+. "$SCRIPTS_DIR/lib/microvm-image.sh"
+microvm_check_tools() { :; }
+microvm_make_rootfs() { mkdir -p "\$CHROOT_DIR/etc"; }
+microvm_empty_dev() { :; }
+microvm_locate_kernel() { :; }
+microvm_strip_modules() { :; }
+microvm_strip_rootfs() { :; }
+microvm_stage_kernel() { printf 'kernel\n' > "\$STAGE_DIR/vmlinuz"; }
+microvm_pack_initramfs() {
+    (cd "\$CHROOT_DIR" && printf 'initramfs\n' | gzip > "\$STAGE_DIR/initramfs.cpio.gz")
+    dirname "\$STAGE_DIR" > "$TMP/stage-parent"
+    ls -A "$IN" > "$TMP/input-during"
+}
+EOF
+
+# The output is given relative to the repo root, as make passes it.
+OUT="$REPO/build/microvm/lambda-testlink-runtime"
+if ! (cd "$REPO" && PATH="$TMP/bin:$PATH" bash scripts/build-lambda-runtime-microvm-image.sh bin/agent-testlink build/microvm/lambda-testlink-runtime) > "$TMP/log" 2>&1; then
+    fail "build into output dir: $(tr '\n' ' ' < "$TMP/log")"
+else
+    if [ "$(cat "$TMP/stage-parent")" = "$OUT" ]; then
+        pass "stage created in the output directory"
+    else
+        fail "stage created in $(cat "$TMP/stage-parent"), want $OUT"
+    fi
+    if grep -q '^\.stage\.' "$TMP/input-during"; then
+        fail "stage present in the default input directory during the build"
+    else
+        pass "no stage in the default input directory during the build"
+    fi
+    if [ "$(cd "$IN" && ls -A | cksum)" = "$BEFORE" ] && grep -qx 'default image' "$IN/vmlinuz"; then
+        pass "default input directory unchanged"
+    else
+        fail "default input directory changed: $(cd "$IN" && ls -A | tr '\n' ' ')"
+    fi
+    if grep -qx kernel "$OUT/vmlinuz" && gzip -t "$OUT/initramfs.cpio.gz" && [ "$(ls -A "$OUT" | wc -l)" -eq 2 ]; then
+        pass "output directory holds exactly the published pair"
+    else
+        fail "output directory: $(ls -A "$OUT" | tr '\n' ' ')"
+    fi
+fi
+
+if [ "$fails" -ne 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi
@@ -260,6 +260,14 @@ out() {
    ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "$@" 2>/dev/null || true
 }

+# remote_join_cmd prints the command that joins one node. The token is not in it: it arrives on
+# stdin and is read after sudo has logged its command, so sudo's log never holds it. The other
+# values are positional arguments of a literal script, never spliced into it.
+remote_join_cmd() {
+    local script='exec spx admin join --force --node "$1" --bind "$2" --cluster-bind "$2" --advertise "$3" --host "$4" --token "$(cat)" --region "$5" --az "$6"'
+    printf 'sudo bash -c %q _ %q %q %q %q %q %q' "$script" "$@"
+}
+
 SETUP_OVN="/usr/local/share/spinifex/setup-ovn.sh"

 # --- Preflight -------------------------------------------------------------
@@ -652,14 +660,18 @@ fi
 [ -n "$EXT_IFACE" ] && init_args+=(--external-iface="$EXT_IFACE")
 [ -n "$EMAIL" ] && init_args+=(--email="$EMAIL")

-INIT_LOG=/tmp/spx-init.log
+# init prints the join token, so its log is root-only and outside the world-writable /tmp.
+INIT_LOG=/var/log/spx-init.log
+
+# redact_tokens hides join tokens in text about to be shown to the operator.
+redact_tokens() { sed -E 's/spx_join_[A-Za-z0-9_-]+/spx_join_[redacted]/g'; }

 log "initializing ${HOSTS[0]} as ${NODE_NAMES[0]} (waiting for $((N - 1)) node(s) to join)"

 # init blocks in a formation server until every node has joined, so it runs
 # detached and the token is read back out of its output.
 on "${HOSTS[0]}" "sudo rm -f $INIT_LOG; \
-    sudo setsid sh -c 'spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
+    sudo setsid sh -c 'umask 077; spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
    exit 0" || fail "${HOSTS[0]}: could not start spx admin init"

 TOKEN=""
@@ -675,7 +687,7 @@ else
        elapsed=$((elapsed + 5))
    done
    if [ -z "$TOKEN" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
        fail "${HOSTS[0]}: spx admin init produced no join token within ${TOKEN_TIMEOUT}s"
    fi
    log "  join token issued"
@@ -691,11 +703,9 @@ fi
 # waiting on a shell that node 2 is still holding.
 JOIN_PIDS=()
 for i in $(seq 1 $((N - 1))); do
-    on "${HOSTS[$i]}" "sudo spx admin join --force \
-        --node ${NODE_NAMES[$i]} \
-        --bind ${LAN_IPS[$i]} --cluster-bind ${LAN_IPS[$i]} --advertise ${WAN_IPS[$i]} \
-        --host ${LAN_IPS[0]}:$PORT --token $TOKEN \
-        --region $REGION --az $AZ" &
+    # The token goes to the node on stdin, so it is in no ssh argument and not in sudo's log.
+    # spx admin join takes it only as --token, so it is still in that process's arguments.
+    on "${HOSTS[$i]}" "$(remote_join_cmd "${NODE_NAMES[$i]}" "${LAN_IPS[$i]}" "${WAN_IPS[$i]}" "${LAN_IPS[0]}:$PORT" "$REGION" "$AZ")" <<<"$TOKEN" &
    JOIN_PIDS+=("$!")
 done

@@ -724,7 +734,7 @@ if ! $DRY_RUN; then
        elapsed=$((elapsed + 5))
    done
    if [ "$elapsed" -ge "$FORMATION_TIMEOUT" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
        fail "${HOSTS[0]}: formation did not complete within ${FORMATION_TIMEOUT}s"
    fi
    log "  formation complete"
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Tests remote_join_cmd from install-node.sh: the join token must reach spx only on stdin, so
+# sudo's log (which holds the command it was given) never contains it, and the other values must
+# arrive as plain arguments even when they hold shell syntax.
+# preflight runs this with sh
+[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
+set -euo pipefail
+here="$(cd "$(dirname "$0")" && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+mkdir "$work/bin"
+cd "$work"
+
+cat > "$work/bin/sudo" <<'SH'
+#!/bin/sh
+echo "$*" >> "$FAKE_SUDO_LOG"
+exec "$@"
+SH
+cat > "$work/bin/spx" <<'SH'
+#!/bin/sh
+for a in "$@"; do printf '<%s>' "$a"; done > "$FAKE_SPX_ARGV"
+SH
+chmod +x "$work/bin/sudo" "$work/bin/spx"
+export FAKE_SUDO_LOG="$work/sudo.log" FAKE_SPX_ARGV="$work/spx.argv" PATH="$work/bin:$PATH"
+
+eval "$(sed -n '/^remote_join_cmd()/,/^}/p' "$here/install-node.sh")"
+
+fails=0
+check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
+
+TOKEN=spx_join_FAKETOKEN-abc_1
+cmd="$(remote_join_cmd node2 10.0.0.2 203.0.113.2 '10.0.0.1:4432;touch PWNED' us-east-1 'az $(touch PWNED2)')"
+bash -c "$cmd" <<<"$TOKEN"
+
+check "the token is not in the command" "$(printf '%s' "$cmd" | grep -c FAKETOKEN || true)" 0
+check "the token is not in sudo's log" "$(grep -c FAKETOKEN "$FAKE_SUDO_LOG" || true)" 0
+check "spx got the token" "$(grep -o '<--token><[^>]*>' "$FAKE_SPX_ARGV")" "<--token><$TOKEN>"
+check "spx got the other values as given" "$(cat "$FAKE_SPX_ARGV")" "<admin><join><--force><--node><node2><--bind><10.0.0.2><--cluster-bind><10.0.0.2><--advertise><203.0.113.2><--host><10.0.0.1:4432;touch PWNED><--token><$TOKEN><--region><us-east-1><--az><az \$(touch PWNED2)>"
+check "no value was run as a command" "$(find "$work" -maxdepth 1 -name 'PWNED*' | wc -l | tr -d ' ')" 0
+
+[ "$fails" -eq 0 ]
@@ -0,0 +1,111 @@
+#!/bin/sh
+# lambda-testlink-build_test.sh — the test-link make targets refuse a malformed
+# image digest and an unsupported host or GOARCH before touching any output.
+#
+# Every case is refused on the recipe's first lines, so nothing is compiled.
+set -eu
+
+LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+REPO_ROOT=$(dirname -- "$(dirname -- "$LIB_DIR")")
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- make build-lambda-testlink / build-lambda-testlink-image guards"
+
+PWN="$TMP/pwn"
+OUT="$TMP/out"
+HEX=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+mkdir -p "$OUT"
+printf 'old daemon\n' > "$OUT/spx"
+printf 'old initializer\n' > "$OUT/hardkill.so"
+PAIR=$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)
+
+# unchanged <name> — the existing pair is intact, no scratch was made and no
+# injected command ran.
+unchanged() {
+    if [ -e "$PWN" ]; then
+        fail "$1: injected command ran"
+        rm -f "$PWN"
+    elif [ -e "$OUT.tmp" ]; then
+        fail "$1: scratch directory created"
+    elif [ "$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)" != "$PAIR" ]; then
+        fail "$1: existing pair replaced"
+    else
+        pass "$1"
+    fi
+}
+
+# refused <name> <want> <log> <status> — make failed with the expected message.
+refused() {
+    if [ "$4" -eq 0 ]; then
+        fail "$1: make succeeded"
+    elif ! grep -q "$2" "$3"; then
+        fail "$1: wrong refusal: $(tr '\n' ' ' < "$3")"
+    else
+        unchanged "$1"
+    fi
+}
+
+testlink() {
+    make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" "$@" > "$TMP/log" 2>&1
+}
+
+DIGEST_MSG="LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"
+NL='
+'
+for v in '' "\$(shell touch $PWN)" "\$\$(touch $PWN)" "\`touch $PWN\`" "$HEX; touch $PWN" \
+    "$HEX\" -X main.x=\"y" "$HEX -X main.x=y" "$HEX${NL}touch $PWN" \
+    0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF "${HEX%?}" "${HEX}0"; do
+    # GNU make itself expands a command-line $(shell ...) while parsing, for
+    # any target, so that one literal is tested only from the environment.
+    case "$v" in
+    "\$(shell"*) ;;
+    *)
+        status=0
+        testlink LAMBDA_IMAGE_DIGEST="$v" || status=$?
+        refused "command-line digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
+        ;;
+    esac
+    status=0
+    LAMBDA_IMAGE_DIGEST="$v" testlink || status=$?
+    refused "environment digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
+done
+
+status=0
+env -u LAMBDA_IMAGE_DIGEST make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" > "$TMP/log" 2>&1 || status=$?
+refused "digest unset" "$DIGEST_MSG" "$TMP/log" "$status"
+
+ARCH_MSG="supports only an x86_64 host building linux/amd64"
+SHIM="$TMP/shim"
+mkdir -p "$SHIM"
+printf '#!/bin/sh\necho aarch64\n' > "$SHIM/uname"
+chmod +x "$SHIM/uname"
+
+status=0
+GOARCH=arm64 testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
+refused "GOARCH=arm64" "$ARCH_MSG" "$TMP/log" "$status"
+status=0
+PATH="$SHIM:$PATH" testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
+refused "aarch64 host" "$ARCH_MSG" "$TMP/log" "$status"
+
+# The image target refuses before building qboot or any agent binary.
+for c in "GOARCH=arm64" "PATH=$SHIM:$PATH"; do
+    status=0
+    env "$c" make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink-image > "$TMP/log" 2>&1 || status=$?
+    if [ "$status" -eq 0 ]; then
+        fail "image target [${c%%=*}]: make succeeded"
+    elif ! grep -q "$ARCH_MSG" "$TMP/log" || grep -q "qboot\|go build" "$TMP/log"; then
+        fail "image target [${c%%=*}]: did not refuse first: $(tr '\n' ' ' < "$TMP/log")"
+    else
+        pass "image target [${c%%=*}] refused first"
+    fi
+done
+
+if [ "$fails" -ne 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi
@@ -0,0 +1,10 @@
+.terraform/
+terraform.tfstate
+terraform.tfstate.*
+*.tfplan
+*.auto.tfvars
+crash.log
+terraform.tfvars
+*.pem
+*.log
+.terraform.lock.hcl
@@ -0,0 +1,137 @@
+# latitude-spx
+
+Builds a Spinifex deployment on latitude.sh bare metal from your laptop with `deploy-latitude.sh`.
+The commands, flags, step markers and state layout follow `vultr-spx`.
+
+**Status: not yet run end to end.**
+The script was written from the plan doc findings (`docs/development/feature/latitude-bm.md`, L1 to L26) and from the provider schema.
+No latitude.sh API call or host run was made while building it, so treat the first `up` as the test.
+One node has been the only shape run live, and only in part (see Unverified). The three-node flow, including the floating pool, has not been run: Ben takes that over.
+
+## What it creates
+
+| Resource | Notes |
+|---|---|
+| Project | `spinifex-<name>`, unless `--project` names an existing one (then it is never deleted) |
+| SSH key | `spinifex-<name>`, account scoped |
+| User data | `spinifex-<name>`, host preparation only (no `{{ }}`, latitude.sh substitutes them) |
+| Servers | `m4-metal-medium`, site `DAL`, Ubuntu 26.04, hourly billing, hostnames `spx-<name>-nodeN` |
+| Elastic IP | one by default (`elastic_ip_count` in `variables.tf`, 1 to 5; set `TF_VAR_elastic_ip_count` to change it). With one node it is a static pool address and the count must stay 1, because a second pool is never tried when the first is full; with three nodes the addresses form the `latitude` floating pool, which Spinifex moves between servers. `nginx-webserver` needs one address and runs on one node; `nginx-alb` needs two public addresses and is a three-node workbook |
+| Virtual network | three nodes only: `spinifex-<name>`, one VLAN assignment per server |
+
+## Requirements
+
+OpenTofu (or Terraform) 1.5 or later, `aws` CLI, `python3`, `curl`, `ssh`, an SSH key pair.
+
+| Variable | When | Meaning |
+|---|---|---|
+| `LATITUDESH_AUTH_TOKEN` | always | the account API token; used by the provider and this script only, never sent to a node |
+| `LATITUDESH_NODE_TOKEN` | `--nodes 3` | a second, different, writable API key, copied to each node (`/etc/spinifex/latitude/api-token`, owned by `spinifex-daemon` with mode 0400, in a root-owned 0711 directory; the other services share the `spinifex` group, so no group may read it; ownership and mode are set on every `up`; a symlink or any other owner there is refused) so the daemon can move Elastic IPs |
+
+The script refuses a node key it can see is read-only (it lists `/auth/api_keys` and matches the key's last five characters). If the key is not among the account's keys, write authority is first proven by the first Elastic IP move.
+
+Exposure: a latitude.sh API key is team-wide. It is not scoped to Elastic IPs, so anyone who reads it from a node can do whatever the key's owner can in the team (the API exposes `read_only` and `allowed_ips` on a key, and no narrower scope). Create the node key with `allowed_ips` set to the nodes' public addresses, and revoke it after `destroy`.
+
+Nodes are reached as `ubuntu` with the deployment's SSH key, and remote commands run through passwordless `sudo`: latitude.sh puts the key on the image's login user and root login is not enabled.
+
+## Usage
+
+```
+export LATITUDESH_AUTH_TOKEN=...        # from the latitude.sh dashboard
+./deploy-latitude.sh up --name demo     # one node
+
+export LATITUDESH_NODE_TOKEN=...        # a second, writable key
+./deploy-latitude.sh up --name demo3 --nodes 3
+./deploy-latitude.sh status --name demo
+./deploy-latitude.sh ssh --name demo node1 'systemctl status spinifex.target'
+./deploy-latitude.sh credentials --name demo
+./deploy-latitude.sh stop node1 --name demo
+./deploy-latitude.sh start node1 --name demo
+./deploy-latitude.sh destroy --name demo
+```
+
+Options: `--name`, `--site` (alias `--region`), `--plan`, `--os`, `--project ID|SLUG`, `--distro TARBALL`, `--channel`, `--spx-region`, `--spx-az`, `--ssh-key`, `--image`, `--data-disk-id`, `--vpc-subnet` (the VLAN's /24, three nodes), `--nodes 1|3`, `--yes`.
+`./deploy-latitude.sh --help` lists them with defaults.
+
+Three nodes need `--distro` with a tarball built from this branch (`make distro-amd64`) until a release carries the latitude provider.
+A released spx rejects `source="latitude"` and its daemon restarts in a loop, so `up --nodes 3` without `--distro` stops in preflight.
+Before the pool is written, each node's `/usr/local/bin/spx` is checked for the `latitude_elastic_ip_ids` key name; spx has no validate subcommand, so this is a string search of the binary, not a run of its config check.
+
+## What `up` does
+
+`terraform` then, per node: `host` (wait for SSH as `ubuntu` and cloud-init, `/dev/kvm`), `disk`, `vlan` (three nodes), `install`, `form`, `start`; then `pool` (one node) or `cluster` (three nodes: `install-node.sh` forms the cluster); three nodes then install the node token on every `up` and add the `latitude` pool (`pool`); then `restart` (three nodes wait for the current daemon to log "floating pool ready" for the pool), `public`, `image` and the AWS profile.
+Each step writes `step.<name>[.nodeN]` in the state directory, and a rerun resumes at the first step without one.
+
+- **Data disk.** `host-data-disk.sh` picks the blank NVMe by content and by-id EUI, never by `nvmeXn1` (L1, L11).
+A candidate is at least 100 GB with no signature, partitions, holders or mounts, and not in the root filesystem's ancestry; with none it stops and lists the disks.
+With several (LON2 has two 1.7 TB and one spare 447 GB blank NVMe next to the root disk) it takes the largest, the lowest by-id path among equals, prints what it chose and the blank disks it left, and never touches the others (`--data-disk-id` chooses another).
+It mounts by UUID with `nofail` and the script records the UUID in `data_uuid.<node>`.
+- **Mount check.** Before anything writes under `/var/lib/spinifex`, and on every run, the mounted filesystem UUID must equal the recorded one.
+The units that write there also get a `RequiresMountsFor=/var/lib/spinifex` drop-in.
+The mount is by UUID in fstab, so renamed disks do not change it, and there is no check at service start.
+- **Firewall.** `setup.sh --firewall on` runs before Spinifex first starts.
+- **One node.** The Elastic IP is a static pool (`gateway_ip = "192.0.2.1"`, the node's next hop as `gateway`, no `prefix_len`) with the IMDS addresses moved off the ones latitude.sh routes.
+- **Three nodes.** The `vlan` step makes `eno2.<vid>` with `ip link` and persists it as systemd-networkd files for that link plus a drop-in on the `eno2` unit.
+It never runs `netplan apply`, refuses a subnet that overlaps the SSH peer or a route, and removes the VLAN again if the public interface's address, default route or the route to the SSH peer differ afterwards.
+`form_cluster` then puts the cluster and Geneve planes on that VLAN with `install-node.sh` (`--user ubuntu`); its output is not kept, as it can hold the join token.
+- **Floating pool (three nodes).** The `latitude` pool names the Elastic IP IDs and the project ID from the Terraform outputs, and each node's `provider_host_id` is its server ID. `spinifex.toml` is edited idempotently and read back. Failover moves an address with one `PATCH /elastic_ips/{id}` and waits for it to be active; latitude.sh has no unassign, so a node that loses an address is not told to release it.
+- **State** is in `~/.spinifex/latitude/<name>/`: Terraform state, `ids` (every `latitudesh_*` object in the state, merged on each apply and before each destroy), `step.*` markers, logs and the data disk UUIDs.
+- **AWS profile** `spinifex-latitude-<name>` is replaced on each `up`, with the CA at `~/.spinifex/latitude-<name>-ca.pem`.
+The secret key is written to `~/.aws/credentials` (0600, other profiles kept, the old file saved beside it) and never appears in a process argument.
+
+## Token handling
+
+`LATITUDESH_AUTH_TOKEN` is read from the environment by the provider and by the script's API calls.
+It is not a Terraform variable, is on no command line (curl reads it from stdin), is not sent to a node, and is written to no file.
+`LATITUDESH_NODE_TOKEN` is checked with curl reading it from stdin and reaches each node on stdin of the install script, so it is in no argument on either side.
+`TF_LOG` is unset by the script.
+The join token that `install-node.sh` creates is shared with Vultr and OCI: its init log is root-only, and `spx admin join` still takes it as `--token` on the joining node (tracked separately).
+
+## Recovery
+
+If SSH is lost once the firewall is armed, use the serial console.
+`up` enables the server's out-of-band connection with the deployment's SSH key and prints the `ssh` command it returns before it arms the firewall, falling back to the Out-of-Band tab in the dashboard.
+Logging in there needs an OS user password, which you set yourself while SSH still works.
+A fresh deployment that cannot be recovered is cheaper to `destroy` and `up` again.
+The VLAN step prints the console command first.
+It refuses a subnet that holds the SSH peer or overlaps an existing route, and removes the VLAN again if the route to the SSH peer or the public interface changes.
+
+## destroy
+
+`destroy` asks you to type the deployment name (`--yes` skips only that), then records every ID in the Terraform state and stops without destroying anything if it cannot.
+It runs `terraform destroy`, fetches every recorded ID by itself (2xx present, 404 gone, anything else fails and keeps the records), and deletes what remains of those.
+Servers, keys, user data, virtual networks and projects whose names match the deployment but that are not recorded are listed by ID and deleted only after you type `delete-untracked` for exactly those IDs, which `--yes` never supplies; an ID that shows up later needs its own confirmation, and without a terminal they are kept and `destroy` exits 1.
+Elastic IPs are touched only by recorded ID.
+Elastic IP release is asynchronous and only works once the address is active or in error, so the sweep retries.
+Because the Elastic IP list ignores pagination, `destroy` ends by asking you to check the dashboard for leftovers and type `checked`.
+An `up` that failed while creating can leave an object nothing recorded, such as an Elastic IP; that dashboard check is the only thing that finds it.
+
+## Unverified
+
+Nothing in the three-node flow has run against real servers. For one node, only the parts the person ran have been exercised, and the script has changed since.
+
+- The OS slug `ubuntu_26_04_x64_lts` (the `latitudesh_operating_system` data source fails the plan if it is wrong).
+- That the deploy key is on `ubuntu` and `sudo -n` works for it, and that `sudo` resets nothing the script needs (the SSH peer address is read before `sudo` for that reason).
+- The power action request (`POST /servers/{id}/actions` with `power_off` and `power_on`).
+- Resolving `--project` as a slug through `GET /projects/{slug}`, and the `in_stock` shape of `GET /plans` used by the stock check, which only warns when it cannot read it.
+- The `GET` and `DELETE` paths for `/user_data/{id}` and `/virtual_networks/{id}`.
+- That the Spinifex API certificate names the node's public address, which the profile check needs.
+- That the private NIC is `eno2`, its tagging and MTU (the script uses 1500), and that the networkd files persist the VLAN across a reboot.
+- `GET /auth/api_keys` and its `token_last_slice` and `read_only` fields, which the node key check relies on; and that a key with `allowed_ips` set to the nodes' addresses still works from them.
+- The Elastic IP move itself: `PATCH /elastic_ips/{id}` with `server_id`, its 202 then `active`, the error codes, and how long a move takes; the client waits up to its move timeout and makes one write per call.
+- That the daemon reads the node token and logs "floating pool ready", then fails over when a node stops (`stop node1`) and the address lands on another node.
+- `install-node.sh` as `--user ubuntu` on latitude.sh servers.
+
+## Not exercised
+
+Any real API call from the three-node flow, `destroy`, `host-data-disk.sh` on a host and the VLAN step on a server.
+Guest launch, public reachability of the Elastic IP, the Terraform workbooks and failover are not part of `up`; prove them by hand.
+`e2e-latitude.sh` and removing the old `install-node.sh` in this directory are not done.
+
+## Tests
+
+`sh node-scripts_test.sh` checks the node token install (install, unchanged, rotation, modes, an older install converging, symlinks and owners refused; run it as root too, with `sudo sh node-scripts_test.sh`, to check that only the daemon's user can read the file) and the readiness check (old success then current failure) with stand-in tools.
+`bash deploy-argv_test.sh` checks that neither API token, nor the five characters used to find the node key, is an argument of any external command in the node token paths, and that the SSH peer address comes from the same connection after sudo's environment reset.
+`scripts/install-node_test.sh` (part of `make preflight`) checks that the join token reaches `spx admin join` only on stdin, so sudo's log never holds it.
+The first two are not part of `make preflight`.
+The Go side: `cd spinifex && GOWORK=off go test ./providers/cloud/latitude/ ./domains/network/external/floating/ ./bootstrap/config/ ./runtime/daemon/`.
@@ -0,0 +1,23 @@
+#cloud-config
+# Host preparation only. Spinifex is installed and formed over SSH by deploy-latitude.sh.
+# latitude.sh substitutes double-brace variables in user data, so this file holds none.
+write_files:
+  - path: /etc/cloud/cloud.cfg.d/99-spinifex-network.cfg
+    content: |
+      network: {config: disabled}
+  # The image's netplan lists eno2 with no address and without optional, so boot waits two minutes
+  # for it. --any returns once one link, eno1, is online; it changes the wait, not the network.
+  # UNVERIFIED: the binary path is the Ubuntu one and the drop-in first applies on the next boot.
+  - path: /etc/systemd/system/systemd-networkd-wait-online.service.d/10-spinifex-any.conf
+    content: |
+      [Service]
+      ExecStart=
+      ExecStart=/usr/lib/systemd/systemd-networkd-wait-online --any
+# runcmd carries on past a failing command, so each line stands alone.
+runcmd:
+  - [sh, -c, "command -v ufw >/dev/null && ufw --force disable || true"]
+  - [sh, -c, "systemctl disable --now unattended-upgrades.service apt-daily-upgrade.timer apt-daily.timer || true"]
+  # The image ships empty apt lists and no git, which the workbooks need.
+  - [sh, -c, "apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git || true"]
+  - [sh, -c, "systemctl daemon-reload || true"]
+  - [sh, -c, "touch /var/lib/cloud/spinifex-host-ready"]
@@ -0,0 +1,70 @@
+locals {
+  hostname_prefix = "spx-${var.deployment_name}-node"
+  project         = var.project != "" ? var.project : latitudesh_project.deploy[0].slug
+  project_id      = var.project != "" ? var.project_id : latitudesh_project.deploy[0].id
+}
+
+# Both lookups fail the plan when a slug does not exist, which is the only check of the OS slug.
+data "latitudesh_operating_system" "os" {
+  slug = var.operating_system
+}
+
+data "latitudesh_plan" "plan" {
+  slug = var.plan
+}
+
+resource "latitudesh_project" "deploy" {
+  count             = var.project == "" ? 1 : 0
+  name              = "spinifex-${var.deployment_name}"
+  description       = "Spinifex deployment ${var.deployment_name}"
+  environment       = "Development"
+  provisioning_type = "on_demand"
+}
+
+# The key is team scoped. The server login user is ubuntu, not root.
+resource "latitudesh_ssh_key" "deploy" {
+  name       = "spinifex-${var.deployment_name}"
+  public_key = trimspace(file(pathexpand(var.ssh_public_key_path)))
+}
+
+# Host preparation only. Latitude substitutes double-brace variables inside user data, so the file
+# must hold none. Spinifex is installed over SSH by deploy-latitude.sh.
+resource "latitudesh_user_data" "host" {
+  description = "spinifex-${var.deployment_name}"
+  content     = filebase64("${path.module}/cloud-init/user-data.yaml")
+}
+
+# billing must be hourly: the default is monthly, which bills upfront with no refund.
+resource "latitudesh_server" "node" {
+  count            = var.node_count
+  project          = local.project
+  hostname         = "${local.hostname_prefix}${count.index + 1}"
+  site             = var.site
+  plan             = data.latitudesh_plan.plan.slug
+  operating_system = data.latitudesh_operating_system.os.slug
+  billing          = "hourly"
+  ssh_keys         = [latitudesh_ssh_key.deploy.id]
+  user_data        = latitudesh_user_data.host.id
+
+  timeouts = {
+    create = "60m"
+  }
+}
+
+# Elastic IPs are bought here and never created or deleted by Spinifex. Each starts on a server, spread
+# over the nodes, and server_id is ignored afterwards so a move by Spinifex is not undone by a later apply.
+resource "latitudesh_elastic_ip" "pool" {
+  count = var.elastic_ip_count
+  # The provider documents ID or slug but posts the value as project_id unresolved: a slug gave 404.
+  project   = local.project_id
+  server_id = latitudesh_server.node[count.index % var.node_count].id
+
+  lifecycle {
+    ignore_changes = [server_id]
+  }
+}
+
+moved {
+  from = latitudesh_elastic_ip.node1
+  to   = latitudesh_elastic_ip.pool[0]
+}
@@ -0,0 +1,228 @@
+#!/usr/bin/env bash
+# Checks that neither API token is ever an argument of an external command in the node token
+# paths of deploy-latitude.sh. External commands are stand-ins that record their arguments and
+# their standard input; the test fails if a token is in the first and proves it is in the second.
+# preflight runs this with sh
+[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
+set -euo pipefail
+exec </dev/null
+here="$(cd "$(dirname "$0")" && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+mkdir "$work/bin" "$work/home"
+real_python="$(command -v python3)"
+
+AUTH=FAKEAUTHTOKEN9
+NODE=FAKENODETOKEN7
+
+# Every command the script runs outside bash records "name args" and, for the first line of
+# standard input, "name stdin: ...".
+for c in curl ssh scp aws tofu ssh-keygen; do
+    cat > "$work/bin/$c" <<'SH'
+#!/bin/sh
+echo "$(basename "$0") $*" >> "$FAKE_ARGV"
+if [ ! -t 0 ]; then
+    in="$(cat)"
+    [ -z "$in" ] || echo "$(basename "$0") stdin: $in" >> "$FAKE_STDIN"
+fi
+case "$(basename "$0")" in
+    curl) printf '{"data":[{"attributes":{"token_last_slice":"%s","read_only":false}}]}\n200' "$(printf '%s' "$FAKE_NODE" | tail -c 5)" ;;
+    ssh)
+        if [ -n "${FAKE_SSH_EVAL:-}" ]; then
+            # Behave like a login on a node: skip the options and host, run the rest with the
+            # connection variable of this connection, and let sudo reset the environment.
+            while [ "$#" -gt 0 ]; do case "$1" in ubuntu@*) shift; break ;; *) shift ;; esac; done
+            SSH_CONNECTION="$FAKE_SSH_CONNECTION" bash -c "$*"
+        else
+            echo installed
+        fi ;;
+esac
+SH
+    chmod +x "$work/bin/$c"
+done
+cat > "$work/bin/sudo" <<'SH'
+#!/bin/sh
+while [ "$#" -gt 0 ]; do case "$1" in -*) shift ;; *) break ;; esac; done
+exec env -i PATH="$PATH" "$@"
+SH
+chmod +x "$work/bin/sudo"
+cat > "$work/bin/python3" <<SH
+#!/bin/sh
+echo "python3 \$*" >> "\$FAKE_ARGV"
+exec "$real_python" "\$@"
+SH
+chmod +x "$work/bin/python3"
+
+export FAKE_ARGV="$work/argv" FAKE_STDIN="$work/stdin" FAKE_NODE="$NODE" PATH="$work/bin:$PATH" HOME="$work/home"
+: > "$FAKE_ARGV"
+: > "$FAKE_STDIN"
+export LATITUDESH_AUTH_TOKEN="$AUTH" LATITUDESH_NODE_TOKEN="$NODE"
+
+# shellcheck source=deploy-latitude.sh
+source "$here/deploy-latitude.sh"
+
+# shellcheck disable=SC2034 # read by the sourced functions
+NODES=3 IP=192.0.2.10 NODE_NAME=node1
+check_node_token >/dev/null
+install_key >/dev/null 2>&1
+
+fails=0
+for tok in "$AUTH" "$NODE" "${NODE: -5}"; do
+    if grep -q "$tok" "$FAKE_ARGV"; then
+        echo "FAIL token $tok is in a command's arguments:"
+        grep "$tok" "$FAKE_ARGV"
+        fails=$((fails + 1))
+    else
+        echo "ok   $tok is in no argument"
+    fi
+done
+for what in "curl stdin:.*Bearer $NODE" "ssh stdin: $NODE"; do
+    if grep -q "$what" "$FAKE_STDIN"; then
+        echo "ok   the check can see the token where it is sent ($what)"
+    else
+        echo "FAIL the token never reached stdin ($what): the test would pass without checking anything"
+        fails=$((fails + 1))
+    fi
+done
+
+# The peer address comes from the same connection that runs the command, whichever source
+# address that connection has, and survives sudo's environment reset.
+export FAKE_SSH_EVAL=1
+for src in 198.51.100.7 203.0.113.99; do
+    export FAKE_SSH_CONNECTION="$src 51000 192.0.2.10 22"
+    got="$(on 'printf %s "$SSH_PEER"')"
+    if [ "$got" = "$src" ]; then echo "ok   peer $src reaches the command"; else echo "FAIL peer for $src was '$got'"; fails=$((fails + 1)); fi
+done
+
+# A three-node run needs a build that knows the latitude source: a release would be refused by its
+# own config check. The guard needs --distro, and the node is checked before it gets the pool.
+check_result() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
+check_result "three nodes without --distro stop" "$( (NODES=3 DISTRO="" require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
+check_result "three nodes with --distro go on" "$( (NODES=3 DISTRO=/tmp/x.tar.gz require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+check_result "one node without --distro goes on" "$( (NODES=1 DISTRO="" require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+check_result "one node with one address goes on" "$( (NODES=1 TF_VAR_elastic_ip_count=1 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+check_result "one node with no setting goes on" "$( (NODES=1 TF_VAR_elastic_ip_count="" require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+check_result "one node with two addresses stops" "$( (NODES=1 TF_VAR_elastic_ip_count=2 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
+check_result "three nodes with three addresses go on" "$( (NODES=3 TF_VAR_elastic_ip_count=3 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+printf 'binary with latitude_elastic_ip_ids inside\n' > "$work/spx-new"
+printf 'binary of a release\n' > "$work/spx-old"
+SPX_BIN="$work/spx-new"
+check_result "a build that knows latitude is accepted" "$( (node_knows_latitude) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+# shellcheck disable=SC2034 # read by the sourced functions
+SPX_BIN="$work/spx-old"
+check_result "a release is refused" "$( (node_knows_latitude) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
+# The profile spx writes under sudo is in the ubuntu user's home, so node-side reads name that
+# directory instead of relying on root's HOME.
+mkdir "$work/awsdir"
+printf '[other]\naws_access_key_id = X\n[spinifex]\naws_access_key_id = AK1\naws_secret_access_key = SK1\n[z]\nq = 1\n' > "$work/awsdir/credentials"
+node_ip() { printf 192.0.2.10; }
+# shellcheck disable=SC2034 # read by the sourced functions
+NODE_AWS_DIR="$work/awsdir"
+check_result "credentials come from the node's AWS dir" "$(cmd_credentials | tr '\n' ' ')" "[spinifex] aws_access_key_id = AK1 aws_secret_access_key = SK1 "
+
+# A failed API call logs its status and a documented code, never the response text, which can
+# echo the request's headers.
+mkdir "$work/bin2"
+cat > "$work/bin2/curl" <<'SH'
+#!/bin/sh
+cat > /dev/null
+printf '{"errors":[{"code":"%s","detail":"Authorization: Bearer %s"}]}\n401' "$FAKE_CODE" "$FAKE_LEAK"
+SH
+chmod +x "$work/bin2/curl"
+logged="$(FAKE_CODE=UNAUTHORIZED FAKE_LEAK="$AUTH" PATH="$work/bin2:$PATH" latitude GET /servers 2>&1 || true)"
+check_result "a failed call names its status and code" "$(grep -c 'returned 401 (UNAUTHORIZED)' <<<"$logged" || true)" 1
+check_result "a failed call never logs the token" "$(grep -c "$AUTH" <<<"$logged" || true)" 0
+logged="$(FAKE_CODE="$AUTH" FAKE_LEAK="$AUTH" PATH="$work/bin2:$PATH" latitude GET /servers 2>&1 || true)"
+check_result "an undocumented code is not logged" "$(grep -c "$AUTH" <<<"$logged" || true)" 0
+
+# A recorded ID that now names another deployment's object is not ours to delete.
+# shellcheck disable=SC2034 # read by the sourced functions
+NAME=demo
+latitude() { printf '%s' "$FAKE_LAT"; }
+orig_expected_project_id="$(declare -f expected_project_id)"
+expected_project_id() { echo proj_1; }
+srv() {
+    local frag=',"project":{"id":"proj_1"}'
+    [ "$#" -lt 2 ] || frag="$2"
+    printf '{"data":{"id":"sv_1","attributes":{"hostname":"%s"%s}}}' "$1" "$frag"
+}
+FAKE_LAT="$(srv spx-demo-node2)"
+check_result "a recorded server with our hostname is ours" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" ours
+FAKE_LAT="$(srv spx-other-node1)"
+check_result "a recorded server with another hostname is not" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
+FAKE_LAT="$(srv spx-demo2-node1)"
+check_result "a hostname that only starts like ours is not" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
+FAKE_LAT="$(srv spx-demo-node1 ',"project":{"id":"proj_other"}')"
+check_result "our hostname in another project is not ours" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
+FAKE_LAT="$(srv spx-demo-node1 '')"
+check_result "a server with no project is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
+FAKE_LAT="$(srv spx-demo-node1 ',"project":{}')"
+check_result "a project with no id is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
+FAKE_LAT="$(srv spx-demo-node1 ',"project":null')"
+check_result "a null project is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
+expected_project_id() { return 1; }
+FAKE_LAT="$(srv spx-demo-node1)"
+check_result "no known project id is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
+expected_project_id() { echo proj_1; }
+FAKE_LAT='not json'
+check_result "an unreadable object is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
+KNOWN_IDS="$work/ids"
+printf 'server sv_1\n' > "$KNOWN_IDS"
+lat_exists() { return 0; }
+FAKE_LAT="$(srv spx-other-node1)"
+check_result "destroy stops before Terraform when a recorded ID was reused" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
+FAKE_LAT="$(srv spx-demo-node1 ',"project":{"id":"proj_other"}')"
+check_result "destroy stops when the same hostname is in another project" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
+FAKE_LAT="$(srv spx-demo-node1 '')"
+check_result "destroy stops when the project cannot be read" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
+FAKE_LAT="$(srv spx-demo-node1)"
+check_result "destroy goes on when the recorded server is ours" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" went
+leftovers_out="$( (lat_match() { return 0; }; FAKE_LAT="$(srv spx-other-node1)" leftovers) 2>/dev/null || true)"
+check_result "a reused ID is not listed as a leftover to delete" "$(grep -c sv_1 <<<"$leftovers_out" || true)" 0
+
+# After terraform destroy the state's project output is gone; the remembered ID still lets the
+# sweep delete a matching leftover and keep a foreign one.
+eval "$orig_expected_project_id"
+TF_BIN="$work/tofu"
+printf '#!/bin/sh\nif [ -f "%s/gone" ]; then exit 1; fi\nprintf proj_1\n' "$work" > "$TF_BIN"
+chmod +x "$TF_BIN"
+EXPECTED_PROJECT=""
+remember_project_id
+check_result "the project ID is remembered while the state holds it" "$EXPECTED_PROJECT" proj_1
+: > "$work/gone"
+check_result "the state no longer answers after destroy" "$("$TF_BIN" output >/dev/null 2>&1 && echo up || echo gone)" gone
+printf 'server sv_1\nserver sv_2\n' > "$KNOWN_IDS"
+lat_match() { return 1; }
+latitude() { if [ "$2" = /servers/sv_1 ]; then srv spx-demo-node1; else srv spx-other-node1 ',"project":{"id":"proj_other"}'; fi; }
+swept="$(leftovers 2>/dev/null || true)"
+check_result "after destroy a matching leftover is still listed" "$(grep -c 'server sv_1 ' <<<"$swept" || true)" 1
+check_result "after destroy a foreign one is kept out" "$(grep -c 'server sv_2 ' <<<"$swept" || true)" 0
+EXPECTED_PROJECT=""
+swept="$(leftovers 2>/dev/null || true)"
+check_result "without the remembered ID a leftover is unknown, not matched" "$(grep -c 'server sv_1 sv_1 (unknown)' <<<"$swept" || true)" 1
+expected_project_id() { echo proj_1; }
+
+# Cleanup never deletes an object whose ownership could not be read, recorded or not.
+deleted="$work/deleted"
+: > "$deleted"
+latitude() { printf '%s %s\n' "$1" "$2" >> "$deleted"; }
+printf 'server sv_1\nserver sv_2\n' > "$KNOWN_IDS"
+delete_left $'server sv_1 sv_1 (unknown)\nserver sv_2 spx-demo-node2\n' "" >/dev/null 2>&1
+check_result "an unreadable recorded object is not deleted" "$(grep -c sv_1 "$deleted" || true)" 0
+check_result "a verified recorded object is deleted" "$(grep -c 'DELETE .*sv_2' "$deleted" || true)" 1
+
+# Both deployment sizes write the metadata remap.
+cfg="$work/spinifex.toml"
+extract() { sed -n "/^$1()/,/^}/p" "$here/deploy-latitude.sh" | sed -n "/<<'PY'/,/^PY\$/p" | sed '1d;$d' | sed "s|/etc/spinifex/spinifex.toml|$cfg|"; }
+printf '[network]\nexternal_mode = "nat"\n\n[bootstrap]\ny = 1\n' > "$cfg"
+extract configure_pool | python3 -I - 203.0.113.1 203.0.113.254
+check_result "one node gets the metadata remap" "$(grep -c 'imds_host_meta_ip = "169.254.42.254"' "$cfg")" 1
+printf '[network]\nexternal_mode = "nat"\n\n[nodes.node1]\nx = 1\n\n[nodes.node2]\nx = 1\n\n[nodes.node3]\nx = 1\n\n[bootstrap]\ny = 1\n' > "$cfg"
+extract configure_latitude_pool | python3 -I - proj_1 eip_1,eip_2 node1=sv_1,node2=sv_2,node3=sv_3
+check_result "three nodes get the metadata remap" "$(grep -c 'imds_host_dns_ip  = "169.254.42.253"' "$cfg")" 1
+check_result "three nodes get the latitude pool" "$(grep -c 'source                  = "latitude"' "$cfg")" 1
+extract configure_latitude_pool | python3 -I - proj_1 eip_1,eip_2 node1=sv_1,node2=sv_2,node3=sv_3
+check_result "writing the config again changes nothing" "$(grep -c 'imds_host_meta_ip' "$cfg")" 1
+unset FAKE_SSH_EVAL
+
+[ "$fails" -eq 0 ]
@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# host-data-disk.sh — format and mount the blank data disk at /var/lib/spinifex. Run as root ON the host.
+#
+#   host-data-disk.sh [/dev/disk/by-id/nvme-eui.<id>]
+#
+# The disk is chosen by content, never by its nvmeXn1 name, which swaps between boots. A candidate is a
+# whole-disk by-id EUI link whose device is not in the root filesystem's ancestry, has no partitions,
+# holders, swap or mounts, no filesystem signature and no partition table. With none the script stops
+# and lists the disks. With several it takes the largest, and the lowest by-id path among equals, and
+# prints what it chose and the blank disks it left alone; pass the by-id path to choose another.
+# It never reformats: a rerun on a host already done prints the UUID and exits. The last line of stdout
+# is DATA_UUID=<uuid>, which the later install phases need.
+
+set -euo pipefail
+
+MP=/var/lib/spinifex
+LABEL=spinifex-data
+FSTAB_BAK=/etc/fstab.pre-spinifex
+MIN_BYTES="${MIN_DATA_BYTES:-100000000000}"
+WANT="${1:-}"
+
+die() { echo "STOP: $*" >&2; exit 1; }
+[ "$(id -u)" = 0 ] || die "run as root"
+
+# Resume: the mount is already there and is ours.
+if mp_uuid="$(findmnt --mountpoint "$MP" -no UUID 2>/dev/null)" && [ -n "$mp_uuid" ]; then
+    src="$(findmnt --mountpoint "$MP" -no SOURCE)"
+    [ "$(blkid -s LABEL -o value "$src")" = "$LABEL" ] || die "$MP is mounted from $src, which is not labelled $LABEL; not touching it"
+    grep -q "UUID=${mp_uuid}[[:space:]]" /etc/fstab || die "$MP is mounted but /etc/fstab has no UUID=$mp_uuid entry"
+    echo "already done: $MP is mounted from $src (UUID $mp_uuid)"
+    echo "DATA_UUID=$mp_uuid"
+    exit 0
+fi
+if grep -q "$MP" /etc/fstab; then die "/etc/fstab mentions $MP but nothing is mounted there; fix that by hand first"; fi
+[ ! -e "$MP" ] || [ -z "$(ls -A "$MP")" ] || die "$MP exists and is not empty"
+
+root_src="$(findmnt -no SOURCE /)" || die "cannot find the root source"
+root_anc="$(lsblk -s -nro NAME "$root_src")" || die "cannot read the root ancestry"
+[ -n "$root_anc" ] || die "empty root ancestry"
+
+# blank_reason ID prints why the disk is not a candidate and returns 1, or returns 0.
+blank_reason() {
+    local id="$1" dev name size kids hold sw mnt rc w
+    dev="$(readlink -e "$id")" || { echo "link does not resolve"; return 1; }
+    [ -b "$dev" ] || { echo "not a block device"; return 1; }
+    name="$(basename "$dev")"
+    if grep -qx "$name" <<<"$root_anc"; then echo "in the root ancestry"; return 1; fi
+    size="$(blockdev --getsize64 "$id")" || { echo "size unreadable"; return 1; }
+    [ "$size" -ge "$MIN_BYTES" ] || { echo "only $size bytes"; return 1; }
+    kids="$(lsblk -nro NAME "$id")" || { echo "children unreadable"; return 1; }
+    [ "$kids" = "$name" ] || { echo "has children"; return 1; }
+    hold="$(ls -A "/sys/block/$name/holders/")" || { echo "holders unreadable"; return 1; }
+    [ -z "$hold" ] || { echo "has holders"; return 1; }
+    sw="$(swapon --show=NAME --noheadings)" || { echo "swapon failed"; return 1; }
+    if grep -q "$name" <<<"$sw"; then echo "is swap"; return 1; fi
+    mnt="$(lsblk -nro MOUNTPOINTS "$id")" || { echo "mounts unreadable"; return 1; }
+    [ -z "${mnt//[[:space:]]/}" ] || { echo "is mounted"; return 1; }
+    set +e; blkid -p "$id" >/dev/null 2>&1; rc=$?; set -e
+    [ "$rc" -eq 2 ] || { echo "blkid -p says it has a signature (rc $rc)"; return 1; }
+    w="$(wipefs -n "$id")" || { echo "wipefs failed"; return 1; }
+    [ -z "$w" ] || { echo "wipefs finds signatures"; return 1; }
+    return 0
+}
+
+# choose_data_disk ID... prints the largest disk's ID, the lowest path among equal sizes, and says on
+# stderr which it left. The choice depends only on size and path, so every node of one plan agrees.
+choose_data_disk() {
+    local chosen rest
+    chosen="$(for id in "$@"; do printf '%s %s\n' "$(blockdev --getsize64 "$id")" "$id"; done |
+        sort -k1,1nr -k2,2 | head -n1)"
+    chosen="${chosen#* }"
+    if [ "$#" -gt 1 ]; then
+        rest="$(printf '%s\n' "$@" | grep -vxF "$chosen" | sort | tr '\n' ' ')"
+        echo "chose $chosen ($(blockdev --getsize64 "$chosen") bytes), the largest of $# blank disks; left untouched: ${rest% }" >&2
+    fi
+    printf '%s\n' "$chosen"
+}
+
+candidates=()
+shopt -s nullglob
+for id in /dev/disk/by-id/nvme-eui.*; do
+    case "$id" in *-part*) continue ;; esac
+    if [ -n "$WANT" ] && [ "$id" != "$WANT" ]; then continue; fi
+    if why="$(blank_reason "$id")"; then
+        candidates+=("$id")
+    else
+        echo "skip $id: $why"
+    fi
+done
+shopt -u nullglob
+
+if [ "${#candidates[@]}" -eq 0 ]; then
+    lsblk -o NAME,SIZE,TYPE,MODEL,SERIAL,FSTYPE,MOUNTPOINTS >&2 || true
+    die "no blank data disk among the by-id EUI disks"
+fi
+ID="$(choose_data_disk "${candidates[@]}")"
+DEV="$(readlink -e "$ID")"
+echo "CHECKS PASSED: $ID -> $DEV"
+
+mkdir -p "$MP"
+mkfs.ext4 -q -L "$LABEL" "$ID"
+U="$(blkid -p -o value -s UUID "$ID")" || die "blkid UUID failed"
+[ -n "$U" ] || die "no UUID after mkfs"
+[ "$(blkid -p -o value -s TYPE "$ID")" = ext4 ] || die "not ext4 after mkfs"
+
+NEW="$(mktemp /etc/fstab.spinifex.XXXXXX)"
+cp /etc/fstab "$NEW"
+echo "UUID=$U $MP ext4 defaults,nofail,x-systemd.device-timeout=30s 0 2" >> "$NEW"
+findmnt --verify --tab-file "$NEW" || { rm -f "$NEW"; die "staged fstab failed verify, /etc/fstab untouched"; }
+BAK="$FSTAB_BAK.$(date +%s)"
+cp -a /etc/fstab "$BAK"
+trap 'cp -a "$BAK" /etc/fstab; systemctl daemon-reload; echo "ROLLED BACK /etc/fstab from $BAK" >&2' EXIT
+chmod 644 "$NEW"
+mv "$NEW" /etc/fstab
+systemctl daemon-reload
+mount "$MP"
+[ "$(findmnt --mountpoint "$MP" -no UUID)" = "$U" ] || die "wrong UUID at $MP"
+[ "$(findmnt --mountpoint "$MP" -no FSTYPE)" = ext4 ] || die "wrong fstype at $MP"
+[ "$(findmnt --mountpoint "$MP" -no MAJ:MIN | tr -d ' ')" = "$(lsblk -dno MAJ:MIN "$ID" | tr -d ' ')" ] || die "wrong device at $MP"
+trap - EXIT
+findmnt --mountpoint "$MP" -o TARGET,SOURCE,MAJ:MIN,UUID,FSTYPE,OPTIONS
+echo "STORAGE OK"
+echo "DATA_UUID=$U"
@@ -0,0 +1,14 @@
+# A private network needs two or more servers, so a single node has none. The VLAN carries the cluster
+# and Geneve traffic; its address is set over SSH, since the VID is only known after apply.
+resource "latitudesh_virtual_network" "lan" {
+  count       = var.node_count > 1 ? 1 : 0
+  project     = local.project_id
+  site        = var.site
+  description = "spinifex-${var.deployment_name}"
+}
+
+resource "latitudesh_vlan_assignment" "node" {
+  count              = var.node_count > 1 ? var.node_count : 0
+  server_id          = latitudesh_server.node[count.index].id
+  virtual_network_id = latitudesh_virtual_network.lan[0].id
+}
@@ -0,0 +1,48 @@
+#!/bin/sh
+# Runs as root on a node. Reads the API token on stdin and installs it for the daemon.
+# The directory belongs to root and is not writable by anyone else, so no service can create or
+# swap an entry in it. The file belongs to the daemon's user alone (mode 0400): the other
+# services share the spinifex group, so group access would hand them the team-wide token. The
+# temporary file has an unpredictable name and is created exclusively. A symlink, or an owner
+# other than root or the daemon's user, where the directory or the file should be is refused.
+# Ownership and mode are set on every run, so an older install converges. The file's content is
+# replaced only when the token differs. Prints installed or unchanged.
+set -eu
+dir="${API_TOKEN_DIR:-/etc/spinifex/latitude}"
+root="${API_TOKEN_ROOT-root}"
+rootgroup="${API_TOKEN_ROOT_GROUP-root}"
+user="${API_TOKEN_USER-spinifex-daemon}"
+umask 077
+
+fail() { echo "$*" >&2; exit 1; }
+owner_of() { stat -c %U "$1"; }
+
+[ ! -L "$dir" ] || fail "$dir is a symlink"
+if [ ! -e "$dir" ]; then mkdir -p "$dir"; fi
+[ -d "$dir" ] || fail "$dir is not a directory"
+[ "$(owner_of "$dir")" = "$root" ] || fail "$dir is not owned by $root"
+chown "$root:$rootgroup" "$dir"
+chmod 0711 "$dir"
+
+target="$dir/api-token"
+[ ! -L "$target" ] || fail "$target is a symlink"
+if [ -e "$target" ]; then
+    [ -f "$target" ] || fail "$target is not a regular file"
+    holder="$(owner_of "$target")"
+    [ "$holder" = "$root" ] || [ "$holder" = "$user" ] || fail "$target is owned by $holder"
+fi
+
+tmp="$(mktemp "$dir/.api-token.XXXXXX")"
+trap 'rm -f "$tmp"' EXIT
+cat > "$tmp"
+[ -s "$tmp" ] || fail "empty token on stdin"
+if [ -f "$target" ] && cmp -s "$tmp" "$target"; then
+    chown "$user" "$target"
+    chmod 0400 "$target"
+    echo unchanged
+    exit 0
+fi
+chown "$user" "$tmp"
+chmod 0400 "$tmp"
+mv -f "$tmp" "$target"
+echo installed
@@ -0,0 +1,17 @@
+#!/bin/sh
+# Runs as root on a node with the pool name as $1. Reads only the journal of the daemon's
+# current invocation, so a success logged before a restart cannot hide a failure after it.
+# Prints ready, refused or waiting.
+set -eu
+pool="${1:?pool name}"
+id="$(systemctl show -p InvocationID --value spinifex-daemon)"
+[ -n "$id" ] || { echo waiting; exit 0; }
+log="$(journalctl "_SYSTEMD_INVOCATION_ID=$id" -u spinifex-daemon --no-pager -o cat 2>/dev/null || true)"
+named() { grep -F -e "pool=$pool" -e "\"pool\":\"$pool\""; }
+if printf '%s\n' "$log" | grep -F "refused the pool's first call" | named >/dev/null; then
+    echo refused
+elif printf '%s\n' "$log" | grep -F "floating pool ready" | named >/dev/null; then
+    echo ready
+else
+    echo waiting
+fi
@@ -0,0 +1,119 @@
+#!/bin/sh
+# Tests node-install-api-token.sh and node-pool-state.sh with stand-in tools.
+set -eu
+here="$(cd "$(dirname "$0")" && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+fails=0
+check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
+
+# token install: A, A again, then B across a rotation; the file ends as B, owned by the daemon
+# user alone (0400) in a 0711 directory that only root may change
+me="$(id -un)"
+export API_TOKEN_DIR="$work/tok" API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
+install_token() { sh "$here/node-install-api-token.sh"; }
+check "first install" "$(printf 'TOKENA' | install_token)" installed
+check "same token" "$(printf 'TOKENA' | install_token)" unchanged
+check "rotation" "$(printf 'TOKENB' | install_token)" installed
+check "file holds B" "$(cat "$work/tok/api-token")" TOKENB
+check "file mode and owner" "$(stat -c '%U %a' "$work/tok/api-token")" "$me 400"
+check "dir mode and owner" "$(stat -c '%U %a' "$work/tok")" "$me 711"
+check "no temp left after install" "$(ls -A "$work/tok")" api-token
+check "empty refused" "$(printf '' | install_token 2>/dev/null || echo failed)" failed
+check "file kept after refusal" "$(cat "$work/tok/api-token")" TOKENB
+
+# an older install converges on the unchanged path too
+chmod 0640 "$work/tok/api-token"
+chmod 0755 "$work/tok"
+check "unchanged path repairs" "$(printf 'TOKENB' | install_token)" unchanged
+check "mode repaired on unchanged" "$(stat -c %a "$work/tok/api-token")" 400
+check "dir mode repaired on unchanged" "$(stat -c %a "$work/tok")" 711
+
+# who can read it: only the daemon's identity. This needs two real identities, so it runs as root only.
+if [ "$(id -u)" = 0 ]; then
+    chmod 0711 "$work"
+    export API_TOKEN_ROOT=root API_TOKEN_ROOT_GROUP=root API_TOKEN_USER=daemon
+    for mode in installed unchanged; do
+        [ "$mode" = installed ] && rm -f "$work/tok/api-token"
+        check "$mode path" "$(printf 'TOKENB' | install_token)" "$mode"
+        check "daemon reads ($mode)" "$(setpriv --reuid=daemon --regid=daemon --clear-groups cat "$work/tok/api-token")" TOKENB
+        check "another identity in the shared group cannot read ($mode)" \
+            "$(setpriv --reuid=nobody --regid=daemon --clear-groups cat "$work/tok/api-token" 2>/dev/null || echo denied)" denied
+    done
+    export API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
+else
+    echo "skip reading as the daemon and as another service: needs root"
+fi
+
+# a link planted where the file or the directory belongs is refused, and nothing it points at is touched
+echo SENTINEL > "$work/sentinel"
+rm -f "$work/tok/api-token"
+ln -s "$work/sentinel" "$work/tok/api-token"
+check "symlinked file refused" "$(printf 'TOKENC' | sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
+check "symlink target untouched" "$(cat "$work/sentinel")" SENTINEL
+ln -s "$work/tok" "$work/tok-link"
+check "symlinked dir refused" "$(printf 'TOKENC' | API_TOKEN_DIR="$work/tok-link" sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
+check "dir owned by someone else refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=nobody-such install_token 2>/dev/null || echo refused)" refused
+rm -f "$work/tok/api-token"
+printf 'TOKENB' | install_token >/dev/null
+check "file owned by a stranger refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=$me API_TOKEN_USER=nobody-such install_token 2>/dev/null || echo refused)" refused
+check "no temp left" "$(ls -A "$work/tok")" api-token
+
+# data disk choice: the largest blank disk, the lowest path among equals, the rest left alone
+choose_dir="$work/choose"
+mkdir -p "$choose_dir/bin"
+cat > "$choose_dir/bin/blockdev" <<'SH'
+#!/bin/sh
+# blockdev --getsize64 <id>: the fixture's size for that disk
+cat "$CHOOSE_DIR/size.$(basename "$2")"
+SH
+chmod +x "$choose_dir/bin/blockdev"
+eval "$(sed -n '/^choose_data_disk()/,/^}/p' "$here/host-data-disk.sh")"
+size_of() { printf '%s' "$2" > "$choose_dir/size.$1"; }
+pick() { CHOOSE_DIR="$choose_dir" PATH="$choose_dir/bin:$PATH" choose_data_disk "$@" 2>"$choose_dir/err"; }
+size_of nvme-eui.0b 1700000000000
+size_of nvme-eui.0a 1700000000000
+size_of nvme-eui.09 447000000000
+size_of nvme-eui.0c 1900000000000
+check "largest of three wins" "$(pick /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0c)" /dev/disk/by-id/nvme-eui.0c
+check "equal sizes: the lowest path" "$(pick /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0a)" /dev/disk/by-id/nvme-eui.0a
+check "order of the list does not matter" "$(pick /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0a /dev/disk/by-id/nvme-eui.09)" /dev/disk/by-id/nvme-eui.0a
+check "what was left is said" "$(grep -c 'left untouched: /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b' "$choose_dir/err")" 1
+check "one disk is just chosen" "$(pick /dev/disk/by-id/nvme-eui.09)" /dev/disk/by-id/nvme-eui.09
+check "one disk says nothing" "$(wc -c < "$choose_dir/err" | tr -d ' ')" 0
+
+# pool state: the stand-in journal serves only the lines of the invocation asked for
+mkdir "$work/bin"
+cat > "$work/bin/systemctl" <<'SH'
+#!/bin/sh
+cat "$FAKE_DIR/invocation"
+SH
+cat > "$work/bin/journalctl" <<'SH'
+#!/bin/sh
+for a in "$@"; do
+    case "$a" in _SYSTEMD_INVOCATION_ID=*) id="${a#_SYSTEMD_INVOCATION_ID=}" ;; esac
+done
+cat "$FAKE_DIR/journal.$id" 2>/dev/null
+SH
+chmod +x "$work/bin/systemctl" "$work/bin/journalctl"
+export FAKE_DIR="$work" PATH="$work/bin:$PATH"
+state() { sh "$here/node-pool-state.sh" latitude; }
+
+echo old > "$work/invocation"
+echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.old"
+check "current success" "$(state)" ready
+
+echo new > "$work/invocation"
+echo 'ERROR floating: the provider refused the pool'"'"'s first call; public addresses will fail until fixed pool=latitude' > "$work/journal.new"
+check "old success then current failure" "$(state)" refused
+
+echo 'INFO starting' > "$work/journal.new"
+check "current invocation not yet ready" "$(state)" waiting
+
+echo 'INFO floating pool ready pool=other source=latitude' > "$work/journal.new"
+check "another pool does not count" "$(state)" waiting
+
+echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.new"
+check "current success" "$(state)" ready
+
+[ "$fails" -eq 0 ]
@@ -0,0 +1,56 @@
+output "deployment_name" {
+  value = var.deployment_name
+}
+
+output "site" {
+  value = var.site
+}
+
+output "project" {
+  value = local.project
+}
+
+# provider_host_id is the server ID. public_ip is the server's own /31 address, which SSH and the
+# Spinifex API use; the Elastic IP is separate.
+output "nodes" {
+  value = [for n in latitudesh_server.node : {
+    name             = trimprefix(n.hostname, "spx-${var.deployment_name}-")
+    provider_host_id = n.id
+    public_ip        = n.primary_ipv4
+  }]
+}
+
+# The first address, for a single node's static pool.
+output "elastic_ip" {
+  value = latitudesh_elastic_ip.pool[0].address
+}
+
+# IDs as well as addresses: destroy deletes only these IDs, and the floating pool lists them.
+output "elastic_ips" {
+  value = [for e in latitudesh_elastic_ip.pool : { id = e.id, address = e.address }]
+}
+
+# The VLAN ID of the cluster network; null for one node, which has no private network.
+output "vid" {
+  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].vid
+}
+
+output "ssh_key_id" {
+  value = latitudesh_ssh_key.deploy.id
+}
+
+output "user_data_id" {
+  value = latitudesh_user_data.host.id
+}
+
+output "project_id" {
+  value = local.project_id
+}
+
+output "project_created" {
+  value = length(latitudesh_project.deploy) > 0
+}
+
+output "virtual_network_id" {
+  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].id
+}
@@ -0,0 +1,4 @@
+# The token comes from LATITUDESH_AUTH_TOKEN in the environment, which the provider reads
+# itself (its auth_token argument documents that variable). It is never a variable here, so it
+# is not written to the state or to a tfvars file. The default project is set per resource.
+provider "latitudesh" {}
@@ -0,0 +1,5 @@
+deployment_name     = "l1"
+site                = "DAL"
+plan                = "m4-metal-medium"
+node_count          = 1
+ssh_public_key_path = "~/.ssh/id_ed25519.pub"
@@ -0,0 +1,63 @@
+variable "deployment_name" {
+  type        = string
+  description = "Names every resource and the server hostnames, so two deployments in one account never collide."
+  validation {
+    # A hostname is at most 32 characters: "spx-" + name + "-node1".
+    condition     = can(regex("^[a-z0-9][a-z0-9-]{0,15}$", var.deployment_name))
+    error_message = "deployment_name must be 1-16 lowercase letters, digits or hyphens."
+  }
+}
+
+variable "project" {
+  type        = string
+  default     = ""
+  description = "ID or slug of an existing latitude.sh project to deploy into. Empty creates a project spinifex-<deployment_name> and removes it on destroy."
+}
+
+variable "project_id" {
+  type        = string
+  default     = ""
+  description = "ID of the project named by project. The Elastic IP create sends it as project_id without resolving a slug (provider 4.8.0), so it must be an ID. Unused when project is empty."
+}
+
+variable "site" {
+  type        = string
+  default     = "DAL"
+  description = "latitude.sh site slug. Pick one with stock for the plan and Elastic IP support; not SAO."
+}
+
+variable "plan" {
+  type        = string
+  default     = "m4-metal-medium"
+  description = "Server plan slug (the pricing page calls it m4.metal.medium)."
+}
+
+variable "operating_system" {
+  type        = string
+  default     = "ubuntu_26_04_x64_lts"
+  description = "OS slug. UNVERIFIED: the 26.04 slug follows the 24.04 pattern (ubuntu_24_04_x64_lts) and is checked by a data source at plan time."
+}
+
+variable "node_count" {
+  type    = number
+  default = 1
+  validation {
+    condition     = contains([1, 3], var.node_count)
+    error_message = "node_count must be 1 or 3."
+  }
+}
+
+variable "elastic_ip_count" {
+  type        = number
+  default     = 1
+  description = "Elastic IPs to buy, each 4 USD a month. Every one starts on a server and keeps it unless Spinifex moves it. latitude.sh allows 5 per team."
+  validation {
+    condition     = var.elastic_ip_count >= 1 && var.elastic_ip_count <= 5 && floor(var.elastic_ip_count) == var.elastic_ip_count
+    error_message = "elastic_ip_count must be a whole number from 1 to 5."
+  }
+}
+
+variable "ssh_public_key_path" {
+  type    = string
+  default = "~/.ssh/id_ed25519.pub"
+}
@@ -0,0 +1,7 @@
+terraform {
+  required_version = ">= 1.5.0"
+  required_providers {
+    # Pinned exactly: 4.8.0 is the version whose schema these files were written against.
+    latitudesh = { source = "latitudesh/latitudesh", version = "= 4.8.0" }
+  }
+}
@@ -212,7 +212,7 @@ func (s *Server) response(w http.ResponseWriter, r *http.Request) {
 	}
 	if len(body) > MaxResponseBytes {
 		msg := "Response payload size exceeded maximum allowed payload size (" + strconv.Itoa(MaxResponseBytes) + " bytes)."
-		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeResponseTooLarge})
+		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeUnhandled})
 		writeError(w, http.StatusRequestEntityTooLarge, errTypeTooLarge, msg)
 		return
 	}
@@ -170,7 +170,7 @@ func TestOversizeResponse(t *testing.T) {

 	got := <-out
 	require.NoError(t, got.err)
-	assert.Equal(t, "Function.ResponseSizeTooLarge", got.res.FunctionError)
+	assert.Equal(t, "Unhandled", got.res.FunctionError)
 	assert.Equal(t, "Function.ResponseSizeTooLarge", errType(t, got.res.Payload))
 }

@@ -111,7 +111,7 @@ type AWSConfig struct {
 // ExternalPool defines a range of routable IPs that Spinifex manages for public subnets.
 type ExternalPool struct {
 	Name       string   `mapstructure:"name"`        // Pool identifier (e.g., "wan", "dc1-primary")
-	Source     string   `mapstructure:"source"`      // IP source: "static" (default), "dhcp", "oci", "exoscale" or "vultr"
+	Source     string   `mapstructure:"source"`      // IP source: "static" (default), "dhcp", "oci", "exoscale", "vultr" or "latitude"
 	BindBridge string   `mapstructure:"bind_bridge"` // Linux bridge for DHCP DORA (source=dhcp only)
 	DHCPMAC    string   `mapstructure:"dhcp_mac"`    // DHCP client MAC strategy: "derived" (default) or "interface" (source=dhcp only)
 	RangeStart string   `mapstructure:"range_start"` // First IP in range (static source only)
@@ -155,6 +155,17 @@ type ExternalPool struct {
 	VultrRegion  string `mapstructure:"vultr_region"`   // Vultr region ID, e.g. "ord"; never derived from the Spinifex region
 	VultrKeyFile string `mapstructure:"vultr_key_file"` // API key file (optional; see vultr.DefaultKeyFile)
 	LabelPrefix  string `mapstructure:"label_prefix"`   // Marks this deployment's addresses, e.g. "spinifex-prod"
+
+	// Latitude names the project, token file and fixed set of Elastic IPs
+	// (source=latitude only). Each node's server ID is provider_host_id on the node.
+	LatitudeProjectID    string   `mapstructure:"latitude_project_id"`     // latitude.sh project ID the Elastic IPs belong to
+	LatitudeKeyFile      string   `mapstructure:"latitude_key_file"`       // API token file (optional; see latitude.DefaultKeyFile)
+	LatitudeElasticIPIDs []string `mapstructure:"latitude_elastic_ip_ids"` // Elastic IP IDs the pool hands out
+}
+
+// hasLatitudeKeys reports whether any latitude_* key is set.
+func (p ExternalPool) hasLatitudeKeys() bool {
+	return p.LatitudeProjectID != "" || p.LatitudeKeyFile != "" || len(p.LatitudeElasticIPIDs) > 0
 }

 // hasVultrKeys reports whether any vultr_* key, or the label prefix only a
@@ -251,7 +262,7 @@ type Config struct {

 	// ProviderHostID is this node's instance ID at its cloud provider, which a
 	// floating public address is attached to. Required on every node when a
-	// source="vultr" pool exists.
+	// source="vultr" or source="latitude" pool exists.
 	ProviderHostID string `json:"ProviderHostID,omitempty" mapstructure:"provider_host_id"`

 	Daemon      DaemonConfig      `json:"Daemon" mapstructure:"daemon"`
@@ -807,6 +818,53 @@ func validateVultrPool(cc *ClusterConfig, p ExternalPool) error {
 	return nil
 }

+// MaxLatitudeElasticIPs bounds a latitude pool: latitude.sh allows five Elastic IPs per team, and
+// every allocation reads each configured address from the API.
+const MaxLatitudeElasticIPs = 5
+
+// validateLatitudePool checks a source="latitude" pool. The Elastic IPs are
+// bought ahead of time and named here, the API picks nothing, so the range,
+// gateway and label keys mean nothing, and every node must name its server.
+func validateLatitudePool(cc *ClusterConfig, p ExternalPool) error {
+	if p.LatitudeProjectID == "" || len(p.LatitudeElasticIPIDs) == 0 {
+		return fmt.Errorf("config: [[network.external_pools]] %q: source=\"latitude\" requires latitude_project_id and latitude_elastic_ip_ids", p.Name)
+	}
+	if len(p.LatitudeElasticIPIDs) > MaxLatitudeElasticIPs {
+		return fmt.Errorf("config: [[network.external_pools]] %q: latitude_elastic_ip_ids lists %d addresses, at most %d are allowed", p.Name, len(p.LatitudeElasticIPIDs), MaxLatitudeElasticIPs)
+	}
+	seen := map[string]struct{}{}
+	for _, id := range p.LatitudeElasticIPIDs {
+		if id == "" {
+			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_elastic_ip_ids has an empty entry", p.Name)
+		}
+		if _, dup := seen[id]; dup {
+			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_elastic_ip_ids lists %q twice", p.Name, id)
+		}
+		seen[id] = struct{}{}
+	}
+	if p.LatitudeKeyFile != "" && !filepath.IsAbs(p.LatitudeKeyFile) {
+		return fmt.Errorf("config: [[network.external_pools]] %q: latitude_key_file must be an absolute path, got %q", p.Name, p.LatitudeKeyFile)
+	}
+	if p.BindBridge != "" || p.DHCPMAC != "" {
+		return fmt.Errorf("config: [[network.external_pools]] %q: bind_bridge/dhcp_mac are only valid with source=\"dhcp\"", p.Name)
+	}
+	if p.RangeStart != "" || p.RangeEnd != "" || p.Gateway != "" || p.GatewayIP != "" {
+		return fmt.Errorf("config: [[network.external_pools]] %q: range_start/range_end/gateway/gateway_ip not allowed with source=\"latitude\" (the Elastic IPs are listed by ID)", p.Name)
+	}
+	if p.GwLrpRangeStart != "" || p.GwLrpRangeEnd != "" {
+		return fmt.Errorf("config: [[network.external_pools]] %q: gw_lrp_range_start/gw_lrp_range_end not allowed with source=\"latitude\" (a gateway address per VPC would use up the Elastic IP limit)", p.Name)
+	}
+	if cc.Network.ExternalMode != "nat" {
+		return fmt.Errorf("config: [[network.external_pools]] %q: source=\"latitude\" requires [network] external_mode = \"nat\" (routed); a routed Elastic IP is delivered to the server's own MAC", p.Name)
+	}
+	for name, n := range cc.Nodes {
+		if n.ProviderHostID == "" {
+			return fmt.Errorf("config: [nodes.%s] provider_host_id is required with a source=\"latitude\" pool (%q): it is the latitude.sh server ID the node's Elastic IPs move to", name, p.Name)
+		}
+	}
+	return nil
+}
+
 // validateSingleEBSProvider refuses nodes whose explicit [nodes.<name>.ebs]
 // providers differ. A peer stanza without one is unknown, not the default: the
 // generated config omits it, and the cluster's KV lock is the authoritative check.
@@ -879,6 +937,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		end   netip.Addr
 	}
 	var ranges []poolRange
+	// latitudeIDs maps each Elastic IP ID to the pool naming it. Every pool keeps its own
+	// record of what it handed out, so an ID in two pools would be given to two guests.
+	latitudeIDs := map[string]string{}
 	for _, p := range cc.Network.ExternalPools {
 		if p.Source != "oci" && (p.OCICompartmentID != "" || p.OCIVNICID != "" || p.OCIVNICIface != "" || p.OCISubnetID != "" || p.OCIPublicIPPool != "" || p.OCIConfigFile != "" || p.OCIConfigProfile != "") {
 			return fmt.Errorf("config: [[network.external_pools]] %q: oci_* keys are only valid with source=\"oci\"", p.Name)
@@ -886,6 +947,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		if p.Source != "exoscale" && p.hasExoscaleKeys() {
 			return fmt.Errorf("config: [[network.external_pools]] %q: exoscale_* keys are only valid with source=\"exoscale\"", p.Name)
 		}
+		if p.Source != "latitude" && p.hasLatitudeKeys() {
+			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_* keys are only valid with source=\"latitude\"", p.Name)
+		}
 		if p.Source != "vultr" && p.hasVultrKeys() {
 			return fmt.Errorf("config: [[network.external_pools]] %q: vultr_* and label_prefix keys are only valid with source=\"vultr\"", p.Name)
 		}
@@ -985,8 +1049,19 @@ func validateClusterConfig(cc *ClusterConfig) error {
 				return err
 			}
 			continue
+		case "latitude":
+			if err := validateLatitudePool(cc, p); err != nil {
+				return err
+			}
+			for _, id := range p.LatitudeElasticIPIDs {
+				if other, dup := latitudeIDs[id]; dup {
+					return fmt.Errorf("config: [[network.external_pools]] %q: Elastic IP %q is also named by pool %q; each Elastic IP may belong to one pool", p.Name, id, other)
+				}
+				latitudeIDs[id] = p.Name
+			}
+			continue
 		default:
-			return fmt.Errorf("config: [[network.external_pools]] %q: source=%q unsupported; use \"static\", \"dhcp\", \"oci\", \"exoscale\" or \"vultr\"", p.Name, p.Source)
+			return fmt.Errorf("config: [[network.external_pools]] %q: source=%q unsupported; use \"static\", \"dhcp\", \"oci\", \"exoscale\", \"vultr\" or \"latitude\"", p.Name, p.Source)
 		}
 		if p.RangeStart == "" || p.RangeEnd == "" {
 			continue
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"

 	"github.com/spf13/viper"
@@ -1550,6 +1551,83 @@ func TestLoadConfig_NetworkPoolVultrRejections(t *testing.T) {
 	}
 }

+const latitudePoolBase = `
+source = "latitude"
+latitude_project_id = "proj_1"
+latitude_elastic_ip_ids = ["eip_1", "eip_2"]
+`
+
+func TestLoadConfig_NetworkPoolLatitudeSourceAccepted(t *testing.T) {
+	cfg, err := loadVultrPool(t, "nat", latitudePoolBase+"latitude_key_file = \"/etc/spinifex/latitude/api-token\"\n",
+		[2]string{"sv_a", "sv_b"})
+	require.NoError(t, err)
+	p := cfg.Network.ExternalPools[0]
+	assert.Equal(t, "latitude", p.Source)
+	assert.Equal(t, "proj_1", p.LatitudeProjectID)
+	assert.Equal(t, []string{"eip_1", "eip_2"}, p.LatitudeElasticIPIDs)
+	assert.Equal(t, "/etc/spinifex/latitude/api-token", p.LatitudeKeyFile)
+	assert.Equal(t, "sv_a", cfg.Nodes["n1"].ProviderHostID)
+}
+
+func latitudeIDs(n int) string {
+	ids := make([]string, n)
+	for i := range ids {
+		ids[i] = fmt.Sprintf("%q", fmt.Sprintf("eip_%d", i))
+	}
+	return "source = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [" + strings.Join(ids, ", ") + "]\n"
+}
+
+func TestLoadConfig_NetworkPoolLatitudeSetIsCapped(t *testing.T) {
+	cfg, err := loadVultrPool(t, "nat", latitudeIDs(MaxLatitudeElasticIPs), [2]string{"a", "b"})
+	require.NoError(t, err)
+	assert.Len(t, cfg.Network.ExternalPools[0].LatitudeElasticIPIDs, MaxLatitudeElasticIPs)
+
+	_, err = loadVultrPool(t, "nat", latitudeIDs(MaxLatitudeElasticIPs+1), [2]string{"a", "b"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "at most 5 are allowed")
+}
+
+func TestLoadConfig_NetworkPoolLatitudeRejections(t *testing.T) {
+	both := [2]string{"a", "b"}
+	cases := []struct {
+		name, mode, body string
+		nodes            [2]string
+		want             string
+	}{
+		{"pool mode", "pool", latitudePoolBase, both, `requires [network] external_mode = "nat"`},
+		{"no project", "nat", "source = \"latitude\"\nlatitude_elastic_ip_ids = [\"e\"]\n", both, "requires latitude_project_id and latitude_elastic_ip_ids"},
+		{"no ids", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\n", both, "requires latitude_project_id and latitude_elastic_ip_ids"},
+		{"duplicate id", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e\", \"e\"]\n", both, "twice"},
+		{"empty id", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"\"]\n", both, "empty entry"},
+		{"relative key file", "nat", latitudePoolBase + "latitude_key_file = \"token\"\n", both, "absolute path"},
+		{"label prefix", "nat", latitudePoolBase + "label_prefix = \"x\"\n", both, "only valid with source=\"vultr\""},
+		{"range", "nat", latitudePoolBase + "range_start = \"1.2.3.4\"\nrange_end = \"1.2.3.4\"\n", both, "listed by ID"},
+		{"a node without its server id", "nat", latitudePoolBase, [2]string{"a", ""}, "[nodes.n2] provider_host_id is required"},
+		{"latitude keys on static", "nat", "latitude_project_id = \"p\"\n", both, `only valid with source="latitude"`},
+		{"vultr keys on latitude", "nat", latitudePoolBase + "vultr_region = \"ord\"\n", both, `only valid with source="vultr"`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := loadVultrPool(t, tc.mode, tc.body, tc.nodes)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.want)
+		})
+	}
+}
+
+func TestLoadConfig_NetworkPoolLatitudeIDInTwoPoolsIsRejected(t *testing.T) {
+	resetViper(t)
+	body := "node = \"n1\"\n\n[network]\nexternal_mode = \"nat\"\n\n" +
+		"[[network.external_pools]]\nname = \"a\"\nsource = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e1\", \"e2\"]\n\n" +
+		"[[network.external_pools]]\nname = \"b\"\nsource = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e3\", \"e2\"]\n\n" +
+		"[nodes.n1]\nregion = \"us-east-1\"\nprovider_host_id = \"sv_1\"\n"
+	path := filepath.Join(t.TempDir(), "spinifex.toml")
+	require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+	_, err := LoadConfig(path)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), `"e2" is also named by pool "a"`)
+}
+
 func TestLoadConfig_NorthstarNameservers(t *testing.T) {
 	load := func(t *testing.T, block string) (*ClusterConfig, error) {
 		t.Helper()
@@ -0,0 +1,88 @@
+package accountinventory
+
+import (
+	"context"
+	"fmt"
+
+	ec2metadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2spot "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+)
+
+// SpotRequestScanner reads one account's spot requests, failing on any record
+// it cannot read.
+type SpotRequestScanner interface {
+	ScanAccount(ctx context.Context, accountID string) (*ec2spot.AccountScan, error)
+}
+
+// VolumeScanner reads one account's volume documents, failing on any document
+// it cannot read.
+type VolumeScanner interface {
+	ListVolumesStrict(ctx context.Context, accountID string) ([]ec2metadata.Volume, error)
+}
+
+var (
+	_ SpotRequestScanner = (*ec2spot.SpotInstanceServiceImpl)(nil)
+	_ VolumeScanner      = (*ec2metadata.Store)(nil)
+)
+
+// SpotRequestItem is one request the account holds.
+type SpotRequestItem struct {
+	RequestID string
+	State     string
+	Active    bool
+}
+
+// VolumeItem is one volume document the account holds.
+type VolumeItem struct {
+	VolumeID            string
+	State               string
+	AttachedInstance    string
+	DeleteOnTermination bool
+}
+
+// WithCompute adds the spot-request and volume owner stores.
+func (i *Inventory) WithCompute(spot SpotRequestScanner, volumes VolumeScanner) *Inventory {
+	i.spot = spot
+	i.volumes = volumes
+	return i
+}
+
+// SpotRequestInventory returns every spot request the account holds. A scan
+// that cannot be completed is an error, never a shorter list.
+func (i *Inventory) SpotRequestInventory(ctx context.Context, accountID string) ([]SpotRequestItem, error) {
+	if i.spot == nil {
+		return nil, fmt.Errorf("spot request inventory is not configured")
+	}
+	scan, err := i.spot.ScanAccount(ctx, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("spot request inventory incomplete: %w", err)
+	}
+	items := make([]SpotRequestItem, 0, len(scan.Active)+len(scan.Terminal))
+	for _, r := range scan.Active {
+		items = append(items, SpotRequestItem{RequestID: r.RequestID, State: r.State, Active: true})
+	}
+	for _, r := range scan.Terminal {
+		items = append(items, SpotRequestItem{RequestID: r.RequestID, State: r.State})
+	}
+	return items, nil
+}
+
+// VolumeInventory returns every volume document the account holds, including
+// those already marked deleting. An unreadable document is an error.
+func (i *Inventory) VolumeInventory(ctx context.Context, accountID string) ([]VolumeItem, error) {
+	if i.volumes == nil {
+		return nil, fmt.Errorf("volume inventory is not configured")
+	}
+	docs, err := i.volumes.ListVolumesStrict(ctx, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("volume inventory incomplete: %w", err)
+	}
+	items := make([]VolumeItem, 0, len(docs))
+	for _, v := range docs {
+		items = append(items, VolumeItem{
+			VolumeID: v.VolumeID, State: v.State, AttachedInstance: v.AttachedInstance,
+			DeleteOnTermination: v.DeleteOnTermination,
+		})
+	}
+	return items, nil
+}
@@ -0,0 +1,82 @@
+package accountinventory
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	ec2metadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2spot "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+)
+
+type fakeSpot struct {
+	scan *ec2spot.AccountScan
+	err  error
+}
+
+func (f fakeSpot) ScanAccount(context.Context, string) (*ec2spot.AccountScan, error) {
+	return f.scan, f.err
+}
+
+type fakeVolumes struct {
+	docs []ec2metadata.Volume
+	err  error
+}
+
+func (f fakeVolumes) ListVolumesStrict(context.Context, string) ([]ec2metadata.Volume, error) {
+	return f.docs, f.err
+}
+
+func TestSpotRequestInventoryMarksActiveAndTerminal(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(fakeSpot{scan: &ec2spot.AccountScan{
+		Active:   []ec2spot.AccountSpotRequest{{RequestID: "sir-1", State: "active"}},
+		Terminal: []ec2spot.AccountSpotRequest{{RequestID: "sir-2", State: "cancelled"}},
+	}}, nil)
+
+	items, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []SpotRequestItem{
+		{RequestID: "sir-1", State: "active", Active: true},
+		{RequestID: "sir-2", State: "cancelled"},
+	}, items)
+}
+
+func TestSpotRequestInventoryScanFailureIsAnErrorNotAShorterList(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(fakeSpot{err: errList}, nil)
+
+	items, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.ErrorIs(t, err, errList)
+	assert.Nil(t, items)
+}
+
+func TestVolumeInventoryReportsAttachmentAndDeletingDocuments(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(nil, fakeVolumes{docs: []ec2metadata.Volume{
+		{VolumeID: "vol-1", State: "in-use", AttachedInstance: "i-1", DeleteOnTermination: true},
+		{VolumeID: "vol-2", State: "deleting"},
+	}})
+
+	items, err := inv.VolumeInventory(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []VolumeItem{
+		{VolumeID: "vol-1", State: "in-use", AttachedInstance: "i-1", DeleteOnTermination: true},
+		{VolumeID: "vol-2", State: "deleting"},
+	}, items)
+}
+
+func TestVolumeInventoryListFailureIsAnErrorNotAShorterList(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(nil, fakeVolumes{err: errList})
+
+	items, err := inv.VolumeInventory(t.Context(), testAccount)
+	require.ErrorIs(t, err, errList)
+	assert.Nil(t, items)
+}
+
+func TestComputeInventoriesRefuseWhenNotConfigured(t *testing.T) {
+	inv := New(nil, nil, nil, nil)
+	_, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.Error(t, err)
+	_, err = inv.VolumeInventory(t.Context(), testAccount)
+	require.Error(t, err)
+}
@@ -61,6 +61,8 @@ type Inventory struct {
 	groups    PlacementGroupScanner
 	templates LaunchTemplateScanner
 	tags      TagIndexScanner
+	spot      SpotRequestScanner
+	volumes   VolumeScanner
 	now       func() time.Time
 }

@@ -0,0 +1,151 @@
+package instance
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+)
+
+// accountInstanceRecordsSubject serves every non-terminated instance record an
+// account owns from the shared store, whichever node holds the instance.
+const accountInstanceRecordsSubject = "ec2.DescribeAccountInstanceRecords"
+
+// DescribeAccountInstancesStrict lists every non-terminated instance the
+// account owns, or returns a retryable ServiceUnavailable. Existence comes from
+// the shared record store, so an instance no answering node holds is still
+// listed with its recorded state; node answers are joined on for live state.
+// Anything that could make the list shorter than the truth is an error: a
+// configured node that did not answer, a frame from a node outside nodeIDs, a
+// record the owner could not project, or a store that did not answer.
+func DescribeAccountInstancesStrict(ctx context.Context, natsConn *nats.Conn, nodeIDs []string, accountID string, opts ...DescribeOption) (*ec2.DescribeInstancesOutput, error) {
+	if len(nodeIDs) == 0 {
+		slog.ErrorContext(ctx, "DescribeInstances: strict listing refused, configured node set unknown")
+		return nil, incompleteListingError()
+	}
+	cfg := describeConfig{fanoutTimeout: defaultDescribeFanoutTimeout}
+	for _, opt := range opts {
+		opt(&cfg)
+	}
+
+	jsonData, err := json.Marshal(&ec2.DescribeInstancesInput{})
+	if err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "marshal describe input: %v", err)
+	}
+
+	records, ok := queryInstanceBucket(ctx, natsConn, accountInstanceRecordsSubject, jsonData, accountID)
+	if !ok {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, instance record store did not answer")
+		return nil, incompleteListingError()
+	}
+
+	// Collect to the deadline with no responder grace, so a reply from a node
+	// outside the set that lands after the configured nodes is still seen.
+	frames, sum, err := natsmsg.Gather(ctx, natsConn, "ec2.DescribeInstances", jsonData, natsmsg.GatherOpts{
+		Timeout:            cfg.fanoutTimeout,
+		AccountID:          accountID,
+		ExpectedResponders: len(nodeIDs),
+		Mode:               natsmsg.CollectUntilDeadline,
+	})
+	if err != nil {
+		return nil, err
+	}
+	live, complete := judgeIdentityCompleteness(ctx, frames, sum, nodeIDs)
+	if stray := unconfiguredNodes(frames, sum, nodeIDs); len(stray) > 0 {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, frame from a node outside the configured set",
+			"unconfigured_nodes", stray)
+		return nil, incompleteListingError()
+	}
+	if !complete {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, fan-out incomplete", "configured_nodes", len(nodeIDs))
+		return nil, incompleteListingError()
+	}
+
+	joined, nodeOnly, malformed := joinLiveOntoRecords(records, live)
+	if malformed {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, a reservation or instance has no usable identity")
+		return nil, incompleteListingError()
+	}
+	if len(nodeOnly) > 0 {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, node reports instances with no record",
+			"instances", nodeOnly)
+		return nil, incompleteListingError()
+	}
+	return &ec2.DescribeInstancesOutput{Reservations: joined}, nil
+}
+
+// unconfiguredNodes returns the node IDs that answered, with a success or an
+// error, but are not in nodeIDs.
+func unconfiguredNodes(frames []natsmsg.Frame, sum natsmsg.Summary, nodeIDs []string) []string {
+	var stray []string
+	note := func(id string) {
+		if id != "" && !slices.Contains(nodeIDs, id) && !slices.Contains(stray, id) {
+			stray = append(stray, id)
+		}
+	}
+	for _, frame := range frames {
+		note(frame.NodeID)
+	}
+	for id := range sum.Responders {
+		note(id)
+	}
+	slices.Sort(stray)
+	return stray
+}
+
+// joinLiveOntoRecords starts from the recorded instances and replaces each with
+// the answering node's view of it. Instances only a node reports are returned
+// by ID: a lost record write can cause one, and it must not be dropped. A nil
+// reservation, instance or instance ID is reported as malformed, never skipped.
+func joinLiveOntoRecords(records, live []*ec2.Reservation) ([]*ec2.Reservation, []string, bool) {
+	type entry struct {
+		reservation *ec2.Reservation
+		instance    *ec2.Instance
+	}
+	byID := map[string]*entry{}
+	var order []string
+	malformed := false
+	add := func(reservations []*ec2.Reservation) {
+		for _, res := range reservations {
+			if res == nil {
+				malformed = true
+				continue
+			}
+			for _, inst := range res.Instances {
+				if inst == nil || aws.StringValue(inst.InstanceId) == "" {
+					malformed = true
+					continue
+				}
+				id := aws.StringValue(inst.InstanceId)
+				if existing, ok := byID[id]; ok {
+					existing.instance = inst
+					continue
+				}
+				byID[id] = &entry{reservation: res, instance: inst}
+				order = append(order, id)
+			}
+		}
+	}
+	add(records)
+	recorded := len(order)
+	add(live)
+	nodeOnly := slices.Clone(order[recorded:])
+
+	out := make([]*ec2.Reservation, 0, len(order))
+	for _, id := range order {
+		e := byID[id]
+		out = append(out, &ec2.Reservation{
+			ReservationId: e.reservation.ReservationId,
+			OwnerId:       e.reservation.OwnerId,
+			Groups:        e.reservation.Groups,
+			Instances:     []*ec2.Instance{e.instance},
+		})
+	}
+	return mergeReservationsByID(out), nodeOnly, malformed
+}
@@ -0,0 +1,229 @@
+package instance
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func strictInstance(id, state string) *ec2.Instance {
+	return &ec2.Instance{InstanceId: aws.String(id), State: &ec2.InstanceState{Name: aws.String(state)}}
+}
+
+func strictReservations(instances ...*ec2.Instance) *ec2.DescribeInstancesOutput {
+	out := &ec2.DescribeInstancesOutput{}
+	for _, inst := range instances {
+		out.Reservations = append(out.Reservations, &ec2.Reservation{
+			ReservationId: aws.String("r-" + aws.StringValue(inst.InstanceId)),
+			OwnerId:       aws.String("123456789012"),
+			Instances:     []*ec2.Instance{inst},
+		})
+	}
+	return out
+}
+
+func subscribeAccountRecords(t *testing.T, nc *nats.Conn, data []byte) {
+	t.Helper()
+	_, err := nc.Subscribe(accountInstanceRecordsSubject, func(msg *nats.Msg) { _ = msg.Respond(data) })
+	require.NoError(t, err)
+}
+
+const strictAccount = "123456789012"
+
+func strictList(nc *nats.Conn, nodes ...string) (*ec2.DescribeInstancesOutput, error) {
+	return DescribeAccountInstancesStrict(context.Background(), nc, nodes, strictAccount, WithFanoutTimeout(2*time.Second))
+}
+
+// Existence comes from the record store, so an instance no node holds is still
+// listed, with the state the record carries.
+func TestDescribeAccountInstancesStrict_UnheldRecordIsReturnedWithItsRecordedState(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-unheld", "stopped"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.NoError(t, err)
+	require.Len(t, out.Reservations, 1)
+	inst := out.Reservations[0].Instances[0]
+	assert.Equal(t, "i-unheld", aws.StringValue(inst.InstanceId))
+	assert.Equal(t, "stopped", aws.StringValue(inst.State.Name))
+}
+
+// A node's answer replaces the recorded view of the instance it holds.
+func TestDescribeAccountInstancesStrict_NodeAnswersAreJoinedOntoRecords(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-held", "pending"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", mustMarshalDescribeOutput(t,
+		strictReservations(strictInstance("i-held", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.NoError(t, err)
+	states := map[string]string{}
+	for _, res := range out.Reservations {
+		for _, inst := range res.Instances {
+			states[aws.StringValue(inst.InstanceId)] = aws.StringValue(inst.State.Name)
+		}
+	}
+	assert.Equal(t, map[string]string{"i-held": "running"}, states)
+}
+
+// An instance a node reports with no record can come from a lost record write.
+// Returning it or dropping it would both answer for an unproven account.
+func TestDescribeAccountInstancesStrict_NodeOnlyInstanceFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-held", "running"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", mustMarshalDescribeOutput(t,
+		strictReservations(strictInstance("i-held", "running"), strictInstance("i-node-only", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.Error(t, err)
+	assert.Nil(t, out)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorServiceUnavailable, code)
+}
+
+// A frame from a node outside the configured set means the expected set is
+// wrong, and a listing built on it could omit that node's instances.
+func TestDescribeAccountInstancesStrict_FrameFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-on-stray", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// A stray node that answers with an error is still a node outside the set, and
+// its instances are unaccounted for.
+func TestDescribeAccountInstancesStrict_ErrorFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// The configured nodes answer at once, so a reply that lands later but inside
+// the fan-out deadline must still be seen.
+func TestDescribeAccountInstancesStrict_LateFrameFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	late := mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-on-stray", "running")))
+	_, err := nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		time.AfterFunc(600*time.Millisecond, func() {
+			reply := nats.NewMsg(msg.Reply)
+			reply.Data = late
+			reply.Header.Set(natsmsg.NodeIDHeader, "node-stray")
+			_ = msg.RespondMsg(reply)
+		})
+	})
+	require.NoError(t, err)
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// A record with no instance ID cannot be listed or counted, so it must fail
+// the read rather than vanish from it.
+func TestDescribeAccountInstancesStrict_RecordWithoutInstanceIDFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(&ec2.Instance{State: &ec2.InstanceState{Name: aws.String("stopped")}})))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_ConfiguredNodeSilentFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1", "node-2")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_RecordStoreErrorFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_RecordStoreSilentFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_UnknownNodeSetFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc)
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_UnattributedFrameFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
@@ -0,0 +1,63 @@
+package instance
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Pins today: the completeness judge merges the reservations of a frame from a
+// node outside the configured set and still reports the listing complete. The
+// stray node is never named, so a silent unconfigured node cannot be told from
+// one that does not exist.
+func TestCurrentBehaviour_JudgeIdentityCompleteness_FrameFromUnconfiguredNodeIsMerged(t *testing.T) {
+	t.Parallel()
+	stray := mustMarshalDescribeOutput(t, &ec2.DescribeInstancesOutput{
+		Reservations: []*ec2.Reservation{{ReservationId: aws.String("r-stray"), Instances: []*ec2.Instance{{InstanceId: aws.String("i-on-stray")}}}},
+	})
+	frames := []natsmsg.Frame{
+		{NodeID: "node-1", Data: []byte(`{}`)},
+		{NodeID: "node-stray", Data: stray},
+	}
+
+	reservations, complete := judgeIdentityCompleteness(context.Background(), frames, natsmsg.Summary{}, []string{"node-1"})
+
+	assert.True(t, complete, "CURRENT: a node outside nodeIDs does not make the listing incomplete")
+	require.Len(t, reservations, 1)
+	assert.Equal(t, "i-on-stray", *reservations[0].Instances[0].InstanceId,
+		"CURRENT: the unconfigured node's instances are merged into the result")
+}
+
+// Pins today: the fan-out counts every identified frame toward the expected
+// responders, so a stray frame can end collection before a configured node has
+// answered. The listing then refuses, which fails closed, but for a node that
+// was only slow.
+func TestCurrentBehaviour_DescribeAccountInstancesComplete_StrayFrameConsumesAResponderSlot(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeEmptyInstanceBuckets(t, nc)
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", []byte(`{}`))
+	_, err := nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		time.AfterFunc(500*time.Millisecond, func() {
+			reply := nats.NewMsg(msg.Reply)
+			reply.Data = []byte(`{}`)
+			reply.Header.Set(natsmsg.NodeIDHeader, "node-2")
+			_ = msg.RespondMsg(reply)
+		})
+	})
+	require.NoError(t, err)
+	nc.Flush()
+
+	output, err := DescribeAccountInstancesComplete(context.Background(), nc, []string{"node-1", "node-2"}, "123456789012",
+		WithFanoutTimeout(3*time.Second))
+
+	assertIncompleteListing(t, output, err)
+}
@@ -0,0 +1,185 @@
+// Package ebsobligation records the volume exports a detach left in place.
+//
+// A force detach, or a terminate whose unmount did not complete, clears a
+// volume's attachment while the node that served it may still export it. The
+// request cannot reach that node, so it records an UnpublishObligation here and
+// returns; the named node settles it once it answers.
+package ebsobligation
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Bucket holds one obligation per publish a cleared attachment left behind.
+// It has no TTL: an obligation ends only when it is settled or closed.
+const Bucket = "spinifex-ebs-unpublish-obligations"
+
+// BucketVersion is the bucket schema version RunKV stamps.
+const BucketVersion = 1
+
+// SchemaVersion is the version of the UnpublishObligation record.
+const SchemaVersion uint16 = 1
+
+// UnpublishObligation names a publish a cleared attachment left on a node: the
+// node, volume and handle, and the attachment it served. The attachment is what
+// tells it apart from a later publish of the same volume on the same node.
+type UnpublishObligation struct {
+	SchemaVersion uint16    `json:"schema_version"`
+	VolumeID      string    `json:"volume_id"`
+	AccountID     string    `json:"account_id"`
+	Handle        string    `json:"handle,omitempty"` // ProviderHandle when the attachment was cleared
+	NodeID        string    `json:"node_id"`          // the node holding the publish
+	InstanceID    string    `json:"instance_id"`      // the attachment that was cleared
+	AttachedAt    time.Time `json:"attached_at"`
+	RecordedAt    time.Time `json:"recorded_at"`
+	Attempts      int       `json:"attempts"`
+	Error         string    `json:"error,omitempty"`
+}
+
+// keyToken is one NATS subject token a key segment may be: the key is three
+// tokens, so a segment holding a dot would split it and escape the filters.
+var keyToken = regexp.MustCompile(`^[-_=a-zA-Z0-9]+$`)
+
+// Key is the obligation's key, <node>.<volume-id>.<attached-at>, with the
+// attachment time in Unix nanoseconds and an unknown time as 0.
+func Key(o UnpublishObligation) (string, error) {
+	if !keyToken.MatchString(o.NodeID) {
+		return "", fmt.Errorf("unpublish obligation: node %q is not a valid key token", o.NodeID)
+	}
+	if !keyToken.MatchString(o.VolumeID) {
+		return "", fmt.Errorf("unpublish obligation: volume %q is not a valid key token", o.VolumeID)
+	}
+	var nanos int64
+	if !o.AttachedAt.IsZero() {
+		nanos = o.AttachedAt.UnixNano()
+	}
+	return o.NodeID + "." + o.VolumeID + "." + strconv.FormatInt(nanos, 10), nil
+}
+
+// allKeys matches every obligation key and not the bucket's version stamp,
+// which is a single token.
+const allKeys = "*.*.*"
+
+func bucketConfig() kvstore.Config {
+	return kvstore.Config{
+		Name:        Bucket,
+		Description: "EBS exports a cleared attachment left on a node, until that node unpublishes them",
+		History:     1,
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return migrate.DefaultRegistry.RunKV(ctx, Bucket, kv, BucketVersion)
+		},
+		Missing: "EBS unpublish obligation bucket not initialized",
+	}
+}
+
+// Store reads and writes the obligation bucket.
+type Store struct {
+	s *kvstore.Store[UnpublishObligation]
+}
+
+// NewStore returns a Store over js. The bucket is created on first use.
+func NewStore(js jetstream.JetStream) *Store {
+	return &Store{s: kvstore.New[UnpublishObligation](js, bucketConfig())}
+}
+
+// Open creates or opens the bucket now, so a node that cannot reach it fails
+// at startup rather than on its first force detach.
+func (s *Store) Open(ctx context.Context) error {
+	_, err := s.s.KV(ctx)
+	return err
+}
+
+// Record writes o if no obligation with its key exists. An existing key is the
+// same obligation written by an earlier attempt, so it counts as written.
+func (s *Store) Record(ctx context.Context, o UnpublishObligation) error {
+	key, err := Key(o)
+	if err != nil {
+		return err
+	}
+	o.SchemaVersion = SchemaVersion
+	if _, err := s.s.Create(ctx, key, &o); err != nil && !errors.Is(err, kvstore.ErrExists) {
+		return fmt.Errorf("record unpublish obligation %s: %w", key, err)
+	}
+	return nil
+}
+
+// ListNode returns the open obligations naming node.
+func (s *Store) ListNode(ctx context.Context, node string) ([]UnpublishObligation, error) {
+	if !keyToken.MatchString(node) {
+		return nil, fmt.Errorf("unpublish obligation: node %q is not a valid key token", node)
+	}
+	return s.list(ctx, node+".*.*")
+}
+
+// List returns every open obligation.
+func (s *Store) List(ctx context.Context) ([]UnpublishObligation, error) {
+	return s.list(ctx, allKeys)
+}
+
+func (s *Store) list(ctx context.Context, filter string) ([]UnpublishObligation, error) {
+	items, _, err := s.s.Snapshot(ctx, filter)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]UnpublishObligation, 0, len(items))
+	for _, item := range items {
+		out = append(out, item.Value)
+	}
+	return out, nil
+}
+
+// Close removes o. Removing one already gone is not an error.
+func (s *Store) Close(ctx context.Context, o UnpublishObligation) error {
+	key, err := Key(o)
+	if err != nil {
+		return err
+	}
+	return s.s.Delete(ctx, key)
+}
+
+// RecordFailure counts a failed settlement on o and keeps its error, so an
+// operator listing the obligations sees why it is still open.
+func (s *Store) RecordFailure(ctx context.Context, o UnpublishObligation, cause error) error {
+	key, err := Key(o)
+	if err != nil {
+		return err
+	}
+	err = s.s.Mutate(ctx, key, func(cur *UnpublishObligation) (bool, error) {
+		cur.Attempts++
+		cur.Error = cause.Error()
+		return true, nil
+	})
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return nil
+	}
+	return err
+}
+
+// ListOpen reads the open obligations for an operator tool without creating
+// the bucket: a cluster that has never force-detached a volume has none, and
+// that is not an error.
+func ListOpen(ctx context.Context, nc *nats.Conn) ([]UnpublishObligation, error) {
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return nil, fmt.Errorf("jetstream: %w", err)
+	}
+	if _, err := js.KeyValue(ctx, Bucket); errors.Is(err, jetstream.ErrBucketNotFound) {
+		return nil, nil
+	} else if err != nil {
+		return nil, fmt.Errorf("unpublish obligation bucket: %w", err)
+	}
+	cfg := bucketConfig()
+	cfg.AttachOnly = true
+	cfg.OnOpen = nil
+	return (&Store{s: kvstore.New[UnpublishObligation](js, cfg)}).List(ctx)
+}
@@ -0,0 +1,141 @@
+package ebsobligation_test
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newStore(t *testing.T) *ebsobligation.Store {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	store := ebsobligation.NewStore(js)
+	require.NoError(t, store.Open(t.Context()))
+	return store
+}
+
+func obligation(node, volumeID string, attachedAt time.Time) ebsobligation.UnpublishObligation {
+	return ebsobligation.UnpublishObligation{
+		VolumeID: volumeID, AccountID: "123456789012", NodeID: node,
+		InstanceID: "i-1", AttachedAt: attachedAt, RecordedAt: attachedAt.Add(time.Minute),
+	}
+}
+
+func TestKey_NamesNodeVolumeAndAttachment(t *testing.T) {
+	at := time.Unix(1_700_000_000, 5)
+	key, err := ebsobligation.Key(obligation("node-a", "vol-1", at))
+	require.NoError(t, err)
+	assert.Equal(t, "node-a.vol-1.1700000000000000005", key)
+
+	key, err = ebsobligation.Key(obligation("node-a", "vol-1", time.Time{}))
+	require.NoError(t, err)
+	assert.Equal(t, "node-a.vol-1.0", key, "an unknown attachment time is 0, not a huge negative")
+}
+
+// A segment holding a subject separator or wildcard would split the key or
+// let one node's listing match another's.
+func TestKey_RejectsSegmentsThatAreNotOneToken(t *testing.T) {
+	for _, o := range []ebsobligation.UnpublishObligation{
+		obligation("node.a", "vol-1", time.Time{}),
+		obligation("node-a", "vol.1", time.Time{}),
+		obligation("*", "vol-1", time.Time{}),
+		obligation("node-a", ">", time.Time{}),
+		obligation("", "vol-1", time.Time{}),
+	} {
+		_, err := ebsobligation.Key(o)
+		assert.Error(t, err, "%s/%s", o.NodeID, o.VolumeID)
+	}
+}
+
+func TestStore_RecordIsIdempotentPerAttachment(t *testing.T) {
+	store := newStore(t)
+	at := time.Unix(1_700_000_000, 0).UTC()
+	first := obligation("node-a", "vol-1", at)
+	first.Handle = "h1"
+	require.NoError(t, store.Record(t.Context(), first))
+	retry := first
+	retry.RecordedAt = at.Add(time.Hour)
+	require.NoError(t, store.Record(t.Context(), retry), "a retry of the same obligation is not an error")
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", at.Add(time.Second))),
+		"a different attachment of the same volume is a different obligation")
+
+	open, err := store.List(t.Context())
+	require.NoError(t, err)
+	require.Len(t, open, 2)
+	for _, o := range open {
+		assert.Equal(t, ebsobligation.SchemaVersion, o.SchemaVersion)
+		if o.AttachedAt.Equal(at) {
+			assert.True(t, o.RecordedAt.Equal(at.Add(time.Minute)), "the first write is kept")
+		}
+	}
+}
+
+// The bucket's version stamp is a key in the same bucket; listing must not
+// return it as an obligation.
+func TestStore_ListSkipsTheVersionStampAndListNodeIsolatesNodes(t *testing.T) {
+	store := newStore(t)
+	at := time.Unix(1_700_000_000, 0).UTC()
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", at)))
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-2", at)))
+	require.NoError(t, store.Record(t.Context(), obligation("node-ab", "vol-3", at)))
+
+	all, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, all, 3)
+
+	mine, err := store.ListNode(t.Context(), "node-a")
+	require.NoError(t, err)
+	volumes := []string{}
+	for _, o := range mine {
+		volumes = append(volumes, o.VolumeID)
+	}
+	assert.ElementsMatch(t, []string{"vol-1", "vol-2"}, volumes, "a node whose name prefixes another's sees only its own")
+
+	_, err = store.ListNode(t.Context(), "*")
+	assert.Error(t, err)
+}
+
+func TestStore_CloseAndRecordFailure(t *testing.T) {
+	store := newStore(t)
+	o := obligation("node-a", "vol-1", time.Unix(1_700_000_000, 0).UTC())
+	require.NoError(t, store.Record(t.Context(), o))
+
+	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("first")))
+	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("second")))
+	open, err := store.List(t.Context())
+	require.NoError(t, err)
+	require.Len(t, open, 1)
+	assert.Equal(t, 2, open[0].Attempts)
+	assert.Equal(t, "second", open[0].Error)
+
+	require.NoError(t, store.Close(t.Context(), o))
+	require.NoError(t, store.Close(t.Context(), o), "closing one already gone is not an error")
+	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("late")),
+		"a failure recorded after another sweep closed it does not resurrect it")
+	open, err = store.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, open)
+}
+
+func TestListOpen_MissingBucketIsEmptyAndNotCreated(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+
+	open, err := ebsobligation.ListOpen(t.Context(), nc)
+	require.NoError(t, err)
+	assert.Empty(t, open)
+	_, err = js.KeyValue(t.Context(), ebsobligation.Bucket)
+	assert.Error(t, err, "an operator listing must not create the bucket")
+
+	store := ebsobligation.NewStore(js)
+	require.NoError(t, store.Open(t.Context()))
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", time.Unix(1_700_000_000, 0).UTC())))
+	open, err = ebsobligation.ListOpen(t.Context(), nc)
+	require.NoError(t, err)
+	require.Len(t, open, 1)
+	assert.Equal(t, "vol-1", open[0].VolumeID)
+}
@@ -0,0 +1,89 @@
+package instance
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func recordsService(store *vmmock.StateStore) *InstanceServiceImpl {
+	return &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store, config: &config.Config{}}
+}
+
+func recordIDs(out *ec2.DescribeInstancesOutput) map[string]string {
+	ids := map[string]string{}
+	for _, res := range out.Reservations {
+		for _, inst := range res.Instances {
+			ids[*inst.InstanceId] = *inst.State.Name
+		}
+	}
+	return ids
+}
+
+// The record read answers for instances the stopped listing leaves out: a
+// running or drain-stopped record that no node need be holding.
+func TestDescribeAccountInstanceRecords_ReturnsEveryNonTerminatedRecordOfTheAccount(t *testing.T) {
+	owner := "111122223333"
+	store := &vmmock.StateStore{
+		Stopped: map[string]*vm.VM{"i-stopped": nilRecordVM("i-stopped", owner, true, true, vm.StateStopped)},
+		Saved: map[string]map[string]*vm.VM{"node-gone": {
+			"i-running": nilRecordVM("i-running", owner, true, true, vm.StateRunning),
+			"i-other":   nilRecordVM("i-other", "999999999999", true, true, vm.StateRunning),
+		}},
+	}
+
+	out, err := recordsService(store).DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{"i-stopped": "stopped", "i-running": "running"}, recordIDs(out),
+		"the other account's record is not returned")
+}
+
+// A record that cannot be projected fails the read; it is never skipped, since
+// the caller takes existence from this answer.
+func TestDescribeAccountInstanceRecords_NilReservationOrInstanceFails(t *testing.T) {
+	owner := "111122223333"
+	for name, bad := range map[string]*vm.VM{
+		"nil reservation": nilRecordVM("i-bad", owner, false, true, vm.StateRunning),
+		"nil instance":    nilRecordVM("i-bad", owner, true, false, vm.StateRunning),
+	} {
+		t.Run(name, func(t *testing.T) {
+			store := &vmmock.StateStore{
+				Stopped: map[string]*vm.VM{"i-good": nilRecordVM("i-good", owner, true, true, vm.StateStopped)},
+				Saved:   map[string]map[string]*vm.VM{"node-1": {"i-bad": bad}},
+			}
+
+			out, err := recordsService(store).DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+			require.Error(t, err)
+			assert.Nil(t, out, "no shorter list is returned")
+			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+		})
+	}
+}
+
+func TestDescribeAccountInstanceRecords_StoreErrorFails(t *testing.T) {
+	store := &vmmock.StateStore{ListLiveErr: errors.New("kv down")}
+
+	out, err := recordsService(store).DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, "111122223333")
+
+	require.Error(t, err)
+	assert.Nil(t, out)
+}
+
+func TestDescribeAccountInstanceRecords_NoStoreFails(t *testing.T) {
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), config: &config.Config{}}
+
+	out, err := svc.DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, "111122223333")
+
+	require.Error(t, err)
+	assert.Nil(t, out)
+}
@@ -0,0 +1,64 @@
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func nilRecordVM(id, owner string, withReservation, withInstance bool, status vm.InstanceState) *vm.VM {
+	v := &vm.VM{ID: id, InstanceType: "t3.micro", Status: status, AccountID: owner}
+	if withReservation {
+		v.Reservation = &ec2.Reservation{ReservationId: aws.String("r-" + id), OwnerId: aws.String(owner)}
+	}
+	if withInstance {
+		v.Instance = &ec2.Instance{InstanceId: aws.String(id), InstanceType: aws.String("t3.micro")}
+	}
+	return v
+}
+
+// Pins today: the node path drops an instance whose Reservation or Instance is
+// nil without a log line or an error, so it reads as absent from a listing the
+// caller may treat as complete.
+func TestCurrentBehaviour_DescribeInstances_NodePathSkipsNilReservationAndInstanceSilently(t *testing.T) {
+	owner := "111122223333"
+	vms := map[string]*vm.VM{
+		"i-good":    nilRecordVM("i-good", owner, true, true, vm.StateRunning),
+		"i-no-res":  nilRecordVM("i-no-res", owner, false, true, vm.StateRunning),
+		"i-no-inst": nilRecordVM("i-no-inst", owner, true, false, vm.StateRunning),
+	}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(vms), config: &config.Config{}}
+
+	out, err := svc.DescribeInstances(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+	require.NoError(t, err, "CURRENT: no error for the two skipped instances")
+	require.Len(t, out.Reservations, 1)
+	require.Len(t, out.Reservations[0].Instances, 1)
+	assert.Equal(t, "i-good", *out.Reservations[0].Instances[0].InstanceId, "CURRENT: only the complete record is returned")
+}
+
+// Pins today: the KV path skips the same records, with a warning, and still
+// answers success.
+func TestCurrentBehaviour_DescribeStoppedInstances_KVPathSkipsNilReservationAndInstance(t *testing.T) {
+	owner := "111122223333"
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{
+		"i-good":    nilRecordVM("i-good", owner, true, true, vm.StateStopped),
+		"i-no-res":  nilRecordVM("i-no-res", owner, false, true, vm.StateStopped),
+		"i-no-inst": nilRecordVM("i-no-inst", owner, true, false, vm.StateStopped),
+	}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store, config: &config.Config{}}
+
+	out, err := svc.DescribeStoppedInstances(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+	require.NoError(t, err, "CURRENT: no error for the two skipped instances")
+	require.Len(t, out.Reservations, 1)
+	require.Len(t, out.Reservations[0].Instances, 1)
+	assert.Equal(t, "i-good", *out.Reservations[0].Instances[0].InstanceId, "CURRENT: only the complete record is returned")
+}
@@ -106,6 +106,8 @@ type GPUClaimer interface {
 type StoppedInstanceStore interface {
 	LoadStoppedInstance(instanceID string) (*vm.VM, error)
 	ListStoppedInstances() ([]*vm.VM, error)
+	ListNonTerminatedInstances() ([]*vm.VM, error)
+	ListTerminatedInPrimary() ([]*vm.VM, error)
 	ListTerminatedInstances() ([]*vm.VM, error)
 	WriteStoppedInstance(instanceID string, instance *vm.VM) error
 	DeleteStoppedInstance(instanceID string) error
@@ -139,6 +141,17 @@ type InstanceTagWriter interface {
 	DeleteAllTags(ctx context.Context, accountID, resourceID string) error
 }

+// OwedUnpublish names the publish a terminate leaves in place when its unmount
+// did not complete: the node still exporting the volume and the instance it
+// served. The zero value means the unmount completed and nothing is owed.
+type OwedUnpublish struct {
+	NodeID     string
+	InstanceID string
+}
+
+// Owed reports whether a publish is left in place.
+func (o OwedUnpublish) Owed() bool { return o.NodeID != "" }
+
 // VolumeDeleter deletes EBS volumes. Implemented by domains/ec2/volume's
 // VolumeServiceImpl (used by the daemon).
 type VolumeDeleter interface {
@@ -147,14 +160,16 @@ type VolumeDeleter interface {
 	// DeleteVolumeOnTerminate deletes a DeleteOnTermination volume as part of
 	// an instance terminate: it clears any stale attachment (terminate
 	// implies detach) before deleting, so a still-attached boot volume left
-	// over from Stop is not rejected by DeleteVolume's in-use guard.
-	DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error
+	// over from Stop is not rejected by DeleteVolume's in-use guard. When owed
+	// names a publish, an unpublish obligation is recorded before the clear.
+	DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed OwedUnpublish) error

 	// DetachVolumeOnTerminate clears a still-attached volume's attachment on
 	// terminate without deleting it, matching AWS semantics for a
 	// DeleteOnTermination=false volume: terminate still implies detach, it
 	// just leaves the volume behind as available rather than deleting it.
-	DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error
+	// When owed names a publish, an unpublish obligation is recorded first.
+	DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed OwedUnpublish) error
 }

 // VolumeCreator creates the volumes a launch names beyond the root device, and
@@ -1,6 +1,7 @@
 package instance

 import (
+	"cmp"
 	"context"
 	"encoding/base64"
 	"encoding/json"
@@ -2234,7 +2235,18 @@ func (s *InstanceServiceImpl) DescribeStoppedInstances(ctx context.Context, inpu
 	if s.stoppedStore == nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListStoppedInstances, 80, "stopped", "DescribeStoppedInstances")
+	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListStoppedInstances, 80, "stopped", "DescribeStoppedInstances", false)
+}
+
+// DescribeAccountInstanceRecords returns every non-terminated instance record
+// the account owns, whichever node holds it. It is the existence source for the
+// strict account inventory, so a record it cannot project fails the call rather
+// than being skipped.
+func (s *InstanceServiceImpl) DescribeAccountInstanceRecords(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
+	if s.stoppedStore == nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListNonTerminatedInstances, 0, "pending", "DescribeAccountInstanceRecords", true)
 }

 // DescribeTerminatedInstances returns terminated instances from the terminated KV bucket.
@@ -2242,10 +2254,10 @@ func (s *InstanceServiceImpl) DescribeTerminatedInstances(ctx context.Context, i
 	if s.stoppedStore == nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListTerminatedInstances, 48, "terminated", "DescribeTerminatedInstances")
+	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListTerminatedInstances, 48, "terminated", "DescribeTerminatedInstances", false)
 }

-func (s *InstanceServiceImpl) describeInstancesFromKV(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string, listFn func() ([]*vm.VM, error), fallbackCode int64, fallbackName, opName string) (*ec2.DescribeInstancesOutput, error) {
+func (s *InstanceServiceImpl) describeInstancesFromKV(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string, listFn func() ([]*vm.VM, error), fallbackCode int64, fallbackName, opName string, strict bool) (*ec2.DescribeInstancesOutput, error) {
 	instanceIDFilter, err := ParseInstanceIDFilter(input.InstanceIds)
 	if err != nil {
 		return nil, err
@@ -2273,6 +2285,11 @@ func (s *InstanceServiceImpl) describeInstancesFromKV(ctx context.Context, input
 			continue
 		}
 		if instance.Reservation == nil || instance.Instance == nil {
+			if strict {
+				slog.ErrorContext(ctx, opName+": record has nil Reservation/Instance, refusing a listing that would omit it",
+					"instanceId", instance.ID)
+				return nil, errors.New(awserrors.ErrorServerInternal)
+			}
 			slog.WarnContext(ctx, opName+": skipping instance with nil Reservation/Instance (data integrity issue)",
 				"instanceId", instance.ID)
 			continue
@@ -3031,6 +3048,15 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 	// stopped-path delete failure.
 	var lastErr error

+	// A volume whose seal failed at stop is still exported on the node that
+	// stopped it, so clearing its attachment owes that node an unpublish.
+	owedFor := func(volumeID string) OwedUnpublish {
+		if instance.LastNode == "" || !slices.Contains(instance.UnpublishOwed, volumeID) {
+			return OwedUnpublish{}
+		}
+		return OwedUnpublish{NodeID: instance.LastNode, InstanceID: instance.ID}
+	}
+
 	for _, ebsRequest := range instance.EBSRequests.Requests {
 		// Internal volumes (EFI) are always cleaned up via ebs.delete.
 		if ebsRequest.EFI {
@@ -3061,7 +3087,7 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 				continue
 			}
 			// A volume already gone is the detach's own goal, not a failure.
-			if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID); err != nil &&
+			if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID, owedFor(ebsRequest.Name)); err != nil &&
 				!awserrors.IsNotFound(err) && !objectstore.IsNoSuchKeyError(err) {
 				slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to detach volume", "name", ebsRequest.Name, "err", err)
 				lastErr = err
@@ -3080,7 +3106,7 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 		// clears a Boot volume, runtime/daemon/vm_adapters.go) before deleting, so
 		// DeleteVolume's in-use guard does not reject an otherwise-legitimate
 		// delete. The resulting error is surfaced into lastErr, not swallowed.
-		if err := s.volumeDeleter.DeleteVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID); err != nil {
+		if err := s.volumeDeleter.DeleteVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID, owedFor(ebsRequest.Name)); err != nil {
 			slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to delete volume", "name", ebsRequest.Name, "err", err)
 			lastErr = err
 		}
@@ -3889,3 +3915,57 @@ func (s *InstanceServiceImpl) AttachVolumeToStopped(ctx context.Context, input *
 		State:      aws.String(ec2.VolumeAttachmentStateAttaching),
 	}, nil
 }
+
+// AccountTeardownOutstanding reports instances not terminated with every dependent done.
+// Terminated records expire after 1h, so it is never sole evidence an account is clear.
+// An unreadable record or one with no instance ID fails the read.
+func (s *InstanceServiceImpl) AccountTeardownOutstanding(accountID string) ([]OutstandingInstance, error) {
+	if s.stoppedStore == nil {
+		return nil, errors.New("instance record store not configured")
+	}
+	live, err := s.stoppedStore.ListNonTerminatedInstances()
+	if err != nil {
+		return nil, fmt.Errorf("list non-terminated instances: %w", err)
+	}
+	primaryTerminated, err := s.stoppedStore.ListTerminatedInPrimary()
+	if err != nil {
+		return nil, fmt.Errorf("list terminated instances in primary store: %w", err)
+	}
+	terminated, err := s.stoppedStore.ListTerminatedInstances()
+	if err != nil {
+		return nil, fmt.Errorf("list terminated instances: %w", err)
+	}
+
+	// An instance can sit in both stores while a terminate is mid-flight. The
+	// non-terminated record is listed first and wins, so it is never reported
+	// as torn down on the strength of the terminated copy.
+	byID := map[string]OutstandingInstance{}
+	for _, store := range []struct {
+		name    string
+		records []*vm.VM
+	}{{"non-terminated", live}, {"primary terminated", primaryTerminated}, {"terminated", terminated}} {
+		for _, v := range store.records {
+			if v == nil || v.ID == "" {
+				return nil, fmt.Errorf("%s instance store holds a record with no instance ID", store.name)
+			}
+			if !IsInstanceVisible(accountID, v.AccountID) {
+				continue
+			}
+			if v.Status == vm.StateTerminated && v.TeardownComplete() {
+				continue
+			}
+			if _, seen := byID[v.ID]; seen {
+				continue
+			}
+			byID[v.ID] = OutstandingInstance{
+				InstanceID: v.ID,
+				State:      string(v.Status),
+				Teardown:   maps.Clone(v.Teardown),
+			}
+		}
+	}
+
+	return slices.SortedFunc(maps.Values(byID), func(a, b OutstandingInstance) int {
+		return cmp.Compare(a.InstanceID, b.InstanceID)
+	}), nil
+}
@@ -1038,7 +1038,7 @@ func TestDescribeInstancesFromKV_FilterExcludesOnlyInstance_ReturnsZeroReservati
 	input := &ec2.DescribeInstancesInput{
 		Filters: []*ec2.Filter{{Name: aws.String("instance-type"), Values: []*string{aws.String("t3.nano")}}},
 	}
-	out, err := svc.describeInstancesFromKV(context.Background(), input, awsidentifiers.GlobalAccountID, listFn, 80, "stopped", "DescribeStoppedInstances")
+	out, err := svc.describeInstancesFromKV(context.Background(), input, awsidentifiers.GlobalAccountID, listFn, 80, "stopped", "DescribeStoppedInstances", false)
 	require.NoError(t, err)
 	assert.Empty(t, out.Reservations, "a reservation with every instance filtered out must not appear in the response")
 }
@@ -1734,6 +1734,7 @@ type fakeVolumeDeleter struct {
 	calls    []string
 	deleted  []string
 	detached []string
+	owed     []OwedUnpublish
 	err      error
 }

@@ -1749,7 +1750,8 @@ func (f *fakeVolumeDeleter) DeleteVolume(_ context.Context, input *ec2.DeleteVol

 // DeleteVolumeOnTerminate mirrors DeleteVolume's call/error bookkeeping; the
 // stopped-instance terminate path calls this instead of DeleteVolume.
-func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID, _ string) error {
+func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID, _ string, owed OwedUnpublish) error {
+	f.owed = append(f.owed, owed)
 	f.calls = append(f.calls, volumeID)
 	if f.err != nil {
 		return f.err
@@ -1761,7 +1763,8 @@ func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID,
 // DetachVolumeOnTerminate mirrors DeleteVolumeOnTerminate's call/error
 // bookkeeping but records into detached instead of deleted; the non-DoT
 // Boot-volume branch calls this instead of deleting.
-func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _ string) error {
+func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _ string, owed OwedUnpublish) error {
+	f.owed = append(f.owed, owed)
 	f.calls = append(f.calls, volumeID)
 	if f.err != nil {
 		return f.err
@@ -2009,6 +2012,45 @@ func TestTerminateStoppedInstance_UserVolumeDeleted(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, []string{"vol-user-001"}, vd.deleted, "only DeleteOnTermination=true volumes deleted")
 	assert.Equal(t, []string{"vol-keep-001"}, vd.detached, "terminate implies detach even for a volume that survives it")
+	assert.Equal(t, []OwedUnpublish{{}, {}}, vd.owed, "a stop whose unmount completed owes no unpublish")
+}
+
+// A stop whose seal failed left those volumes exported on the node that
+// stopped the instance, so terminate names that node for each of them alone.
+func TestTerminateStoppedInstance_FailedSealOwesTheStoppingNodeAnUnpublish(t *testing.T) {
+	id := "i-seal-failed"
+	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", LastNode: "node-a",
+		UnpublishOwed: []string{"vol-root-001", "vol-keep-001"}}
+	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+		{Name: "vol-root-001", Boot: true, DeleteOnTermination: true},
+		{Name: "vol-keep-001", DeleteOnTermination: false},
+		{Name: "vol-sealed-001", DeleteOnTermination: false},
+	}
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
+	vd := &fakeVolumeDeleter{}
+	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
+
+	_, err := svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
+	require.NoError(t, err)
+
+	owed := OwedUnpublish{NodeID: "node-a", InstanceID: id}
+	assert.Equal(t, []string{"vol-root-001", "vol-keep-001", "vol-sealed-001"}, vd.calls)
+	assert.Equal(t, []OwedUnpublish{owed, owed, {}}, vd.owed)
+}
+
+// With no node recorded there is no node to name; the terminate owes nothing
+// rather than naming an empty node.
+func TestTerminateStoppedInstance_NoLastNodeOwesNothing(t *testing.T) {
+	id := "i-seal-failed-nonode"
+	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", UnpublishOwed: []string{"vol-keep-001"}}
+	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-keep-001"}}
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
+	vd := &fakeVolumeDeleter{}
+	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
+
+	_, err := svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
+	require.NoError(t, err)
+	assert.Equal(t, []OwedUnpublish{{}}, vd.owed)
 }

 // TestTerminateStoppedInstance_StampsTeardownVolumesDone locks the
@@ -18,7 +18,9 @@ import (
 // reconcileStartVolumes drops the data volumes a claimed instance can no longer
 // have, and fails the start when its boot volume is gone.
 func (s *InstanceServiceImpl) reconcileStartVolumes(ctx context.Context, instance *vm.VM) error {
-	if s.metadata == nil {
+	// A direct-boot system guest boots from a kernel, not a root volume, and its
+	// one data volume, if any, is its own store rather than a customer's.
+	if s.metadata == nil || instance.DirectBoot {
 		return nil
 	}

@@ -123,3 +123,20 @@ func TestReconcileStartVolumes_ReattachesStillAvailableUnattachedVolume(t *testi
 	assert.Equal(t, []string{"vol-root", "vol-released"}, requestedNames(inst))
 	assert.Equal(t, []string{"vol-root", "vol-released"}, mappedNames(inst))
 }
+
+// ALB and SQS guests boot from a kernel with no root volume, so a start or a
+// recovery must not demand one; an SQS store's data volume stays named.
+func TestReconcileStartVolumes_DirectBootNeedsNoRootVolume(t *testing.T) {
+	t.Parallel()
+	svc := startVolumesService(t)
+
+	alb := &vm.VM{ID: "i-alb", AccountID: startVolumesAccount, Status: vm.StateStopped, DirectBoot: true,
+		Instance: &ec2.Instance{InstanceId: aws.String("i-alb")}}
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), alb))
+
+	sqs := &vm.VM{ID: "i-sqs", AccountID: startVolumesAccount, Status: vm.StateStopped, DirectBoot: true,
+		Instance: &ec2.Instance{InstanceId: aws.String("i-sqs")}}
+	sqs.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-store-gone", DeviceName: "/dev/vdb"}}
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), sqs))
+	assert.Equal(t, []string{"vol-store-gone"}, requestedNames(sqs), "a system store volume is never pruned")
+}
@@ -0,0 +1,12 @@
+package instance
+
+// OutstandingInstance is an instance of an account that is not yet fully torn
+// down, with the state its record carries.
+type OutstandingInstance struct {
+	InstanceID string
+	// State is the record's lifecycle state, such as running or terminated.
+	State string
+	// Teardown is the per-dependent progress (done, pending or failed) the
+	// record carries. It is empty when no dependent has been recorded.
+	Teardown map[string]string
+}
@@ -0,0 +1,81 @@
+package spotinstance
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+)
+
+// AccountSpotRequest is one persisted request in an account scan.
+type AccountSpotRequest struct {
+	RequestID string
+	State     string
+}
+
+// AccountScan lists every request an account holds in both buckets.
+type AccountScan struct {
+	Active   []AccountSpotRequest
+	Terminal []AccountSpotRequest
+}
+
+// ScanAccount lists the account's requests, or fails. Unlike
+// DescribeSpotInstanceRequests it never skips a record it cannot read: a
+// shorter list would be taken as proof that nothing is left.
+func (s *SpotInstanceServiceImpl) ScanAccount(ctx context.Context, accountID string) (*AccountScan, error) {
+	if accountID == "" {
+		return nil, errors.New("spot request scan needs an account ID")
+	}
+	active, err := scanBucket(ctx, s.activeKV, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("scan active spot requests: %w", err)
+	}
+	terminal, err := scanBucket(ctx, s.terminalKV, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("scan terminal spot requests: %w", err)
+	}
+	return &AccountScan{Active: active, Terminal: terminal}, nil
+}
+
+func scanBucket(ctx context.Context, kv jetstream.KeyValue, accountID string) ([]AccountSpotRequest, error) {
+	keys, err := kv.Keys(ctx)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrNoKeysFound) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	prefix := accountID + "."
+	var out []AccountSpotRequest
+	for _, key := range keys {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+			continue
+		}
+		entry, err := kv.Get(ctx, key)
+		if err != nil {
+			// Expired from the terminal bucket between listing and read.
+			if errors.Is(err, jetstream.ErrKeyNotFound) {
+				continue
+			}
+			return nil, fmt.Errorf("read %s: %w", key, err)
+		}
+		var record SpotRequestRecord
+		if err := json.Unmarshal(entry.Value(), &record); err != nil {
+			return nil, fmt.Errorf("decode %s: %w", key, err)
+		}
+		if record.Request == nil || aws.StringValue(record.Request.SpotInstanceRequestId) == "" {
+			return nil, fmt.Errorf("record %s carries no request", key)
+		}
+		out = append(out, AccountSpotRequest{
+			RequestID: aws.StringValue(record.Request.SpotInstanceRequestId),
+			State:     aws.StringValue(record.Request.State),
+		})
+	}
+	return out, nil
+}
@@ -0,0 +1,84 @@
+package spotinstance
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+)
+
+func TestScanAccountListsActiveAndTerminalForOneAccount(t *testing.T) {
+	svc := setupTestService(t)
+	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), makeSIR("sir-bbb", "i-bbb"))
+	_, err := svc.CancelSpotInstanceRequests(t.Context(), &ec2.CancelSpotInstanceRequestsInput{
+		SpotInstanceRequestIds: aws.StringSlice([]string{"sir-bbb"}),
+	}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.PutSpotInstanceRequests(t.Context(), &PutSpotRequestsInput{Requests: []*ec2.SpotInstanceRequest{makeSIR("sir-other", "i-o")}}, "999999999999")
+	require.NoError(t, err)
+
+	scan, err := svc.ScanAccount(t.Context(), testAccountID)
+	require.NoError(t, err)
+	require.Len(t, scan.Active, 1)
+	assert.Equal(t, "sir-aaa", scan.Active[0].RequestID)
+	require.Len(t, scan.Terminal, 1)
+	assert.Equal(t, "sir-bbb", scan.Terminal[0].RequestID)
+}
+
+func TestScanAccountFailsOnAnUndecodableRecord(t *testing.T) {
+	svc := setupTestService(t)
+	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"))
+	_, err := svc.activeKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-bad"), []byte("{not json"))
+	require.NoError(t, err)
+
+	scan, err := svc.ScanAccount(t.Context(), testAccountID)
+	require.Error(t, err)
+	assert.Nil(t, scan)
+}
+
+func TestScanAccountFailsOnARecordWithoutARequest(t *testing.T) {
+	svc := setupTestService(t)
+	_, err := svc.terminalKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-empty"), []byte(`{"account_id":"x"}`))
+	require.NoError(t, err)
+
+	_, err = svc.ScanAccount(t.Context(), testAccountID)
+	require.Error(t, err)
+}
+
+func TestScanAccountOfAnEmptyAccountIsEmptyNotAnError(t *testing.T) {
+	svc := setupTestService(t)
+	scan, err := svc.ScanAccount(t.Context(), testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, scan.Active)
+	assert.Empty(t, scan.Terminal)
+}
+
+type failingGetKV struct {
+	jetstream.KeyValue
+
+	key string
+}
+
+func (k failingGetKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
+	if key == k.key {
+		return nil, errors.New("injected read failure")
+	}
+	return k.KeyValue.Get(ctx, key)
+}
+
+func TestScanAccountFailsWhenARecordCannotBeRead(t *testing.T) {
+	svc := setupTestService(t)
+	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), makeSIR("sir-bbb", "i-bbb"))
+	svc.activeKV = failingGetKV{KeyValue: svc.activeKV, key: kvutil.AccountKey(testAccountID, "sir-bbb")}
+
+	scan, err := svc.ScanAccount(t.Context(), testAccountID)
+	require.Error(t, err)
+	assert.Nil(t, scan)
+}
@@ -9,6 +9,7 @@ import (
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	ec2volume "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -59,7 +60,7 @@ func TestCurrentBehaviour_AccountTermination_SnapshottedDeleteOnTerminationVolum
 	require.NoError(t, err)
 	require.NoError(t, volumes.UpdateVolumeState(accountID, volumeID, "in-use", instanceID, "/dev/sda1"))

-	err = volumes.DeleteVolumeOnTerminate(ctx, volumeID, accountID)
+	err = volumes.DeleteVolumeOnTerminate(ctx, volumeID, accountID, ec2instance.OwedUnpublish{})
 	assert.EqualError(t, err, awserrors.ErrorVolumeInUse, "the snapshot index refuses the terminate delete")

 	survivor, err := metadata.GetVolume(ctx, accountID, volumeID)
@@ -9,6 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -194,7 +195,7 @@ func TestDeleteVolumeOnTerminate_DeleteReadFailureIsNotSuccess(t *testing.T) {
 	svc.store = failing
 	svc.metadata = ebsmetadata.NewStore(failing, "test-bucket")

-	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
+	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})

 	require.Error(t, err, "a delete-read failure after a successful attachment clear must not be reported as success")
 	assert.False(t, awserrors.IsNotFound(err),
@@ -20,6 +20,7 @@ func newForceDetachService(t *testing.T) *VolumeServiceImpl {
 	t.Helper()
 	svc := newTestVolumeService("ap-southeast-2a")
 	svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}))
+	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-stuck0000000000": "node-a"}}, "node-a", "node-b")
 	return svc
 }

@@ -47,7 +48,11 @@ func TestForceDetachVolumeClearsTheAttachment(t *testing.T) {
 	// The previous instance is reported back: it is the only record of what
 	// the volume was torn away from.
 	assert.Equal(t, "i-stuck0000000000", aws.StringValue(attachment.InstanceId))
-	assert.Equal(t, "detached", aws.StringValue(attachment.State))
+	// The export is still up on the node that served it, so the attachment is
+	// reported mid-teardown, as AWS reports a detach it has not finished.
+	assert.Equal(t, "detaching", aws.StringValue(attachment.State))
+	assert.Equal(t, "/dev/sdf", aws.StringValue(attachment.Device))
+	assert.False(t, aws.TimeValue(attachment.AttachTime).IsZero())

 	meta, err := svc.GetVolumeMetadata("000000000011", "vol-forcedetach1")
 	require.NoError(t, err)
@@ -112,4 +117,8 @@ func TestForceDetachVolumeOnAnUnattachedVolume(t *testing.T) {
 	require.NoError(t, err)

 	assert.Empty(t, aws.StringValue(attachment.InstanceId))
+	assert.Equal(t, "detached", aws.StringValue(attachment.State))
+	open, err := svc.obligations.List(context.Background())
+	require.NoError(t, err)
+	assert.Empty(t, open, "an unattached volume has no publish to undo")
 }
@@ -20,6 +20,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
 	ebspolicy "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/policy"
 	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -59,6 +60,12 @@ type VolumeServiceImpl struct {
 	snapshotKV jetstream.KeyValue
 	provider   ebsprovider.EBSProvider
 	metadata   *ebsmetadata.Store
+
+	// The unpublish obligations a cleared attachment records, and what names
+	// the node holding the publish. Unset, a detach that leaves one fails.
+	obligations   *ebsobligation.Store
+	instanceNodes InstanceNodeLocator
+	members       func() []string
 }

 // SetEBSProvider injects the provider boundary used by the control plane. It
@@ -996,8 +1003,23 @@ func (s *VolumeServiceImpl) UpdateVolumeState(accountID, volumeID, state, attach
 	return nil
 }

-// ModifyVolume modifies an EBS volume (grow-only, requires stopped instance).
+// ModifyVolume modifies an EBS volume (grow-only, refused while attached).
 func (s *VolumeServiceImpl) ModifyVolume(ctx context.Context, input *ec2.ModifyVolumeInput, accountID string) (*ec2.ModifyVolumeOutput, error) {
+	return s.modifyVolume(ctx, input, accountID, "")
+}
+
+// ExpandAttachedToStopped grows a volume still attached to instanceID, which the
+// caller has stopped and holds exclusively. Not reachable from the AWS API.
+func (s *VolumeServiceImpl) ExpandAttachedToStopped(ctx context.Context, accountID, volumeID, instanceID string, sizeGiB int64) (*ec2.ModifyVolumeOutput, error) {
+	if instanceID == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	return s.modifyVolume(ctx, &ec2.ModifyVolumeInput{VolumeId: aws.String(volumeID), Size: aws.Int64(sizeGiB)}, accountID, instanceID)
+}
+
+// modifyVolume grows a volume. An empty stoppedOwner refuses any attached
+// volume; otherwise only a volume attached to that instance is accepted.
+func (s *VolumeServiceImpl) modifyVolume(ctx context.Context, input *ec2.ModifyVolumeInput, accountID, stoppedOwner string) (*ec2.ModifyVolumeOutput, error) {
 	if input == nil || input.VolumeId == nil || *input.VolumeId == "" {
 		return nil, errors.New(awserrors.ErrorInvalidVolumeIDMalformed)
 	}
@@ -1017,7 +1039,9 @@ func (s *VolumeServiceImpl) ModifyVolume(ctx context.Context, input *ec2.ModifyV
 	if input.Size == nil || *input.Size <= originalSize {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}
-	if (meta.AttachedInstance != "" && meta.State == "in-use") || meta.State == string(ebsprovider.VolumeStateCreating) {
+	attached := meta.AttachedInstance != "" && meta.State == "in-use"
+	if meta.State == string(ebsprovider.VolumeStateCreating) || attached != (stoppedOwner != "") ||
+		(attached && meta.AttachedInstance != stoppedOwner) {
 		return nil, errors.New(awserrors.ErrorIncorrectState)
 	}
 	originalType := meta.VolumeType
@@ -1083,7 +1107,7 @@ func (s *VolumeServiceImpl) ModifyVolume(ctx context.Context, input *ec2.ModifyV
 // terminating instance's QEMU has already been asked to shut down — so this
 // is always a metadata-only clear, never a QMP call. Errors from either step
 // are returned, not swallowed.
-func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error {
+func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed ec2instance.OwedUnpublish) error {
 	// Clearing the attachment would overwrite a recorded deletion, so a volume
 	// already owed is left alone: the reclaimer owns it from here, and resetting
 	// it to available would let a second teardown run against the same data.
@@ -1091,6 +1115,9 @@ func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeI
 	if err == nil && meta.State == volumeStateDeleting {
 		return nil
 	}
+	if err := s.recordTerminateObligation(ctx, accountID, volumeID, owed); err != nil {
+		return err
+	}

 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		// No metadata doc means the volume is already gone, per GetVolume's own
@@ -1110,7 +1137,10 @@ func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeI
 // available rather than deleting it. Metadata-only, no QMP — same rationale
 // as DeleteVolumeOnTerminate: there is no live QEMU to hot-unplug on either
 // terminate path.
-func (s *VolumeServiceImpl) DetachVolumeOnTerminate(_ context.Context, volumeID, accountID string) error {
+func (s *VolumeServiceImpl) DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed ec2instance.OwedUnpublish) error {
+	if err := s.recordTerminateObligation(ctx, accountID, volumeID, owed); err != nil {
+		return err
+	}
 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		// No metadata doc means the volume is already gone, per GetVolume's own
 		// contract, so absence here is success, not a teardown failure.
@@ -1129,8 +1159,12 @@ func (s *VolumeServiceImpl) DetachVolumeOnTerminate(_ context.Context, volumeID,
 // The ordinary DetachVolume routes to ec2.cmd.{instanceID} for the QMP unplug,
 // so a volume attached to an instance whose host stopped answering can never be
 // detached and therefore never deleted. This exists for that deadlock and for
-// nothing else: it leaves a live guest holding a device the control plane no
-// longer believes in, which is only safe once that guest is being destroyed.
+// nothing else.
+//
+// That host still exports the volume, and the unpublish has to go to it, so
+// it cannot be made here. The request records an unpublish obligation naming
+// the publish before it clears the attachment, and returns; the node settles
+// it once it answers. A failed record clears nothing.
 func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
 	if input == nil || input.VolumeId == nil || *input.VolumeId == "" {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
@@ -1154,18 +1188,48 @@ func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.De
 	}

 	previous := meta.AttachedInstance
+	if previous != "" {
+		nodes, err := s.publishingNodes(previous)
+		if err != nil {
+			slog.ErrorContext(ctx, "ForceDetachVolume: cannot name the node holding the publish, clearing nothing",
+				"volumeId", volumeID, "instanceId", previous, "err", err)
+			return nil, errors.New(awserrors.ErrorInternalError)
+		}
+		if err := s.recordObligations(ctx, meta, nodes); err != nil {
+			slog.ErrorContext(ctx, "ForceDetachVolume: failed to record the unpublish obligation, clearing nothing",
+				"volumeId", volumeID, "instanceId", previous, "err", err)
+			return nil, errors.New(awserrors.ErrorInternalError)
+		}
+	}
+
 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		slog.ErrorContext(ctx, "ForceDetachVolume: failed to clear attachment", "volumeId", volumeID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	slog.WarnContext(ctx, "ForceDetachVolume: attachment cleared in the control plane only",
+
+	if previous == "" {
+		return &ec2.VolumeAttachment{
+			VolumeId: aws.String(volumeID),
+			State:    aws.String("detached"),
+		}, nil
+	}
+	slog.WarnContext(ctx, "ForceDetachVolume: attachment cleared, unpublish left to the node holding it",
 		"volumeId", volumeID, "previousInstance", previous, "accountId", accountID)

-	return &ec2.VolumeAttachment{
+	// AWS answers a detach with the attachment it is tearing down, in state
+	// detaching; the export here goes when the node settles the obligation.
+	attachment := &ec2.VolumeAttachment{
 		VolumeId:   aws.String(volumeID),
 		InstanceId: aws.String(previous),
-		State:      aws.String("detached"),
-	}, nil
+		State:      aws.String("detaching"),
+	}
+	if meta.DeviceName != "" {
+		attachment.Device = aws.String(meta.DeviceName)
+	}
+	if !meta.AttachedAt.IsZero() {
+		attachment.AttachTime = aws.Time(meta.AttachedAt)
+	}
+	return attachment, nil
 }

 // DeleteVolume records a durable obligation to destroy an EBS volume and
@@ -16,6 +16,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit/ebsfake"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
@@ -1468,6 +1469,60 @@ func TestModifyVolume_AttachedInUse(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorIncorrectState, err.Error())
 }

+func TestExpandAttachedToStopped_GrowsTheNamedInstancesVolume(t *testing.T) {
+	store := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
+	createVolumeInStoreWithMeta(t, svc, store, "vol-stopped", ebsmetadata.Volume{
+		VolumeID:         "vol-stopped",
+		CapacityGiB:      20,
+		State:            "in-use",
+		AttachedInstance: "i-owner",
+		DeviceName:       "/dev/sdf",
+	})
+
+	out, err := svc.ExpandAttachedToStopped(context.Background(), testVolAccountID, "vol-stopped", "i-owner", 30)
+	require.NoError(t, err)
+	assert.Equal(t, int64(30), aws.Int64Value(out.VolumeModification.TargetSize))
+
+	meta, err := svc.GetVolumeMetadata(testVolAccountID, "vol-stopped")
+	require.NoError(t, err)
+	assert.Equal(t, uint64(30), meta.CapacityGiB)
+	assert.Equal(t, "in-use", meta.State, "the grow must not change the attachment")
+	assert.Equal(t, "i-owner", meta.AttachedInstance)
+	assert.Equal(t, "/dev/sdf", meta.DeviceName)
+}
+
+func TestExpandAttachedToStopped_Refusals(t *testing.T) {
+	cases := []struct {
+		name     string
+		meta     ebsmetadata.Volume
+		instance string
+		want     string
+	}{
+		{"attached to another instance", ebsmetadata.Volume{State: "in-use", AttachedInstance: "i-other"}, "i-owner", awserrors.ErrorIncorrectState},
+		{"not attached", ebsmetadata.Volume{State: "available"}, "i-owner", awserrors.ErrorIncorrectState},
+		{"still creating", ebsmetadata.Volume{State: string(ebsprovider.VolumeStateCreating)}, "i-owner", awserrors.ErrorIncorrectState},
+		{"no instance named", ebsmetadata.Volume{State: "in-use", AttachedInstance: "i-owner"}, "", awserrors.ErrorMissingParameter},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			store := objectstore.NewMemoryObjectStore()
+			svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
+			tc.meta.VolumeID = "vol-x"
+			tc.meta.CapacityGiB = 20
+			createVolumeInStoreWithMeta(t, svc, store, "vol-x", tc.meta)
+
+			_, err := svc.ExpandAttachedToStopped(context.Background(), testVolAccountID, "vol-x", tc.instance, 30)
+			require.Error(t, err)
+			assert.Equal(t, tc.want, err.Error())
+
+			meta, err := svc.GetVolumeMetadata(testVolAccountID, "vol-x")
+			require.NoError(t, err)
+			assert.Equal(t, uint64(20), meta.CapacityGiB, "a refused grow must leave the size unchanged")
+		})
+	}
+}
+
 func TestModifyVolume_SuccessfulGrow(t *testing.T) {
 	store := objectstore.NewMemoryObjectStore()
 	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
@@ -1789,7 +1844,7 @@ func TestDeleteVolumeOnTerminate_ClearsAttachmentThenDeletes(t *testing.T) {
 		AttachedInstance: "i-stopped",
 	})

-	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
+	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})
 	require.NoError(t, err, "terminate implies detach: a stale attachment must not block the terminate delete")

 	deleting, err := svc.GetVolumeMetadata(testVolAccountID, volumeID)
@@ -1816,7 +1871,7 @@ func TestDeleteVolumeOnTerminate_AlreadyGoneMetadataIsSuccess(t *testing.T) {
 	svc.snapshotKV = kv

 	// No metadata doc is ever written for this volume ID.
-	err := svc.DeleteVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID)
+	err := svc.DeleteVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID, ec2instance.OwedUnpublish{})
 	require.NoError(t, err, "a volume with no metadata document is already gone, not a teardown failure")
 }

@@ -1829,7 +1884,7 @@ func TestDetachVolumeOnTerminate_AlreadyGoneMetadataIsSuccess(t *testing.T) {
 	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
 	svc.snapshotKV = kv

-	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID)
+	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID, ec2instance.OwedUnpublish{})
 	require.NoError(t, err, "a volume with no metadata document is already gone, not a teardown failure")
 }

@@ -1913,7 +1968,7 @@ func TestDeleteVolume_SecondDeleteIsAcceptedWithoutASecondTeardown(t *testing.T)

 	// DeleteOnTermination arriving for a volume already owed must not reset it
 	// to available, which would let a second teardown through.
-	require.NoError(t, svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID))
+	require.NoError(t, svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{}))
 	meta, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
 	require.NoError(t, err)
 	assert.Equal(t, "deleting", meta.State)
@@ -1961,7 +2016,7 @@ func TestDeleteVolumeOnTerminate_SurfacesDeleteFailure(t *testing.T) {
 	_, err = kv.Put(t.Context(), volumeID, snapData)
 	require.NoError(t, err)

-	err = svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
+	err = svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})
 	require.Error(t, err, "a DeleteVolume failure must be surfaced, not swallowed")
 	assert.Contains(t, err.Error(), awserrors.ErrorVolumeInUse)

@@ -0,0 +1,365 @@
+package volume
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+// InstanceNodeLocator reports the node that last ran an instance. found is
+// false when no record of the instance exists.
+type InstanceNodeLocator interface {
+	InstanceLastNode(instanceID string) (node string, found bool, err error)
+}
+
+// SetUnpublishObligations wires the obligation store a cleared attachment
+// records into, the locator that names the node holding the publish, and the
+// cluster members to name when the instance has no record.
+func (s *VolumeServiceImpl) SetUnpublishObligations(store *ebsobligation.Store, locator InstanceNodeLocator, members func() []string) {
+	s.obligations = store
+	s.instanceNodes = locator
+	s.members = members
+}
+
+// publishingNodes names the node holding instanceID's publish. With no record
+// of the instance there is no node to name, so every member is named and each
+// settles its own.
+func (s *VolumeServiceImpl) publishingNodes(instanceID string) ([]string, error) {
+	if s.instanceNodes == nil {
+		return nil, errors.New("no instance node locator configured")
+	}
+	node, found, err := s.instanceNodes.InstanceLastNode(instanceID)
+	if err != nil {
+		return nil, fmt.Errorf("read instance %s: %w", instanceID, err)
+	}
+	if found && node != "" {
+		return []string{node}, nil
+	}
+	var members []string
+	if s.members != nil {
+		members = s.members()
+	}
+	if len(members) == 0 {
+		return nil, fmt.Errorf("instance %s has no record and no cluster member is known", instanceID)
+	}
+	return members, nil
+}
+
+// recordObligations writes one obligation per node for the attachment meta
+// still carries. Every write must land before the caller clears it.
+func (s *VolumeServiceImpl) recordObligations(ctx context.Context, meta ebsmetadata.Volume, nodes []string) error {
+	if s.obligations == nil {
+		return errors.New("no unpublish obligation store configured")
+	}
+	now := time.Now()
+	for _, node := range nodes {
+		if err := s.obligations.Record(ctx, ebsobligation.UnpublishObligation{
+			VolumeID:   meta.VolumeID,
+			AccountID:  meta.TenantID,
+			Handle:     meta.ProviderHandle,
+			NodeID:     node,
+			InstanceID: meta.AttachedInstance,
+			AttachedAt: meta.AttachedAt,
+			RecordedAt: now,
+		}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// recordTerminateObligation records the obligation for a terminate that clears
+// owed's attachment with its publish still in place. A volume no longer
+// attached to that instance has nothing of it left to clear.
+func (s *VolumeServiceImpl) recordTerminateObligation(ctx context.Context, accountID, volumeID string, owed ec2instance.OwedUnpublish) error {
+	if !owed.Owed() {
+		return nil
+	}
+	meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
+	if objectstore.IsNoSuchKeyError(err) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("read volume %s before recording its unpublish obligation: %w", volumeID, err)
+	}
+	if meta.AttachedInstance != owed.InstanceID {
+		return nil
+	}
+	if err := s.recordObligations(ctx, meta, []string{owed.NodeID}); err != nil {
+		return fmt.Errorf("record unpublish obligation for %s: %w", volumeID, err)
+	}
+	return nil
+}
+
+// ObligationUnpublisher is this node's mounter as the obligation reaper drives
+// it. Its lock is the one every publish and unpublish on this node takes.
+type ObligationUnpublisher interface {
+	// HoldVolume takes volumeID's lock without waiting. false means a publish
+	// or unpublish of it is in flight here, and the sweep comes back later.
+	HoldVolume(volumeID string) (VolumeHold, bool)
+}
+
+// VolumeHold is one volume's publish lock on this node.
+type VolumeHold interface {
+	// PublishedSince reports whether this process published the volume after
+	// t and has not unpublished it since.
+	PublishedSince(t time.Time) bool
+	// Unpublish tears down this node's export of the volume. A volume the
+	// provider holds no export of counts as done.
+	Unpublish(ctx context.Context, accountID, handle string, noSeal bool) error
+	Release()
+}
+
+// obligationAction is what settling an obligation does with the volume as it
+// is now.
+type obligationAction int
+
+const (
+	// obligationWait leaves the obligation open: the attachment it names has
+	// not been cleared yet, or a publish here is newer than it.
+	obligationWait obligationAction = iota
+	// obligationSuperseded closes it without a provider call: a newer
+	// attachment owns the export.
+	obligationSuperseded
+	// obligationSeal unpublishes with a seal: nobody else writes the volume,
+	// so the copy here is the last owner's and its writes are kept.
+	obligationSeal
+	// obligationNoSeal unpublishes without sealing a volume being deleted.
+	obligationNoSeal
+)
+
+func (a obligationAction) String() string {
+	switch a {
+	case obligationSuperseded:
+		return "superseded"
+	case obligationSeal:
+		return "seal"
+	case obligationNoSeal:
+		return "no-seal"
+	default:
+		return "wait"
+	}
+}
+
+// decideObligation is the settlement policy over the volume's current record;
+// meta is nil when the volume has none. It never unpublishes an attachment the
+// volume still carries.
+func decideObligation(o ebsobligation.UnpublishObligation, meta *ebsmetadata.Volume) obligationAction {
+	if meta == nil || meta.State == volumeStateDeleting {
+		return obligationNoSeal
+	}
+	if meta.AttachedInstance != "" {
+		if meta.AttachedInstance != o.InstanceID || meta.AttachedAt.After(o.AttachedAt) {
+			return obligationSuperseded
+		}
+		return obligationWait
+	}
+	return obligationSeal
+}
+
+// obligationSince is the time a publish on this node must follow to be newer
+// than the one an obligation undoes.
+func obligationSince(o ebsobligation.UnpublishObligation) time.Time {
+	if o.AttachedAt.IsZero() {
+		return o.RecordedAt
+	}
+	return o.AttachedAt
+}
+
+// unpublishObligationReaper settles the obligations naming its node through
+// that node's mounter, once the node answers.
+type unpublishObligationReaper struct {
+	svc         *VolumeServiceImpl
+	node        string
+	unpublisher ObligationUnpublisher
+}
+
+var _ vm.Reaper = (*unpublishObligationReaper)(nil)
+
+// NewUnpublishObligationReaper builds the reaper that settles node's
+// obligations through unpublisher, which must be the node's own mounter.
+func (s *VolumeServiceImpl) NewUnpublishObligationReaper(node string, unpublisher ObligationUnpublisher) vm.Reaper {
+	return &unpublishObligationReaper{svc: s, node: node, unpublisher: unpublisher}
+}
+
+func (r *unpublishObligationReaper) Class() string { return "ebs-unpublish-obligation" }
+
+// Scope is node-local: an obligation is settled only by the node it names,
+// because only that node can tear its own export down.
+func (r *unpublishObligationReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
+
+// Sweep settles what it can and reports how many obligations it closed. A
+// failure is recorded on the obligation and retried on the next sweep.
+func (r *unpublishObligationReaper) Sweep(ctx context.Context) (int, error) {
+	if r.svc.obligations == nil || r.unpublisher == nil {
+		return 0, nil
+	}
+	open, err := r.svc.obligations.ListNode(ctx, r.node)
+	if err != nil {
+		return 0, err
+	}
+	closed := 0
+	for _, o := range open {
+		if err := ctx.Err(); err != nil {
+			return closed, err
+		}
+		done, err := r.settle(ctx, o)
+		if err != nil {
+			slog.WarnContext(ctx, "ebs-unpublish-obligation: settlement failed, retrying next sweep",
+				"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "attempts", o.Attempts+1, "err", err)
+			if recErr := r.svc.obligations.RecordFailure(ctx, o, err); recErr != nil {
+				slog.WarnContext(ctx, "ebs-unpublish-obligation: could not record the failure",
+					"volumeId", o.VolumeID, "node", o.NodeID, "err", recErr)
+			}
+			continue
+		}
+		if done {
+			closed++
+		}
+	}
+	return closed, nil
+}
+
+// settle acts on one obligation under the volume's lock, so no publish on this
+// node falls between reading the volume and unpublishing it.
+func (r *unpublishObligationReaper) settle(ctx context.Context, o ebsobligation.UnpublishObligation) (bool, error) {
+	hold, ok := r.unpublisher.HoldVolume(o.VolumeID)
+	if !ok {
+		return false, nil
+	}
+	defer hold.Release()
+
+	var current *ebsmetadata.Volume
+	meta, err := r.svc.metadata.GetVolume(ctx, o.AccountID, o.VolumeID)
+	switch {
+	case err == nil:
+		current = &meta
+	case objectstore.IsNoSuchKeyError(err):
+	default:
+		return false, fmt.Errorf("read volume %s: %w", o.VolumeID, err)
+	}
+
+	action := decideObligation(o, current)
+	// A newer publish here may own the export, so only a superseded obligation,
+	// which sends nothing, closes past it.
+	if action != obligationSuperseded && hold.PublishedSince(obligationSince(o)) {
+		return false, nil
+	}
+	switch action {
+	case obligationWait:
+		return false, nil
+	case obligationSuperseded:
+		slog.WarnContext(ctx, "ebs-unpublish-obligation: a newer attachment owns the volume, closing without unpublishing",
+			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "attachedInstance", current.AttachedInstance)
+	case obligationSeal, obligationNoSeal:
+		if err := hold.Unpublish(ctx, o.AccountID, o.Handle, action == obligationNoSeal); err != nil {
+			return false, err
+		}
+		slog.InfoContext(ctx, "ebs-unpublish-obligation: unpublished the export a cleared attachment left",
+			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "noSeal", action == obligationNoSeal)
+	}
+	if err := r.svc.obligations.Close(ctx, o); err != nil {
+		return false, fmt.Errorf("close obligation: %w", err)
+	}
+	recordObligationClosed(o.NodeID, action.String())
+	return true, nil
+}
+
+// NodeDeparted reports whether node has left the cluster, rather than being
+// down, stale or partitioned, which must not count.
+type NodeDeparted func(ctx context.Context, node string) (bool, error)
+
+// unpublishObligationCloser closes, without a provider call, the obligations of
+// nodes removed from the cluster: such a node holds nothing a member can reach.
+type unpublishObligationCloser struct {
+	svc      *VolumeServiceImpl
+	departed NodeDeparted
+}
+
+var _ vm.Reaper = (*unpublishObligationCloser)(nil)
+
+// NewUnpublishObligationCloser builds the cluster-wide pass that closes the
+// obligations of nodes departed reports as gone.
+func (s *VolumeServiceImpl) NewUnpublishObligationCloser(departed NodeDeparted) vm.Reaper {
+	return &unpublishObligationCloser{svc: s, departed: departed}
+}
+
+func (c *unpublishObligationCloser) Class() string { return "ebs-unpublish-obligation-close" }
+
+// Scope is cluster-wide: membership is the cluster's, and one pass is enough.
+func (c *unpublishObligationCloser) Scope() vm.ReaperScope { return vm.ScopeClusterWide }
+
+func (c *unpublishObligationCloser) Sweep(ctx context.Context) (int, error) {
+	if c.svc.obligations == nil || c.departed == nil {
+		return 0, nil
+	}
+	open, err := c.svc.obligations.List(ctx)
+	if err != nil {
+		return 0, err
+	}
+	gone := map[string]bool{}
+	closed := 0
+	for _, o := range open {
+		departed, seen := gone[o.NodeID]
+		if !seen {
+			departed, err = c.departed(ctx, o.NodeID)
+			if err != nil {
+				slog.WarnContext(ctx, "ebs-unpublish-obligation: cannot tell whether a node left the cluster, keeping its obligations",
+					"node", o.NodeID, "err", err)
+				departed = false
+			}
+			gone[o.NodeID] = departed
+		}
+		if !departed {
+			continue
+		}
+		if err := c.svc.obligations.Close(ctx, o); err != nil {
+			slog.WarnContext(ctx, "ebs-unpublish-obligation: failed to close a departed node's obligation",
+				"volumeId", o.VolumeID, "node", o.NodeID, "err", err)
+			continue
+		}
+		slog.WarnContext(ctx, "ebs-unpublish-obligation: closed for a node removed from the cluster",
+			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID)
+		recordObligationClosed(o.NodeID, "departed")
+		closed++
+	}
+	return closed, nil
+}
+
+const obligationMeterName = "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume/obligation"
+
+var (
+	obligationClosedOnce    sync.Once
+	obligationClosedCounter metric.Int64Counter
+)
+
+// recordObligationClosed counts one closed obligation by the node it named
+// and how it closed.
+func recordObligationClosed(node, outcome string) {
+	obligationClosedOnce.Do(func() {
+		var err error
+		obligationClosedCounter, err = otel.Meter(obligationMeterName).Int64Counter("mulga.ebs.unpublish_obligation.closed",
+			metric.WithDescription("Unpublish obligations closed, by the node they named and how they closed."),
+			metric.WithUnit("{obligation}"))
+		if err != nil {
+			otel.Handle(err)
+		}
+	})
+	if obligationClosedCounter == nil {
+		return
+	}
+	obligationClosedCounter.Add(context.Background(), 1, metric.WithAttributes(
+		attribute.String("node", node), attribute.String("outcome", outcome)))
+}
@@ -0,0 +1,596 @@
+package volume
+
+//test:in-package — the obligation reaper and its policy are unexported, and
+// seeding a volume's metadata document is how an attachment is set up.
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeNodeLocator struct {
+	nodes map[string]string
+	err   error
+}
+
+var _ InstanceNodeLocator = (*fakeNodeLocator)(nil)
+
+func (f *fakeNodeLocator) InstanceLastNode(instanceID string) (string, bool, error) {
+	if f.err != nil {
+		return "", false, f.err
+	}
+	node, found := f.nodes[instanceID]
+	return node, found, nil
+}
+
+// wireObligations gives svc an obligation bucket on a JetStream of its own and
+// returns the connection under it, so a test can cut it.
+func wireObligations(t *testing.T, svc *VolumeServiceImpl, locator InstanceNodeLocator, members ...string) *nats.Conn {
+	t.Helper()
+	_, nc, js := testutil.StartTestJetStream(t)
+	store := ebsobligation.NewStore(js)
+	require.NoError(t, store.Open(t.Context()))
+	svc.SetUnpublishObligations(store, locator, func() []string { return members })
+	return nc
+}
+
+type unpublishCall struct {
+	volumeID, accountID, handle string
+	noSeal                      bool
+}
+
+// fakeUnpublisher stands in for a node's mounter: busy volumes refuse the
+// hold, publishedAt is the last publish this process made of each volume.
+type fakeUnpublisher struct {
+	mu          sync.Mutex
+	busy        map[string]bool
+	publishedAt map[string]time.Time
+	err         error
+	calls       []unpublishCall
+	held        int
+}
+
+var _ ObligationUnpublisher = (*fakeUnpublisher)(nil)
+
+func (f *fakeUnpublisher) HoldVolume(volumeID string) (VolumeHold, bool) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.busy[volumeID] {
+		return nil, false
+	}
+	f.held++
+	return &fakeHold{f: f, volumeID: volumeID}, true
+}
+
+func (f *fakeUnpublisher) unpublishes() []unpublishCall {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]unpublishCall(nil), f.calls...)
+}
+
+type fakeHold struct {
+	f        *fakeUnpublisher
+	volumeID string
+}
+
+var _ VolumeHold = (*fakeHold)(nil)
+
+func (h *fakeHold) PublishedSince(t time.Time) bool {
+	h.f.mu.Lock()
+	defer h.f.mu.Unlock()
+	at, ok := h.f.publishedAt[h.volumeID]
+	return ok && at.After(t)
+}
+
+func (h *fakeHold) Unpublish(_ context.Context, accountID, handle string, noSeal bool) error {
+	h.f.mu.Lock()
+	defer h.f.mu.Unlock()
+	h.f.calls = append(h.f.calls, unpublishCall{volumeID: h.volumeID, accountID: accountID, handle: handle, noSeal: noSeal})
+	return h.f.err
+}
+
+func (h *fakeHold) Release() {
+	h.f.mu.Lock()
+	defer h.f.mu.Unlock()
+	h.f.held--
+}
+
+// putAttachment writes volumeID's document with an explicit attachment, so a
+// test controls AttachedAt rather than racing the clock.
+func putAttachment(t *testing.T, svc *VolumeServiceImpl, volumeID, state, instanceID string, attachedAt time.Time) ebsmetadata.Volume {
+	t.Helper()
+	meta := ebsmetadata.Volume{
+		VolumeID: volumeID, TenantID: testVolAccountID, CapacityGiB: 1,
+		State: state, AvailabilityZone: "ap-southeast-2a", VolumeType: "gp3",
+		AttachedInstance: instanceID, AttachedAt: attachedAt, ProviderHandle: "handle-" + volumeID,
+	}
+	if instanceID != "" {
+		meta.DeviceName = "/dev/sdf"
+	}
+	require.NoError(t, svc.metadata.PutVolume(context.Background(), meta))
+	return meta
+}
+
+func readVolume(t *testing.T, svc *VolumeServiceImpl, volumeID string) ebsmetadata.Volume {
+	t.Helper()
+	meta, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
+	require.NoError(t, err)
+	return meta
+}
+
+func openObligations(t *testing.T, svc *VolumeServiceImpl) []ebsobligation.UnpublishObligation {
+	t.Helper()
+	open, err := svc.obligations.List(context.Background())
+	require.NoError(t, err)
+	return open
+}
+
+func forceDetach(t *testing.T, svc *VolumeServiceImpl, volumeID string) (*ec2.VolumeAttachment, error) {
+	t.Helper()
+	return svc.ForceDetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId: aws.String(volumeID), Force: aws.Bool(true),
+	}, testVolAccountID)
+}
+
+// --- ForceDetachVolume records the obligation ---
+
+func TestForceDetach_RecordsOneObligationNamingThePublish(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a", "node-b")
+	attachedAt := time.Unix(1_700_000_000, 0).UTC()
+	putAttachment(t, svc, "vol-fd-once", "in-use", "i-old", attachedAt)
+
+	_, err := forceDetach(t, svc, "vol-fd-once")
+	require.NoError(t, err)
+
+	open := openObligations(t, svc)
+	require.Len(t, open, 1, "only the node that last ran the instance is named")
+	o := open[0]
+	assert.Equal(t, "vol-fd-once", o.VolumeID)
+	assert.Equal(t, testVolAccountID, o.AccountID)
+	assert.Equal(t, "handle-vol-fd-once", o.Handle)
+	assert.Equal(t, "node-a", o.NodeID)
+	assert.Equal(t, "i-old", o.InstanceID)
+	assert.True(t, attachedAt.Equal(o.AttachedAt), "the obligation names the attachment it undoes")
+	assert.Equal(t, ebsobligation.SchemaVersion, o.SchemaVersion)
+}
+
+// A caller retrying a force detach whose clear failed writes the same key, so
+// the retry leaves one obligation, not two.
+func TestForceDetach_RetryLeavesOneObligation(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a")
+	meta := putAttachment(t, svc, "vol-fd-retry", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
+
+	require.NoError(t, svc.recordObligations(context.Background(), meta, []string{"node-a"}))
+	_, err := forceDetach(t, svc, "vol-fd-retry")
+	require.NoError(t, err)
+
+	assert.Len(t, openObligations(t, svc), 1)
+}
+
+// With no record of the instance there is no node to name, so every member is
+// named and each one settles its own.
+func TestForceDetach_UnknownInstanceNamesEveryMember(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a", "node-b", "node-c")
+	putAttachment(t, svc, "vol-fd-every", "in-use", "i-gone", time.Unix(1_700_000_000, 0).UTC())
+
+	_, err := forceDetach(t, svc, "vol-fd-every")
+	require.NoError(t, err)
+
+	nodes := []string{}
+	for _, o := range openObligations(t, svc) {
+		nodes = append(nodes, o.NodeID)
+	}
+	assert.ElementsMatch(t, []string{"node-a", "node-b", "node-c"}, nodes)
+}
+
+// The attachment is the only record of where the export is. If the obligation
+// cannot be written, clearing it would forget an export, so nothing is cleared.
+func TestForceDetach_ObligationWriteFailureClearsNothing(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	nc := wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a")
+	putAttachment(t, svc, "vol-fd-nowrite", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
+	nc.Close()
+
+	_, err := forceDetach(t, svc, "vol-fd-nowrite")
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInternalError, err.Error())
+	meta := readVolume(t, svc, "vol-fd-nowrite")
+	assert.Equal(t, "in-use", meta.State)
+	assert.Equal(t, "i-old", meta.AttachedInstance)
+}
+
+func TestForceDetach_UnnameableNodeClearsNothing(t *testing.T) {
+	for name, locator := range map[string]InstanceNodeLocator{
+		"locator fails":            &fakeNodeLocator{err: errors.New("instance bucket unreachable")},
+		"no record and no members": &fakeNodeLocator{},
+		"no locator configured":    nil,
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc := newTestVolumeService("ap-southeast-2a")
+			wireObligations(t, svc, locator)
+			putAttachment(t, svc, "vol-fd-noname", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
+
+			_, err := forceDetach(t, svc, "vol-fd-noname")
+
+			require.Error(t, err)
+			assert.Equal(t, awserrors.ErrorInternalError, err.Error())
+			assert.Equal(t, "i-old", readVolume(t, svc, "vol-fd-noname").AttachedInstance)
+			assert.Empty(t, openObligations(t, svc))
+		})
+	}
+}
+
+// providerCallCounter fails the test's expectations if the request path makes
+// any provider call: an unpublish waits on a node that may never answer.
+type providerCallCounter struct {
+	ebsprovider.EBSProvider
+
+	mu        sync.Mutex
+	unpublish int
+	publish   int
+}
+
+func (p *providerCallCounter) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
+	p.mu.Lock()
+	p.unpublish++
+	p.mu.Unlock()
+	<-ctx.Done()
+	return nil, ctx.Err()
+}
+
+func (p *providerCallCounter) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
+	p.mu.Lock()
+	p.publish++
+	p.mu.Unlock()
+	<-ctx.Done()
+	return nil, ctx.Err()
+}
+
+// TestS8_ForceDetachRecordsTheObligationAndReturns pins that a force detach
+// never waits on the node holding the export: it decides its errors, records
+// the obligation and returns, and the unpublish is left to that node's reaper.
+func TestS8_ForceDetachRecordsTheObligationAndReturns(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	provider := &providerCallCounter{EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})}
+	svc.SetEBSProvider(provider)
+	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-wedged": "node-unreachable"}}, "node-unreachable")
+	putAttachment(t, svc, "vol-s8-forcedetach", "in-use", "i-wedged", time.Unix(1_700_000_000, 0).UTC())
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	attachment, err := svc.ForceDetachVolume(ctx, &ec2.DetachVolumeInput{
+		VolumeId: aws.String("vol-s8-forcedetach"), Force: aws.Bool(true),
+	}, testVolAccountID)
+	require.NoError(t, err)
+	require.NoError(t, ctx.Err(), "the request returned on its own, not on the deadline")
+
+	assert.Equal(t, "detaching", aws.StringValue(attachment.State))
+	provider.mu.Lock()
+	assert.Zero(t, provider.unpublish, "the request path makes no unpublish call")
+	assert.Zero(t, provider.publish)
+	provider.mu.Unlock()
+
+	open := openObligations(t, svc)
+	require.Len(t, open, 1, "the unpublish the request did not wait for is recorded")
+	assert.Equal(t, "node-unreachable", open[0].NodeID)
+	assert.Empty(t, readVolume(t, svc, "vol-s8-forcedetach").AttachedInstance)
+}
+
+// --- Terminate paths ---
+
+func TestTerminate_OwedUnpublishRecordsOneObligationBeforeTheClear(t *testing.T) {
+	for name, terminate := range map[string]func(*VolumeServiceImpl, string, ec2instance.OwedUnpublish) error{
+		"detach": func(svc *VolumeServiceImpl, id string, owed ec2instance.OwedUnpublish) error {
+			return svc.DetachVolumeOnTerminate(context.Background(), id, testVolAccountID, owed)
+		},
+		"delete": func(svc *VolumeServiceImpl, id string, owed ec2instance.OwedUnpublish) error {
+			return svc.DeleteVolumeOnTerminate(context.Background(), id, testVolAccountID, owed)
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc := newTestVolumeService("ap-southeast-2a")
+			svc.snapshotKV = setupTestVolumeKV(t)
+			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+			seedTestProviderVolume(t, svc, "vol-term-owed", 1)
+			putAttachment(t, svc, "vol-term-owed", "in-use", "i-term", time.Unix(1_700_000_000, 0).UTC())
+
+			require.NoError(t, terminate(svc, "vol-term-owed", ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"}))
+
+			open := openObligations(t, svc)
+			require.Len(t, open, 1)
+			assert.Equal(t, "node-a", open[0].NodeID)
+			assert.Equal(t, "i-term", open[0].InstanceID)
+		})
+	}
+}
+
+// A terminate whose unmount completed owes nothing, and an attachment that is
+// already some other instance's is not this terminate's to undo.
+func TestTerminate_NothingOwedRecordsNoObligation(t *testing.T) {
+	for name, tc := range map[string]struct {
+		attachedTo string
+		owed       ec2instance.OwedUnpublish
+	}{
+		"unmount completed":   {attachedTo: "i-term", owed: ec2instance.OwedUnpublish{}},
+		"attached to another": {attachedTo: "i-other", owed: ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc := newTestVolumeService("ap-southeast-2a")
+			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+			putAttachment(t, svc, "vol-term-none", "in-use", tc.attachedTo, time.Unix(1_700_000_000, 0).UTC())
+
+			require.NoError(t, svc.DetachVolumeOnTerminate(context.Background(), "vol-term-none", testVolAccountID, tc.owed))
+
+			assert.Empty(t, openObligations(t, svc))
+		})
+	}
+}
+
+func TestTerminate_ObligationWriteFailureClearsNothing(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	nc := wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+	putAttachment(t, svc, "vol-term-nowrite", "in-use", "i-term", time.Unix(1_700_000_000, 0).UTC())
+	nc.Close()
+
+	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-term-nowrite", testVolAccountID,
+		ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"})
+
+	require.Error(t, err)
+	assert.Equal(t, "i-term", readVolume(t, svc, "vol-term-nowrite").AttachedInstance)
+}
+
+// --- The reaper ---
+
+func recordObligation(t *testing.T, svc *VolumeServiceImpl, node, volumeID, instanceID string, attachedAt time.Time) ebsobligation.UnpublishObligation {
+	t.Helper()
+	o := ebsobligation.UnpublishObligation{
+		VolumeID: volumeID, AccountID: testVolAccountID, Handle: "handle-" + volumeID,
+		NodeID: node, InstanceID: instanceID, AttachedAt: attachedAt, RecordedAt: attachedAt.Add(time.Minute),
+	}
+	require.NoError(t, svc.obligations.Record(context.Background(), o))
+	return o
+}
+
+func sweepObligations(t *testing.T, svc *VolumeServiceImpl, node string, unpublisher ObligationUnpublisher) int {
+	t.Helper()
+	closed, err := svc.NewUnpublishObligationReaper(node, unpublisher).Sweep(context.Background())
+	require.NoError(t, err)
+	return closed
+}
+
+// TestS7_ObligationReaperNeverUnpublishesACurrentAttachment pins that the
+// reaper, which runs with no customer asking, never tears down an export a
+// live attachment uses and never removes or rewrites the volume.
+func TestS7_ObligationReaperNeverUnpublishesACurrentAttachment(t *testing.T) {
+	cleared := time.Unix(1_700_000_000, 0).UTC()
+	for name, tc := range map[string]struct {
+		state       string
+		attachedTo  string
+		attachedAt  time.Time
+		publishedAt time.Time // a publish this process made, zero if none
+		stillOpen   bool
+	}{
+		// The volume moved to another instance, possibly on this very node.
+		"attached to another instance": {state: "in-use", attachedTo: "i-new", attachedAt: cleared.Add(time.Hour)},
+		// The same instance was attached again after the clear.
+		"same instance attached again": {state: "in-use", attachedTo: "i-old", attachedAt: cleared.Add(time.Hour)},
+		// The clear has not landed: the obligation was written first.
+		"attachment not yet cleared": {state: "in-use", attachedTo: "i-old", attachedAt: cleared, stillOpen: true},
+		// A hot attach published here and has not yet written in-use.
+		"newer publish here, record not yet in-use": {state: "available", publishedAt: cleared.Add(time.Hour), stillOpen: true},
+		// Force detach, then attach again on this node: the new publish is here.
+		"attached again here after a newer publish": {state: "in-use", attachedTo: "i-new", attachedAt: cleared.Add(time.Hour), publishedAt: cleared.Add(time.Hour)},
+		"same instance attached again here":         {state: "in-use", attachedTo: "i-old", attachedAt: cleared.Add(time.Hour), publishedAt: cleared.Add(time.Hour)},
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc := newTestVolumeService("ap-southeast-2a")
+			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+			before := putAttachment(t, svc, "vol-s7-current", tc.state, tc.attachedTo, tc.attachedAt)
+			recordObligation(t, svc, "node-a", "vol-s7-current", "i-old", cleared)
+			unpublisher := &fakeUnpublisher{publishedAt: map[string]time.Time{}}
+			if !tc.publishedAt.IsZero() {
+				unpublisher.publishedAt["vol-s7-current"] = tc.publishedAt
+			}
+
+			sweepObligations(t, svc, "node-a", unpublisher)
+
+			assert.Empty(t, unpublisher.unpublishes(), "a current attachment's export must never be unpublished")
+			assert.Zero(t, unpublisher.held, "every hold is released")
+			after := readVolume(t, svc, "vol-s7-current")
+			assert.Equal(t, before.State, after.State, "the reaper never rewrites the volume")
+			assert.Equal(t, before.AttachedInstance, after.AttachedInstance)
+			if tc.stillOpen {
+				assert.Len(t, openObligations(t, svc), 1, "the obligation waits for its attachment to be gone")
+			} else {
+				assert.Empty(t, openObligations(t, svc), "a superseded obligation closes without a provider call")
+			}
+		})
+	}
+}
+
+// TestS7_ObligationReaperSealsLiveDataAndDropsOnlyDeletingVolumes pins that
+// the reaper keeps a surviving volume's writes and only drops them unsealed
+// for a volume that is being deleted or is gone.
+func TestS7_ObligationReaperSealsLiveDataAndDropsOnlyDeletingVolumes(t *testing.T) {
+	cleared := time.Unix(1_700_000_000, 0).UTC()
+	for name, tc := range map[string]struct {
+		state  string // empty for no document
+		noSeal bool
+	}{
+		"available keeps its writes": {state: "available", noSeal: false},
+		"deleting is dropped":        {state: volumeStateDeleting, noSeal: true},
+		"absent is dropped":          {state: "", noSeal: true},
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc := newTestVolumeService("ap-southeast-2a")
+			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+			if tc.state != "" {
+				putAttachment(t, svc, "vol-s7-settle", tc.state, "", time.Time{})
+			}
+			recordObligation(t, svc, "node-a", "vol-s7-settle", "i-old", cleared)
+			unpublisher := &fakeUnpublisher{}
+
+			assert.Equal(t, 1, sweepObligations(t, svc, "node-a", unpublisher))
+
+			assert.Equal(t, []unpublishCall{{volumeID: "vol-s7-settle", accountID: testVolAccountID, handle: "handle-vol-s7-settle", noSeal: tc.noSeal}},
+				unpublisher.unpublishes())
+			assert.Empty(t, openObligations(t, svc))
+			if tc.state != "" {
+				assert.Equal(t, tc.state, readVolume(t, svc, "vol-s7-settle").State, "the reaper never rewrites the volume")
+			}
+		})
+	}
+}
+
+// TestS7_ObligationReaperNeverTreatsAnUnreadableVolumeAsGone pins that a read
+// failing for a reason other than absence leaves the obligation open: taking
+// it for a deleted volume would drop live writes unsealed.
+func TestS7_ObligationReaperNeverTreatsAnUnreadableVolumeAsGone(t *testing.T) {
+	memory := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", memory)
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+	putAttachment(t, svc, "vol-s7-unread", "available", "", time.Time{})
+	recordObligation(t, svc, "node-a", "vol-s7-unread", "i-old", time.Unix(1_700_000_000, 0).UTC())
+	failing := &getFailingStore{MemoryObjectStore: memory, failKey: metadataKey(t, "vol-s7-unread")}
+	svc.metadata = ebsmetadata.NewStore(failing, "test-bucket")
+	unpublisher := &fakeUnpublisher{}
+
+	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
+
+	assert.Empty(t, unpublisher.unpublishes())
+	open := openObligations(t, svc)
+	require.Len(t, open, 1)
+	assert.Equal(t, 1, open[0].Attempts, "the failure is counted where an operator can see it")
+	assert.Contains(t, open[0].Error, "reconstruction failed")
+}
+
+// End to end through the service: a force detach followed by a new attachment
+// of the same volume on the same node must leave that new export alone.
+func TestS7_ForceDetachThenReattachOnTheSameNodeKeepsTheNewExport(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a", "i-new": "node-a"}}, "node-a")
+	putAttachment(t, svc, "vol-s7-reattach", "in-use", "i-old", time.Now().Add(-time.Hour).UTC())
+
+	_, err := forceDetach(t, svc, "vol-s7-reattach")
+	require.NoError(t, err)
+	require.NoError(t, svc.UpdateVolumeState(testVolAccountID, "vol-s7-reattach", "in-use", "i-new", "/dev/sdg"))
+	unpublisher := &fakeUnpublisher{}
+
+	assert.Equal(t, 1, sweepObligations(t, svc, "node-a", unpublisher))
+
+	assert.Empty(t, unpublisher.unpublishes(), "the new attachment's export must survive the old obligation")
+	assert.Equal(t, "i-new", readVolume(t, svc, "vol-s7-reattach").AttachedInstance)
+	assert.Empty(t, openObligations(t, svc))
+}
+
+func TestObligationReaper_SettlesOnlyItsOwnNode(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a", "node-b")
+	putAttachment(t, svc, "vol-own", "available", "", time.Time{})
+	recordObligation(t, svc, "node-b", "vol-own", "i-old", time.Unix(1_700_000_000, 0).UTC())
+	unpublisher := &fakeUnpublisher{}
+
+	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
+
+	assert.Empty(t, unpublisher.unpublishes(), "only the node holding the export can tear it down")
+	assert.Len(t, openObligations(t, svc), 1)
+}
+
+// A publish or unpublish of the volume in flight on this node holds its lock;
+// the sweep skips it rather than waiting behind a possibly wedged call.
+func TestObligationReaper_SkipsABusyVolume(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+	putAttachment(t, svc, "vol-busy", "available", "", time.Time{})
+	recordObligation(t, svc, "node-a", "vol-busy", "i-old", time.Unix(1_700_000_000, 0).UTC())
+	unpublisher := &fakeUnpublisher{busy: map[string]bool{"vol-busy": true}}
+
+	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
+
+	assert.Empty(t, unpublisher.unpublishes())
+	open := openObligations(t, svc)
+	require.Len(t, open, 1)
+	assert.Zero(t, open[0].Attempts, "a busy volume is not a failure")
+}
+
+func TestObligationReaper_FailedUnpublishStaysOpenAndCounts(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+	putAttachment(t, svc, "vol-fail", "available", "", time.Time{})
+	recordObligation(t, svc, "node-a", "vol-fail", "i-old", time.Unix(1_700_000_000, 0).UTC())
+	unpublisher := &fakeUnpublisher{err: errors.New("seal refused")}
+
+	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
+	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
+
+	open := openObligations(t, svc)
+	require.Len(t, open, 1)
+	assert.Equal(t, 2, open[0].Attempts)
+	assert.Equal(t, "seal refused", open[0].Error)
+	assert.Zero(t, unpublisher.held, "a failed unpublish still releases the hold")
+}
+
+// An obligation with no attachment time falls back to when it was recorded, so
+// a publish here after the clear still counts as newer.
+func TestObligationSince_FallsBackToRecordedAt(t *testing.T) {
+	recorded := time.Unix(1_700_000_000, 0)
+	assert.Equal(t, recorded, obligationSince(ebsobligation.UnpublishObligation{RecordedAt: recorded}))
+	attached := recorded.Add(-time.Hour)
+	assert.Equal(t, attached, obligationSince(ebsobligation.UnpublishObligation{AttachedAt: attached, RecordedAt: recorded}))
+}
+
+// --- The cluster-wide close ---
+
+func TestObligationCloser_ClosesOnlyDepartedNodes(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+	at := time.Unix(1_700_000_000, 0).UTC()
+	recordObligation(t, svc, "node-gone", "vol-c1", "i-1", at)
+	recordObligation(t, svc, "node-gone", "vol-c2", "i-2", at)
+	recordObligation(t, svc, "node-down", "vol-c3", "i-3", at)
+	recordObligation(t, svc, "node-unknown", "vol-c4", "i-4", at)
+	asked := map[string]int{}
+	departed := func(_ context.Context, node string) (bool, error) {
+		asked[node]++
+		switch node {
+		case "node-gone":
+			return true, nil
+		case "node-unknown":
+			return false, errors.New("heartbeat bucket unreachable")
+		default:
+			return false, nil
+		}
+	}
+
+	closed, err := svc.NewUnpublishObligationCloser(departed).Sweep(context.Background())
+	require.NoError(t, err)
+
+	assert.Equal(t, 2, closed)
+	remaining := []string{}
+	for _, o := range openObligations(t, svc) {
+		remaining = append(remaining, o.NodeID)
+	}
+	assert.ElementsMatch(t, []string{"node-down", "node-unknown"}, remaining,
+		"a node that is down, or whose membership cannot be read, keeps its obligations")
+	assert.Equal(t, 1, asked["node-gone"], "membership is asked once per node per sweep")
+}
@@ -33,19 +33,19 @@ func BatchDeleteImage(ctx context.Context, deleter ImageDeleter, accountID strin
 	if err := ValidateRepositoryScope(req.RepositoryName, req.RegistryID, accountID); err != nil {
 		return nil, err
 	}
-	if len(req.ImageIDs) > maxImageBatch {
-		return nil, MaxItemsError("imageIds", maxImageBatch)
+	if err := validateImageIDs(req.ImageIDs, true); err != nil {
+		return nil, err
 	}

 	deleted := make([]*ecr.ImageIdentifier, 0, len(req.ImageIDs))
 	failures := make([]*ecr.ImageFailure, 0)
 	for _, imageID := range req.ImageIDs {
-		if imageID.ImageDigest == "" && imageID.ImageTag == "" {
+		if imageID.ImageDigest == "" && imageID.ImageTag == nil {
 			failures = append(failures, imageFailure(imageID, ecr.ImageFailureCodeMissingDigestAndTag, "no digest or tag specified"))
 			continue
 		}

-		digest, err := deleter.DeleteImage(ctx, accountID, req.RepositoryName, imageID.ImageTag, imageID.ImageDigest)
+		digest, err := deleter.DeleteImage(ctx, accountID, req.RepositoryName, imageID.tag(), imageID.ImageDigest)
 		if errors.Is(err, ecrregistry.ErrImageNotFound) {
 			failures = append(failures, imageFailure(imageID, ecr.ImageFailureCodeImageNotFound, ImageNotFoundReason))
 			continue
@@ -56,8 +56,8 @@ func BatchDeleteImage(ctx context.Context, deleter ImageDeleter, accountID strin
 		}

 		deletedImage := &ecr.ImageIdentifier{ImageDigest: aws.String(digest)}
-		if imageID.ImageTag != "" {
-			deletedImage.ImageTag = aws.String(imageID.ImageTag)
+		if imageID.ImageTag != nil {
+			deletedImage.ImageTag = imageID.ImageTag
 		}
 		deleted = append(deleted, deletedImage)
 	}
@@ -36,8 +36,13 @@ func BatchGetImage(ctx context.Context, reader ManifestReader, accountID string,
 	if err := ValidateRepositoryScope(req.RepositoryName, req.RegistryID, accountID); err != nil {
 		return nil, err
 	}
-	if len(req.ImageIDs) > maxImageBatch {
-		return nil, MaxItemsError("imageIds", maxImageBatch)
+	if err := validateImageIDs(req.ImageIDs, true); err != nil {
+		return nil, err
+	}
+	if req.AcceptedMediaTypes != nil {
+		if err := validateListLength("acceptedMediaTypes", len(req.AcceptedMediaTypes), 1, maxMediaTypeListItems); err != nil {
+			return nil, err
+		}
 	}

 	images := make([]*ecr.Image, 0, len(req.ImageIDs))
@@ -45,7 +50,7 @@ func BatchGetImage(ctx context.Context, reader ManifestReader, accountID string,
 	for _, imageID := range req.ImageIDs {
 		reference := imageID.ImageDigest
 		if reference == "" {
-			reference = imageID.ImageTag
+			reference = imageID.tag()
 		}
 		if reference == "" {
 			failures = append(failures, imageFailure(imageID, ecr.ImageFailureCodeMissingDigestAndTag, "no digest or tag specified"))
@@ -69,8 +74,8 @@ func BatchGetImage(ctx context.Context, reader ManifestReader, accountID string,
 			ImageManifest:          aws.String(string(manifest)),
 			ImageManifestMediaType: aws.String(mediaType),
 		}
-		if imageID.ImageTag != "" {
-			image.ImageId.ImageTag = aws.String(imageID.ImageTag)
+		if imageID.ImageTag != nil {
+			image.ImageId.ImageTag = imageID.ImageTag
 		}
 		images = append(images, image)
 	}
@@ -87,8 +92,8 @@ func imageFailure(imageID imageIdentifier, code, reason string) *ecr.ImageFailur
 	if imageID.ImageDigest != "" {
 		failure.ImageId.ImageDigest = aws.String(imageID.ImageDigest)
 	}
-	if imageID.ImageTag != "" {
-		failure.ImageId.ImageTag = aws.String(imageID.ImageTag)
+	if imageID.ImageTag != nil {
+		failure.ImageId.ImageTag = imageID.ImageTag
 	}
 	return failure
 }
@@ -10,10 +10,15 @@ import (
 	ecrregistry "github.com/mulgadc/spinifex/spinifex/domains/ecr/registry"
 )

-// imageIdentifier is the AWS JSON 1.1 {imageDigest, imageTag} pair.
+// imageIdentifier is the AWS JSON 1.1 {imageDigest, imageTag} pair. ImageTag
+// is a pointer so an empty tag can be told apart from an omitted one.
 type imageIdentifier struct {
-	ImageDigest string `json:"imageDigest"`
-	ImageTag    string `json:"imageTag"`
+	ImageDigest string  `json:"imageDigest"`
+	ImageTag    *string `json:"imageTag"`
+}
+
+func (id imageIdentifier) tag() string {
+	return aws.StringValue(id.ImageTag)
 }

 type describeImagesRequest struct {
@@ -36,6 +41,9 @@ func DescribeImages(ctx context.Context, catalog ImageCatalog, accountID string,
 	if err := ValidateRepositoryScope(req.RepositoryName, req.RegistryID, accountID); err != nil {
 		return nil, err
 	}
+	if err := validateImageIDs(req.ImageIDs, false); err != nil {
+		return nil, err
+	}
 	if err := req.Filter.validate(); err != nil {
 		return nil, err
 	}
@@ -53,7 +61,7 @@ func DescribeImages(ctx context.Context, catalog ImageCatalog, accountID string,
 			if image.ImageDigest != "" && image.ImageDigest == record.Digest {
 				return true
 			}
-			if image.ImageTag != "" && slices.Contains(record.Tags, image.ImageTag) {
+			if image.ImageTag != nil && slices.Contains(record.Tags, *image.ImageTag) {
 				return true
 			}
 		}
@@ -82,7 +90,7 @@ func DescribeImages(ctx context.Context, catalog ImageCatalog, accountID string,
 	}
 	if len(req.ImageIDs) > 0 && len(details) == 0 {
 		first := req.ImageIDs[0]
-		return nil, ImageNotFoundError(accountID, req.RepositoryName, first.ImageDigest, first.ImageTag)
+		return nil, ImageNotFoundError(accountID, req.RepositoryName, first.ImageDigest, first.tag())
 	}

 	return &ecr.DescribeImagesOutput{ImageDetails: details}, nil
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"log/slog"

 	"github.com/aws/aws-sdk-go/service/ecr"
@@ -40,6 +41,13 @@ func DescribeRepositories(ctx context.Context, store RepositoryStore, endpoint R
 		return nil, errors.New(awserrors.ErrorAccessDenied)
 	}

+	switch {
+	case req.RepositoryNames != nil && len(req.RepositoryNames) == 0:
+		return nil, ConstraintError("RepositoryNameList", "Must not be empty")
+	case len(req.RepositoryNames) > maxRepositoryNameList:
+		return nil, ConstraintError("RepositoryNameList", fmt.Sprintf("Must have length less than or equal to %d", maxRepositoryNameList))
+	}
+
 	names := req.RepositoryNames
 	if len(names) == 0 {
 		var err error
@@ -0,0 +1,92 @@
+package awsapi
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ecr"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The messages below are AWS's, captured from real ECR refusals.
+const (
+	awsImageIDsNull     = "Invalid parameter at 'imageIds' failed to satisfy constraint: 'Member must not be null'"
+	awsImageIDsEmpty    = "Invalid parameter at 'imageIds' failed to satisfy constraint: 'Member must have length greater than or equal to 1'"
+	awsImageIDsTooMany  = "Invalid parameter at 'imageIds' failed to satisfy constraint: 'Member must have length less than or equal to 100'"
+	awsImageTagEmpty    = "Invalid parameter at 'imageIds.2.member.imageTag' failed to satisfy constraint: 'Member must have length greater than or equal to 1'"
+	awsImageTagTooLong  = "Invalid parameter at 'imageIds.1.member.imageTag' failed to satisfy constraint: 'Member must have length less than or equal to 300'"
+	awsMediaTypesEmpty  = "Invalid parameter at 'acceptedMediaTypes' failed to satisfy constraint: 'Member must have length greater than or equal to 1'"
+	awsMediaTypesTooMny = "Invalid parameter at 'acceptedMediaTypes' failed to satisfy constraint: 'Member must have length less than or equal to 100'"
+)
+
+func requireECRInvalidParameter(t *testing.T, err error, want string) {
+	t.Helper()
+	require.Error(t, err)
+	code, message, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorECRInvalidParameter, code)
+	assert.Equal(t, want, message)
+}
+
+func imageIDCases() map[string]struct{ ids, want string } {
+	long := strings.Repeat("a", 301)
+	return map[string]struct{ ids, want string }{
+		"omitted":      {"", awsImageIDsNull},
+		"empty":        {`,"imageIds":[]`, awsImageIDsEmpty},
+		"too many":     {`,"imageIds":[` + repeatImageIdentifier(101) + `]`, awsImageIDsTooMany},
+		"empty tag":    {`,"imageIds":[{"imageTag":"v1"},{"imageDigest":"sha256:x","imageTag":""}]`, awsImageTagEmpty},
+		"tag too long": {`,"imageIds":[{"imageTag":"` + long + `"}]`, awsImageTagTooLong},
+	}
+}
+
+func TestBatchImageActions_RefuseImageIDsOutsideModel(t *testing.T) {
+	for name, tc := range imageIDCases() {
+		body := []byte(`{"repositoryName":"team/app"` + tc.ids + `}`)
+		t.Run("BatchGetImage "+name, func(t *testing.T) {
+			_, err := BatchGetImage(context.Background(), fakeManifestReader{}, "123456789012", body)
+			requireECRInvalidParameter(t, err, tc.want)
+		})
+		t.Run("BatchDeleteImage "+name, func(t *testing.T) {
+			_, err := BatchDeleteImage(context.Background(), fakeImageDeleter{}, "123456789012", body)
+			requireECRInvalidParameter(t, err, tc.want)
+		})
+	}
+}
+
+func TestBatchGetImage_RefusesAcceptedMediaTypesOutsideModel(t *testing.T) {
+	many := strings.TrimSuffix(strings.Repeat(`"m",`, 101), ",")
+	for want, types := range map[string]string{awsMediaTypesEmpty: `[]`, awsMediaTypesTooMny: `[` + many + `]`} {
+		body := []byte(`{"repositoryName":"team/app","imageIds":[{"imageTag":"v1"}],"acceptedMediaTypes":` + types + `}`)
+		_, err := BatchGetImage(context.Background(), fakeManifestReader{}, "123456789012", body)
+		requireECRInvalidParameter(t, err, want)
+	}
+}
+
+// DescribeImages makes imageIds optional, so only an omitted list lists all.
+func TestDescribeImages_RefusesImageIDsOutsideModel(t *testing.T) {
+	for name, tc := range imageIDCases() {
+		if tc.want == awsImageIDsNull {
+			continue
+		}
+		t.Run(name, func(t *testing.T) {
+			_, err := DescribeImages(context.Background(), nil, "123456789012", []byte(`{"repositoryName":"team/app"`+tc.ids+`}`))
+			requireECRInvalidParameter(t, err, tc.want)
+		})
+	}
+}
+
+func TestDescribeRepositories_RefusesRepositoryNamesOutsideModel(t *testing.T) {
+	store := ecr.NewNATSMetaStore(newRepositoryActionTestConn(t))
+	many := strings.TrimSuffix(strings.Repeat(`"team/app",`, 101), ",")
+	cases := map[string]string{
+		`[]`:             "Invalid parameter at 'RepositoryNameList' failed to satisfy constraint: 'Must not be empty'",
+		`[` + many + `]`: "Invalid parameter at 'RepositoryNameList' failed to satisfy constraint: 'Must have length less than or equal to 100'",
+	}
+	for names, want := range cases {
+		_, err := DescribeRepositories(context.Background(), store, repositoryActionEndpoint(), repositoryActionTestAccount, []byte(`{"repositoryNames":`+names+`}`))
+		requireECRInvalidParameter(t, err, want)
+	}
+}
@@ -18,6 +18,11 @@ const (
 	minLifecyclePolicyTextLength  = 100
 	maxLifecyclePolicyTextLength  = 30720
 	maxRepositoryPolicyTextLength = 10240
+
+	minImageTagLength     = 1
+	maxImageTagLength     = 300
+	maxMediaTypeListItems = 100
+	maxRepositoryNameList = 100
 )

 // ValidateRepositoryName returns InvalidParameterException naming the first
@@ -52,6 +57,42 @@ func MaxItemsError(field string, limit int) error {
 	return ConstraintError(field, fmt.Sprintf("Member must have length less than or equal to %d", limit))
 }

+// validateListLength refuses a list field whose member count falls outside
+// [minLen, maxLen], the bounds of field's model shape.
+func validateListLength(field string, n, minLen, maxLen int) error {
+	if n < minLen {
+		return ConstraintError(field, fmt.Sprintf("Member must have length greater than or equal to %d", minLen))
+	}
+	if n > maxLen {
+		return MaxItemsError(field, maxLen)
+	}
+	return nil
+}
+
+// validateImageIDs checks imageIds against ImageIdentifierList: 1-100 members,
+// each tag 1-300 characters. An omitted list is refused only when required.
+func validateImageIDs(ids []imageIdentifier, required bool) error {
+	if ids == nil {
+		if required {
+			return RequiredParameterError("imageIds")
+		}
+		return nil
+	}
+	if err := validateListLength("imageIds", len(ids), 1, maxImageBatch); err != nil {
+		return err
+	}
+	for i, id := range ids {
+		if id.ImageTag == nil {
+			continue
+		}
+		field := fmt.Sprintf("imageIds.%d.member.imageTag", i+1)
+		if err := ValidateTextLength(field, *id.ImageTag, minImageTagLength, maxImageTagLength); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // ValidateTextLength refuses text whose length in characters falls outside
 // [minLen, maxLen], the bounds of field's model shape.
 func ValidateTextLength(field, text string, minLen, maxLen int) error {
@@ -76,6 +76,9 @@ func DescribeClusters(ctx context.Context, nc *nats.Conn, accountID string, body
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkClustersInclude(input.Include); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).DescribeClusters(ctx, input, accountID)
 }

@@ -134,6 +137,9 @@ func ListTaskDefinitions(ctx context.Context, nc *nats.Conn, accountID string, b
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkListTaskDefinitions(input); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).ListTaskDefinitions(ctx, input, accountID)
 }

@@ -190,6 +196,9 @@ func DescribeContainerInstances(ctx context.Context, nc *nats.Conn, accountID st
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkContainerInstancesInclude(input.Include); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).DescribeContainerInstances(ctx, input, accountID)
 }

@@ -313,6 +322,9 @@ func DescribeTasks(ctx context.Context, nc *nats.Conn, accountID string, body []
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkTasksInclude(input.Include); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).DescribeTasks(ctx, input, accountID)
 }

@@ -323,6 +335,9 @@ func ListTasks(ctx context.Context, nc *nats.Conn, accountID string, body []byte
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkListTasks(input); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).ListTasks(ctx, input, accountID)
 }

@@ -391,6 +406,9 @@ func DescribeServices(ctx context.Context, nc *nats.Conn, accountID string, body
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkServicesInclude(input.Include); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).DescribeServices(ctx, input, accountID)
 }

@@ -401,6 +419,9 @@ func ListServices(ctx context.Context, nc *nats.Conn, accountID string, body []b
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkListServices(body, input); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).ListServices(ctx, input, accountID)
 }

@@ -493,6 +514,9 @@ func CreateCapacityProvider(ctx context.Context, nc *nats.Conn, accountID string
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkAutoScalingGroupProvider(input.AutoScalingGroupProvider); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).CreateCapacityProvider(ctx, input, accountID)
 }

@@ -504,6 +528,9 @@ func DescribeCapacityProviders(ctx context.Context, nc *nats.Conn, accountID str
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkCapacityProvidersInclude(input.Include); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).DescribeCapacityProviders(ctx, input, accountID)
 }

@@ -0,0 +1,59 @@
+package awsapi
+
+import (
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// Messages AWS answers, as ClientException, for each autoScalingGroupProvider
+// break, before it looks up the Auto Scaling group.
+const (
+	msgManagedDraining              = "The specified managed draining setting is invalid. Specify ENABLED or DISABLED and try again."
+	msgManagedTerminationProtection = "The specified managed termination protection setting is invalid. Specify ENABLED or DISABLED and try again."
+	msgManagedScalingStatus         = "The specified managed scaling status setting is invalid. Specify ENABLED or DISABLED and try again."
+	msgTargetCapacity               = "The managed scaling target capacity value is invalid. Specify an integer greater than 0 and less than or equal to 100 and try again."
+	msgMinimumScalingStepSize       = "The specified minimum scaling step size is invalid. Specify an integer between 1 and 10000 and try again."
+	msgMaximumScalingStepSize       = "The specified maximum scaling step size is invalid. Specify an integer between 1 and 10000 and try again."
+	msgInstanceWarmupPeriod         = "The managed scaling instance warmup period value is invalid. Specify an integer greater than or equal to 0 and less than or equal to 9999 and try again."
+)
+
+func validEnabledDisabled(v *string) bool {
+	return v == nil || *v == "ENABLED" || *v == "DISABLED"
+}
+
+func validRange(v *int64, lo, hi int64) bool {
+	return v == nil || (*v >= lo && *v <= hi)
+}
+
+// checkAutoScalingGroupProvider refuses an enum or range break inside
+// autoScalingGroupProvider. AWS refuses an instanceWarmupPeriod of 10000 although
+// the model allows it, so the upper bound here is 9999.
+func checkAutoScalingGroupProvider(p *ecs.AutoScalingGroupProvider) error {
+	if p == nil {
+		return nil
+	}
+	fail := func(msg string) error { return awserrors.Errorf(awserrors.ErrorECSClientException, "%s", msg) }
+	if !validEnabledDisabled(p.ManagedDraining) {
+		return fail(msgManagedDraining)
+	}
+	if !validEnabledDisabled(p.ManagedTerminationProtection) {
+		return fail(msgManagedTerminationProtection)
+	}
+	ms := p.ManagedScaling
+	if ms == nil {
+		return nil
+	}
+	switch {
+	case !validEnabledDisabled(ms.Status):
+		return fail(msgManagedScalingStatus)
+	case !validRange(ms.TargetCapacity, 1, 100):
+		return fail(msgTargetCapacity)
+	case !validRange(ms.MinimumScalingStepSize, 1, 10000):
+		return fail(msgMinimumScalingStepSize)
+	case !validRange(ms.MaximumScalingStepSize, 1, 10000):
+		return fail(msgMaximumScalingStepSize)
+	case !validRange(ms.InstanceWarmupPeriod, 0, 9999):
+		return fail(msgInstanceWarmupPeriod)
+	}
+	return nil
+}
@@ -0,0 +1,51 @@
+package awsapi
+
+import (
+	"context"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Each break answers AWS's ClientException and message before dispatch. A valid
+// request reaches dispatch, which fails here only because nc is nil.
+func TestCreateCapacityProviderAutoScalingGroupProvider(t *testing.T) {
+	body := func(asg string) []byte {
+		return []byte(`{"name":"cp","autoScalingGroupProvider":{"autoScalingGroupArn":"arn:aws:autoscaling:x"` + asg + `}}`)
+	}
+	refused := []struct {
+		name, asg, msg string
+	}{
+		{"managedDraining", `,"managedDraining":"BOGUS"`, msgManagedDraining},
+		{"managedTerminationProtection", `,"managedTerminationProtection":"enabled"`, msgManagedTerminationProtection},
+		{"status", `,"managedScaling":{"status":"BOGUS"}`, msgManagedScalingStatus},
+		{"targetCapacity/0", `,"managedScaling":{"targetCapacity":0}`, msgTargetCapacity},
+		{"targetCapacity/101", `,"managedScaling":{"targetCapacity":101}`, msgTargetCapacity},
+		{"minimumScalingStepSize/0", `,"managedScaling":{"minimumScalingStepSize":0}`, msgMinimumScalingStepSize},
+		{"maximumScalingStepSize/10001", `,"managedScaling":{"maximumScalingStepSize":10001}`, msgMaximumScalingStepSize},
+		{"instanceWarmupPeriod/-1", `,"managedScaling":{"instanceWarmupPeriod":-1}`, msgInstanceWarmupPeriod},
+		{"instanceWarmupPeriod/10000", `,"managedScaling":{"instanceWarmupPeriod":10000}`, msgInstanceWarmupPeriod},
+	}
+	for _, tc := range refused {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := CreateCapacityProvider(context.Background(), nil, "123456789012", body(tc.asg), nil)
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "err = %v", err)
+			assert.Equal(t, awserrors.ErrorECSClientException, code)
+			assert.Equal(t, tc.msg, msg)
+		})
+	}
+
+	valid := []string{
+		``,
+		`,"managedDraining":"ENABLED","managedTerminationProtection":"DISABLED","managedScaling":{"status":"ENABLED","targetCapacity":1,"minimumScalingStepSize":1,"maximumScalingStepSize":10000,"instanceWarmupPeriod":0}`,
+		`,"managedDraining":"DISABLED","managedTerminationProtection":"ENABLED","managedScaling":{"status":"DISABLED","targetCapacity":100,"minimumScalingStepSize":10000,"maximumScalingStepSize":1,"instanceWarmupPeriod":9999}`,
+	}
+	for _, asg := range valid {
+		_, err := CreateCapacityProvider(context.Background(), nil, "123456789012", body(asg), nil)
+		_, _, refusedErr := awserrors.ResolveErrorDetail(err)
+		assert.False(t, refusedErr, "asg %q: err = %v", asg, err)
+	}
+}
@@ -0,0 +1,68 @@
+package awsapi
+
+import (
+	"slices"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// Include values each Describe* operation accepts, in the order AWS lists them
+// in its refusal. Taken from the ECS model rather than the SDK's *_Values().
+var (
+	clusterIncludeValues           = []string{"ATTACHMENTS", "CONFIGURATIONS", "SETTINGS", "STATISTICS", "TAGS"}
+	tagsIncludeValues              = []string{"TAGS"}
+	containerInstanceIncludeValues = []string{"TAGS", "CONTAINER_INSTANCE_HEALTH"}
+)
+
+// firstInvalidInclude returns the first include value outside allowed.
+func firstInvalidInclude(include []*string, allowed []string) (string, bool) {
+	for _, v := range include {
+		if s := aws.StringValue(v); !slices.Contains(allowed, s) {
+			return s, true
+		}
+	}
+	return "", false
+}
+
+func includeOneOf(allowed []string) string {
+	return "[" + strings.Join(allowed, ",") + "]"
+}
+
+func checkClustersInclude(include []*string) error {
+	if _, bad := firstInvalidInclude(include, clusterIncludeValues); bad {
+		return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "include should be one of %s", includeOneOf(clusterIncludeValues))
+	}
+	return nil
+}
+
+func checkServicesInclude(include []*string) error {
+	if _, bad := firstInvalidInclude(include, tagsIncludeValues); bad {
+		return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "include should be one of %s", includeOneOf(tagsIncludeValues))
+	}
+	return nil
+}
+
+func checkTasksInclude(include []*string) error {
+	if _, bad := firstInvalidInclude(include, tagsIncludeValues); bad {
+		return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "include contains an invalid value, should be one of %s", includeOneOf(tagsIncludeValues))
+	}
+	return nil
+}
+
+func checkContainerInstancesInclude(include []*string) error {
+	if _, bad := firstInvalidInclude(include, containerInstanceIncludeValues); bad {
+		return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "include should be one of %s", includeOneOf(containerInstanceIncludeValues))
+	}
+	return nil
+}
+
+// checkCapacityProvidersInclude answers ClientException, as AWS does for this
+// operation alone, naming the refused value.
+func checkCapacityProvidersInclude(include []*string) error {
+	if v, bad := firstInvalidInclude(include, tagsIncludeValues); bad {
+		return awserrors.Errorf(awserrors.ErrorECSClientException, "%s is not a valid value for include. The value must be one of [%s]", v, includeOneOf(tagsIncludeValues))
+	}
+	return nil
+}
@@ -0,0 +1,53 @@
+package awsapi
+
+import (
+	"context"
+	"testing"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type describeHandler func(context.Context, *nats.Conn, string, []byte, PassRoleChecker) (any, error)
+
+// A refused include answers before dispatch with AWS's code and message. A
+// valid one reaches dispatch, which fails here only because nc is nil.
+func TestDescribeInclude(t *testing.T) {
+	cases := []struct {
+		name     string
+		handler  describeHandler
+		valid    []string
+		wantCode string
+		wantMsg  string
+	}{
+		{"DescribeClusters", DescribeClusters, []string{"ATTACHMENTS", "CONFIGURATIONS", "SETTINGS", "STATISTICS", "TAGS"},
+			awserrors.ErrorECSInvalidParameter, "include should be one of [ATTACHMENTS,CONFIGURATIONS,SETTINGS,STATISTICS,TAGS]"},
+		{"DescribeServices", DescribeServices, []string{"TAGS"},
+			awserrors.ErrorECSInvalidParameter, "include should be one of [TAGS]"},
+		{"DescribeTasks", DescribeTasks, []string{"TAGS"},
+			awserrors.ErrorECSInvalidParameter, "include contains an invalid value, should be one of [TAGS]"},
+		{"DescribeContainerInstances", DescribeContainerInstances, []string{"TAGS", "CONTAINER_INSTANCE_HEALTH"},
+			awserrors.ErrorECSInvalidParameter, "include should be one of [TAGS,CONTAINER_INSTANCE_HEALTH]"},
+		{"DescribeCapacityProviders", DescribeCapacityProviders, []string{"TAGS"},
+			awserrors.ErrorECSClientException, "BOGUS is not a valid value for include. The value must be one of [[TAGS]]"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name+"/invalid", func(t *testing.T) {
+			_, err := tc.handler(context.Background(), nil, "123456789012", []byte(`{"include":["TAGS","BOGUS"]}`), nil)
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "err = %v", err)
+			assert.Equal(t, tc.wantCode, code)
+			assert.Equal(t, tc.wantMsg, msg)
+		})
+		for _, v := range tc.valid {
+			t.Run(tc.name+"/"+v, func(t *testing.T) {
+				_, err := tc.handler(context.Background(), nil, "123456789012", []byte(`{"include":["`+v+`"]}`), nil)
+				_, _, refused := awserrors.ResolveErrorDetail(err)
+				assert.False(t, refused, "err = %v", err)
+			})
+		}
+	}
+}
@@ -0,0 +1,62 @@
+package awsapi
+
+import (
+	"encoding/json"
+	"slices"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// Enum sets the List* operations accept, in the order AWS lists them in its
+// refusal. launchType also accepts FMI, which AWS takes but the model omits.
+var (
+	listLaunchTypes             = []string{"EC2", "FARGATE", "EXTERNAL", "FMI", "MANAGED_INSTANCES"}
+	listSchedulingStrategies    = []string{"REPLICA", "DAEMON"}
+	listResourceManagementTypes = []string{"CUSTOMER", "ECS"}
+	listSortOrders              = []string{"ASC", "DESC"}
+	listTaskDefinitionStatuses  = []string{"ACTIVE", "INACTIVE", "DELETE_IN_PROGRESS"}
+	listDesiredStatuses         = []string{"RUNNING", "PENDING", "STOPPED"}
+)
+
+// checkListEnum refuses a set value outside allowed, naming the member as
+// AWS's message does. An omitted value is left to the operation's default.
+func checkListEnum(v *string, label string, allowed []string) error {
+	if v == nil || slices.Contains(allowed, *v) {
+		return nil
+	}
+	return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "%s should be one of [%s]", label, strings.Join(allowed, ","))
+}
+
+func checkListServices(body []byte, input *ecs.ListServicesInput) error {
+	if err := checkListEnum(input.LaunchType, "launch type", listLaunchTypes); err != nil {
+		return err
+	}
+	if err := checkListEnum(input.SchedulingStrategy, "Scheduling strategy", listSchedulingStrategies); err != nil {
+		return err
+	}
+	// The pinned SDK's ListServicesInput predates resourceManagementType, so it
+	// is read off the raw body; a body that fails to parse was already refused.
+	var raw struct {
+		ResourceManagementType *string `json:"resourceManagementType"`
+	}
+	if len(body) > 0 {
+		_ = json.Unmarshal(body, &raw)
+	}
+	return checkListEnum(raw.ResourceManagementType, "Resource management type", listResourceManagementTypes)
+}
+
+func checkListTaskDefinitions(input *ecs.ListTaskDefinitionsInput) error {
+	if err := checkListEnum(input.Sort, "Sort", listSortOrders); err != nil {
+		return err
+	}
+	return checkListEnum(input.Status, "Status", listTaskDefinitionStatuses)
+}
+
+func checkListTasks(input *ecs.ListTasksInput) error {
+	if err := checkListEnum(input.DesiredStatus, "Desired Status", listDesiredStatuses); err != nil {
+		return err
+	}
+	return checkListEnum(input.LaunchType, "launch type", listLaunchTypes)
+}
@@ -0,0 +1,53 @@
+package awsapi
+
+import (
+	"context"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A value outside a List* enum is refused before dispatch with AWS's message.
+// A valid one reaches dispatch, which fails here only because nc is nil.
+func TestListEnums(t *testing.T) {
+	cases := []struct {
+		name    string
+		handler describeHandler
+		member  string
+		valid   []string
+		wantMsg string
+	}{
+		{"ListServices", ListServices, "launchType", []string{"EC2", "FARGATE", "EXTERNAL", "FMI", "MANAGED_INSTANCES"},
+			"launch type should be one of [EC2,FARGATE,EXTERNAL,FMI,MANAGED_INSTANCES]"},
+		{"ListServices", ListServices, "schedulingStrategy", []string{"REPLICA", "DAEMON"},
+			"Scheduling strategy should be one of [REPLICA,DAEMON]"},
+		{"ListServices", ListServices, "resourceManagementType", []string{"CUSTOMER", "ECS"},
+			"Resource management type should be one of [CUSTOMER,ECS]"},
+		{"ListTaskDefinitions", ListTaskDefinitions, "sort", []string{"ASC", "DESC"},
+			"Sort should be one of [ASC,DESC]"},
+		{"ListTaskDefinitions", ListTaskDefinitions, "status", []string{"ACTIVE", "INACTIVE", "DELETE_IN_PROGRESS"},
+			"Status should be one of [ACTIVE,INACTIVE,DELETE_IN_PROGRESS]"},
+		{"ListTasks", ListTasks, "desiredStatus", []string{"RUNNING", "PENDING", "STOPPED"},
+			"Desired Status should be one of [RUNNING,PENDING,STOPPED]"},
+		{"ListTasks", ListTasks, "launchType", []string{"EC2", "FARGATE", "EXTERNAL", "FMI", "MANAGED_INSTANCES"},
+			"launch type should be one of [EC2,FARGATE,EXTERNAL,FMI,MANAGED_INSTANCES]"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name+"/"+tc.member+"/invalid", func(t *testing.T) {
+			_, err := tc.handler(context.Background(), nil, "123456789012", []byte(`{"`+tc.member+`":"BOGUS"}`), nil)
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "err = %v", err)
+			assert.Equal(t, awserrors.ErrorECSInvalidParameter, code)
+			assert.Equal(t, tc.wantMsg, msg)
+		})
+		for _, v := range tc.valid {
+			t.Run(tc.name+"/"+tc.member+"/"+v, func(t *testing.T) {
+				_, err := tc.handler(context.Background(), nil, "123456789012", []byte(`{"`+tc.member+`":"`+v+`"}`), nil)
+				_, _, refused := awserrors.ResolveErrorDetail(err)
+				assert.False(t, refused, "err = %v", err)
+			})
+		}
+	}
+}
@@ -32,8 +32,12 @@ type attrResource[R any] struct {
 	save        func(*R) error
 	accountIDOf func(*R) string
 	defaults    func(*R) map[string]string
-	attrsOf     func(*R) map[string]string
-	setAttrs    func(*R, map[string]string)
+	// accepts overrides the known-key check, which otherwise uses defaults.
+	accepts func(key string) bool
+	// unknownKey builds the error for a rejected key; nil means a bare ValidationError.
+	unknownKey func(key string) error
+	attrsOf    func(*R) map[string]string
+	setAttrs   func(*R, map[string]string)
 	// validate rejects a bad value for a known key before anything is stored.
 	validate func(key, value string) error
 	// afterChange runs once the changed keys are persisted; its error fails the call.
@@ -57,29 +61,36 @@ func (r attrResource[R]) load() (*R, error) {

 // modifyResourceAttributes validates and merges submitted attributes, persisting only
 // on change. rawCount is pre-nil-filter count, used to distinguish "nothing sent"
-// from "everything sent was invalid".
-func modifyResourceAttributes[R any](res attrResource[R], pairs []attrPair, rawCount int) ([]attrPair, error) {
+// from "everything sent was invalid". It returns the submitted pairs and the full merged set.
+func modifyResourceAttributes[R any](res attrResource[R], pairs []attrPair, rawCount int) (submitted, full []attrPair, err error) {
 	if res.arn == "" {
-		return nil, errors.New(awserrors.ErrorMissingParameter)
+		return nil, nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 	rec, err := res.load()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}

-	known := res.defaults(rec)
+	accepts := res.accepts
+	if accepts == nil {
+		known := res.defaults(rec)
+		accepts = func(k string) bool { _, ok := known[k]; return ok }
+	}
 	attrs := res.attrsOf(rec)
-	submitted := make([]attrPair, 0, len(pairs))
+	submitted = make([]attrPair, 0, len(pairs))
 	var changed []string
 	for _, p := range pairs {
-		if _, ok := known[p.Key]; !ok {
+		if !accepts(p.Key) {
 			slog.Warn(res.opName+": rejecting unknown attribute key", "arn", res.arn, "key", p.Key)
-			return nil, errors.New(awserrors.ErrorValidationError)
+			if res.unknownKey != nil {
+				return nil, nil, res.unknownKey(p.Key)
+			}
+			return nil, nil, errors.New(awserrors.ErrorValidationError)
 		}
 		if res.validate != nil {
 			if err := res.validate(p.Key, p.Value); err != nil {
 				slog.Warn(res.opName+": rejecting attribute value", "arn", res.arn, "key", p.Key)
-				return nil, err
+				return nil, nil, err
 			}
 		}
 	}
@@ -102,7 +113,7 @@ func modifyResourceAttributes[R any](res attrResource[R], pairs []attrPair, rawC
 	// landed write.
 	if rawCount > 0 && len(submitted) == 0 {
 		slog.Warn(res.opName+": all submitted attributes were invalid", "arn", res.arn, "submitted_count", rawCount)
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+		return nil, nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}

 	// Skip the NATS/KV write when nothing changed. Terraform re-applies the same
@@ -112,16 +123,16 @@ func modifyResourceAttributes[R any](res attrResource[R], pairs []attrPair, rawC
 		res.setAttrs(rec, attrs)
 		if err := res.save(rec); err != nil {
 			slog.Error(res.opName+": failed to persist", "arn", res.arn, "err", err)
-			return nil, errors.New(awserrors.ErrorServerInternal)
+			return nil, nil, errors.New(awserrors.ErrorServerInternal)
 		}
 		if res.afterChange != nil {
 			if err := res.afterChange(rec, changed); err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 		}
 	}

-	return submitted, nil
+	return submitted, mergedAttributes(res, rec), nil
 }

 // describeResourceAttributes returns the record's attributes merged over the
@@ -136,6 +147,11 @@ func describeResourceAttributes[R any](res attrResource[R]) ([]attrPair, error)
 		return nil, err
 	}

+	return mergedAttributes(res, rec), nil
+}
+
+// mergedAttributes returns rec's stored attributes over its defaults, sorted by key.
+func mergedAttributes[R any](res attrResource[R], rec *R) []attrPair {
 	merged := res.defaults(rec)
 	maps.Copy(merged, res.attrsOf(rec))

@@ -144,7 +160,7 @@ func describeResourceAttributes[R any](res attrResource[R]) ([]attrPair, error)
 	for _, k := range keys {
 		out = append(out, attrPair{Key: k, Value: merged[k]})
 	}
-	return out, nil
+	return out
 }

 // sdkAttrsIn converts an SDK attribute slice into internal pairs, dropping nil
@@ -196,9 +212,15 @@ func (s *ELBv2ServiceImpl) tgAttrResource(ctx context.Context, arn, accountID, o
 			return err
 		},
 		accountIDOf: func(r *TargetGroupRecord) string { return r.AccountID },
-		defaults:    func(*TargetGroupRecord) map[string]string { return DefaultTargetGroupAttributes() },
-		attrsOf:     func(r *TargetGroupRecord) map[string]string { return r.Attributes },
-		setAttrs:    func(r *TargetGroupRecord, m map[string]string) { r.Attributes = m },
+		defaults: func(r *TargetGroupRecord) map[string]string {
+			return DefaultTargetGroupAttributes(r.Protocol, r.TargetType)
+		},
+		accepts: func(key string) bool { return targetGroupAttributeKeys[key] },
+		unknownKey: func(key string) error {
+			return awserrors.Errorf(awserrors.ErrorValidationError, "Target group attribute key '%s' is not recognized", key)
+		},
+		attrsOf:  func(r *TargetGroupRecord) map[string]string { return r.Attributes },
+		setAttrs: func(r *TargetGroupRecord, m map[string]string) { r.Attributes = m },
 	}
 }

@@ -299,11 +321,11 @@ func (s *ELBv2ServiceImpl) ModifyTargetGroupAttributes(ctx context.Context, inpu
 	const op = "ModifyTargetGroupAttributes"
 	arn := aws.StringValue(input.TargetGroupArn)
 	pairs, raw := sdkAttrsIn(input.Attributes, tgAttrKeyVal, arn, op)
-	submitted, err := modifyResourceAttributes(s.tgAttrResource(ctx, arn, accountID, op), pairs, raw)
+	_, full, err := modifyResourceAttributes(s.tgAttrResource(ctx, arn, accountID, op), pairs, raw)
 	if err != nil {
 		return nil, err
 	}
-	return &elbv2.ModifyTargetGroupAttributesOutput{Attributes: sdkAttrsOut(submitted, mkTGAttr)}, nil
+	return &elbv2.ModifyTargetGroupAttributesOutput{Attributes: sdkAttrsOut(full, mkTGAttr)}, nil
 }

 func (s *ELBv2ServiceImpl) DescribeTargetGroupAttributes(ctx context.Context, input *elbv2.DescribeTargetGroupAttributesInput, accountID string) (*elbv2.DescribeTargetGroupAttributesOutput, error) {
@@ -326,7 +348,7 @@ func (s *ELBv2ServiceImpl) ModifyLoadBalancerAttributes(ctx context.Context, inp
 	const op = "ModifyLoadBalancerAttributes"
 	arn := aws.StringValue(input.LoadBalancerArn)
 	pairs, raw := sdkAttrsIn(input.Attributes, lbAttrKeyVal, arn, op)
-	submitted, err := modifyResourceAttributes(s.lbAttrResource(ctx, arn, accountID, op), pairs, raw)
+	submitted, _, err := modifyResourceAttributes(s.lbAttrResource(ctx, arn, accountID, op), pairs, raw)
 	if err != nil {
 		return nil, err
 	}
@@ -5,7 +5,9 @@ import (
 	"encoding/xml"
 	"fmt"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"maps"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -2747,65 +2749,143 @@ func TestUpdateStoredConfig_MissingTargetGroup(t *testing.T) {

 // --- Attribute tests ---

-// TestDefaultTargetGroupAttributes_CoversTerraformKeys guards the full
-// TargetGroupAttribute key set from the ELBv2 API reference. A missing key
-// fails the whole ModifyTargetGroupAttributes call with ValidationError.
-func TestDefaultTargetGroupAttributes_CoversTerraformKeys(t *testing.T) {
-	t.Parallel()
-	attrs := DefaultTargetGroupAttributes()
-
-	expected := map[string]string{
+var (
+	wantHTTPTGAttrs = map[string]string{
 		"deregistration_delay.timeout_seconds":                                           "300",
-		"stickiness.enabled":                                                             "false",
-		"stickiness.type":                                                                "lb_cookie",
-		"stickiness.lb_cookie.duration_seconds":                                          "86400",
+		"load_balancing.algorithm.anomaly_mitigation":                                    "off",
+		"load_balancing.algorithm.type":                                                  "round_robin",
+		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
+		"slow_start.duration_seconds":                                                    "0",
 		"stickiness.app_cookie.cookie_name":                                              "",
 		"stickiness.app_cookie.duration_seconds":                                         "86400",
-		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
-		"load_balancing.algorithm.type":                                                  "round_robin",
-		"load_balancing.algorithm.anomaly_mitigation":                                    "off",
-		"slow_start.duration_seconds":                                                    "0",
-		"lambda.multi_value_headers.enabled":                                             "false",
+		"stickiness.enabled":                                                             "false",
+		"stickiness.lb_cookie.duration_seconds":                                          "86400",
+		"stickiness.type":                                                                "lb_cookie",
 		"target_group_health.dns_failover.minimum_healthy_targets.count":                 "1",
 		"target_group_health.dns_failover.minimum_healthy_targets.percentage":            "off",
 		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.count":      "1",
 		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.percentage": "off",
-		"proxy_protocol_v2.enabled":                                                      "false",
+	}
+	wantTCPTGAttrs = map[string]string{
 		"deregistration_delay.connection_termination.enabled":                            "false",
-		"preserve_client_ip.enabled":                                                     "false",
+		"deregistration_delay.timeout_seconds":                                           "300",
+		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
+		"proxy_protocol_v2.enabled":                                                      "false",
+		"stickiness.enabled":                                                             "false",
+		"stickiness.type":                                                                "source_ip",
+		"target_group_health.dns_failover.minimum_healthy_targets.count":                 "1",
+		"target_group_health.dns_failover.minimum_healthy_targets.percentage":            "off",
+		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.count":      "1",
+		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.percentage": "off",
 		"target_health_state.unhealthy.connection_termination.enabled":                   "true",
 		"target_health_state.unhealthy.draining_interval_seconds":                        "0",
-		"target_failover.on_deregistration":                                              "no_rebalance",
-		"target_failover.on_unhealthy":                                                   "no_rebalance",
 	}
+)

-	for k, v := range expected {
-		got, ok := attrs[k]
-		assert.True(t, ok, "target group default attributes missing key %q — terraform will hit ValidationError", k)
-		assert.Equal(t, v, got, "target group default value mismatch for key %s", k)
+// wantTGAttrs returns the attribute set AWS reports for a new target group.
+func wantTGAttrs(protocol, targetType string) map[string]string {
+	if protocol == "HTTP" {
+		return maps.Clone(wantHTTPTGAttrs)
 	}
+	m := maps.Clone(wantTCPTGAttrs)
+	m["preserve_client_ip.enabled"] = strconv.FormatBool(targetType == "instance")
+	return m
+}
+
+func tgAttrMap(attrs []*elbv2.TargetGroupAttribute) map[string]string {
+	m := make(map[string]string, len(attrs))
+	for _, a := range attrs {
+		m[*a.Key] = *a.Value
+	}
+	return m
+}
+
+var tgAttrCases = []struct{ protocol, targetType string }{
+	{"HTTP", "instance"}, {"HTTP", "ip"}, {"TCP", "instance"}, {"TCP", "ip"},
+}
+
+func createAttrTG(t *testing.T, svc *ELBv2ServiceImpl, protocol, targetType string) *string {
+	t.Helper()
+	out, err := svc.CreateTargetGroup(context.Background(), &elbv2.CreateTargetGroupInput{
+		Name:       aws.String("tg-attr-" + strings.ToLower(protocol) + "-" + targetType),
+		Protocol:   aws.String(protocol),
+		Port:       aws.Int64(80),
+		TargetType: aws.String(targetType),
+	}, testAccountID)
+	require.NoError(t, err)
+	return out.TargetGroups[0].TargetGroupArn
 }

 func TestDescribeTargetGroupAttributes_Defaults(t *testing.T) {
 	t.Parallel()
-	svc := setupTestService(t)
-	tgOut, err := svc.CreateTargetGroup(context.Background(), &elbv2.CreateTargetGroupInput{Name: aws.String("tg-attr-defaults")}, testAccountID)
-	require.NoError(t, err)
+	for _, tc := range tgAttrCases {
+		t.Run(tc.protocol+"/"+tc.targetType, func(t *testing.T) {
+			t.Parallel()
+			svc := setupTestService(t)
+			arn := createAttrTG(t, svc, tc.protocol, tc.targetType)

-	out, err := svc.DescribeTargetGroupAttributes(context.Background(), &elbv2.DescribeTargetGroupAttributesInput{
-		TargetGroupArn: tgOut.TargetGroups[0].TargetGroupArn,
+			out, err := svc.DescribeTargetGroupAttributes(context.Background(), &elbv2.DescribeTargetGroupAttributesInput{
+				TargetGroupArn: arn,
+			}, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, wantTGAttrs(tc.protocol, tc.targetType), tgAttrMap(out.Attributes))
+		})
+	}
+}
+
+func TestModifyTargetGroupAttributes_ReturnsFullSet(t *testing.T) {
+	t.Parallel()
+	for _, tc := range tgAttrCases {
+		t.Run(tc.protocol+"/"+tc.targetType, func(t *testing.T) {
+			t.Parallel()
+			svc := setupTestService(t)
+			arn := createAttrTG(t, svc, tc.protocol, tc.targetType)
+
+			out, err := svc.ModifyTargetGroupAttributes(context.Background(), &elbv2.ModifyTargetGroupAttributesInput{
+				TargetGroupArn: arn,
+				Attributes: []*elbv2.TargetGroupAttribute{
+					{Key: aws.String("deregistration_delay.timeout_seconds"), Value: aws.String("30")},
+				},
+			}, testAccountID)
+			require.NoError(t, err)
+			want := wantTGAttrs(tc.protocol, tc.targetType)
+			want["deregistration_delay.timeout_seconds"] = "30"
+			assert.Equal(t, want, tgAttrMap(out.Attributes))
+		})
+	}
+}
+
+// The load balancer controller writes NLB keys on every target group it manages.
+func TestModifyTargetGroupAttributes_AcceptsOtherProtocolKeys(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	arn := createAttrTG(t, svc, "HTTP", "instance")
+
+	_, err := svc.ModifyTargetGroupAttributes(context.Background(), &elbv2.ModifyTargetGroupAttributesInput{
+		TargetGroupArn: arn,
+		Attributes: []*elbv2.TargetGroupAttribute{
+			{Key: aws.String("proxy_protocol_v2.enabled"), Value: aws.String("false")},
+			{Key: aws.String("target_failover.on_unhealthy"), Value: aws.String("no_rebalance")},
+			{Key: aws.String("target_failover.on_deregistration"), Value: aws.String("no_rebalance")},
+			{Key: aws.String("lambda.multi_value_headers.enabled"), Value: aws.String("false")},
+		},
 	}, testAccountID)
 	require.NoError(t, err)
+}

-	defaults := DefaultTargetGroupAttributes()
-	assert.Len(t, out.Attributes, len(defaults))
-	attrMap := make(map[string]string)
-	for _, a := range out.Attributes {
-		attrMap[*a.Key] = *a.Value
-	}
-	for k, v := range defaults {
-		assert.Equal(t, v, attrMap[k], "default mismatch for key %s", k)
-	}
+func TestModifyTargetGroupAttributes_UnknownKeyMessage(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	arn := createAttrTG(t, svc, "TCP", "ip")
+
+	_, err := svc.ModifyTargetGroupAttributes(context.Background(), &elbv2.ModifyTargetGroupAttributesInput{
+		TargetGroupArn: arn,
+		Attributes:     []*elbv2.TargetGroupAttribute{{Key: aws.String("bogus.key"), Value: aws.String("1")}},
+	}, testAccountID)
+	require.Error(t, err)
+	code, msg, _ := awserrors.ResolveErrorDetail(err)
+	assert.Equal(t, awserrors.ErrorValidationError, code)
+	assert.Equal(t, "Target group attribute key 'bogus.key' is not recognized", msg)
 }

 func TestDescribeLoadBalancerAttributes_ALBDefaults(t *testing.T) {
@@ -3025,6 +3105,14 @@ func echoMap(pairs []echoPair) map[string]string {
 	return m
 }

+// modifyLen is the number of pairs k's Modify returns after n were submitted.
+func (k attrKind) modifyLen(n int) int {
+	if k.modifySetLen > 0 {
+		return k.modifySetLen
+	}
+	return n
+}
+
 // attrArnPtr returns nil for an empty ARN so the MissingArn cases exercise the
 // real "no ARN supplied" path rather than an empty-string ARN.
 func attrArnPtr(arn string) *string {
@@ -3046,6 +3134,7 @@ type attrKind struct {
 	defaultKey        string // a default present after create
 	defaultVal        string
 	unknownKey        string // a key the handler must reject
+	modifySetLen      int    // size of the full set Modify returns; 0 when it echoes the submitted pairs

 	create   func(t *testing.T, svc *ELBv2ServiceImpl, name string) string
 	modify   func(svc *ELBv2ServiceImpl, arn, accountID string, attrs []rawAttr) ([]echoPair, error)
@@ -3062,7 +3151,8 @@ func attrKinds() []attrKind {
 			keyA:       "deregistration_delay.timeout_seconds", valA: "45", valA2: "30",
 			keyB: "stickiness.enabled", valB: "true",
 			defaultKey: "stickiness.type", defaultVal: "lb_cookie",
-			unknownKey: "stickness.enabled", // typo of "stickiness"
+			unknownKey:   "stickness.enabled", // typo of "stickiness"
+			modifySetLen: len(wantHTTPTGAttrs),
 			create: func(t *testing.T, svc *ELBv2ServiceImpl, name string) string {
 				out, err := svc.CreateTargetGroup(context.Background(), &elbv2.CreateTargetGroupInput{Name: aws.String(name)}, testAccountID)
 				require.NoError(t, err)
@@ -3179,7 +3269,7 @@ func TestAttributeRoundTrip(t *testing.T) {
 			// pass a length check.
 			echoed, err := k.modify(svc, arn, testAccountID, kvAttrs([2]string{k.keyA, k.valA}, [2]string{k.keyB, k.valB}))
 			require.NoError(t, err)
-			require.Len(t, echoed, 2)
+			require.Len(t, echoed, k.modifyLen(2))
 			em := echoMap(echoed)
 			assert.Equal(t, k.valA, em[k.keyA])
 			assert.Equal(t, k.valB, em[k.keyB])
@@ -3298,10 +3388,9 @@ func TestAttributeSkipsInvalidEntries(t *testing.T) {
 				kvAttr(k.keyB, k.valB),
 			})
 			require.NoError(t, err)
-			// Only the one valid attribute should be returned.
-			require.Len(t, echoed, 1)
-			assert.Equal(t, k.keyB, echoed[0].Key)
-			assert.Equal(t, k.valB, echoed[0].Val)
+			// The invalid elements are dropped and the valid one applied.
+			require.Len(t, echoed, k.modifyLen(1))
+			assert.Equal(t, k.valB, echoMap(echoed)[k.keyB])

 			// The valid attribute should have been persisted.
 			desc, err := k.describe(svc, arn, testAccountID)
@@ -3387,7 +3476,7 @@ func TestAttributeNoopSkipsPersist(t *testing.T) {
 			// Second modify with identical values — must skip the Put.
 			echoed, err := k.modify(svc, arn, testAccountID, kvAttrs([2]string{k.keyA, k.valA}))
 			require.NoError(t, err)
-			require.Len(t, echoed, 1)
+			require.Len(t, echoed, k.modifyLen(1))
 			assert.Equal(t, revBefore, k.revision(t, svc, arn), "identical modify must not increment KV revision")

 			// Empty attribute list is also a no-op.
@@ -3410,7 +3499,8 @@ func TestAttributeRejectsUnknownKey(t *testing.T) {
 			arn := k.create(t, svc, "attr-unknown-key")

 			_, err := k.modify(svc, arn, testAccountID, kvAttrs([2]string{k.unknownKey, "true"}))
-			assert.EqualError(t, err, awserrors.ErrorValidationError)
+			code, _ := awserrors.ResolveErrorCode(err)
+			assert.Equal(t, awserrors.ErrorValidationError, code)

 			// The rejected key must not have been persisted.
 			desc, err := k.describe(svc, arn, testAccountID)
@@ -342,9 +342,18 @@ func DefaultLoadBalancerAttributes(lbType string) map[string]string {
 	}
 }

-// DefaultTargetGroupAttributes returns the default attribute set for target groups.
-func DefaultTargetGroupAttributes() map[string]string {
-	return maps.Clone(targetGroupAttributeDefaults)
+// DefaultTargetGroupAttributes returns the attributes AWS reports for a new
+// target group of this protocol and target type. HTTPS uses the HTTP set and
+// every other protocol the TCP set.
+func DefaultTargetGroupAttributes(protocol, targetType string) map[string]string {
+	if protocol == ProtocolHTTP || protocol == ProtocolHTTPS {
+		return maps.Clone(httpTargetGroupAttributeDefaults)
+	}
+	attrs := maps.Clone(tcpTargetGroupAttributeDefaults)
+	if targetType == TargetTypeInstance {
+		attrs["preserve_client_ip.enabled"] = "true"
+	}
+	return attrs
 }

 // Default attribute sets are package-level vars; callers receive a clone so
@@ -395,32 +404,56 @@ var (
 		"secondary_ips.auto_assigned.per_subnet": "0",
 	}

-	targetGroupAttributeDefaults = map[string]string{
+	// HTTP and HTTPS target groups, as AWS reports them for instance and ip targets.
+	httpTargetGroupAttributeDefaults = map[string]string{
 		"deregistration_delay.timeout_seconds":                                           "300",
-		"stickiness.enabled":                                                             "false",
-		"stickiness.type":                                                                "lb_cookie",
-		"stickiness.lb_cookie.duration_seconds":                                          "86400",
-		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
-		"load_balancing.algorithm.type":                                                  "round_robin",
-		"slow_start.duration_seconds":                                                    "0",
 		"load_balancing.algorithm.anomaly_mitigation":                                    "off",
+		"load_balancing.algorithm.type":                                                  "round_robin",
+		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
+		"slow_start.duration_seconds":                                                    "0",
 		"stickiness.app_cookie.cookie_name":                                              "",
 		"stickiness.app_cookie.duration_seconds":                                         "86400",
-		"lambda.multi_value_headers.enabled":                                             "false",
+		"stickiness.enabled":                                                             "false",
+		"stickiness.lb_cookie.duration_seconds":                                          "86400",
+		"stickiness.type":                                                                "lb_cookie",
 		"target_group_health.dns_failover.minimum_healthy_targets.count":                 "1",
 		"target_group_health.dns_failover.minimum_healthy_targets.percentage":            "off",
 		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.count":      "1",
 		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.percentage": "off",
-		// NLB target-group defaults. The AWS Load Balancer Controller always
-		// writes these on NLB target groups (e.g. ingress-nginx); without them as
-		// known keys the modify is rejected and LBC loops on the Service forever.
-		"proxy_protocol_v2.enabled":                                    "false",
-		"deregistration_delay.connection_termination.enabled":          "false",
-		"preserve_client_ip.enabled":                                   "false",
-		"target_health_state.unhealthy.connection_termination.enabled": "true",
-		"target_health_state.unhealthy.draining_interval_seconds":      "0",
-		// GWLB-only; AWS requires both to hold the same value.
-		"target_failover.on_deregistration": "no_rebalance",
-		"target_failover.on_unhealthy":      "no_rebalance",
 	}
+
+	// TCP-family target groups; preserve_client_ip.enabled is set by target type.
+	tcpTargetGroupAttributeDefaults = map[string]string{
+		"deregistration_delay.connection_termination.enabled":                            "false",
+		"deregistration_delay.timeout_seconds":                                           "300",
+		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
+		"preserve_client_ip.enabled":                                                     "false",
+		"proxy_protocol_v2.enabled":                                                      "false",
+		"stickiness.enabled":                                                             "false",
+		"stickiness.type":                                                                "source_ip",
+		"target_group_health.dns_failover.minimum_healthy_targets.count":                 "1",
+		"target_group_health.dns_failover.minimum_healthy_targets.percentage":            "off",
+		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.count":      "1",
+		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.percentage": "off",
+		"target_health_state.unhealthy.connection_termination.enabled":                   "true",
+		"target_health_state.unhealthy.draining_interval_seconds":                        "0",
+	}
+
+	// Keys ModifyTargetGroupAttributes accepts on any target group. The load
+	// balancer controller writes the NLB keys on every NLB group, so this is the
+	// union of the protocol sets plus the lambda and GWLB keys.
+	targetGroupAttributeKeys = func() map[string]bool {
+		keys := map[string]bool{
+			"lambda.multi_value_headers.enabled": true,
+			"target_failover.on_deregistration":  true,
+			"target_failover.on_unhealthy":       true,
+		}
+		for k := range httpTargetGroupAttributeDefaults {
+			keys[k] = true
+		}
+		for k := range tcpTargetGroupAttributeDefaults {
+			keys[k] = true
+		}
+		return keys
+	}()
 )
@@ -167,6 +167,17 @@ func ResourceARNs(action, accountID string, input any, svc iamdomain.IAMService)
 	if resource == "" {
 		resource = anyResource
 	}
+	if action == "UpdateUser" && resource != anyResource {
+		newName, err := stringField(input, "NewUserName")
+		if err != nil {
+			return nil, err
+		}
+		if newName != "" {
+			// AWS also authorizes the rename target, at the user's current path.
+			prefix := resource[:strings.LastIndex(resource, "/")+1]
+			return []string{resource, prefix + newName}, nil
+		}
+	}
 	return []string{resource}, nil
 }

@@ -32,6 +32,7 @@ type iamAuthzPolicyService struct {
 	canonicalErr  error
 	deleteRoleErr error
 	deleted       []string
+	renamed       []string
 }

 func (s *iamAuthzPolicyService) CanonicalResourceARN(_ string, _ arn.IAMResourceType, name string) (string, error) {
@@ -53,6 +54,11 @@ func (s *iamAuthzPolicyService) DeleteRole(_ string, input *iam.DeleteRoleInput)
 	return &iam.DeleteRoleOutput{}, nil
 }

+func (s *iamAuthzPolicyService) UpdateUser(_ string, input *iam.UpdateUserInput) (*iam.UpdateUserOutput, error) {
+	s.renamed = append(s.renamed, *input.UserName+"->"+*input.NewUserName)
+	return &iam.UpdateUserOutput{}, nil
+}
+
 // iamGateway pairs a caller holding statements with the service they act on.
 type iamGateway struct {
 	h   *gwtest.Harness
@@ -249,3 +255,34 @@ func TestAddRoleToInstanceProfile_MissingRoleStaysValidationFault(t *testing.T)
 	err := dispatchIAM(t, gw, "Action=AddRoleToInstanceProfile&InstanceProfileName=web")
 	require.EqualError(t, err, awserrors.ErrorMissingParameter)
 }
+
+// TestIAMRequest_UpdateUserRenameAuthorizesSourceAndTarget pins AWS's rule that
+// a rename needs iam:UpdateUser on both the user and user/<NewUserName>, the
+// target taking the user's current path.
+func TestIAMRequest_UpdateUserRenameAuthorizesSourceAndTarget(t *testing.T) {
+	users := map[string]string{"alice": "arn:aws:iam::123456789012:user/teams/alice"}
+	const (
+		source = "arn:aws:iam::*:user/teams/alice"
+		target = "arn:aws:iam::*:user/teams/bob"
+		body   = "Action=UpdateUser&UserName=alice&NewUserName=bob"
+	)
+
+	t.Run("source-only grant is denied", func(t *testing.T) {
+		gw, svc := iamScopedGateway(users, statement("Allow", "iam:UpdateUser", source))
+		assertDenied(t, dispatchIAM(t, gw, body))
+		assert.Empty(t, svc.renamed)
+	})
+
+	t.Run("target-only grant is denied", func(t *testing.T) {
+		gw, svc := iamScopedGateway(users, statement("Allow", "iam:UpdateUser", target))
+		assertDenied(t, dispatchIAM(t, gw, body))
+		assert.Empty(t, svc.renamed)
+	})
+
+	t.Run("grant on both succeeds", func(t *testing.T) {
+		gw, svc := iamScopedGateway(users, statement("Allow", "iam:UpdateUser", source),
+			statement("Allow", "iam:UpdateUser", target))
+		require.NoError(t, dispatchIAM(t, gw, body))
+		assert.Equal(t, []string{"alice->bob"}, svc.renamed)
+	})
+}
--- a/Show More
+++ b/Show More