test: stub QEMU for the test-link package so it runs on hosts without it

The test link now passes the stored handler to the guest as _HANDLER and the
validated qualifier into the invoked-function ARN. Functions with environment
variables are refused before any presign, grant or dispatch, since delivering
them needs a separate secret-transport design.

.github/CODEOWNERS

@@ -1,15 +0,0 @@
-# CODEOWNERS — GitHub PR review enforcement
-#
-# Format: <path pattern>  <owner> [<owner> …]
-# Order matters — last matching pattern wins.
-#
-# Set the matching branch protection rule on `dev` and `main` to
-# "Require review from Code Owners" or this file is advisory only.
-
-# Network layer is governed by its invariant tests.
-# Changes here must respect those clauses
-# see spinifex/domains/network/invariants/
-# mulga docs/development/proposals/region-local-architecture/0006-spinifex-network-layer-contract.md (proposal, not accepted)
-/spinifex/domains/network/          @joshsiv-mulga
-/spinifex/runtime/roles/vpcd/ @joshsiv-mulga
-/spinifex/domains/network/invariants/ @joshsiv-mulga

.github/pull_request_template.md

@@ -6,20 +6,6 @@
 
 <!-- The notable changes, as bullets. -->
 
-## Account-scoped lifecycle impact
-
-<!-- Complete this section only when the PR changes account-scoped resource lifecycle, account termination, resource cleanup, or a lifecycle boundary. Otherwise write: N/A. -->
-
-- Lifecycle owner and resource family:
-- Owner-local ordering, recovery, retained-state, or force impact:
-- Cross-domain dependency or ownership impact:
-- Evidence added or rechecked:
-- ADR-0008 lifecycle register / ADR-0001 migration-record / bead updates:
-
-## Mulga design
-
-<!-- One line, so a reviewer knows whether the design records in the private mulga repository need to change. Write either "Updated: <mulga PR or bead id>" or "Not needed: <reason>". This is a reminder, not a gate. -->
-
 ## Community Contribution License
 
 - [ ] All community contributions in this pull request are licensed to Mulga under the terms of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). By creating this pull request I represent that I have the right to license these contributions to Mulga under the Apache License 2.0.

.github/scripts/hypervisor-disk-preflight_test.sh

@@ -1,87 +0,0 @@
-#!/bin/sh
-# hypervisor-disk-preflight_test.sh — drives hypervisor-disk-preflight.sh against
-# a stub ssh that answers for each hypervisor from a canned file, so every
-# verdict is checked without a real host.
-set -eu
-
-SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-PREFLIGHT="$SCRIPT_DIR/hypervisor-disk-preflight.sh"
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-# The stub answers "<root-kb> <pool-kb> <path>" from $FAKE/<host>, or exits 255
-# when that host has no file, the way ssh does for a host it cannot reach.
-mkdir -p "$TMP/bin" "$TMP/hosts"
-cat > "$TMP/bin/ssh" <<'STUB'
-#!/bin/sh
-for a in "$@"; do case $a in *@*) host=${a#*@} ;; esac; done
-cat >/dev/null
-if [ -f "$FAKE/$host" ]; then cat "$FAKE/$host"; exit 0; fi
-echo "ssh: Could not resolve hostname $host" >&2
-exit 255
-STUB
-chmod +x "$TMP/bin/ssh"
-PATH="$TMP/bin:$PATH"
-FAKE="$TMP/hosts"
-export PATH FAKE
-
-cat > "$TMP/env.tfvars" <<'TFVARS'
-nodes = [
-  { name = "wattle", wan_ip = "192.0.2.1", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
-  # { name = "retired", wan_ip = "192.0.2.9", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
-  { name = "banksia", wan_ip = "192.0.2.2", storage_pool = "nvme1-pool", base_image = "/x.qcow2" },
-]
-TFVARS
-
-GiB=1048576
-plenty="$((50 * GiB)) $((200 * GiB)) /mnt/drive1/pool"
-
-# run <label> <want-rc> <want-text>
-run() {
-    rc=0
-    out=$(bash "$PREFLIGHT" "$TMP/env.tfvars" 2>&1) || rc=$?
-    if [ "$rc" -ne "$2" ]; then
-        fail "$1: exit $rc, want $2"
-        echo "$out" | sed 's/^/        /'
-        return
-    fi
-    case $out in
-        *"$3"*) pass "$1" ;;
-        *) fail "$1: output lacks '$3'"; echo "$out" | sed 's/^/        /' ;;
-    esac
-}
-
-echo "-- hypervisor-disk-preflight.sh"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$plenty" > "$FAKE/banksia"
-run "every host has room" 0 "banksia: root fs 51200 MiB free"
-
-echo "0 $((200 * GiB)) /mnt/drive1/pool" > "$FAKE/wattle"
-run "a full root filesystem fails fast" 1 "::error::hypervisor wattle has 0 MiB free on the filesystem holding /etc/libvirt/qemu"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$((50 * GiB)) $((3 * GiB)) /mnt/nvme1/pool" > "$FAKE/banksia"
-run "a nearly full pool fails fast" 1 "::error::hypervisor banksia has 3 GiB free in storage pool nvme1-pool"
-
-echo "$plenty" > "$FAKE/banksia"
-rm -f "$FAKE/wattle"
-run "an unreachable host is reported, not failed" 0 "::warning::hypervisor disk preflight: cannot reach wattle"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$((50 * GiB)) - -" > "$FAKE/banksia"
-run "a pool libvirt does not know is reported, not failed" 0 "pool nvme1-pool (-) not found"
-
-echo "0 0 /x" > "$FAKE/retired"
-echo "$plenty" > "$FAKE/banksia"
-run "a commented-out node is never checked" 0 "wattle: root fs"
-
-if [ "$fails" -ne 0 ]; then
-    echo "$fails failure(s)"
-    exit 1
-fi
-echo "  hypervisor-disk-preflight ok"

.github/scripts/mulga-design-check.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY and
+# never fails the job: the check is a prompt, not a gate.
+set -euo pipefail
+shopt -s nocasematch
+
+section_re='^#{1,6}[[:space:]]*Mulga design[[:space:]]*$'
+line_re='^[[:space:]]*(Updated|Not needed)[[:space:]]*:[[:space:]]*(.*)$'
+placeholder_re='<[^<>]*>'
+
+filled() {
+  local body=$1 line value kept inside=0 pre rest
+  body=${body//$'\r'/}
+  while [[ $body == *'<!--'*'-->'* ]]; do
+    pre=${body%%'<!--'*}
+    rest=${body#*'<!--'}
+    body=$pre${rest#*'-->'}
+  done
+  while IFS= read -r line; do
+    if [[ $line =~ $section_re ]]; then
+      inside=1
+    elif ((inside)) && [[ ${line#"${line%%[![:space:]]*}"} == '#'* ]]; then
+      return 1
+    elif ((inside)) && [[ $line =~ $line_re ]]; then
+      value=${BASH_REMATCH[2]} kept=''
+      while [[ $value =~ $placeholder_re ]]; do
+        kept+=${value%%"${BASH_REMATCH[0]}"*}
+        value=${value#*"${BASH_REMATCH[0]}"}
+      done
+      [[ $kept$value == *[![:space:]]* ]] && return 0
+    fi
+  done <<<"$body"
+  return 1
+}
+
+if filled "${PR_BODY:-}"; then
+  echo "The Mulga design line is filled in."
+else
+  echo '::warning title=Mulga design::Add a "## Mulga design" section with one line: "Updated: <mulga PR>" or "Not needed: <reason>".'
+fi

.github/scripts/mulga_design_check.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python3
-"""Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY; never fails the job."""
-
-import os
-import re
-
-COMMENT_RE = re.compile(r"<!--.*?-->", re.S)
-SECTION_RE = re.compile(r"^#{1,6}\s*Mulga design\s*$", re.I)
-LINE_RE = re.compile(r"^\s*(Updated|Not needed)\s*:\s*(.*?)\s*$", re.I)
-PLACEHOLDER_RE = re.compile(r"<[^<>]*>")
-
-
-def filled(body):
-    inside = False
-    for line in COMMENT_RE.sub("", body or "").splitlines():
-        if SECTION_RE.match(line):
-            inside = True
-        elif inside and line.lstrip().startswith("#"):
-            return False
-        elif inside:
-            m = LINE_RE.match(line)
-            if m and PLACEHOLDER_RE.sub("", m.group(2)).strip():
-                return True
-    return False
-
-
-def main():
-    if filled(os.environ.get("PR_BODY", "")):
-        print("The Mulga design line is filled in.")
-    else:
-        print('::warning title=Mulga design::Add a line under "## Mulga design": "Updated: <mulga PR or bead id>" or "Not needed: <reason>".')
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

.github/scripts/test_mulga_design_check.py

@@ -1,23 +0,0 @@
-import unittest
-
-from mulga_design_check import filled
-
-
-class FilledTest(unittest.TestCase):
-    def test_accepts_either_form(self):
-        self.assertTrue(filled("## Mulga design\nUpdated: mulga#275"))
-        self.assertTrue(filled("## Mulga design\n\nnot needed: internal refactor\n## Next"))
-
-    def test_rejects_unanswered(self):
-        self.assertFalse(filled(""))
-        self.assertFalse(filled("## Mulga design\n<!-- Updated: x -->"))
-        self.assertFalse(filled("## Mulga design\nUpdated: <mulga PR or bead id>"))
-        self.assertFalse(filled("## Mulga design\nUpdated:"))
-
-    def test_scoped_to_the_section(self):
-        self.assertFalse(filled("Updated: mulga#1"))
-        self.assertFalse(filled("## Mulga design\n## Other\nUpdated: mulga#1"))
-
-
-if __name__ == "__main__":
-    unittest.main()

.github/scripts/wait-for-memory_test.sh

@@ -1,47 +0,0 @@
-#!/bin/sh
-# wait-for-memory_test.sh — drives wait-for-memory.sh against a fake meminfo
-# that a background writer changes mid-wait, so each verdict is checked without
-# a loaded host.
-set -eu
-
-SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-WAIT="$SCRIPT_DIR/wait-for-memory.sh"
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-meminfo() { printf 'MemTotal:       65536000 kB\nMemAvailable:   %s kB\n' "$(( $1 * 1024 ))" > "$TMP/meminfo"; }
-MEMINFO="$TMP/meminfo"
-MEM_WAIT_POLL_S=1
-export MEMINFO MEM_WAIT_POLL_S
-
-meminfo 30000
-if MEM_WAIT_TIMEOUT_S=5 bash "$WAIT" 26624 > "$TMP/out" 2>&1; then
-  pass "enough memory returns at once"
-else
-  fail "enough memory returns at once: $(cat "$TMP/out")"
-fi
-
-meminfo 21577
-( sleep 2; meminfo 30000 ) &
-if MEM_WAIT_TIMEOUT_S=20 bash "$WAIT" 26624 > "$TMP/out" 2>&1 && grep -q waiting "$TMP/out"; then
-  pass "memory freed mid-wait is admitted"
-else
-  fail "memory freed mid-wait is admitted: $(cat "$TMP/out")"
-fi
-wait
-
-meminfo 21577
-if MEM_WAIT_TIMEOUT_S=2 bash "$WAIT" 26624 > "$TMP/out" 2>&1; then
-  fail "memory that never frees fails: exited 0"
-elif grep -q '::error::need 26624 MiB available, have 21577 MiB' "$TMP/out" && grep -q 'RSS' "$TMP/out"; then
-  pass "memory that never frees fails and names the biggest processes"
-else
-  fail "memory that never frees fails and names the biggest processes: $(cat "$TMP/out")"
-fi
-
-[ "$fails" -eq 0 ] || { echo "wait-for-memory: $fails failed"; exit 1; }
-echo "wait-for-memory: all passed"

.github/workflows/e2e-nightly.yml

@@ -1017,7 +1017,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 
       - uses: actions/checkout@v7.0.1
@@ -1275,7 +1275,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 
       - uses: actions/checkout@v7.0.1
@@ -1423,7 +1423,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Predastore
@@ -1547,7 +1547,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Predastore

.github/workflows/mulga-design.yml

@@ -2,7 +2,8 @@ name: Mulga design
 
 # Reminds the author to say whether the Mulga design records need to change.
 # It reads only the pull request body, uses no secrets, and warns without ever
-# failing: the check is a prompt, not a gate.
+# failing: the check is a prompt, not a gate. Fork PRs are skipped because their
+# authors cannot see the design records.
 
 on:
   pull_request:
@@ -16,7 +17,7 @@ permissions:
 jobs:
   mulga_design:
     name: Mulga design line
-    if: ${{ !endsWith(github.actor, '[bot]') }}
+    if: ${{ !endsWith(github.actor, '[bot]') && github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: ubuntu-26.04
     steps:
       - name: Checkout Code
@@ -28,6 +29,4 @@ jobs:
         env:
           PR_BODY: ${{ github.event.pull_request.body }}
         working-directory: .github/scripts
-        run: |
-          python3 -m unittest -q test_mulga_design_check
-          python3 mulga_design_check.py
+        run: bash mulga-design-check.sh

.github/workflows/perf-3node.yml

@@ -113,7 +113,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - uses: actions/checkout@v7.0.1

.github/workflows/publish-system-ami.yml

@@ -84,7 +84,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Viperblock
         uses: actions/checkout@v7.0.1

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Predastore
         uses: actions/checkout@v7.0.1
@@ -85,7 +85,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Predastore
         uses: actions/checkout@v7.0.1
@@ -335,7 +335,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Predastore
         uses: actions/checkout@v7.0.1

.github/workflows/segscan-oracle.yml

@@ -34,7 +34,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock

.github/workflows/spinifex.yml

@@ -36,7 +36,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock
@@ -166,7 +166,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock
@@ -268,7 +268,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock

.gitignore

@@ -27,6 +27,7 @@ build/microvm/sqs-worker/.stage.*/
 build/microvm/lambda-runtime/vmlinuz
 build/microvm/lambda-runtime/initramfs.cpio.gz
 build/microvm/lambda-runtime/.stage.*/
+build/microvm/lambda-testlink-runtime/
 build/qboot/
 !spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/

Makefile

@@ -42,11 +42,15 @@ ifdef QUIET
   _COVQ  = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' | grep -v 'coverage: 0\.0%' || true; }; exit $${PIPESTATUS[0]}
   _RACEQ = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' || true; }; exit $${PIPESTATUS[0]}
   _SECQ  = >
+  _HDR   = :
+  _RUNSH = sh "$$t" >"$$log" 2>&1 || { echo "-- $$t"; cat "$$log"; exit 1; }
 else
   _Q     =
   _COVQ  =
   _RACEQ =
   _SECQ  = 2>&1 | tee
+  _HDR   = echo -e
+  _RUNSH = echo "-- $$t"; sh "$$t" || exit 1
 endif
 
 build: go_build build-installer build-lb-agent generate-aws-model-coverage
@@ -78,6 +82,35 @@ go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
 	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
+# Controlled-test only: builds the tagged daemon and its hard-kill initializer
+# together, pinning the initializer's digest into that daemon build. The daemon
+# keeps its symbols so the pin can be read back from the binary.
+# LAMBDA_IMAGE_DIGEST is the test-link runtime image digest that
+# build-lambda-testlink-image prints; it is read raw and validated before use.
+LAMBDA_TESTLINK_DIR := ./bin/lambda-testlink
+LAMBDA_TESTLINK_PKG := github.com/mulgadc/spinifex/spinifex/runtime/daemon
+LAMBDA_TESTLINK_PIN := $(LAMBDA_TESTLINK_PKG).lambdaHardKillSHA256
+LAMBDA_DIGEST_RE := ^[0-9a-f]{64}$$
+build-lambda-testlink: override export LAMBDA_IMAGE_DIGEST := $(value LAMBDA_IMAGE_DIGEST)
+build-lambda-testlink:
+	@[[ "$$LAMBDA_IMAGE_DIGEST" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"; exit 1; }
+	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
+	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
+	@rm -f $(LAMBDA_TESTLINK_DIR)/hardkill.so $(LAMBDA_TESTLINK_DIR)/$(GO_PROJECT_NAME)
+	@mkdir -p $(LAMBDA_TESTLINK_DIR).tmp
+	@rm -f $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME)
+	cc -std=c17 -O2 -Wall -Wextra -Werror -shared -fPIC -o $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so spinifex/runtime/compute/lambda/testlink/hardkill/hardkill.c -lrt
+	scripts/check-glibc-ceiling.sh 2.34 $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so
+	@digest=$$(sha256sum $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so | cut -d' ' -f1); \
+	test $${#digest} -eq 64 || { echo "initializer digest missing"; exit 1; }; \
+	echo "initializer sha256 $$digest"; \
+	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
+	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ && \
+	go build $(GO_BUILD_MOD) -tags lambdatestlink -ldflags "$(subst -s -w ,,$(LDFLAGS)) $$pin" -o $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	@mkdir -p $(LAMBDA_TESTLINK_DIR)
+	mv $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) $(LAMBDA_TESTLINK_DIR)/
+	@rmdir $(LAMBDA_TESTLINK_DIR).tmp
+
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
 	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
@@ -193,6 +226,24 @@ bin/lambda-agent: go.mod go.sum $$(call go_local_srcs,./cmd/lambda-agent)
 	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/lambda-agent
 
 build-lambda-runtime-microvm-image: $(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) ## Build the Lambda runtime microVM kernel and initramfs (not shipped)
+	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz absent
+
+# Controlled-test only: a separately built guest whose agent has the
+# lambdatestlink tag, in its own outputs, never touching bin/lambda-agent or the
+# default image. It prints the digest to pass as LAMBDA_IMAGE_DIGEST.
+LAMBDA_TESTLINK_IMAGE_DIR := $(MICROVM_OUT_DIR)/lambda-testlink-runtime
+build-lambda-testlink-image: ## Build the test-link Lambda guest image and print its digest (not shipped)
+	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
+	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
+	$(MAKE) --no-print-directory $(MICROVM_OUT_DIR)/qboot.rom
+	@mkdir -p bin $(LAMBDA_TESTLINK_IMAGE_DIR)
+	GOWORK=off GOFLAGS= CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -tags lambdatestlink -ldflags '-s -w' -o bin/lambda-agent-testlink ./cmd/lambda-agent
+	./scripts/build-lambda-runtime-microvm-image.sh bin/lambda-agent-testlink $(LAMBDA_TESTLINK_IMAGE_DIR)
+	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_TESTLINK_IMAGE_DIR)/initramfs.cpio.gz present
+	install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom $(LAMBDA_TESTLINK_IMAGE_DIR)/qboot.rom
+	@digest=$$(GOWORK=off go run -tags lambdatestlink ./cmd/lambda-image-digest $(LAMBDA_TESTLINK_IMAGE_DIR)) && \
+	[[ "$$digest" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "image digest not computed"; exit 1; }; \
+	echo "LAMBDA_IMAGE_DIGEST=$$digest"
 
 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
@@ -240,27 +291,25 @@ preflight:
 NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh scripts/lib/stage-sqs-store-microvm.sh
 
 test-build-scripts:
-	@echo -e "\n....Running build/scripts/**/*_test.sh...."
-	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running build/scripts/**/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find build/scripts -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
 	done
-	@echo -e "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
-	@for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
+		$(_RUNSH); \
 	done
-	@echo -e "\n....Running .github/scripts/*_test.sh...."
-	@for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running .github/scripts/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
 	done
-	shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh .github/scripts/hypervisor-disk-preflight_test.sh \
-		.github/scripts/wait-for-memory.sh .github/scripts/wait-for-memory_test.sh
-	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
-	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
-	shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
-	@echo "  test-build-scripts ok"
+	$(_Q)shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh \
+		.github/scripts/wait-for-memory.sh \
+		.github/scripts/mulga-design-check.sh
+	@$(_HDR) "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
+	$(_Q)shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
+	$(_Q)shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
+	@$(_HDR) "  test-build-scripts ok"
 
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
@@ -327,7 +376,10 @@ test-cover:
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
 	$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ ./spinifex/runtime/compute/lambda/testlink/ ./spinifex/domains/network/host/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink(Executor|Reply)_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink,lambdatestlinkseam -timeout 120s -run '^TestLambdaTestLinkSeam_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink' ./spinifex/runtime/daemon/ $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -538,7 +590,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-lambda-testlink build-lambda-testlink-image build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -70,8 +70,8 @@ The platform serves **405 AWS API operations** across the services below. See ex
 | **RDS** | Databases | PostgreSQL and MariaDB, with automated backups | Available |
 | **Route 53** | DNS | [Northstar](https://github.com/mulgadc/northstar): authoritative DNS with hosted zones | Available |
 | **Bedrock** | AI deployment | vLLM for text generation and Hugging Face TEI for embeddings and reranking | Q4 2026 |
-| **Lambda** | Serverless functions | | Q1 2027 |
-| **SQS** | Message queues | | Q1 2027 |
+| **Lambda** | Serverless functions | QEMU `microvm` execution environments | Q1 2027 |
+| **SQS** | Message queues | A dedicated NATS JetStream store in a per-Region QEMU `microvm` guest | Q1 2027 |
 
 Roadmap services ship under the same AWS API surface, so code written for AWS today works the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 

cmd/lambda-agent/connect.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 )
 
@@ -29,20 +30,20 @@ func defaultStartup() startup {
 
 // connect loads the identity, seals fw_cfg whether or not the load succeeded,
 // then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
-func (s startup) connect(ctx context.Context) (*tls.Conn, int, error) {
+func (s startup) connect(ctx context.Context) (*tls.Conn, lambdav1.Scope, int, error) {
 	id, loadErr := hostlink.LoadIdentity(s.fwCfgDir)
 	if err := hostlink.SealFwCfg(s.fwCfgRoot, s.unload); err != nil {
-		return nil, exitRefused, errors.Join(loadErr, err)
+		return nil, "", exitRefused, errors.Join(loadErr, err)
 	}
 	if loadErr != nil {
-		return nil, exitRefused, loadErr
+		return nil, "", exitRefused, loadErr
 	}
 	conn, err := hostlink.Dial(ctx, id)
 	if err != nil {
 		if ctx.Err() != nil {
 			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
 		}
-		return nil, exitLink, err
+		return nil, "", exitLink, err
 	}
-	return conn, 0, nil
+	return conn, id.Scope, 0, nil
 }

cmd/lambda-agent/grant_testlink.go

@@ -0,0 +1,66 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+)
+
+const entryGrant = "test-grant"
+
+// testGrant is the grant the host launched this VM under.
+type testGrant struct {
+	grantID, attemptID, environmentID string
+	generation                        uint64
+}
+
+// readGrant reads the test-grant fw_cfg entry: grant ID, attempt ID,
+// environment ID and generation, one per line.
+func readGrant(fwCfgDir string) (testGrant, error) {
+	b, err := os.ReadFile(filepath.Join(fwCfgDir, entryGrant, "raw"))
+	if err != nil {
+		return testGrant{}, fmt.Errorf("read %s: %w", entryGrant, err)
+	}
+	lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n")
+	if len(lines) != 4 {
+		return testGrant{}, errors.New(entryGrant + " entry malformed")
+	}
+	g := testGrant{grantID: lines[0], attemptID: lines[1], environmentID: lines[2]}
+	g.generation, err = strconv.ParseUint(lines[3], 10, 64)
+	if err != nil || g.generation == 0 || g.grantID == "" || g.attemptID == "" || g.environmentID == "" {
+		return testGrant{}, errors.New(entryGrant + " entry malformed")
+	}
+	return g, nil
+}
+
+// checkBinding refuses a run unless the grant names the authenticated environment
+// generation, the request carries that grant and its deadline is still ahead. The
+// clock check is a local guard only: it cannot detect skew or a superseded grant.
+func checkBinding(scope lambdav1.Scope, g testGrant, req hostlink.Request, now time.Time) error {
+	parts := strings.Split(string(scope), "/")
+	if len(parts) != 5 {
+		return errors.New("authenticated scope malformed")
+	}
+	if parts[3] != g.environmentID || parts[4] != strconv.FormatUint(g.generation, 10) {
+		return errors.New("grant is not for the authenticated environment generation")
+	}
+	if req.GrantID != g.grantID || req.AttemptID != g.attemptID {
+		return errors.New("request does not carry this VM's grant")
+	}
+	if req.DeadlineMs <= 0 {
+		return errors.New("request has no deadline")
+	}
+	if !now.Before(time.UnixMilli(req.DeadlineMs)) {
+		return errors.New("request deadline already passed")
+	}
+	return nil
+}

cmd/lambda-agent/helpers_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
+const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/1"
 
 // guest is a fake fw_cfg tree; unload removes it as unloading the module would.
 type guest struct {

cmd/lambda-agent/package_testlink.go

@@ -18,6 +18,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode"
 
 	"golang.org/x/sys/unix"
 )
@@ -28,6 +29,7 @@ import (
 const (
 	entryPackageSize   = "test-package-size"
 	entryPackageSHA256 = "test-package-sha256"
+	entryHandler       = "test-handler"
 
 	devMaxZipBytes    = 16 << 20
 	devMaxUnzipBytes  = 64 << 20
@@ -37,8 +39,9 @@ const (
 )
 
 type packageSpec struct {
-	size   int64
-	sha256 [sha256.Size]byte
+	size    int64
+	sha256  [sha256.Size]byte
+	handler string
 }
 
 type unpackLimits struct {
@@ -76,6 +79,13 @@ func readPackageSpec(fwCfgDir string) (packageSpec, error) {
 		return spec, errors.New("package sha256 is not 64 hex digits")
 	}
 	copy(spec.sha256[:], b)
+	if spec.handler, err = read(entryHandler); err != nil {
+		return spec, err
+	}
+	if spec.handler == "" || len(spec.handler) > 128 || strings.IndexFunc(spec.handler, unicode.IsSpace) >= 0 ||
+		strings.ContainsRune(spec.handler, 0) {
+		return spec, errors.New("handler must be 1..128 characters with no whitespace")
+	}
 	return spec, nil
 }
 

cmd/lambda-agent/package_testlink_test.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"syscall"
 	"testing"
 
@@ -165,19 +166,23 @@ func TestUnpackChecksSizeAndHash(t *testing.T) {
 func TestReadPackageSpecBounds(t *testing.T) {
 	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 	cases := map[string]struct {
-		size, hash string
-		want       string
+		size, hash, handler string
+		want                string
 	}{
-		"valid":     {"10", sum, ""},
-		"zero":      {"0", sum, "outside"},
-		"too large": {"16777217", sum, "outside"},
-		"not hex":   {"10", "zz", "64 hex digits"},
-		"short":     {"10", sum[:62], "64 hex digits"},
+		"valid":            {"10", sum, "bootstrap.h", ""},
+		"zero":             {"0", sum, "bootstrap.h", "outside"},
+		"too large":        {"16777217", sum, "bootstrap.h", "outside"},
+		"not hex":          {"10", "zz", "bootstrap.h", "64 hex digits"},
+		"short":            {"10", sum[:62], "bootstrap.h", "64 hex digits"},
+		"empty handler":    {"10", sum, "", "handler"},
+		"handler space":    {"10", sum, "a b", "handler"},
+		"128-char handler": {"10", sum, strings.Repeat("h", 128), ""},
+		"129-char handler": {"10", sum, strings.Repeat("h", 129), "handler"},
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
 			dir := t.TempDir()
-			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash, entryHandler: tc.handler} {
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
 				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
 			}
@@ -185,6 +190,7 @@ func TestReadPackageSpecBounds(t *testing.T) {
 			if tc.want == "" {
 				require.NoError(t, err)
 				assert.Equal(t, int64(10), spec.size)
+				assert.Equal(t, tc.handler, spec.handler)
 				return
 			}
 			assert.ErrorContains(t, err, tc.want)

cmd/lambda-agent/start_tcp.go

@@ -25,7 +25,7 @@ func start(ctx context.Context) int {
 
 // run authenticates to the host and holds the connection until it closes.
 func (s startup) run(ctx context.Context) (int, error) {
-	conn, code, err := s.connect(ctx)
+	conn, _, code, err := s.connect(ctx)
 	if err != nil {
 		return code, err
 	}

cmd/lambda-agent/start_testlink.go

@@ -5,7 +5,6 @@ package main
 import (
 	"context"
 	"errors"
-	"flag"
 	"fmt"
 	"log/slog"
 	"net"
@@ -24,8 +23,6 @@ const (
 	taskRoot    = "/var/task"
 )
 
-var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler passed to the bootstrap")
-
 // testlink relays one invocation over the test carrier on the authenticated
 // connection; the carrier is not the guest protocol.
 type testlink struct {
@@ -54,13 +51,14 @@ func start(ctx context.Context) int {
 func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
-	conn, code, err := t.connect(parent)
+	grant, grantErr := readGrant(t.fwCfgDir)
+	conn, scope, code, err := t.connect(parent)
 	if err != nil {
-		return code, errors.Join(err, specErr)
+		return code, errors.Join(err, specErr, grantErr)
 	}
 	defer conn.Close()
-	if specErr != nil {
-		return exitRefused, specErr
+	if err := errors.Join(specErr, grantErr); err != nil {
+		return exitRefused, err
 	}
 	link := hostlink.NewTestStream(conn)
 
@@ -76,6 +74,10 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
+	// Nothing of the tenant's is installed or run for an unbound request.
+	if err := checkBinding(scope, grant, first, time.Now()); err != nil {
+		return exitRefused, err
+	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -93,7 +95,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	defer cancel(nil)
 	cmd, err := t.startBootstrap(ctx, []string{
 		"AWS_LAMBDA_RUNTIME_API=" + ln.Addr().String(),
-		"_HANDLER=" + *handler,
+		"_HANDLER=" + spec.handler,
 		"LAMBDA_TASK_ROOT=" + taskRoot,
 		"PATH=/bin:/sbin:/usr/bin",
 	})

cmd/lambda-agent/start_testlink_test.go

@@ -25,12 +25,18 @@ import (
 
 const helperEnv = "LAMBDA_AGENT_TEST_BOOTSTRAP"
 
+// helperHandlerOut names the file the helper bootstrap writes its _HANDLER to.
+const helperHandlerOut = "LAMBDA_AGENT_TEST_HANDLER_OUT"
+
 // TestHelperBootstrap is the fake function bootstrap, run as a subprocess: it
 // echoes one invocation through the Runtime API, then waits for the next.
 func TestHelperBootstrap(t *testing.T) {
 	if os.Getenv(helperEnv) != "1" {
 		return
 	}
+	if out := os.Getenv(helperHandlerOut); out != "" {
+		require.NoError(t, os.WriteFile(out, []byte(os.Getenv("_HANDLER")), 0o600))
+	}
 	base := "http://" + os.Getenv("AWS_LAMBDA_RUNTIME_API") + "/2018-06-01/runtime/invocation/"
 	resp, err := http.Get(base + "next")
 	require.NoError(t, err)
@@ -44,11 +50,19 @@ func TestHelperBootstrap(t *testing.T) {
 	_, _ = http.Get(base + "next")
 }
 
+// testGrantEntry grants g-1 for attempt a-1 on the fixture scope's env-1/1.
+const testGrantEntry = "g-1\na-1\nenv-1\n1\n"
+
+// testHandler is the handler the fixture's host configures.
+const testHandler = "bootstrap.fixture"
+
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
 	task    string
 	sealed  bool
+	// handlerOut receives the _HANDLER the bootstrap process itself saw.
+	handlerOut string
 }
 
 // testlink builds a guest whose fw_cfg also carries a controlled package,
@@ -61,9 +75,16 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	sum := sha256.Sum256(zipBytes)
 	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
 	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
+	if _, ok := entries[entryHandler]; !ok {
+		entries[entryHandler] = []byte(testHandler)
+	}
+	if _, ok := entries[entryGrant]; !ok {
+		entries[entryGrant] = []byte(testGrantEntry)
+	}
 	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
+	b.handlerOut = filepath.Join(t.TempDir(), "handler")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -77,7 +98,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			b.n++
 			b.started <- struct{}{}
 			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestHelperBootstrap$")
-			cmd.Env = append(env, helperEnv+"=1")
+			cmd.Env = append(env, helperEnv+"=1", helperHandlerOut+"="+b.handlerOut)
 			return cmd, cmd.Start()
 		},
 	}
@@ -132,6 +153,9 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
+	seen, err := os.ReadFile(b.handlerOut)
+	require.NoError(t, err, "bootstrap process did not report its handler")
+	assert.Equal(t, testHandler, string(seen), "bootstrap process not given the configured handler")
 	assert.True(t, b.sealed, "task dir not made read-only")
 	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
 }
@@ -218,7 +242,8 @@ func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
 
 	conn := h.guest(t)
 	defer conn.Close()
-	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1",
+		DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli()}))
 	res := wait(t, done, "agent to stop")
 	assert.Equal(t, exitRefused, res.code)
 	assert.ErrorContains(t, res.err, "install package")
@@ -244,3 +269,87 @@ func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
 	_, err := os.Stat(tl.fwCfgRoot)
 	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
 }
+
+// A request that does not carry this VM's grant, or a grant for another
+// environment generation than the one authenticated, installs and runs
+// nothing of the tenant's.
+func TestTestlinkUnboundRequestNeverStartsBootstrap(t *testing.T) {
+	cases := map[string]struct {
+		grant            string
+		grantID, attempt string
+		deadlineMs       int64
+		want             string
+	}{
+		"frame with no deadline":      {testGrantEntry, "g-1", "a-1", -1, "no deadline"},
+		"frame with expired deadline": {testGrantEntry, "g-1", "a-1", time.Now().Add(-time.Second).UnixMilli(), "deadline already passed"},
+		"frame for another grant":     {testGrantEntry, "g-2", "a-1", 0, "does not carry this VM's grant"},
+		"frame for another attempt":   {testGrantEntry, "g-1", "a-2", 0, "does not carry this VM's grant"},
+		"frame with no grant":         {testGrantEntry, "", "", 0, "does not carry this VM's grant"},
+		"grant for another env":       {"g-1\na-1\nenv-2\n1\n", "g-1", "a-1", 0, "not for the authenticated environment"},
+		"grant for another gen":       {"g-1\na-1\nenv-1\n2\n", "g-1", "a-1", 0, "not for the authenticated environment"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			h := newHost(t)
+			h.accept(ctx)
+			var b bootstrapStarts
+			entries := h.entries()
+			entries[entryGrant] = []byte(tc.grant)
+			done := runTestlinkAsync(ctx, b.testlink(t, entries))
+			conn := h.guest(t)
+			defer conn.Close()
+			deadline := time.Now().Add(5 * time.Second).UnixMilli()
+			switch {
+			case tc.deadlineMs < 0:
+				deadline = 0
+			case tc.deadlineMs > 0:
+				deadline = tc.deadlineMs
+			}
+			req := hostlink.Request{GrantID: tc.grantID, AttemptID: tc.attempt, RequestID: "r-1",
+				DeadlineMs: deadline, Payload: []byte(`{}`)}
+			require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
+
+			res := wait(t, done, "agent to stop")
+			assert.Equal(t, exitRefused, res.code)
+			assert.ErrorContains(t, res.err, tc.want)
+			assert.Zero(t, b.n)
+			assert.False(t, b.sealed)
+			assert.NoFileExists(t, filepath.Join(b.task, "bootstrap"), "package installed")
+		})
+	}
+}
+
+// A missing or malformed grant entry refuses before any frame is awaited.
+func TestTestlinkBadGrantEntryRefuses(t *testing.T) {
+	cases := map[string]*string{
+		"missing":       nil,
+		"too few lines": ptr("g-1\na-1\nenv-1\n"),
+		"generation 0":  ptr("g-1\na-1\nenv-1\n0\n"),
+		"empty grant":   ptr("\na-1\nenv-1\n1\n"),
+	}
+	for name, grant := range cases {
+		t.Run(name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			h := newHost(t)
+			h.accept(ctx)
+			var b bootstrapStarts
+			entries := h.entries()
+			if grant != nil {
+				entries[entryGrant] = []byte(*grant)
+			}
+			tl := b.testlink(t, entries)
+			if grant == nil {
+				require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryGrant)))
+			}
+			res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
+			assert.Equal(t, exitRefused, res.code)
+			assert.ErrorContains(t, res.err, entryGrant)
+			assert.Zero(t, b.n)
+		})
+	}
+}
+
+func ptr(s string) *string { return &s }

cmd/lambda-image-agent-tag/main.go

@@ -0,0 +1,221 @@
+// Command lambda-image-agent-tag checks whether the lambda-agent archived in a
+// built Lambda guest initramfs carries the lambdatestlink build tag. Build
+// information is recorded by the build; it is not proof of how the agent behaves.
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"debug/buildinfo"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+const (
+	agentPath    = "usr/local/bin/lambda-agent"
+	maxArchive   = 1 << 30
+	maxAgent     = 256 << 20
+	maxName      = 4096
+	newcHeader   = 110
+	trailerName  = "TRAILER!!!"
+	modeTypeMask = 0o170000
+	modeDir      = 0o040000
+	modeRegular  = 0o100000
+)
+
+var agentParents = []string{"usr", "usr/local", "usr/local/bin"}
+
+func main() {
+	if len(os.Args) != 3 || (os.Args[2] != "present" && os.Args[2] != "absent") {
+		fmt.Fprintln(os.Stderr, "usage: lambda-image-agent-tag <initramfs.cpio.gz> present|absent")
+		os.Exit(2)
+	}
+	if err := run(os.Args[1], os.Args[2] == "present"); err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[1], err)
+		os.Exit(1)
+	}
+}
+
+func run(path string, want bool) error {
+	fi, err := os.Lstat(path)
+	if err != nil {
+		return err
+	}
+	if !fi.Mode().IsRegular() {
+		return errors.New("not a regular file")
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	agent, err := archivedAgent(f)
+	if err != nil {
+		return err
+	}
+	return checkTag(agent, want)
+}
+
+// archivedAgent returns the single agent entry after checking listed names,
+// entry types, the agent's link count and zero-only padding after the trailer.
+// It is not a full kernel-equivalent parser; 070702 CRCs, for one, are unchecked.
+func archivedAgent(r io.Reader) ([]byte, error) {
+	zr, err := gzip.NewReader(bufio.NewReader(r))
+	if err != nil {
+		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
+	}
+	raw, err := io.ReadAll(io.LimitReader(zr, maxArchive+1))
+	if err != nil {
+		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
+	}
+	if len(raw) > maxArchive {
+		return nil, fmt.Errorf("archive larger than %d bytes", maxArchive)
+	}
+
+	seen := map[string]bool{}
+	var agent []byte
+	off := 0
+	for {
+		e, next, err := parseEntry(raw, off)
+		if err != nil {
+			return nil, fmt.Errorf("entry at offset %d: %w", off, err)
+		}
+		off = next
+		if e.name == trailerName {
+			break
+		}
+		if !canonical(e.name) {
+			return nil, fmt.Errorf("entry name %q is not canonical", e.name)
+		}
+		if seen[e.name] {
+			return nil, fmt.Errorf("entry %q appears more than once", e.name)
+		}
+		seen[e.name] = true
+		for _, p := range agentParents {
+			if e.name == p && e.mode&modeTypeMask != modeDir {
+				return nil, fmt.Errorf("%s is not a directory", p)
+			}
+		}
+		if e.name == agentPath {
+			for _, p := range agentParents {
+				if !seen[p] {
+					return nil, fmt.Errorf("%s has no %s directory entry before it", agentPath, p)
+				}
+			}
+			if e.mode&modeTypeMask != modeRegular {
+				return nil, fmt.Errorf("%s is not a regular file", agentPath)
+			}
+			if e.nlink != 1 {
+				return nil, fmt.Errorf("%s has %d links, want 1", agentPath, e.nlink)
+			}
+			if len(e.data) > maxAgent {
+				return nil, fmt.Errorf("%s is larger than %d bytes", agentPath, maxAgent)
+			}
+			agent = e.data
+		}
+	}
+	if i := bytes.IndexFunc(raw[off:], func(r rune) bool { return r != 0 }); i >= 0 {
+		return nil, fmt.Errorf("data after the archive trailer at offset %d", off+i)
+	}
+	if agent == nil {
+		return nil, fmt.Errorf("no %s entry", agentPath)
+	}
+	return agent, nil
+}
+
+type entry struct {
+	name  string
+	mode  uint64
+	nlink uint64
+	data  []byte
+}
+
+func parseEntry(raw []byte, off int) (entry, int, error) {
+	if len(raw)-off < newcHeader {
+		return entry{}, 0, errors.New("truncated header")
+	}
+	h := raw[off : off+newcHeader]
+	if m := string(h[:6]); m != "070701" && m != "070702" {
+		return entry{}, 0, fmt.Errorf("magic %q is not newc", m)
+	}
+	field := func(i int) (uint64, error) {
+		return strconv.ParseUint(string(h[6+8*i:14+8*i]), 16, 32)
+	}
+	mode, err1 := field(1)
+	nlink, err2 := field(4)
+	size, err3 := field(6)
+	nameSize, err4 := field(11)
+	if err := errors.Join(err1, err2, err3, err4); err != nil {
+		return entry{}, 0, fmt.Errorf("bad header field: %w", err)
+	}
+	if nameSize < 2 || nameSize > maxName {
+		return entry{}, 0, fmt.Errorf("name size %d out of range", nameSize)
+	}
+	nameEnd := off + newcHeader + int(nameSize)
+	dataOff := align4(nameEnd)
+	dataEnd := dataOff + int(size)
+	if dataEnd > len(raw) || align4(dataEnd) > len(raw) {
+		return entry{}, 0, errors.New("truncated entry")
+	}
+	name := raw[off+newcHeader : nameEnd]
+	if name[len(name)-1] != 0 || bytes.IndexByte(name[:len(name)-1], 0) >= 0 {
+		return entry{}, 0, errors.New("name is not one NUL-terminated string")
+	}
+	return entry{name: string(name[:len(name)-1]), mode: mode, nlink: nlink, data: raw[dataOff:dataEnd]}, align4(dataEnd), nil
+}
+
+func align4(n int) int { return (n + 3) &^ 3 }
+
+// canonical accepts "." and relative names with no empty, "." or ".." component.
+func canonical(name string) bool {
+	if name == "." {
+		return true
+	}
+	if name == "" || strings.HasPrefix(name, "/") {
+		return false
+	}
+	for c := range strings.SplitSeq(name, "/") {
+		if c == "" || c == "." || c == ".." {
+			return false
+		}
+	}
+	return true
+}
+
+func checkTag(agent []byte, want bool) error {
+	info, err := buildinfo.Read(bytes.NewReader(agent))
+	if err != nil {
+		return fmt.Errorf("no Go build information in %s: %w", agentPath, err)
+	}
+	return decide(info, want)
+}
+
+// decide refuses build information with no recorded settings, since their
+// absence does not show the tag was absent.
+func decide(info *buildinfo.BuildInfo, want bool) error {
+	if len(info.Settings) == 0 {
+		return fmt.Errorf("%s records no build settings", agentPath)
+	}
+	if hasTag(info, "lambdatestlink") != want {
+		return fmt.Errorf("%s: lambdatestlink build tag present=%v, want %v", agentPath, !want, want)
+	}
+	return nil
+}
+
+func hasTag(info *buildinfo.BuildInfo, tag string) bool {
+	for _, s := range info.Settings {
+		if s.Key != "-tags" {
+			continue
+		}
+		if slices.Contains(strings.Split(s.Value, ","), tag) {
+			return true
+		}
+	}
+	return false
+}

cmd/lambda-image-agent-tag/main_test.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"debug/buildinfo"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type fileEntry struct {
+	name  string
+	mode  uint32
+	nlink uint32
+	data  string
+}
+
+func dir(name string) fileEntry { return fileEntry{name: name, mode: modeDir | 0o755, nlink: 2} }
+func file(name, data string) fileEntry {
+	return fileEntry{name: name, mode: modeRegular | 0o755, nlink: 1, data: data}
+}
+func symlink(name, target string) fileEntry {
+	return fileEntry{name: name, mode: 0o120777, nlink: 1, data: target}
+}
+
+// newc writes entries and a trailer in the format the kernel unpacks.
+func newc(entries ...fileEntry) []byte {
+	var b bytes.Buffer
+	for i, e := range append(entries, fileEntry{name: trailerName, nlink: 1}) {
+		fmt.Fprintf(&b, "070701%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X",
+			i+1, e.mode, 0, 0, e.nlink, 0, len(e.data), 0, 0, 0, 0, len(e.name)+1, 0)
+		b.WriteString(e.name)
+		b.WriteByte(0)
+		pad(&b)
+		b.WriteString(e.data)
+		pad(&b)
+	}
+	return b.Bytes()
+}
+
+func pad(b *bytes.Buffer) {
+	for b.Len()%4 != 0 {
+		b.WriteByte(0)
+	}
+}
+
+func gz(parts ...[]byte) []byte {
+	var b bytes.Buffer
+	for _, p := range parts {
+		w := gzip.NewWriter(&b)
+		_, _ = w.Write(p)
+		_ = w.Close()
+	}
+	return b.Bytes()
+}
+
+func withAgent(agent fileEntry) []byte {
+	return newc(dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent, file("init", "#!/bin/sh"))
+}
+
+func TestArchivedAgent_ReturnsTheOneArchivedAgent(t *testing.T) {
+	got, err := archivedAgent(bytes.NewReader(gz(withAgent(file(agentPath, "agent-bytes")))))
+	require.NoError(t, err)
+	require.Equal(t, "agent-bytes", string(got))
+
+	padded := append(withAgent(file(agentPath, "agent-bytes")), make([]byte, 1024)...)
+	got, err = archivedAgent(bytes.NewReader(gz(padded)))
+	require.NoError(t, err, "zero padding after the trailer")
+	require.Equal(t, "agent-bytes", string(got))
+}
+
+func TestArchivedAgent_Refuses(t *testing.T) {
+	parents := []fileEntry{dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin")}
+	agent := file(agentPath, "untagged")
+	small := newc(append(parents, agent)...)
+	second := newc(append(parents[1:], file(agentPath, "tagged"))...)
+	// The second archive starts inside what would be the first one's final block.
+	sameBlock := append(append([]byte{}, small...), second...)
+	for len(sameBlock)%512 != 0 {
+		sameBlock = append(sameBlock, 0)
+	}
+
+	for name, archive := range map[string][]byte{
+		"aliased duplicate":            gz(newc(append(parents, agent, file("././"+agentPath, "tagged"))...)),
+		"leading slash alias":          gz(newc(append(parents, agent, file("/"+agentPath, "tagged"))...)),
+		"dot-dot component":            gz(newc(append(parents, agent, file("usr/local/../local/bin/lambda-agent", "x"))...)),
+		"empty component":              gz(newc(append(parents, agent, file("usr//local/bin/lambda-agent", "x"))...)),
+		"exact duplicate":              gz(newc(append(parents, agent, file(agentPath, "tagged"))...)),
+		"duplicate directory":          gz(newc(dir("usr"), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent)),
+		"missing directory entry":      gz(newc(dir("usr"), dir("usr/local/bin"), agent)),
+		"directory after the agent":    gz(newc(dir("usr"), dir("usr/local"), agent, dir("usr/local/bin"))),
+		"parent is a symlink":          gz(newc(dir("usr"), dir("usr/local"), symlink("usr/local/bin", "elsewhere"), agent)),
+		"agent is a symlink":           gz(newc(append(parents, symlink(agentPath, "/bin/sh"))...)),
+		"hard-linked agent":            gz(newc(append(parents, fileEntry{name: agentPath, mode: modeRegular | 0o755, nlink: 2, data: "x"})...)),
+		"no agent":                     gz(newc(parents...)),
+		"second archive appended":      gz(small, second),
+		"second archive in same block": gz(sameBlock),
+		"non-zero byte after trailer":  gz(append(append([]byte{}, small...), 0, 0, 1)),
+		"truncated archive":            gz(small[:len(small)-20]),
+		"not newc":                     gz([]byte("not cpio at all, but long enough to fill one newc header: " + string(make([]byte, 120)))),
+		"not gzip":                     small,
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, err := archivedAgent(bytes.NewReader(archive))
+			require.Error(t, err)
+		})
+	}
+}
+
+func TestRun_RefusesASymlinkedImagePath(t *testing.T) {
+	d := t.TempDir()
+	img := filepath.Join(d, "initramfs.cpio.gz")
+	require.NoError(t, os.WriteFile(img, gz(withAgent(file(agentPath, "x"))), 0o644))
+	require.NoError(t, os.Symlink(img, filepath.Join(d, "link")))
+	require.ErrorContains(t, run(filepath.Join(d, "link"), false), "not a regular file")
+}
+
+func TestHasTag_MatchesWholeTagsOnly(t *testing.T) {
+	info := func(tags string) *buildinfo.BuildInfo {
+		return &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}, {Key: "-tags", Value: tags}}}
+	}
+	require.True(t, hasTag(info("lambdatestlink,fips140v1.0"), "lambdatestlink"))
+	require.False(t, hasTag(info("fips140v1.0"), "lambdatestlink"))
+	require.False(t, hasTag(info("lambdatestlinkx,xlambdatestlink"), "lambdatestlink"))
+	require.False(t, hasTag(&buildinfo.BuildInfo{}, "lambdatestlink"))
+}
+
+// The test binary carries real build information, so checkTag reads it and
+// refuses the opposite answer; bytes with no build information are refused.
+func TestCheckTag_ReadsRealBuildInformation(t *testing.T) {
+	exe, err := os.Executable()
+	require.NoError(t, err)
+	raw, err := os.ReadFile(exe)
+	require.NoError(t, err)
+	info, err := buildinfo.Read(bytes.NewReader(raw))
+	require.NoError(t, err)
+	tagged := hasTag(info, "lambdatestlink")
+	require.NoError(t, checkTag(raw, tagged))
+	require.Error(t, checkTag(raw, !tagged))
+	require.Error(t, checkTag([]byte("not a Go binary"), false))
+}
+
+func TestDecide_RefusesMissingBuildSettings(t *testing.T) {
+	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, false), "no build settings")
+	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, true), "no build settings")
+	untagged := &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}}}
+	require.NoError(t, decide(untagged, false))
+	require.Error(t, decide(untagged, true))
+}

cmd/lambda-image-digest/main.go

@@ -0,0 +1,26 @@
+//go:build lambdatestlink
+
+// Command lambda-image-digest prints the test-link runtime image digest of one
+// directory, selecting the boot files the same way the daemon does. The digest
+// identifies the bytes; it does not authenticate where they came from.
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Fprintln(os.Stderr, "usage: lambda-image-digest <image-dir>")
+		os.Exit(2)
+	}
+	digest, err := testlink.DirDigest(os.Args[1])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	fmt.Println(digest)
+}

docs/testing/e2e-coverage.md

@@ -110,7 +110,7 @@ Three entry points, in the order they run:
 
 ### `iam` — everything IAM and STS
 
-Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
+Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, a user rename needing `iam:UpdateUser` on both the old and the new name, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
 
 ### `cert` — the cluster CA and TLS identity
 

scripts/build-lambda-runtime-microvm-image.sh

@@ -3,24 +3,28 @@
 # + initramfs: a guest running the Lambda guest agent, with virtio-net for its
 # host control connection. It is not shipped.
 #
+# Usage: build-lambda-runtime-microvm-image.sh [agent-binary [output-dir]]
+#
 # Inputs:
 #   $REPO_ROOT/build/microvm/lambda-runtime/{init.sh,inittab,run-agent}
-#   $REPO_ROOT/bin/lambda-agent  (make bin/lambda-agent)
+#   agent binary, default $REPO_ROOT/bin/lambda-agent (make bin/lambda-agent)
 #
-# Outputs:
-#   $REPO_ROOT/build/microvm/lambda-runtime/vmlinuz
-#   $REPO_ROOT/build/microvm/lambda-runtime/initramfs.cpio.gz
+# Outputs, in output-dir (default $REPO_ROOT/build/microvm/lambda-runtime):
+#   vmlinuz
+#   initramfs.cpio.gz
 #
 # Rootfs strategy and module stripping: scripts/lib/microvm-image.sh.
 # shellcheck disable=SC2034 # variables are consumed by the sourced library
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-BUILD_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
-INIT_SH="$BUILD_DIR/init.sh"
-INITTAB="$BUILD_DIR/inittab"
-RUN_AGENT="$BUILD_DIR/run-agent"
-LAMBDA_AGENT_BIN="$REPO_ROOT/bin/lambda-agent"
+INPUT_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
+INIT_SH="$INPUT_DIR/init.sh"
+INITTAB="$INPUT_DIR/inittab"
+RUN_AGENT="$INPUT_DIR/run-agent"
+LAMBDA_AGENT_BIN="${1:-$REPO_ROOT/bin/lambda-agent}"
+# The library stages beside BUILD_DIR, so it names the output, not the inputs.
+BUILD_DIR="${2:-$INPUT_DIR}"
 TAG="[build-lambda-runtime-microvm-image]"
 
 ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
@@ -38,7 +42,8 @@ MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
 source "$REPO_ROOT/scripts/lib/microvm-image.sh"
 
 echo "$TAG repo root: $REPO_ROOT"
-echo "$TAG build dir: $BUILD_DIR"
+echo "$TAG input dir: $INPUT_DIR"
+echo "$TAG output dir: $BUILD_DIR"
 
 microvm_check_tools
 
@@ -57,6 +62,9 @@ if ! file -b "$LAMBDA_AGENT_BIN" | grep -q "statically linked"; then
     exit 1
 fi
 
+# Absolute, because packing the initramfs runs from inside the chroot.
+mkdir -p "$BUILD_DIR"
+BUILD_DIR=$(cd "$BUILD_DIR" && pwd)
 microvm_init_dirs
 microvm_make_rootfs
 microvm_empty_dev

scripts/build-lambda-runtime-microvm-image_test.sh

@@ -0,0 +1,81 @@
+#!/bin/sh
+# build-lambda-runtime-microvm-image_test.sh — a build into a chosen output
+# directory stages and publishes there, and leaves the default input directory
+# untouched.
+#
+# The script runs in a fixture repo against the real library, with only the
+# rootfs, kernel and packing steps replaced so no image is downloaded.
+set -eu
+
+SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- build-lambda-runtime-microvm-image.sh output directory"
+
+REPO="$TMP/repo"
+IN="$REPO/build/microvm/lambda-runtime"
+mkdir -p "$REPO/scripts/lib" "$IN" "$REPO/bin" "$TMP/bin"
+cp "$SCRIPTS_DIR/build-lambda-runtime-microvm-image.sh" "$REPO/scripts/"
+for f in init.sh inittab run-agent; do printf '%s\n' "$f" > "$IN/$f"; done
+printf 'default image\n' > "$IN/vmlinuz"
+printf '#!/bin/sh\n' > "$REPO/bin/agent-testlink"
+chmod +x "$REPO/bin/agent-testlink"
+printf '#!/bin/sh\necho "ELF 64-bit, statically linked"\n' > "$TMP/bin/file"
+chmod +x "$TMP/bin/file"
+BEFORE=$(cd "$IN" && ls -A | cksum)
+
+# The real library, with the steps that need a rootfs or kernel replaced.
+# Packing runs from inside the chroot as the real step does, and records where
+# the stage is while it still exists.
+cat > "$REPO/scripts/lib/microvm-image.sh" <<EOF
+. "$SCRIPTS_DIR/lib/microvm-image.sh"
+microvm_check_tools() { :; }
+microvm_make_rootfs() { mkdir -p "\$CHROOT_DIR/etc"; }
+microvm_empty_dev() { :; }
+microvm_locate_kernel() { :; }
+microvm_strip_modules() { :; }
+microvm_strip_rootfs() { :; }
+microvm_stage_kernel() { printf 'kernel\n' > "\$STAGE_DIR/vmlinuz"; }
+microvm_pack_initramfs() {
+    (cd "\$CHROOT_DIR" && printf 'initramfs\n' | gzip > "\$STAGE_DIR/initramfs.cpio.gz")
+    dirname "\$STAGE_DIR" > "$TMP/stage-parent"
+    ls -A "$IN" > "$TMP/input-during"
+}
+EOF
+
+# The output is given relative to the repo root, as make passes it.
+OUT="$REPO/build/microvm/lambda-testlink-runtime"
+if ! (cd "$REPO" && PATH="$TMP/bin:$PATH" bash scripts/build-lambda-runtime-microvm-image.sh bin/agent-testlink build/microvm/lambda-testlink-runtime) > "$TMP/log" 2>&1; then
+    fail "build into output dir: $(tr '\n' ' ' < "$TMP/log")"
+else
+    if [ "$(cat "$TMP/stage-parent")" = "$OUT" ]; then
+        pass "stage created in the output directory"
+    else
+        fail "stage created in $(cat "$TMP/stage-parent"), want $OUT"
+    fi
+    if grep -q '^\.stage\.' "$TMP/input-during"; then
+        fail "stage present in the default input directory during the build"
+    else
+        pass "no stage in the default input directory during the build"
+    fi
+    if [ "$(cd "$IN" && ls -A | cksum)" = "$BEFORE" ] && grep -qx 'default image' "$IN/vmlinuz"; then
+        pass "default input directory unchanged"
+    else
+        fail "default input directory changed: $(cd "$IN" && ls -A | tr '\n' ' ')"
+    fi
+    if grep -qx kernel "$OUT/vmlinuz" && gzip -t "$OUT/initramfs.cpio.gz" && [ "$(ls -A "$OUT" | wc -l)" -eq 2 ]; then
+        pass "output directory holds exactly the published pair"
+    else
+        fail "output directory: $(ls -A "$OUT" | tr '\n' ' ')"
+    fi
+fi
+
+if [ "$fails" -ne 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi

scripts/lib/lambda-testlink-build_test.sh

@@ -0,0 +1,111 @@
+#!/bin/sh
+# lambda-testlink-build_test.sh — the test-link make targets refuse a malformed
+# image digest and an unsupported host or GOARCH before touching any output.
+#
+# Every case is refused on the recipe's first lines, so nothing is compiled.
+set -eu
+
+LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+REPO_ROOT=$(dirname -- "$(dirname -- "$LIB_DIR")")
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- make build-lambda-testlink / build-lambda-testlink-image guards"
+
+PWN="$TMP/pwn"
+OUT="$TMP/out"
+HEX=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+mkdir -p "$OUT"
+printf 'old daemon\n' > "$OUT/spx"
+printf 'old initializer\n' > "$OUT/hardkill.so"
+PAIR=$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)
+
+# unchanged <name> — the existing pair is intact, no scratch was made and no
+# injected command ran.
+unchanged() {
+    if [ -e "$PWN" ]; then
+        fail "$1: injected command ran"
+        rm -f "$PWN"
+    elif [ -e "$OUT.tmp" ]; then
+        fail "$1: scratch directory created"
+    elif [ "$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)" != "$PAIR" ]; then
+        fail "$1: existing pair replaced"
+    else
+        pass "$1"
+    fi
+}
+
+# refused <name> <want> <log> <status> — make failed with the expected message.
+refused() {
+    if [ "$4" -eq 0 ]; then
+        fail "$1: make succeeded"
+    elif ! grep -q "$2" "$3"; then
+        fail "$1: wrong refusal: $(tr '\n' ' ' < "$3")"
+    else
+        unchanged "$1"
+    fi
+}
+
+testlink() {
+    make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" "$@" > "$TMP/log" 2>&1
+}
+
+DIGEST_MSG="LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"
+NL='
+'
+for v in '' "\$(shell touch $PWN)" "\$\$(touch $PWN)" "\`touch $PWN\`" "$HEX; touch $PWN" \
+    "$HEX\" -X main.x=\"y" "$HEX -X main.x=y" "$HEX${NL}touch $PWN" \
+    0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF "${HEX%?}" "${HEX}0"; do
+    # GNU make itself expands a command-line $(shell ...) while parsing, for
+    # any target, so that one literal is tested only from the environment.
+    case "$v" in
+    "\$(shell"*) ;;
+    *)
+        status=0
+        testlink LAMBDA_IMAGE_DIGEST="$v" || status=$?
+        refused "command-line digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
+        ;;
+    esac
+    status=0
+    LAMBDA_IMAGE_DIGEST="$v" testlink || status=$?
+    refused "environment digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
+done
+
+status=0
+env -u LAMBDA_IMAGE_DIGEST make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" > "$TMP/log" 2>&1 || status=$?
+refused "digest unset" "$DIGEST_MSG" "$TMP/log" "$status"
+
+ARCH_MSG="supports only an x86_64 host building linux/amd64"
+SHIM="$TMP/shim"
+mkdir -p "$SHIM"
+printf '#!/bin/sh\necho aarch64\n' > "$SHIM/uname"
+chmod +x "$SHIM/uname"
+
+status=0
+GOARCH=arm64 testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
+refused "GOARCH=arm64" "$ARCH_MSG" "$TMP/log" "$status"
+status=0
+PATH="$SHIM:$PATH" testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
+refused "aarch64 host" "$ARCH_MSG" "$TMP/log" "$status"
+
+# The image target refuses before building qboot or any agent binary.
+for c in "GOARCH=arm64" "PATH=$SHIM:$PATH"; do
+    status=0
+    env "$c" make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink-image > "$TMP/log" 2>&1 || status=$?
+    if [ "$status" -eq 0 ]; then
+        fail "image target [${c%%=*}]: make succeeded"
+    elif ! grep -q "$ARCH_MSG" "$TMP/log" || grep -q "qboot\|go build" "$TMP/log"; then
+        fail "image target [${c%%=*}]: did not refuse first: $(tr '\n' ' ' < "$TMP/log")"
+    else
+        pass "image target [${c%%=*}] refused first"
+    fi
+done
+
+if [ "$fails" -ne 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi

spinifex/agents/lambda/runtimeapi/server.go

@@ -212,7 +212,7 @@ func (s *Server) response(w http.ResponseWriter, r *http.Request) {
 	}
 	if len(body) > MaxResponseBytes {
 		msg := "Response payload size exceeded maximum allowed payload size (" + strconv.Itoa(MaxResponseBytes) + " bytes)."
-		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeResponseTooLarge})
+		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeUnhandled})
 		writeError(w, http.StatusRequestEntityTooLarge, errTypeTooLarge, msg)
 		return
 	}

spinifex/agents/lambda/runtimeapi/server_test.go

@@ -170,7 +170,7 @@ func TestOversizeResponse(t *testing.T) {
 
 	got := <-out
 	require.NoError(t, got.err)
-	assert.Equal(t, "Function.ResponseSizeTooLarge", got.res.FunctionError)
+	assert.Equal(t, "Unhandled", got.res.FunctionError)
 	assert.Equal(t, "Function.ResponseSizeTooLarge", errType(t, got.res.Payload))
 }
 

spinifex/domains/ec2/accountinventory/compute.go

@@ -0,0 +1,88 @@
+package accountinventory
+
+import (
+	"context"
+	"fmt"
+
+	ec2metadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2spot "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+)
+
+// SpotRequestScanner reads one account's spot requests, failing on any record
+// it cannot read.
+type SpotRequestScanner interface {
+	ScanAccount(ctx context.Context, accountID string) (*ec2spot.AccountScan, error)
+}
+
+// VolumeScanner reads one account's volume documents, failing on any document
+// it cannot read.
+type VolumeScanner interface {
+	ListVolumesStrict(ctx context.Context, accountID string) ([]ec2metadata.Volume, error)
+}
+
+var (
+	_ SpotRequestScanner = (*ec2spot.SpotInstanceServiceImpl)(nil)
+	_ VolumeScanner      = (*ec2metadata.Store)(nil)
+)
+
+// SpotRequestItem is one request the account holds.
+type SpotRequestItem struct {
+	RequestID string
+	State     string
+	Active    bool
+}
+
+// VolumeItem is one volume document the account holds.
+type VolumeItem struct {
+	VolumeID            string
+	State               string
+	AttachedInstance    string
+	DeleteOnTermination bool
+}
+
+// WithCompute adds the spot-request and volume owner stores.
+func (i *Inventory) WithCompute(spot SpotRequestScanner, volumes VolumeScanner) *Inventory {
+	i.spot = spot
+	i.volumes = volumes
+	return i
+}
+
+// SpotRequestInventory returns every spot request the account holds. A scan
+// that cannot be completed is an error, never a shorter list.
+func (i *Inventory) SpotRequestInventory(ctx context.Context, accountID string) ([]SpotRequestItem, error) {
+	if i.spot == nil {
+		return nil, fmt.Errorf("spot request inventory is not configured")
+	}
+	scan, err := i.spot.ScanAccount(ctx, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("spot request inventory incomplete: %w", err)
+	}
+	items := make([]SpotRequestItem, 0, len(scan.Active)+len(scan.Terminal))
+	for _, r := range scan.Active {
+		items = append(items, SpotRequestItem{RequestID: r.RequestID, State: r.State, Active: true})
+	}
+	for _, r := range scan.Terminal {
+		items = append(items, SpotRequestItem{RequestID: r.RequestID, State: r.State})
+	}
+	return items, nil
+}
+
+// VolumeInventory returns every volume document the account holds, including
+// those already marked deleting. An unreadable document is an error.
+func (i *Inventory) VolumeInventory(ctx context.Context, accountID string) ([]VolumeItem, error) {
+	if i.volumes == nil {
+		return nil, fmt.Errorf("volume inventory is not configured")
+	}
+	docs, err := i.volumes.ListVolumesStrict(ctx, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("volume inventory incomplete: %w", err)
+	}
+	items := make([]VolumeItem, 0, len(docs))
+	for _, v := range docs {
+		items = append(items, VolumeItem{
+			VolumeID: v.VolumeID, State: v.State, AttachedInstance: v.AttachedInstance,
+			DeleteOnTermination: v.DeleteOnTermination,
+		})
+	}
+	return items, nil
+}

spinifex/domains/ec2/accountinventory/compute_test.go

@@ -0,0 +1,82 @@
+package accountinventory
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	ec2metadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2spot "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+)
+
+type fakeSpot struct {
+	scan *ec2spot.AccountScan
+	err  error
+}
+
+func (f fakeSpot) ScanAccount(context.Context, string) (*ec2spot.AccountScan, error) {
+	return f.scan, f.err
+}
+
+type fakeVolumes struct {
+	docs []ec2metadata.Volume
+	err  error
+}
+
+func (f fakeVolumes) ListVolumesStrict(context.Context, string) ([]ec2metadata.Volume, error) {
+	return f.docs, f.err
+}
+
+func TestSpotRequestInventoryMarksActiveAndTerminal(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(fakeSpot{scan: &ec2spot.AccountScan{
+		Active:   []ec2spot.AccountSpotRequest{{RequestID: "sir-1", State: "active"}},
+		Terminal: []ec2spot.AccountSpotRequest{{RequestID: "sir-2", State: "cancelled"}},
+	}}, nil)
+
+	items, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []SpotRequestItem{
+		{RequestID: "sir-1", State: "active", Active: true},
+		{RequestID: "sir-2", State: "cancelled"},
+	}, items)
+}
+
+func TestSpotRequestInventoryScanFailureIsAnErrorNotAShorterList(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(fakeSpot{err: errList}, nil)
+
+	items, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.ErrorIs(t, err, errList)
+	assert.Nil(t, items)
+}
+
+func TestVolumeInventoryReportsAttachmentAndDeletingDocuments(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(nil, fakeVolumes{docs: []ec2metadata.Volume{
+		{VolumeID: "vol-1", State: "in-use", AttachedInstance: "i-1", DeleteOnTermination: true},
+		{VolumeID: "vol-2", State: "deleting"},
+	}})
+
+	items, err := inv.VolumeInventory(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []VolumeItem{
+		{VolumeID: "vol-1", State: "in-use", AttachedInstance: "i-1", DeleteOnTermination: true},
+		{VolumeID: "vol-2", State: "deleting"},
+	}, items)
+}
+
+func TestVolumeInventoryListFailureIsAnErrorNotAShorterList(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(nil, fakeVolumes{err: errList})
+
+	items, err := inv.VolumeInventory(t.Context(), testAccount)
+	require.ErrorIs(t, err, errList)
+	assert.Nil(t, items)
+}
+
+func TestComputeInventoriesRefuseWhenNotConfigured(t *testing.T) {
+	inv := New(nil, nil, nil, nil)
+	_, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.Error(t, err)
+	_, err = inv.VolumeInventory(t.Context(), testAccount)
+	require.Error(t, err)
+}

spinifex/domains/ec2/accountinventory/inventory.go

@@ -61,6 +61,8 @@ type Inventory struct {
 	groups    PlacementGroupScanner
 	templates LaunchTemplateScanner
 	tags      TagIndexScanner
+	spot      SpotRequestScanner
+	volumes   VolumeScanner
 	now       func() time.Time
 }
 

spinifex/domains/ec2/awsapi/instance/DescribeAccountInstancesStrict.go

@@ -0,0 +1,151 @@
+package instance
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+)
+
+// accountInstanceRecordsSubject serves every non-terminated instance record an
+// account owns from the shared store, whichever node holds the instance.
+const accountInstanceRecordsSubject = "ec2.DescribeAccountInstanceRecords"
+
+// DescribeAccountInstancesStrict lists every non-terminated instance the
+// account owns, or returns a retryable ServiceUnavailable. Existence comes from
+// the shared record store, so an instance no answering node holds is still
+// listed with its recorded state; node answers are joined on for live state.
+// Anything that could make the list shorter than the truth is an error: a
+// configured node that did not answer, a frame from a node outside nodeIDs, a
+// record the owner could not project, or a store that did not answer.
+func DescribeAccountInstancesStrict(ctx context.Context, natsConn *nats.Conn, nodeIDs []string, accountID string, opts ...DescribeOption) (*ec2.DescribeInstancesOutput, error) {
+	if len(nodeIDs) == 0 {
+		slog.ErrorContext(ctx, "DescribeInstances: strict listing refused, configured node set unknown")
+		return nil, incompleteListingError()
+	}
+	cfg := describeConfig{fanoutTimeout: defaultDescribeFanoutTimeout}
+	for _, opt := range opts {
+		opt(&cfg)
+	}
+
+	jsonData, err := json.Marshal(&ec2.DescribeInstancesInput{})
+	if err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "marshal describe input: %v", err)
+	}
+
+	records, ok := queryInstanceBucket(ctx, natsConn, accountInstanceRecordsSubject, jsonData, accountID)
+	if !ok {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, instance record store did not answer")
+		return nil, incompleteListingError()
+	}
+
+	// Collect to the deadline with no responder grace, so a reply from a node
+	// outside the set that lands after the configured nodes is still seen.
+	frames, sum, err := natsmsg.Gather(ctx, natsConn, "ec2.DescribeInstances", jsonData, natsmsg.GatherOpts{
+		Timeout:            cfg.fanoutTimeout,
+		AccountID:          accountID,
+		ExpectedResponders: len(nodeIDs),
+		Mode:               natsmsg.CollectUntilDeadline,
+	})
+	if err != nil {
+		return nil, err
+	}
+	live, complete := judgeIdentityCompleteness(ctx, frames, sum, nodeIDs)
+	if stray := unconfiguredNodes(frames, sum, nodeIDs); len(stray) > 0 {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, frame from a node outside the configured set",
+			"unconfigured_nodes", stray)
+		return nil, incompleteListingError()
+	}
+	if !complete {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, fan-out incomplete", "configured_nodes", len(nodeIDs))
+		return nil, incompleteListingError()
+	}
+
+	joined, nodeOnly, malformed := joinLiveOntoRecords(records, live)
+	if malformed {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, a reservation or instance has no usable identity")
+		return nil, incompleteListingError()
+	}
+	if len(nodeOnly) > 0 {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, node reports instances with no record",
+			"instances", nodeOnly)
+		return nil, incompleteListingError()
+	}
+	return &ec2.DescribeInstancesOutput{Reservations: joined}, nil
+}
+
+// unconfiguredNodes returns the node IDs that answered, with a success or an
+// error, but are not in nodeIDs.
+func unconfiguredNodes(frames []natsmsg.Frame, sum natsmsg.Summary, nodeIDs []string) []string {
+	var stray []string
+	note := func(id string) {
+		if id != "" && !slices.Contains(nodeIDs, id) && !slices.Contains(stray, id) {
+			stray = append(stray, id)
+		}
+	}
+	for _, frame := range frames {
+		note(frame.NodeID)
+	}
+	for id := range sum.Responders {
+		note(id)
+	}
+	slices.Sort(stray)
+	return stray
+}
+
+// joinLiveOntoRecords starts from the recorded instances and replaces each with
+// the answering node's view of it. Instances only a node reports are returned
+// by ID: a lost record write can cause one, and it must not be dropped. A nil
+// reservation, instance or instance ID is reported as malformed, never skipped.
+func joinLiveOntoRecords(records, live []*ec2.Reservation) ([]*ec2.Reservation, []string, bool) {
+	type entry struct {
+		reservation *ec2.Reservation
+		instance    *ec2.Instance
+	}
+	byID := map[string]*entry{}
+	var order []string
+	malformed := false
+	add := func(reservations []*ec2.Reservation) {
+		for _, res := range reservations {
+			if res == nil {
+				malformed = true
+				continue
+			}
+			for _, inst := range res.Instances {
+				if inst == nil || aws.StringValue(inst.InstanceId) == "" {
+					malformed = true
+					continue
+				}
+				id := aws.StringValue(inst.InstanceId)
+				if existing, ok := byID[id]; ok {
+					existing.instance = inst
+					continue
+				}
+				byID[id] = &entry{reservation: res, instance: inst}
+				order = append(order, id)
+			}
+		}
+	}
+	add(records)
+	recorded := len(order)
+	add(live)
+	nodeOnly := slices.Clone(order[recorded:])
+
+	out := make([]*ec2.Reservation, 0, len(order))
+	for _, id := range order {
+		e := byID[id]
+		out = append(out, &ec2.Reservation{
+			ReservationId: e.reservation.ReservationId,
+			OwnerId:       e.reservation.OwnerId,
+			Groups:        e.reservation.Groups,
+			Instances:     []*ec2.Instance{e.instance},
+		})
+	}
+	return mergeReservationsByID(out), nodeOnly, malformed
+}

spinifex/domains/ec2/awsapi/instance/DescribeAccountInstancesStrict_test.go

@@ -0,0 +1,229 @@
+package instance
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func strictInstance(id, state string) *ec2.Instance {
+	return &ec2.Instance{InstanceId: aws.String(id), State: &ec2.InstanceState{Name: aws.String(state)}}
+}
+
+func strictReservations(instances ...*ec2.Instance) *ec2.DescribeInstancesOutput {
+	out := &ec2.DescribeInstancesOutput{}
+	for _, inst := range instances {
+		out.Reservations = append(out.Reservations, &ec2.Reservation{
+			ReservationId: aws.String("r-" + aws.StringValue(inst.InstanceId)),
+			OwnerId:       aws.String("123456789012"),
+			Instances:     []*ec2.Instance{inst},
+		})
+	}
+	return out
+}
+
+func subscribeAccountRecords(t *testing.T, nc *nats.Conn, data []byte) {
+	t.Helper()
+	_, err := nc.Subscribe(accountInstanceRecordsSubject, func(msg *nats.Msg) { _ = msg.Respond(data) })
+	require.NoError(t, err)
+}
+
+const strictAccount = "123456789012"
+
+func strictList(nc *nats.Conn, nodes ...string) (*ec2.DescribeInstancesOutput, error) {
+	return DescribeAccountInstancesStrict(context.Background(), nc, nodes, strictAccount, WithFanoutTimeout(2*time.Second))
+}
+
+// Existence comes from the record store, so an instance no node holds is still
+// listed, with the state the record carries.
+func TestDescribeAccountInstancesStrict_UnheldRecordIsReturnedWithItsRecordedState(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-unheld", "stopped"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.NoError(t, err)
+	require.Len(t, out.Reservations, 1)
+	inst := out.Reservations[0].Instances[0]
+	assert.Equal(t, "i-unheld", aws.StringValue(inst.InstanceId))
+	assert.Equal(t, "stopped", aws.StringValue(inst.State.Name))
+}
+
+// A node's answer replaces the recorded view of the instance it holds.
+func TestDescribeAccountInstancesStrict_NodeAnswersAreJoinedOntoRecords(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-held", "pending"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", mustMarshalDescribeOutput(t,
+		strictReservations(strictInstance("i-held", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.NoError(t, err)
+	states := map[string]string{}
+	for _, res := range out.Reservations {
+		for _, inst := range res.Instances {
+			states[aws.StringValue(inst.InstanceId)] = aws.StringValue(inst.State.Name)
+		}
+	}
+	assert.Equal(t, map[string]string{"i-held": "running"}, states)
+}
+
+// An instance a node reports with no record can come from a lost record write.
+// Returning it or dropping it would both answer for an unproven account.
+func TestDescribeAccountInstancesStrict_NodeOnlyInstanceFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-held", "running"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", mustMarshalDescribeOutput(t,
+		strictReservations(strictInstance("i-held", "running"), strictInstance("i-node-only", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.Error(t, err)
+	assert.Nil(t, out)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorServiceUnavailable, code)
+}
+
+// A frame from a node outside the configured set means the expected set is
+// wrong, and a listing built on it could omit that node's instances.
+func TestDescribeAccountInstancesStrict_FrameFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-on-stray", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// A stray node that answers with an error is still a node outside the set, and
+// its instances are unaccounted for.
+func TestDescribeAccountInstancesStrict_ErrorFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// The configured nodes answer at once, so a reply that lands later but inside
+// the fan-out deadline must still be seen.
+func TestDescribeAccountInstancesStrict_LateFrameFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	late := mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-on-stray", "running")))
+	_, err := nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		time.AfterFunc(600*time.Millisecond, func() {
+			reply := nats.NewMsg(msg.Reply)
+			reply.Data = late
+			reply.Header.Set(natsmsg.NodeIDHeader, "node-stray")
+			_ = msg.RespondMsg(reply)
+		})
+	})
+	require.NoError(t, err)
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// A record with no instance ID cannot be listed or counted, so it must fail
+// the read rather than vanish from it.
+func TestDescribeAccountInstancesStrict_RecordWithoutInstanceIDFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(&ec2.Instance{State: &ec2.InstanceState{Name: aws.String("stopped")}})))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_ConfiguredNodeSilentFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1", "node-2")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_RecordStoreErrorFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_RecordStoreSilentFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_UnknownNodeSetFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc)
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_UnattributedFrameFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}

spinifex/domains/ec2/awsapi/instance/DescribeInstances_current_behaviour_test.go

@@ -0,0 +1,63 @@
+package instance
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Pins today: the completeness judge merges the reservations of a frame from a
+// node outside the configured set and still reports the listing complete. The
+// stray node is never named, so a silent unconfigured node cannot be told from
+// one that does not exist.
+func TestCurrentBehaviour_JudgeIdentityCompleteness_FrameFromUnconfiguredNodeIsMerged(t *testing.T) {
+	t.Parallel()
+	stray := mustMarshalDescribeOutput(t, &ec2.DescribeInstancesOutput{
+		Reservations: []*ec2.Reservation{{ReservationId: aws.String("r-stray"), Instances: []*ec2.Instance{{InstanceId: aws.String("i-on-stray")}}}},
+	})
+	frames := []natsmsg.Frame{
+		{NodeID: "node-1", Data: []byte(`{}`)},
+		{NodeID: "node-stray", Data: stray},
+	}
+
+	reservations, complete := judgeIdentityCompleteness(context.Background(), frames, natsmsg.Summary{}, []string{"node-1"})
+
+	assert.True(t, complete, "CURRENT: a node outside nodeIDs does not make the listing incomplete")
+	require.Len(t, reservations, 1)
+	assert.Equal(t, "i-on-stray", *reservations[0].Instances[0].InstanceId,
+		"CURRENT: the unconfigured node's instances are merged into the result")
+}
+
+// Pins today: the fan-out counts every identified frame toward the expected
+// responders, so a stray frame can end collection before a configured node has
+// answered. The listing then refuses, which fails closed, but for a node that
+// was only slow.
+func TestCurrentBehaviour_DescribeAccountInstancesComplete_StrayFrameConsumesAResponderSlot(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeEmptyInstanceBuckets(t, nc)
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", []byte(`{}`))
+	_, err := nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		time.AfterFunc(500*time.Millisecond, func() {
+			reply := nats.NewMsg(msg.Reply)
+			reply.Data = []byte(`{}`)
+			reply.Header.Set(natsmsg.NodeIDHeader, "node-2")
+			_ = msg.RespondMsg(reply)
+		})
+	})
+	require.NoError(t, err)
+	nc.Flush()
+
+	output, err := DescribeAccountInstancesComplete(context.Background(), nc, []string{"node-1", "node-2"}, "123456789012",
+		WithFanoutTimeout(3*time.Second))
+
+	assertIncompleteListing(t, output, err)
+}

spinifex/domains/ec2/ebs/obligation/obligation.go

@@ -0,0 +1,185 @@
+// Package ebsobligation records the volume exports a detach left in place.
+//
+// A force detach, or a terminate whose unmount did not complete, clears a
+// volume's attachment while the node that served it may still export it. The
+// request cannot reach that node, so it records an UnpublishObligation here and
+// returns; the named node settles it once it answers.
+package ebsobligation
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Bucket holds one obligation per publish a cleared attachment left behind.
+// It has no TTL: an obligation ends only when it is settled or closed.
+const Bucket = "spinifex-ebs-unpublish-obligations"
+
+// BucketVersion is the bucket schema version RunKV stamps.
+const BucketVersion = 1
+
+// SchemaVersion is the version of the UnpublishObligation record.
+const SchemaVersion uint16 = 1
+
+// UnpublishObligation names a publish a cleared attachment left on a node: the
+// node, volume and handle, and the attachment it served. The attachment is what
+// tells it apart from a later publish of the same volume on the same node.
+type UnpublishObligation struct {
+	SchemaVersion uint16    `json:"schema_version"`
+	VolumeID      string    `json:"volume_id"`
+	AccountID     string    `json:"account_id"`
+	Handle        string    `json:"handle,omitempty"` // ProviderHandle when the attachment was cleared
+	NodeID        string    `json:"node_id"`          // the node holding the publish
+	InstanceID    string    `json:"instance_id"`      // the attachment that was cleared
+	AttachedAt    time.Time `json:"attached_at"`
+	RecordedAt    time.Time `json:"recorded_at"`
+	Attempts      int       `json:"attempts"`
+	Error         string    `json:"error,omitempty"`
+}
+
+// keyToken is one NATS subject token a key segment may be: the key is three
+// tokens, so a segment holding a dot would split it and escape the filters.
+var keyToken = regexp.MustCompile(`^[-_=a-zA-Z0-9]+$`)
+
+// Key is the obligation's key, <node>.<volume-id>.<attached-at>, with the
+// attachment time in Unix nanoseconds and an unknown time as 0.
+func Key(o UnpublishObligation) (string, error) {
+	if !keyToken.MatchString(o.NodeID) {
+		return "", fmt.Errorf("unpublish obligation: node %q is not a valid key token", o.NodeID)
+	}
+	if !keyToken.MatchString(o.VolumeID) {
+		return "", fmt.Errorf("unpublish obligation: volume %q is not a valid key token", o.VolumeID)
+	}
+	var nanos int64
+	if !o.AttachedAt.IsZero() {
+		nanos = o.AttachedAt.UnixNano()
+	}
+	return o.NodeID + "." + o.VolumeID + "." + strconv.FormatInt(nanos, 10), nil
+}
+
+// allKeys matches every obligation key and not the bucket's version stamp,
+// which is a single token.
+const allKeys = "*.*.*"
+
+func bucketConfig() kvstore.Config {
+	return kvstore.Config{
+		Name:        Bucket,
+		Description: "EBS exports a cleared attachment left on a node, until that node unpublishes them",
+		History:     1,
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return migrate.DefaultRegistry.RunKV(ctx, Bucket, kv, BucketVersion)
+		},
+		Missing: "EBS unpublish obligation bucket not initialized",
+	}
+}
+
+// Store reads and writes the obligation bucket.
+type Store struct {
+	s *kvstore.Store[UnpublishObligation]
+}
+
+// NewStore returns a Store over js. The bucket is created on first use.
+func NewStore(js jetstream.JetStream) *Store {
+	return &Store{s: kvstore.New[UnpublishObligation](js, bucketConfig())}
+}
+
+// Open creates or opens the bucket now, so a node that cannot reach it fails
+// at startup rather than on its first force detach.
+func (s *Store) Open(ctx context.Context) error {
+	_, err := s.s.KV(ctx)
+	return err
+}
+
+// Record writes o if no obligation with its key exists. An existing key is the
+// same obligation written by an earlier attempt, so it counts as written.
+func (s *Store) Record(ctx context.Context, o UnpublishObligation) error {
+	key, err := Key(o)
+	if err != nil {
+		return err
+	}
+	o.SchemaVersion = SchemaVersion
+	if _, err := s.s.Create(ctx, key, &o); err != nil && !errors.Is(err, kvstore.ErrExists) {
+		return fmt.Errorf("record unpublish obligation %s: %w", key, err)
+	}
+	return nil
+}
+
+// ListNode returns the open obligations naming node.
+func (s *Store) ListNode(ctx context.Context, node string) ([]UnpublishObligation, error) {
+	if !keyToken.MatchString(node) {
+		return nil, fmt.Errorf("unpublish obligation: node %q is not a valid key token", node)
+	}
+	return s.list(ctx, node+".*.*")
+}
+
+// List returns every open obligation.
+func (s *Store) List(ctx context.Context) ([]UnpublishObligation, error) {
+	return s.list(ctx, allKeys)
+}
+
+func (s *Store) list(ctx context.Context, filter string) ([]UnpublishObligation, error) {
+	items, _, err := s.s.Snapshot(ctx, filter)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]UnpublishObligation, 0, len(items))
+	for _, item := range items {
+		out = append(out, item.Value)
+	}
+	return out, nil
+}
+
+// Close removes o. Removing one already gone is not an error.
+func (s *Store) Close(ctx context.Context, o UnpublishObligation) error {
+	key, err := Key(o)
+	if err != nil {
+		return err
+	}
+	return s.s.Delete(ctx, key)
+}
+
+// RecordFailure counts a failed settlement on o and keeps its error, so an
+// operator listing the obligations sees why it is still open.
+func (s *Store) RecordFailure(ctx context.Context, o UnpublishObligation, cause error) error {
+	key, err := Key(o)
+	if err != nil {
+		return err
+	}
+	err = s.s.Mutate(ctx, key, func(cur *UnpublishObligation) (bool, error) {
+		cur.Attempts++
+		cur.Error = cause.Error()
+		return true, nil
+	})
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return nil
+	}
+	return err
+}
+
+// ListOpen reads the open obligations for an operator tool without creating
+// the bucket: a cluster that has never force-detached a volume has none, and
+// that is not an error.
+func ListOpen(ctx context.Context, nc *nats.Conn) ([]UnpublishObligation, error) {
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return nil, fmt.Errorf("jetstream: %w", err)
+	}
+	if _, err := js.KeyValue(ctx, Bucket); errors.Is(err, jetstream.ErrBucketNotFound) {
+		return nil, nil
+	} else if err != nil {
+		return nil, fmt.Errorf("unpublish obligation bucket: %w", err)
+	}
+	cfg := bucketConfig()
+	cfg.AttachOnly = true
+	cfg.OnOpen = nil
+	return (&Store{s: kvstore.New[UnpublishObligation](js, cfg)}).List(ctx)
+}

spinifex/domains/ec2/ebs/obligation/obligation_test.go

@@ -0,0 +1,141 @@
+package ebsobligation_test
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newStore(t *testing.T) *ebsobligation.Store {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	store := ebsobligation.NewStore(js)
+	require.NoError(t, store.Open(t.Context()))
+	return store
+}
+
+func obligation(node, volumeID string, attachedAt time.Time) ebsobligation.UnpublishObligation {
+	return ebsobligation.UnpublishObligation{
+		VolumeID: volumeID, AccountID: "123456789012", NodeID: node,
+		InstanceID: "i-1", AttachedAt: attachedAt, RecordedAt: attachedAt.Add(time.Minute),
+	}
+}
+
+func TestKey_NamesNodeVolumeAndAttachment(t *testing.T) {
+	at := time.Unix(1_700_000_000, 5)
+	key, err := ebsobligation.Key(obligation("node-a", "vol-1", at))
+	require.NoError(t, err)
+	assert.Equal(t, "node-a.vol-1.1700000000000000005", key)
+
+	key, err = ebsobligation.Key(obligation("node-a", "vol-1", time.Time{}))
+	require.NoError(t, err)
+	assert.Equal(t, "node-a.vol-1.0", key, "an unknown attachment time is 0, not a huge negative")
+}
+
+// A segment holding a subject separator or wildcard would split the key or
+// let one node's listing match another's.
+func TestKey_RejectsSegmentsThatAreNotOneToken(t *testing.T) {
+	for _, o := range []ebsobligation.UnpublishObligation{
+		obligation("node.a", "vol-1", time.Time{}),
+		obligation("node-a", "vol.1", time.Time{}),
+		obligation("*", "vol-1", time.Time{}),
+		obligation("node-a", ">", time.Time{}),
+		obligation("", "vol-1", time.Time{}),
+	} {
+		_, err := ebsobligation.Key(o)
+		assert.Error(t, err, "%s/%s", o.NodeID, o.VolumeID)
+	}
+}
+
+func TestStore_RecordIsIdempotentPerAttachment(t *testing.T) {
+	store := newStore(t)
+	at := time.Unix(1_700_000_000, 0).UTC()
+	first := obligation("node-a", "vol-1", at)
+	first.Handle = "h1"
+	require.NoError(t, store.Record(t.Context(), first))
+	retry := first
+	retry.RecordedAt = at.Add(time.Hour)
+	require.NoError(t, store.Record(t.Context(), retry), "a retry of the same obligation is not an error")
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", at.Add(time.Second))),
+		"a different attachment of the same volume is a different obligation")
+
+	open, err := store.List(t.Context())
+	require.NoError(t, err)
+	require.Len(t, open, 2)
+	for _, o := range open {
+		assert.Equal(t, ebsobligation.SchemaVersion, o.SchemaVersion)
+		if o.AttachedAt.Equal(at) {
+			assert.True(t, o.RecordedAt.Equal(at.Add(time.Minute)), "the first write is kept")
+		}
+	}
+}
+
+// The bucket's version stamp is a key in the same bucket; listing must not
+// return it as an obligation.
+func TestStore_ListSkipsTheVersionStampAndListNodeIsolatesNodes(t *testing.T) {
+	store := newStore(t)
+	at := time.Unix(1_700_000_000, 0).UTC()
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", at)))
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-2", at)))
+	require.NoError(t, store.Record(t.Context(), obligation("node-ab", "vol-3", at)))
+
+	all, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, all, 3)
+
+	mine, err := store.ListNode(t.Context(), "node-a")
+	require.NoError(t, err)
+	volumes := []string{}
+	for _, o := range mine {
+		volumes = append(volumes, o.VolumeID)
+	}
+	assert.ElementsMatch(t, []string{"vol-1", "vol-2"}, volumes, "a node whose name prefixes another's sees only its own")
+
+	_, err = store.ListNode(t.Context(), "*")
+	assert.Error(t, err)
+}
+
+func TestStore_CloseAndRecordFailure(t *testing.T) {
+	store := newStore(t)
+	o := obligation("node-a", "vol-1", time.Unix(1_700_000_000, 0).UTC())
+	require.NoError(t, store.Record(t.Context(), o))
+
+	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("first")))
+	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("second")))
+	open, err := store.List(t.Context())
+	require.NoError(t, err)
+	require.Len(t, open, 1)
+	assert.Equal(t, 2, open[0].Attempts)
+	assert.Equal(t, "second", open[0].Error)
+
+	require.NoError(t, store.Close(t.Context(), o))
+	require.NoError(t, store.Close(t.Context(), o), "closing one already gone is not an error")
+	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("late")),
+		"a failure recorded after another sweep closed it does not resurrect it")
+	open, err = store.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, open)
+}
+
+func TestListOpen_MissingBucketIsEmptyAndNotCreated(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+
+	open, err := ebsobligation.ListOpen(t.Context(), nc)
+	require.NoError(t, err)
+	assert.Empty(t, open)
+	_, err = js.KeyValue(t.Context(), ebsobligation.Bucket)
+	assert.Error(t, err, "an operator listing must not create the bucket")
+
+	store := ebsobligation.NewStore(js)
+	require.NoError(t, store.Open(t.Context()))
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", time.Unix(1_700_000_000, 0).UTC())))
+	open, err = ebsobligation.ListOpen(t.Context(), nc)
+	require.NoError(t, err)
+	require.Len(t, open, 1)
+	assert.Equal(t, "vol-1", open[0].VolumeID)
+}

spinifex/domains/ec2/instance/describe_account_records_test.go

@@ -0,0 +1,89 @@
+package instance
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func recordsService(store *vmmock.StateStore) *InstanceServiceImpl {
+	return &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store, config: &config.Config{}}
+}
+
+func recordIDs(out *ec2.DescribeInstancesOutput) map[string]string {
+	ids := map[string]string{}
+	for _, res := range out.Reservations {
+		for _, inst := range res.Instances {
+			ids[*inst.InstanceId] = *inst.State.Name
+		}
+	}
+	return ids
+}
+
+// The record read answers for instances the stopped listing leaves out: a
+// running or drain-stopped record that no node need be holding.
+func TestDescribeAccountInstanceRecords_ReturnsEveryNonTerminatedRecordOfTheAccount(t *testing.T) {
+	owner := "111122223333"
+	store := &vmmock.StateStore{
+		Stopped: map[string]*vm.VM{"i-stopped": nilRecordVM("i-stopped", owner, true, true, vm.StateStopped)},
+		Saved: map[string]map[string]*vm.VM{"node-gone": {
+			"i-running": nilRecordVM("i-running", owner, true, true, vm.StateRunning),
+			"i-other":   nilRecordVM("i-other", "999999999999", true, true, vm.StateRunning),
+		}},
+	}
+
+	out, err := recordsService(store).DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{"i-stopped": "stopped", "i-running": "running"}, recordIDs(out),
+		"the other account's record is not returned")
+}
+
+// A record that cannot be projected fails the read; it is never skipped, since
+// the caller takes existence from this answer.
+func TestDescribeAccountInstanceRecords_NilReservationOrInstanceFails(t *testing.T) {
+	owner := "111122223333"
+	for name, bad := range map[string]*vm.VM{
+		"nil reservation": nilRecordVM("i-bad", owner, false, true, vm.StateRunning),
+		"nil instance":    nilRecordVM("i-bad", owner, true, false, vm.StateRunning),
+	} {
+		t.Run(name, func(t *testing.T) {
+			store := &vmmock.StateStore{
+				Stopped: map[string]*vm.VM{"i-good": nilRecordVM("i-good", owner, true, true, vm.StateStopped)},
+				Saved:   map[string]map[string]*vm.VM{"node-1": {"i-bad": bad}},
+			}
+
+			out, err := recordsService(store).DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+			require.Error(t, err)
+			assert.Nil(t, out, "no shorter list is returned")
+			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+		})
+	}
+}
+
+func TestDescribeAccountInstanceRecords_StoreErrorFails(t *testing.T) {
+	store := &vmmock.StateStore{ListLiveErr: errors.New("kv down")}
+
+	out, err := recordsService(store).DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, "111122223333")
+
+	require.Error(t, err)
+	assert.Nil(t, out)
+}
+
+func TestDescribeAccountInstanceRecords_NoStoreFails(t *testing.T) {
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), config: &config.Config{}}
+
+	out, err := svc.DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, "111122223333")
+
+	require.Error(t, err)
+	assert.Nil(t, out)
+}

spinifex/domains/ec2/instance/describe_nil_records_current_behaviour_test.go

@@ -0,0 +1,64 @@
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func nilRecordVM(id, owner string, withReservation, withInstance bool, status vm.InstanceState) *vm.VM {
+	v := &vm.VM{ID: id, InstanceType: "t3.micro", Status: status, AccountID: owner}
+	if withReservation {
+		v.Reservation = &ec2.Reservation{ReservationId: aws.String("r-" + id), OwnerId: aws.String(owner)}
+	}
+	if withInstance {
+		v.Instance = &ec2.Instance{InstanceId: aws.String(id), InstanceType: aws.String("t3.micro")}
+	}
+	return v
+}
+
+// Pins today: the node path drops an instance whose Reservation or Instance is
+// nil without a log line or an error, so it reads as absent from a listing the
+// caller may treat as complete.
+func TestCurrentBehaviour_DescribeInstances_NodePathSkipsNilReservationAndInstanceSilently(t *testing.T) {
+	owner := "111122223333"
+	vms := map[string]*vm.VM{
+		"i-good":    nilRecordVM("i-good", owner, true, true, vm.StateRunning),
+		"i-no-res":  nilRecordVM("i-no-res", owner, false, true, vm.StateRunning),
+		"i-no-inst": nilRecordVM("i-no-inst", owner, true, false, vm.StateRunning),
+	}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(vms), config: &config.Config{}}
+
+	out, err := svc.DescribeInstances(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+	require.NoError(t, err, "CURRENT: no error for the two skipped instances")
+	require.Len(t, out.Reservations, 1)
+	require.Len(t, out.Reservations[0].Instances, 1)
+	assert.Equal(t, "i-good", *out.Reservations[0].Instances[0].InstanceId, "CURRENT: only the complete record is returned")
+}
+
+// Pins today: the KV path skips the same records, with a warning, and still
+// answers success.
+func TestCurrentBehaviour_DescribeStoppedInstances_KVPathSkipsNilReservationAndInstance(t *testing.T) {
+	owner := "111122223333"
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{
+		"i-good":    nilRecordVM("i-good", owner, true, true, vm.StateStopped),
+		"i-no-res":  nilRecordVM("i-no-res", owner, false, true, vm.StateStopped),
+		"i-no-inst": nilRecordVM("i-no-inst", owner, true, false, vm.StateStopped),
+	}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store, config: &config.Config{}}
+
+	out, err := svc.DescribeStoppedInstances(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+	require.NoError(t, err, "CURRENT: no error for the two skipped instances")
+	require.Len(t, out.Reservations, 1)
+	require.Len(t, out.Reservations[0].Instances, 1)
+	assert.Equal(t, "i-good", *out.Reservations[0].Instances[0].InstanceId, "CURRENT: only the complete record is returned")
+}

spinifex/domains/ec2/instance/service.go

@@ -106,6 +106,8 @@ type GPUClaimer interface {
 type StoppedInstanceStore interface {
 	LoadStoppedInstance(instanceID string) (*vm.VM, error)
 	ListStoppedInstances() ([]*vm.VM, error)
+	ListNonTerminatedInstances() ([]*vm.VM, error)
+	ListTerminatedInPrimary() ([]*vm.VM, error)
 	ListTerminatedInstances() ([]*vm.VM, error)
 	WriteStoppedInstance(instanceID string, instance *vm.VM) error
 	DeleteStoppedInstance(instanceID string) error
@@ -139,6 +141,17 @@ type InstanceTagWriter interface {
 	DeleteAllTags(ctx context.Context, accountID, resourceID string) error
 }
 
+// OwedUnpublish names the publish a terminate leaves in place when its unmount
+// did not complete: the node still exporting the volume and the instance it
+// served. The zero value means the unmount completed and nothing is owed.
+type OwedUnpublish struct {
+	NodeID     string
+	InstanceID string
+}
+
+// Owed reports whether a publish is left in place.
+func (o OwedUnpublish) Owed() bool { return o.NodeID != "" }
+
 // VolumeDeleter deletes EBS volumes. Implemented by domains/ec2/volume's
 // VolumeServiceImpl (used by the daemon).
 type VolumeDeleter interface {
@@ -147,14 +160,16 @@ type VolumeDeleter interface {
 	// DeleteVolumeOnTerminate deletes a DeleteOnTermination volume as part of
 	// an instance terminate: it clears any stale attachment (terminate
 	// implies detach) before deleting, so a still-attached boot volume left
-	// over from Stop is not rejected by DeleteVolume's in-use guard.
-	DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error
+	// over from Stop is not rejected by DeleteVolume's in-use guard. When owed
+	// names a publish, an unpublish obligation is recorded before the clear.
+	DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed OwedUnpublish) error
 
 	// DetachVolumeOnTerminate clears a still-attached volume's attachment on
 	// terminate without deleting it, matching AWS semantics for a
 	// DeleteOnTermination=false volume: terminate still implies detach, it
 	// just leaves the volume behind as available rather than deleting it.
-	DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error
+	// When owed names a publish, an unpublish obligation is recorded first.
+	DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed OwedUnpublish) error
 }
 
 // VolumeCreator creates the volumes a launch names beyond the root device, and

spinifex/domains/ec2/instance/service_impl.go

@@ -1,6 +1,7 @@
 package instance
 
 import (
+	"cmp"
 	"context"
 	"encoding/base64"
 	"encoding/json"
@@ -2234,7 +2235,18 @@ func (s *InstanceServiceImpl) DescribeStoppedInstances(ctx context.Context, inpu
 	if s.stoppedStore == nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListStoppedInstances, 80, "stopped", "DescribeStoppedInstances")
+	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListStoppedInstances, 80, "stopped", "DescribeStoppedInstances", false)
+}
+
+// DescribeAccountInstanceRecords returns every non-terminated instance record
+// the account owns, whichever node holds it. It is the existence source for the
+// strict account inventory, so a record it cannot project fails the call rather
+// than being skipped.
+func (s *InstanceServiceImpl) DescribeAccountInstanceRecords(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
+	if s.stoppedStore == nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListNonTerminatedInstances, 0, "pending", "DescribeAccountInstanceRecords", true)
 }
 
 // DescribeTerminatedInstances returns terminated instances from the terminated KV bucket.
@@ -2242,10 +2254,10 @@ func (s *InstanceServiceImpl) DescribeTerminatedInstances(ctx context.Context, i
 	if s.stoppedStore == nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListTerminatedInstances, 48, "terminated", "DescribeTerminatedInstances")
+	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListTerminatedInstances, 48, "terminated", "DescribeTerminatedInstances", false)
 }
 
-func (s *InstanceServiceImpl) describeInstancesFromKV(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string, listFn func() ([]*vm.VM, error), fallbackCode int64, fallbackName, opName string) (*ec2.DescribeInstancesOutput, error) {
+func (s *InstanceServiceImpl) describeInstancesFromKV(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string, listFn func() ([]*vm.VM, error), fallbackCode int64, fallbackName, opName string, strict bool) (*ec2.DescribeInstancesOutput, error) {
 	instanceIDFilter, err := ParseInstanceIDFilter(input.InstanceIds)
 	if err != nil {
 		return nil, err
@@ -2273,6 +2285,11 @@ func (s *InstanceServiceImpl) describeInstancesFromKV(ctx context.Context, input
 			continue
 		}
 		if instance.Reservation == nil || instance.Instance == nil {
+			if strict {
+				slog.ErrorContext(ctx, opName+": record has nil Reservation/Instance, refusing a listing that would omit it",
+					"instanceId", instance.ID)
+				return nil, errors.New(awserrors.ErrorServerInternal)
+			}
 			slog.WarnContext(ctx, opName+": skipping instance with nil Reservation/Instance (data integrity issue)",
 				"instanceId", instance.ID)
 			continue
@@ -3031,6 +3048,15 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 	// stopped-path delete failure.
 	var lastErr error
 
+	// A volume whose seal failed at stop is still exported on the node that
+	// stopped it, so clearing its attachment owes that node an unpublish.
+	owedFor := func(volumeID string) OwedUnpublish {
+		if instance.LastNode == "" || !slices.Contains(instance.UnpublishOwed, volumeID) {
+			return OwedUnpublish{}
+		}
+		return OwedUnpublish{NodeID: instance.LastNode, InstanceID: instance.ID}
+	}
+
 	for _, ebsRequest := range instance.EBSRequests.Requests {
 		// Internal volumes (EFI) are always cleaned up via ebs.delete.
 		if ebsRequest.EFI {
@@ -3061,7 +3087,7 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 				continue
 			}
 			// A volume already gone is the detach's own goal, not a failure.
-			if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID); err != nil &&
+			if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID, owedFor(ebsRequest.Name)); err != nil &&
 				!awserrors.IsNotFound(err) && !objectstore.IsNoSuchKeyError(err) {
 				slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to detach volume", "name", ebsRequest.Name, "err", err)
 				lastErr = err
@@ -3080,7 +3106,7 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 		// clears a Boot volume, runtime/daemon/vm_adapters.go) before deleting, so
 		// DeleteVolume's in-use guard does not reject an otherwise-legitimate
 		// delete. The resulting error is surfaced into lastErr, not swallowed.
-		if err := s.volumeDeleter.DeleteVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID); err != nil {
+		if err := s.volumeDeleter.DeleteVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID, owedFor(ebsRequest.Name)); err != nil {
 			slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to delete volume", "name", ebsRequest.Name, "err", err)
 			lastErr = err
 		}
@@ -3889,3 +3915,57 @@ func (s *InstanceServiceImpl) AttachVolumeToStopped(ctx context.Context, input *
 		State:      aws.String(ec2.VolumeAttachmentStateAttaching),
 	}, nil
 }
+
+// AccountTeardownOutstanding reports instances not terminated with every dependent done.
+// Terminated records expire after 1h, so it is never sole evidence an account is clear.
+// An unreadable record or one with no instance ID fails the read.
+func (s *InstanceServiceImpl) AccountTeardownOutstanding(accountID string) ([]OutstandingInstance, error) {
+	if s.stoppedStore == nil {
+		return nil, errors.New("instance record store not configured")
+	}
+	live, err := s.stoppedStore.ListNonTerminatedInstances()
+	if err != nil {
+		return nil, fmt.Errorf("list non-terminated instances: %w", err)
+	}
+	primaryTerminated, err := s.stoppedStore.ListTerminatedInPrimary()
+	if err != nil {
+		return nil, fmt.Errorf("list terminated instances in primary store: %w", err)
+	}
+	terminated, err := s.stoppedStore.ListTerminatedInstances()
+	if err != nil {
+		return nil, fmt.Errorf("list terminated instances: %w", err)
+	}
+
+	// An instance can sit in both stores while a terminate is mid-flight. The
+	// non-terminated record is listed first and wins, so it is never reported
+	// as torn down on the strength of the terminated copy.
+	byID := map[string]OutstandingInstance{}
+	for _, store := range []struct {
+		name    string
+		records []*vm.VM
+	}{{"non-terminated", live}, {"primary terminated", primaryTerminated}, {"terminated", terminated}} {
+		for _, v := range store.records {
+			if v == nil || v.ID == "" {
+				return nil, fmt.Errorf("%s instance store holds a record with no instance ID", store.name)
+			}
+			if !IsInstanceVisible(accountID, v.AccountID) {
+				continue
+			}
+			if v.Status == vm.StateTerminated && v.TeardownComplete() {
+				continue
+			}
+			if _, seen := byID[v.ID]; seen {
+				continue
+			}
+			byID[v.ID] = OutstandingInstance{
+				InstanceID: v.ID,
+				State:      string(v.Status),
+				Teardown:   maps.Clone(v.Teardown),
+			}
+		}
+	}
+
+	return slices.SortedFunc(maps.Values(byID), func(a, b OutstandingInstance) int {
+		return cmp.Compare(a.InstanceID, b.InstanceID)
+	}), nil
+}

spinifex/domains/ec2/instance/service_impl_test.go

@@ -1038,7 +1038,7 @@ func TestDescribeInstancesFromKV_FilterExcludesOnlyInstance_ReturnsZeroReservati
 	input := &ec2.DescribeInstancesInput{
 		Filters: []*ec2.Filter{{Name: aws.String("instance-type"), Values: []*string{aws.String("t3.nano")}}},
 	}
-	out, err := svc.describeInstancesFromKV(context.Background(), input, awsidentifiers.GlobalAccountID, listFn, 80, "stopped", "DescribeStoppedInstances")
+	out, err := svc.describeInstancesFromKV(context.Background(), input, awsidentifiers.GlobalAccountID, listFn, 80, "stopped", "DescribeStoppedInstances", false)
 	require.NoError(t, err)
 	assert.Empty(t, out.Reservations, "a reservation with every instance filtered out must not appear in the response")
 }
@@ -1734,6 +1734,7 @@ type fakeVolumeDeleter struct {
 	calls    []string
 	deleted  []string
 	detached []string
+	owed     []OwedUnpublish
 	err      error
 }
 
@@ -1749,7 +1750,8 @@ func (f *fakeVolumeDeleter) DeleteVolume(_ context.Context, input *ec2.DeleteVol
 
 // DeleteVolumeOnTerminate mirrors DeleteVolume's call/error bookkeeping; the
 // stopped-instance terminate path calls this instead of DeleteVolume.
-func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID, _ string) error {
+func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID, _ string, owed OwedUnpublish) error {
+	f.owed = append(f.owed, owed)
 	f.calls = append(f.calls, volumeID)
 	if f.err != nil {
 		return f.err
@@ -1761,7 +1763,8 @@ func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID,
 // DetachVolumeOnTerminate mirrors DeleteVolumeOnTerminate's call/error
 // bookkeeping but records into detached instead of deleted; the non-DoT
 // Boot-volume branch calls this instead of deleting.
-func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _ string) error {
+func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _ string, owed OwedUnpublish) error {
+	f.owed = append(f.owed, owed)
 	f.calls = append(f.calls, volumeID)
 	if f.err != nil {
 		return f.err
@@ -2009,6 +2012,45 @@ func TestTerminateStoppedInstance_UserVolumeDeleted(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, []string{"vol-user-001"}, vd.deleted, "only DeleteOnTermination=true volumes deleted")
 	assert.Equal(t, []string{"vol-keep-001"}, vd.detached, "terminate implies detach even for a volume that survives it")
+	assert.Equal(t, []OwedUnpublish{{}, {}}, vd.owed, "a stop whose unmount completed owes no unpublish")
+}
+
+// A stop whose seal failed left those volumes exported on the node that
+// stopped the instance, so terminate names that node for each of them alone.
+func TestTerminateStoppedInstance_FailedSealOwesTheStoppingNodeAnUnpublish(t *testing.T) {
+	id := "i-seal-failed"
+	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", LastNode: "node-a",
+		UnpublishOwed: []string{"vol-root-001", "vol-keep-001"}}
+	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+		{Name: "vol-root-001", Boot: true, DeleteOnTermination: true},
+		{Name: "vol-keep-001", DeleteOnTermination: false},
+		{Name: "vol-sealed-001", DeleteOnTermination: false},
+	}
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
+	vd := &fakeVolumeDeleter{}
+	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
+
+	_, err := svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
+	require.NoError(t, err)
+
+	owed := OwedUnpublish{NodeID: "node-a", InstanceID: id}
+	assert.Equal(t, []string{"vol-root-001", "vol-keep-001", "vol-sealed-001"}, vd.calls)
+	assert.Equal(t, []OwedUnpublish{owed, owed, {}}, vd.owed)
+}
+
+// With no node recorded there is no node to name; the terminate owes nothing
+// rather than naming an empty node.
+func TestTerminateStoppedInstance_NoLastNodeOwesNothing(t *testing.T) {
+	id := "i-seal-failed-nonode"
+	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", UnpublishOwed: []string{"vol-keep-001"}}
+	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-keep-001"}}
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
+	vd := &fakeVolumeDeleter{}
+	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
+
+	_, err := svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
+	require.NoError(t, err)
+	assert.Equal(t, []OwedUnpublish{{}}, vd.owed)
 }
 
 // TestTerminateStoppedInstance_StampsTeardownVolumesDone locks the

spinifex/domains/ec2/instance/teardown_outstanding.go

@@ -0,0 +1,12 @@
+package instance
+
+// OutstandingInstance is an instance of an account that is not yet fully torn
+// down, with the state its record carries.
+type OutstandingInstance struct {
+	InstanceID string
+	// State is the record's lifecycle state, such as running or terminated.
+	State string
+	// Teardown is the per-dependent progress (done, pending or failed) the
+	// record carries. It is empty when no dependent has been recorded.
+	Teardown map[string]string
+}

spinifex/domains/ec2/spotinstance/account_scan.go

@@ -0,0 +1,81 @@
+package spotinstance
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+)
+
+// AccountSpotRequest is one persisted request in an account scan.
+type AccountSpotRequest struct {
+	RequestID string
+	State     string
+}
+
+// AccountScan lists every request an account holds in both buckets.
+type AccountScan struct {
+	Active   []AccountSpotRequest
+	Terminal []AccountSpotRequest
+}
+
+// ScanAccount lists the account's requests, or fails. Unlike
+// DescribeSpotInstanceRequests it never skips a record it cannot read: a
+// shorter list would be taken as proof that nothing is left.
+func (s *SpotInstanceServiceImpl) ScanAccount(ctx context.Context, accountID string) (*AccountScan, error) {
+	if accountID == "" {
+		return nil, errors.New("spot request scan needs an account ID")
+	}
+	active, err := scanBucket(ctx, s.activeKV, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("scan active spot requests: %w", err)
+	}
+	terminal, err := scanBucket(ctx, s.terminalKV, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("scan terminal spot requests: %w", err)
+	}
+	return &AccountScan{Active: active, Terminal: terminal}, nil
+}
+
+func scanBucket(ctx context.Context, kv jetstream.KeyValue, accountID string) ([]AccountSpotRequest, error) {
+	keys, err := kv.Keys(ctx)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrNoKeysFound) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	prefix := accountID + "."
+	var out []AccountSpotRequest
+	for _, key := range keys {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+			continue
+		}
+		entry, err := kv.Get(ctx, key)
+		if err != nil {
+			// Expired from the terminal bucket between listing and read.
+			if errors.Is(err, jetstream.ErrKeyNotFound) {
+				continue
+			}
+			return nil, fmt.Errorf("read %s: %w", key, err)
+		}
+		var record SpotRequestRecord
+		if err := json.Unmarshal(entry.Value(), &record); err != nil {
+			return nil, fmt.Errorf("decode %s: %w", key, err)
+		}
+		if record.Request == nil || aws.StringValue(record.Request.SpotInstanceRequestId) == "" {
+			return nil, fmt.Errorf("record %s carries no request", key)
+		}
+		out = append(out, AccountSpotRequest{
+			RequestID: aws.StringValue(record.Request.SpotInstanceRequestId),
+			State:     aws.StringValue(record.Request.State),
+		})
+	}
+	return out, nil
+}

spinifex/domains/ec2/spotinstance/account_scan_test.go

@@ -0,0 +1,84 @@
+package spotinstance
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+)
+
+func TestScanAccountListsActiveAndTerminalForOneAccount(t *testing.T) {
+	svc := setupTestService(t)
+	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), makeSIR("sir-bbb", "i-bbb"))
+	_, err := svc.CancelSpotInstanceRequests(t.Context(), &ec2.CancelSpotInstanceRequestsInput{
+		SpotInstanceRequestIds: aws.StringSlice([]string{"sir-bbb"}),
+	}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.PutSpotInstanceRequests(t.Context(), &PutSpotRequestsInput{Requests: []*ec2.SpotInstanceRequest{makeSIR("sir-other", "i-o")}}, "999999999999")
+	require.NoError(t, err)
+
+	scan, err := svc.ScanAccount(t.Context(), testAccountID)
+	require.NoError(t, err)
+	require.Len(t, scan.Active, 1)
+	assert.Equal(t, "sir-aaa", scan.Active[0].RequestID)
+	require.Len(t, scan.Terminal, 1)
+	assert.Equal(t, "sir-bbb", scan.Terminal[0].RequestID)
+}
+
+func TestScanAccountFailsOnAnUndecodableRecord(t *testing.T) {
+	svc := setupTestService(t)
+	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"))
+	_, err := svc.activeKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-bad"), []byte("{not json"))
+	require.NoError(t, err)
+
+	scan, err := svc.ScanAccount(t.Context(), testAccountID)
+	require.Error(t, err)
+	assert.Nil(t, scan)
+}
+
+func TestScanAccountFailsOnARecordWithoutARequest(t *testing.T) {
+	svc := setupTestService(t)
+	_, err := svc.terminalKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-empty"), []byte(`{"account_id":"x"}`))
+	require.NoError(t, err)
+
+	_, err = svc.ScanAccount(t.Context(), testAccountID)
+	require.Error(t, err)
+}
+
+func TestScanAccountOfAnEmptyAccountIsEmptyNotAnError(t *testing.T) {
+	svc := setupTestService(t)
+	scan, err := svc.ScanAccount(t.Context(), testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, scan.Active)
+	assert.Empty(t, scan.Terminal)
+}
+
+type failingGetKV struct {
+	jetstream.KeyValue
+
+	key string
+}
+
+func (k failingGetKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
+	if key == k.key {
+		return nil, errors.New("injected read failure")
+	}
+	return k.KeyValue.Get(ctx, key)
+}
+
+func TestScanAccountFailsWhenARecordCannotBeRead(t *testing.T) {
+	svc := setupTestService(t)
+	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), makeSIR("sir-bbb", "i-bbb"))
+	svc.activeKV = failingGetKV{KeyValue: svc.activeKV, key: kvutil.AccountKey(testAccountID, "sir-bbb")}
+
+	scan, err := svc.ScanAccount(t.Context(), testAccountID)
+	require.Error(t, err)
+	assert.Nil(t, scan)
+}

spinifex/domains/ec2/volume/account_termination_current_behaviour_test.go

@@ -9,6 +9,7 @@ import (
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	ec2volume "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -59,7 +60,7 @@ func TestCurrentBehaviour_AccountTermination_SnapshottedDeleteOnTerminationVolum
 	require.NoError(t, err)
 	require.NoError(t, volumes.UpdateVolumeState(accountID, volumeID, "in-use", instanceID, "/dev/sda1"))
 
-	err = volumes.DeleteVolumeOnTerminate(ctx, volumeID, accountID)
+	err = volumes.DeleteVolumeOnTerminate(ctx, volumeID, accountID, ec2instance.OwedUnpublish{})
 	assert.EqualError(t, err, awserrors.ErrorVolumeInUse, "the snapshot index refuses the terminate delete")
 
 	survivor, err := metadata.GetVolume(ctx, accountID, volumeID)

spinifex/domains/ec2/volume/delete_volume_unreadable_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -194,7 +195,7 @@ func TestDeleteVolumeOnTerminate_DeleteReadFailureIsNotSuccess(t *testing.T) {
 	svc.store = failing
 	svc.metadata = ebsmetadata.NewStore(failing, "test-bucket")
 
-	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
+	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})
 
 	require.Error(t, err, "a delete-read failure after a successful attachment clear must not be reported as success")
 	assert.False(t, awserrors.IsNotFound(err),

spinifex/domains/ec2/volume/force_detach_test.go

@@ -20,6 +20,7 @@ func newForceDetachService(t *testing.T) *VolumeServiceImpl {
 	t.Helper()
 	svc := newTestVolumeService("ap-southeast-2a")
 	svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}))
+	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-stuck0000000000": "node-a"}}, "node-a", "node-b")
 	return svc
 }
 
@@ -47,7 +48,11 @@ func TestForceDetachVolumeClearsTheAttachment(t *testing.T) {
 	// The previous instance is reported back: it is the only record of what
 	// the volume was torn away from.
 	assert.Equal(t, "i-stuck0000000000", aws.StringValue(attachment.InstanceId))
-	assert.Equal(t, "detached", aws.StringValue(attachment.State))
+	// The export is still up on the node that served it, so the attachment is
+	// reported mid-teardown, as AWS reports a detach it has not finished.
+	assert.Equal(t, "detaching", aws.StringValue(attachment.State))
+	assert.Equal(t, "/dev/sdf", aws.StringValue(attachment.Device))
+	assert.False(t, aws.TimeValue(attachment.AttachTime).IsZero())
 
 	meta, err := svc.GetVolumeMetadata("000000000011", "vol-forcedetach1")
 	require.NoError(t, err)
@@ -112,4 +117,8 @@ func TestForceDetachVolumeOnAnUnattachedVolume(t *testing.T) {
 	require.NoError(t, err)
 
 	assert.Empty(t, aws.StringValue(attachment.InstanceId))
+	assert.Equal(t, "detached", aws.StringValue(attachment.State))
+	open, err := svc.obligations.List(context.Background())
+	require.NoError(t, err)
+	assert.Empty(t, open, "an unattached volume has no publish to undo")
 }

spinifex/domains/ec2/volume/service_impl.go

@@ -20,6 +20,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
 	ebspolicy "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/policy"
 	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -59,6 +60,12 @@ type VolumeServiceImpl struct {
 	snapshotKV jetstream.KeyValue
 	provider   ebsprovider.EBSProvider
 	metadata   *ebsmetadata.Store
+
+	// The unpublish obligations a cleared attachment records, and what names
+	// the node holding the publish. Unset, a detach that leaves one fails.
+	obligations   *ebsobligation.Store
+	instanceNodes InstanceNodeLocator
+	members       func() []string
 }
 
 // SetEBSProvider injects the provider boundary used by the control plane. It
@@ -1083,7 +1090,7 @@ func (s *VolumeServiceImpl) ModifyVolume(ctx context.Context, input *ec2.ModifyV
 // terminating instance's QEMU has already been asked to shut down — so this
 // is always a metadata-only clear, never a QMP call. Errors from either step
 // are returned, not swallowed.
-func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error {
+func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed ec2instance.OwedUnpublish) error {
 	// Clearing the attachment would overwrite a recorded deletion, so a volume
 	// already owed is left alone: the reclaimer owns it from here, and resetting
 	// it to available would let a second teardown run against the same data.
@@ -1091,6 +1098,9 @@ func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeI
 	if err == nil && meta.State == volumeStateDeleting {
 		return nil
 	}
+	if err := s.recordTerminateObligation(ctx, accountID, volumeID, owed); err != nil {
+		return err
+	}
 
 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		// No metadata doc means the volume is already gone, per GetVolume's own
@@ -1110,7 +1120,10 @@ func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeI
 // available rather than deleting it. Metadata-only, no QMP — same rationale
 // as DeleteVolumeOnTerminate: there is no live QEMU to hot-unplug on either
 // terminate path.
-func (s *VolumeServiceImpl) DetachVolumeOnTerminate(_ context.Context, volumeID, accountID string) error {
+func (s *VolumeServiceImpl) DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed ec2instance.OwedUnpublish) error {
+	if err := s.recordTerminateObligation(ctx, accountID, volumeID, owed); err != nil {
+		return err
+	}
 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		// No metadata doc means the volume is already gone, per GetVolume's own
 		// contract, so absence here is success, not a teardown failure.
@@ -1129,8 +1142,12 @@ func (s *VolumeServiceImpl) DetachVolumeOnTerminate(_ context.Context, volumeID,
 // The ordinary DetachVolume routes to ec2.cmd.{instanceID} for the QMP unplug,
 // so a volume attached to an instance whose host stopped answering can never be
 // detached and therefore never deleted. This exists for that deadlock and for
-// nothing else: it leaves a live guest holding a device the control plane no
-// longer believes in, which is only safe once that guest is being destroyed.
+// nothing else.
+//
+// That host still exports the volume, and the unpublish has to go to it, so
+// it cannot be made here. The request records an unpublish obligation naming
+// the publish before it clears the attachment, and returns; the node settles
+// it once it answers. A failed record clears nothing.
 func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
 	if input == nil || input.VolumeId == nil || *input.VolumeId == "" {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
@@ -1154,18 +1171,48 @@ func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.De
 	}
 
 	previous := meta.AttachedInstance
+	if previous != "" {
+		nodes, err := s.publishingNodes(previous)
+		if err != nil {
+			slog.ErrorContext(ctx, "ForceDetachVolume: cannot name the node holding the publish, clearing nothing",
+				"volumeId", volumeID, "instanceId", previous, "err", err)
+			return nil, errors.New(awserrors.ErrorInternalError)
+		}
+		if err := s.recordObligations(ctx, meta, nodes); err != nil {
+			slog.ErrorContext(ctx, "ForceDetachVolume: failed to record the unpublish obligation, clearing nothing",
+				"volumeId", volumeID, "instanceId", previous, "err", err)
+			return nil, errors.New(awserrors.ErrorInternalError)
+		}
+	}
+
 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		slog.ErrorContext(ctx, "ForceDetachVolume: failed to clear attachment", "volumeId", volumeID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	slog.WarnContext(ctx, "ForceDetachVolume: attachment cleared in the control plane only",
+
+	if previous == "" {
+		return &ec2.VolumeAttachment{
+			VolumeId: aws.String(volumeID),
+			State:    aws.String("detached"),
+		}, nil
+	}
+	slog.WarnContext(ctx, "ForceDetachVolume: attachment cleared, unpublish left to the node holding it",
 		"volumeId", volumeID, "previousInstance", previous, "accountId", accountID)
 
-	return &ec2.VolumeAttachment{
+	// AWS answers a detach with the attachment it is tearing down, in state
+	// detaching; the export here goes when the node settles the obligation.
+	attachment := &ec2.VolumeAttachment{
 		VolumeId:   aws.String(volumeID),
 		InstanceId: aws.String(previous),
-		State:      aws.String("detached"),
-	}, nil
+		State:      aws.String("detaching"),
+	}
+	if meta.DeviceName != "" {
+		attachment.Device = aws.String(meta.DeviceName)
+	}
+	if !meta.AttachedAt.IsZero() {
+		attachment.AttachTime = aws.Time(meta.AttachedAt)
+	}
+	return attachment, nil
 }
 
 // DeleteVolume records a durable obligation to destroy an EBS volume and

spinifex/domains/ec2/volume/service_impl_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit/ebsfake"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
@@ -1789,7 +1790,7 @@ func TestDeleteVolumeOnTerminate_ClearsAttachmentThenDeletes(t *testing.T) {
 		AttachedInstance: "i-stopped",
 	})
 
-	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
+	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})
 	require.NoError(t, err, "terminate implies detach: a stale attachment must not block the terminate delete")
 
 	deleting, err := svc.GetVolumeMetadata(testVolAccountID, volumeID)
@@ -1816,7 +1817,7 @@ func TestDeleteVolumeOnTerminate_AlreadyGoneMetadataIsSuccess(t *testing.T) {
 	svc.snapshotKV = kv
 
 	// No metadata doc is ever written for this volume ID.
-	err := svc.DeleteVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID)
+	err := svc.DeleteVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID, ec2instance.OwedUnpublish{})
 	require.NoError(t, err, "a volume with no metadata document is already gone, not a teardown failure")
 }
 
@@ -1829,7 +1830,7 @@ func TestDetachVolumeOnTerminate_AlreadyGoneMetadataIsSuccess(t *testing.T) {
 	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
 	svc.snapshotKV = kv
 
-	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID)
+	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID, ec2instance.OwedUnpublish{})
 	require.NoError(t, err, "a volume with no metadata document is already gone, not a teardown failure")
 }
 
@@ -1913,7 +1914,7 @@ func TestDeleteVolume_SecondDeleteIsAcceptedWithoutASecondTeardown(t *testing.T)
 
 	// DeleteOnTermination arriving for a volume already owed must not reset it
 	// to available, which would let a second teardown through.
-	require.NoError(t, svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID))
+	require.NoError(t, svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{}))
 	meta, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
 	require.NoError(t, err)
 	assert.Equal(t, "deleting", meta.State)
@@ -1961,7 +1962,7 @@ func TestDeleteVolumeOnTerminate_SurfacesDeleteFailure(t *testing.T) {
 	_, err = kv.Put(t.Context(), volumeID, snapData)
 	require.NoError(t, err)
 
-	err = svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
+	err = svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})
 	require.Error(t, err, "a DeleteVolume failure must be surfaced, not swallowed")
 	assert.Contains(t, err.Error(), awserrors.ErrorVolumeInUse)
 

spinifex/domains/ec2/volume/unpublish_obligation.go

@@ -0,0 +1,365 @@
+package volume
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+// InstanceNodeLocator reports the node that last ran an instance. found is
+// false when no record of the instance exists.
+type InstanceNodeLocator interface {
+	InstanceLastNode(instanceID string) (node string, found bool, err error)
+}
+
+// SetUnpublishObligations wires the obligation store a cleared attachment
+// records into, the locator that names the node holding the publish, and the
+// cluster members to name when the instance has no record.
+func (s *VolumeServiceImpl) SetUnpublishObligations(store *ebsobligation.Store, locator InstanceNodeLocator, members func() []string) {
+	s.obligations = store
+	s.instanceNodes = locator
+	s.members = members
+}
+
+// publishingNodes names the node holding instanceID's publish. With no record
+// of the instance there is no node to name, so every member is named and each
+// settles its own.
+func (s *VolumeServiceImpl) publishingNodes(instanceID string) ([]string, error) {
+	if s.instanceNodes == nil {
+		return nil, errors.New("no instance node locator configured")
+	}
+	node, found, err := s.instanceNodes.InstanceLastNode(instanceID)
+	if err != nil {
+		return nil, fmt.Errorf("read instance %s: %w", instanceID, err)
+	}
+	if found && node != "" {
+		return []string{node}, nil
+	}
+	var members []string
+	if s.members != nil {
+		members = s.members()
+	}
+	if len(members) == 0 {
+		return nil, fmt.Errorf("instance %s has no record and no cluster member is known", instanceID)
+	}
+	return members, nil
+}
+
+// recordObligations writes one obligation per node for the attachment meta
+// still carries. Every write must land before the caller clears it.
+func (s *VolumeServiceImpl) recordObligations(ctx context.Context, meta ebsmetadata.Volume, nodes []string) error {
+	if s.obligations == nil {
+		return errors.New("no unpublish obligation store configured")
+	}
+	now := time.Now()
+	for _, node := range nodes {
+		if err := s.obligations.Record(ctx, ebsobligation.UnpublishObligation{
+			VolumeID:   meta.VolumeID,
+			AccountID:  meta.TenantID,
+			Handle:     meta.ProviderHandle,
+			NodeID:     node,
+			InstanceID: meta.AttachedInstance,
+			AttachedAt: meta.AttachedAt,
+			RecordedAt: now,
+		}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// recordTerminateObligation records the obligation for a terminate that clears
+// owed's attachment with its publish still in place. A volume no longer
+// attached to that instance has nothing of it left to clear.
+func (s *VolumeServiceImpl) recordTerminateObligation(ctx context.Context, accountID, volumeID string, owed ec2instance.OwedUnpublish) error {
+	if !owed.Owed() {
+		return nil
+	}
+	meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
+	if objectstore.IsNoSuchKeyError(err) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("read volume %s before recording its unpublish obligation: %w", volumeID, err)
+	}
+	if meta.AttachedInstance != owed.InstanceID {
+		return nil
+	}
+	if err := s.recordObligations(ctx, meta, []string{owed.NodeID}); err != nil {
+		return fmt.Errorf("record unpublish obligation for %s: %w", volumeID, err)
+	}
+	return nil
+}
+
+// ObligationUnpublisher is this node's mounter as the obligation reaper drives
+// it. Its lock is the one every publish and unpublish on this node takes.
+type ObligationUnpublisher interface {
+	// HoldVolume takes volumeID's lock without waiting. false means a publish
+	// or unpublish of it is in flight here, and the sweep comes back later.
+	HoldVolume(volumeID string) (VolumeHold, bool)
+}
+
+// VolumeHold is one volume's publish lock on this node.
+type VolumeHold interface {
+	// PublishedSince reports whether this process published the volume after
+	// t and has not unpublished it since.
+	PublishedSince(t time.Time) bool
+	// Unpublish tears down this node's export of the volume. A volume the
+	// provider holds no export of counts as done.
+	Unpublish(ctx context.Context, accountID, handle string, noSeal bool) error
+	Release()
+}
+
+// obligationAction is what settling an obligation does with the volume as it
+// is now.
+type obligationAction int
+
+const (
+	// obligationWait leaves the obligation open: the attachment it names has
+	// not been cleared yet, or a publish here is newer than it.
+	obligationWait obligationAction = iota
+	// obligationSuperseded closes it without a provider call: a newer
+	// attachment owns the export.
+	obligationSuperseded
+	// obligationSeal unpublishes with a seal: nobody else writes the volume,
+	// so the copy here is the last owner's and its writes are kept.
+	obligationSeal
+	// obligationNoSeal unpublishes without sealing a volume being deleted.
+	obligationNoSeal
+)
+
+func (a obligationAction) String() string {
+	switch a {
+	case obligationSuperseded:
+		return "superseded"
+	case obligationSeal:
+		return "seal"
+	case obligationNoSeal:
+		return "no-seal"
+	default:
+		return "wait"
+	}
+}
+
+// decideObligation is the settlement policy over the volume's current record;
+// meta is nil when the volume has none. It never unpublishes an attachment the
+// volume still carries.
+func decideObligation(o ebsobligation.UnpublishObligation, meta *ebsmetadata.Volume) obligationAction {
+	if meta == nil || meta.State == volumeStateDeleting {
+		return obligationNoSeal
+	}
+	if meta.AttachedInstance != "" {
+		if meta.AttachedInstance != o.InstanceID || meta.AttachedAt.After(o.AttachedAt) {
+			return obligationSuperseded
+		}
+		return obligationWait
+	}
+	return obligationSeal
+}
+
+// obligationSince is the time a publish on this node must follow to be newer
+// than the one an obligation undoes.
+func obligationSince(o ebsobligation.UnpublishObligation) time.Time {
+	if o.AttachedAt.IsZero() {
+		return o.RecordedAt
+	}
+	return o.AttachedAt
+}
+
+// unpublishObligationReaper settles the obligations naming its node through
+// that node's mounter, once the node answers.
+type unpublishObligationReaper struct {
+	svc         *VolumeServiceImpl
+	node        string
+	unpublisher ObligationUnpublisher
+}
+
+var _ vm.Reaper = (*unpublishObligationReaper)(nil)
+
+// NewUnpublishObligationReaper builds the reaper that settles node's
+// obligations through unpublisher, which must be the node's own mounter.
+func (s *VolumeServiceImpl) NewUnpublishObligationReaper(node string, unpublisher ObligationUnpublisher) vm.Reaper {
+	return &unpublishObligationReaper{svc: s, node: node, unpublisher: unpublisher}
+}
+
+func (r *unpublishObligationReaper) Class() string { return "ebs-unpublish-obligation" }
+
+// Scope is node-local: an obligation is settled only by the node it names,
+// because only that node can tear its own export down.
+func (r *unpublishObligationReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
+
+// Sweep settles what it can and reports how many obligations it closed. A
+// failure is recorded on the obligation and retried on the next sweep.
+func (r *unpublishObligationReaper) Sweep(ctx context.Context) (int, error) {
+	if r.svc.obligations == nil || r.unpublisher == nil {
+		return 0, nil
+	}
+	open, err := r.svc.obligations.ListNode(ctx, r.node)
+	if err != nil {
+		return 0, err
+	}
+	closed := 0
+	for _, o := range open {
+		if err := ctx.Err(); err != nil {
+			return closed, err
+		}
+		done, err := r.settle(ctx, o)
+		if err != nil {
+			slog.WarnContext(ctx, "ebs-unpublish-obligation: settlement failed, retrying next sweep",
+				"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "attempts", o.Attempts+1, "err", err)
+			if recErr := r.svc.obligations.RecordFailure(ctx, o, err); recErr != nil {
+				slog.WarnContext(ctx, "ebs-unpublish-obligation: could not record the failure",
+					"volumeId", o.VolumeID, "node", o.NodeID, "err", recErr)
+			}
+			continue
+		}
+		if done {
+			closed++
+		}
+	}
+	return closed, nil
+}
+
+// settle acts on one obligation under the volume's lock, so no publish on this
+// node falls between reading the volume and unpublishing it.
+func (r *unpublishObligationReaper) settle(ctx context.Context, o ebsobligation.UnpublishObligation) (bool, error) {
+	hold, ok := r.unpublisher.HoldVolume(o.VolumeID)
+	if !ok {
+		return false, nil
+	}
+	defer hold.Release()
+
+	var current *ebsmetadata.Volume
+	meta, err := r.svc.metadata.GetVolume(ctx, o.AccountID, o.VolumeID)
+	switch {
+	case err == nil:
+		current = &meta
+	case objectstore.IsNoSuchKeyError(err):
+	default:
+		return false, fmt.Errorf("read volume %s: %w", o.VolumeID, err)
+	}
+
+	action := decideObligation(o, current)
+	// A newer publish here may own the export, so only a superseded obligation,
+	// which sends nothing, closes past it.
+	if action != obligationSuperseded && hold.PublishedSince(obligationSince(o)) {
+		return false, nil
+	}
+	switch action {
+	case obligationWait:
+		return false, nil
+	case obligationSuperseded:
+		slog.WarnContext(ctx, "ebs-unpublish-obligation: a newer attachment owns the volume, closing without unpublishing",
+			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "attachedInstance", current.AttachedInstance)
+	case obligationSeal, obligationNoSeal:
+		if err := hold.Unpublish(ctx, o.AccountID, o.Handle, action == obligationNoSeal); err != nil {
+			return false, err
+		}
+		slog.InfoContext(ctx, "ebs-unpublish-obligation: unpublished the export a cleared attachment left",
+			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "noSeal", action == obligationNoSeal)
+	}
+	if err := r.svc.obligations.Close(ctx, o); err != nil {
+		return false, fmt.Errorf("close obligation: %w", err)
+	}
+	recordObligationClosed(o.NodeID, action.String())
+	return true, nil
+}
+
+// NodeDeparted reports whether node has left the cluster, rather than being
+// down, stale or partitioned, which must not count.
+type NodeDeparted func(ctx context.Context, node string) (bool, error)
+
+// unpublishObligationCloser closes, without a provider call, the obligations of
+// nodes removed from the cluster: such a node holds nothing a member can reach.
+type unpublishObligationCloser struct {
+	svc      *VolumeServiceImpl
+	departed NodeDeparted
+}
+
+var _ vm.Reaper = (*unpublishObligationCloser)(nil)
+
+// NewUnpublishObligationCloser builds the cluster-wide pass that closes the
+// obligations of nodes departed reports as gone.
+func (s *VolumeServiceImpl) NewUnpublishObligationCloser(departed NodeDeparted) vm.Reaper {
+	return &unpublishObligationCloser{svc: s, departed: departed}
+}
+
+func (c *unpublishObligationCloser) Class() string { return "ebs-unpublish-obligation-close" }
+
+// Scope is cluster-wide: membership is the cluster's, and one pass is enough.
+func (c *unpublishObligationCloser) Scope() vm.ReaperScope { return vm.ScopeClusterWide }
+
+func (c *unpublishObligationCloser) Sweep(ctx context.Context) (int, error) {
+	if c.svc.obligations == nil || c.departed == nil {
+		return 0, nil
+	}
+	open, err := c.svc.obligations.List(ctx)
+	if err != nil {
+		return 0, err
+	}
+	gone := map[string]bool{}
+	closed := 0
+	for _, o := range open {
+		departed, seen := gone[o.NodeID]
+		if !seen {
+			departed, err = c.departed(ctx, o.NodeID)
+			if err != nil {
+				slog.WarnContext(ctx, "ebs-unpublish-obligation: cannot tell whether a node left the cluster, keeping its obligations",
+					"node", o.NodeID, "err", err)
+				departed = false
+			}
+			gone[o.NodeID] = departed
+		}
+		if !departed {
+			continue
+		}
+		if err := c.svc.obligations.Close(ctx, o); err != nil {
+			slog.WarnContext(ctx, "ebs-unpublish-obligation: failed to close a departed node's obligation",
+				"volumeId", o.VolumeID, "node", o.NodeID, "err", err)
+			continue
+		}
+		slog.WarnContext(ctx, "ebs-unpublish-obligation: closed for a node removed from the cluster",
+			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID)
+		recordObligationClosed(o.NodeID, "departed")
+		closed++
+	}
+	return closed, nil
+}
+
+const obligationMeterName = "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume/obligation"
+
+var (
+	obligationClosedOnce    sync.Once
+	obligationClosedCounter metric.Int64Counter
+)
+
+// recordObligationClosed counts one closed obligation by the node it named
+// and how it closed.
+func recordObligationClosed(node, outcome string) {
+	obligationClosedOnce.Do(func() {
+		var err error
+		obligationClosedCounter, err = otel.Meter(obligationMeterName).Int64Counter("mulga.ebs.unpublish_obligation.closed",
+			metric.WithDescription("Unpublish obligations closed, by the node they named and how they closed."),
+			metric.WithUnit("{obligation}"))
+		if err != nil {
+			otel.Handle(err)
+		}
+	})
+	if obligationClosedCounter == nil {
+		return
+	}
+	obligationClosedCounter.Add(context.Background(), 1, metric.WithAttributes(
+		attribute.String("node", node), attribute.String("outcome", outcome)))
+}

spinifex/domains/ec2/volume/unpublish_obligation_test.go

@@ -0,0 +1,596 @@
+package volume
+
+//test:in-package — the obligation reaper and its policy are unexported, and
+// seeding a volume's metadata document is how an attachment is set up.
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeNodeLocator struct {
+	nodes map[string]string
+	err   error
+}
+
+var _ InstanceNodeLocator = (*fakeNodeLocator)(nil)
+
+func (f *fakeNodeLocator) InstanceLastNode(instanceID string) (string, bool, error) {
+	if f.err != nil {
+		return "", false, f.err
+	}
+	node, found := f.nodes[instanceID]
+	return node, found, nil
+}
+
+// wireObligations gives svc an obligation bucket on a JetStream of its own and
+// returns the connection under it, so a test can cut it.
+func wireObligations(t *testing.T, svc *VolumeServiceImpl, locator InstanceNodeLocator, members ...string) *nats.Conn {
+	t.Helper()
+	_, nc, js := testutil.StartTestJetStream(t)
+	store := ebsobligation.NewStore(js)
+	require.NoError(t, store.Open(t.Context()))
+	svc.SetUnpublishObligations(store, locator, func() []string { return members })
+	return nc
+}
+
+type unpublishCall struct {
+	volumeID, accountID, handle string
+	noSeal                      bool
+}
+
+// fakeUnpublisher stands in for a node's mounter: busy volumes refuse the
+// hold, publishedAt is the last publish this process made of each volume.
+type fakeUnpublisher struct {
+	mu          sync.Mutex
+	busy        map[string]bool
+	publishedAt map[string]time.Time
+	err         error
+	calls       []unpublishCall
+	held        int
+}
+
+var _ ObligationUnpublisher = (*fakeUnpublisher)(nil)
+
+func (f *fakeUnpublisher) HoldVolume(volumeID string) (VolumeHold, bool) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.busy[volumeID] {
+		return nil, false
+	}
+	f.held++
+	return &fakeHold{f: f, volumeID: volumeID}, true
+}
+
+func (f *fakeUnpublisher) unpublishes() []unpublishCall {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]unpublishCall(nil), f.calls...)
+}
+
+type fakeHold struct {
+	f        *fakeUnpublisher
+	volumeID string
+}
+
+var _ VolumeHold = (*fakeHold)(nil)
+
+func (h *fakeHold) PublishedSince(t time.Time) bool {
+	h.f.mu.Lock()
+	defer h.f.mu.Unlock()
+	at, ok := h.f.publishedAt[h.volumeID]
+	return ok && at.After(t)
+}
+
+func (h *fakeHold) Unpublish(_ context.Context, accountID, handle string, noSeal bool) error {
+	h.f.mu.Lock()
+	defer h.f.mu.Unlock()
+	h.f.calls = append(h.f.calls, unpublishCall{volumeID: h.volumeID, accountID: accountID, handle: handle, noSeal: noSeal})
+	return h.f.err
+}
+
+func (h *fakeHold) Release() {
+	h.f.mu.Lock()
+	defer h.f.mu.Unlock()
+	h.f.held--
+}
+
+// putAttachment writes volumeID's document with an explicit attachment, so a
+// test controls AttachedAt rather than racing the clock.
+func putAttachment(t *testing.T, svc *VolumeServiceImpl, volumeID, state, instanceID string, attachedAt time.Time) ebsmetadata.Volume {
+	t.Helper()
+	meta := ebsmetadata.Volume{
+		VolumeID: volumeID, TenantID: testVolAccountID, CapacityGiB: 1,
+		State: state, AvailabilityZone: "ap-southeast-2a", VolumeType: "gp3",
+		AttachedInstance: instanceID, AttachedAt: attachedAt, ProviderHandle: "handle-" + volumeID,
+	}
+	if instanceID != "" {
+		meta.DeviceName = "/dev/sdf"
+	}
+	require.NoError(t, svc.metadata.PutVolume(context.Background(), meta))
+	return meta
+}
+
+func readVolume(t *testing.T, svc *VolumeServiceImpl, volumeID string) ebsmetadata.Volume {
+	t.Helper()
+	meta, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
+	require.NoError(t, err)
+	return meta
+}
+
+func openObligations(t *testing.T, svc *VolumeServiceImpl) []ebsobligation.UnpublishObligation {
+	t.Helper()
+	open, err := svc.obligations.List(context.Background())
+	require.NoError(t, err)
+	return open
+}
+
+func forceDetach(t *testing.T, svc *VolumeServiceImpl, volumeID string) (*ec2.VolumeAttachment, error) {
+	t.Helper()
+	return svc.ForceDetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId: aws.String(volumeID), Force: aws.Bool(true),
+	}, testVolAccountID)
+}
+
+// --- ForceDetachVolume records the obligation ---
+
+func TestForceDetach_RecordsOneObligationNamingThePublish(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a", "node-b")
+	attachedAt := time.Unix(1_700_000_000, 0).UTC()
+	putAttachment(t, svc, "vol-fd-once", "in-use", "i-old", attachedAt)
+
+	_, err := forceDetach(t, svc, "vol-fd-once")
+	require.NoError(t, err)
+
+	open := openObligations(t, svc)
+	require.Len(t, open, 1, "only the node that last ran the instance is named")
+	o := open[0]
+	assert.Equal(t, "vol-fd-once", o.VolumeID)
+	assert.Equal(t, testVolAccountID, o.AccountID)
+	assert.Equal(t, "handle-vol-fd-once", o.Handle)
+	assert.Equal(t, "node-a", o.NodeID)
+	assert.Equal(t, "i-old", o.InstanceID)
+	assert.True(t, attachedAt.Equal(o.AttachedAt), "the obligation names the attachment it undoes")
+	assert.Equal(t, ebsobligation.SchemaVersion, o.SchemaVersion)
+}
+
+// A caller retrying a force detach whose clear failed writes the same key, so
+// the retry leaves one obligation, not two.
+func TestForceDetach_RetryLeavesOneObligation(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a")
+	meta := putAttachment(t, svc, "vol-fd-retry", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
+
+	require.NoError(t, svc.recordObligations(context.Background(), meta, []string{"node-a"}))
+	_, err := forceDetach(t, svc, "vol-fd-retry")
+	require.NoError(t, err)
+
+	assert.Len(t, openObligations(t, svc), 1)
+}
+
+// With no record of the instance there is no node to name, so every member is
+// named and each one settles its own.
+func TestForceDetach_UnknownInstanceNamesEveryMember(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a", "node-b", "node-c")
+	putAttachment(t, svc, "vol-fd-every", "in-use", "i-gone", time.Unix(1_700_000_000, 0).UTC())
+
+	_, err := forceDetach(t, svc, "vol-fd-every")
+	require.NoError(t, err)
+
+	nodes := []string{}
+	for _, o := range openObligations(t, svc) {
+		nodes = append(nodes, o.NodeID)
+	}
+	assert.ElementsMatch(t, []string{"node-a", "node-b", "node-c"}, nodes)
+}
+
+// The attachment is the only record of where the export is. If the obligation
+// cannot be written, clearing it would forget an export, so nothing is cleared.
+func TestForceDetach_ObligationWriteFailureClearsNothing(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	nc := wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a")
+	putAttachment(t, svc, "vol-fd-nowrite", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
+	nc.Close()
+
+	_, err := forceDetach(t, svc, "vol-fd-nowrite")
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInternalError, err.Error())
+	meta := readVolume(t, svc, "vol-fd-nowrite")
+	assert.Equal(t, "in-use", meta.State)
+	assert.Equal(t, "i-old", meta.AttachedInstance)
+}
+
+func TestForceDetach_UnnameableNodeClearsNothing(t *testing.T) {
+	for name, locator := range map[string]InstanceNodeLocator{
+		"locator fails":            &fakeNodeLocator{err: errors.New("instance bucket unreachable")},
+		"no record and no members": &fakeNodeLocator{},
+		"no locator configured":    nil,
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc := newTestVolumeService("ap-southeast-2a")
+			wireObligations(t, svc, locator)
+			putAttachment(t, svc, "vol-fd-noname", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
+
+			_, err := forceDetach(t, svc, "vol-fd-noname")
+
+			require.Error(t, err)
+			assert.Equal(t, awserrors.ErrorInternalError, err.Error())
+			assert.Equal(t, "i-old", readVolume(t, svc, "vol-fd-noname").AttachedInstance)
+			assert.Empty(t, openObligations(t, svc))
+		})
+	}
+}
+
+// providerCallCounter fails the test's expectations if the request path makes
+// any provider call: an unpublish waits on a node that may never answer.
+type providerCallCounter struct {
+	ebsprovider.EBSProvider
+
+	mu        sync.Mutex
+	unpublish int
+	publish   int
+}
+
+func (p *providerCallCounter) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
+	p.mu.Lock()
+	p.unpublish++
+	p.mu.Unlock()
+	<-ctx.Done()
+	return nil, ctx.Err()
+}
+
+func (p *providerCallCounter) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
+	p.mu.Lock()
+	p.publish++
+	p.mu.Unlock()
+	<-ctx.Done()
+	return nil, ctx.Err()
+}
+
+// TestS8_ForceDetachRecordsTheObligationAndReturns pins that a force detach
+// never waits on the node holding the export: it decides its errors, records
+// the obligation and returns, and the unpublish is left to that node's reaper.
+func TestS8_ForceDetachRecordsTheObligationAndReturns(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	provider := &providerCallCounter{EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})}
+	svc.SetEBSProvider(provider)
+	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-wedged": "node-unreachable"}}, "node-unreachable")
+	putAttachment(t, svc, "vol-s8-forcedetach", "in-use", "i-wedged", time.Unix(1_700_000_000, 0).UTC())
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	attachment, err := svc.ForceDetachVolume(ctx, &ec2.DetachVolumeInput{
+		VolumeId: aws.String("vol-s8-forcedetach"), Force: aws.Bool(true),
+	}, testVolAccountID)
+	require.NoError(t, err)
+	require.NoError(t, ctx.Err(), "the request returned on its own, not on the deadline")
+
+	assert.Equal(t, "detaching", aws.StringValue(attachment.State))
+	provider.mu.Lock()
+	assert.Zero(t, provider.unpublish, "the request path makes no unpublish call")
+	assert.Zero(t, provider.publish)
+	provider.mu.Unlock()
+
+	open := openObligations(t, svc)
+	require.Len(t, open, 1, "the unpublish the request did not wait for is recorded")
+	assert.Equal(t, "node-unreachable", open[0].NodeID)
+	assert.Empty(t, readVolume(t, svc, "vol-s8-forcedetach").AttachedInstance)
+}
+
+// --- Terminate paths ---
+
+func TestTerminate_OwedUnpublishRecordsOneObligationBeforeTheClear(t *testing.T) {
+	for name, terminate := range map[string]func(*VolumeServiceImpl, string, ec2instance.OwedUnpublish) error{
+		"detach": func(svc *VolumeServiceImpl, id string, owed ec2instance.OwedUnpublish) error {
+			return svc.DetachVolumeOnTerminate(context.Background(), id, testVolAccountID, owed)
+		},
+		"delete": func(svc *VolumeServiceImpl, id string, owed ec2instance.OwedUnpublish) error {
+			return svc.DeleteVolumeOnTerminate(context.Background(), id, testVolAccountID, owed)
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc := newTestVolumeService("ap-southeast-2a")
+			svc.snapshotKV = setupTestVolumeKV(t)
+			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+			seedTestProviderVolume(t, svc, "vol-term-owed", 1)
+			putAttachment(t, svc, "vol-term-owed", "in-use", "i-term", time.Unix(1_700_000_000, 0).UTC())
+
+			require.NoError(t, terminate(svc, "vol-term-owed", ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"}))
+
+			open := openObligations(t, svc)
+			require.Len(t, open, 1)
+			assert.Equal(t, "node-a", open[0].NodeID)
+			assert.Equal(t, "i-term", open[0].InstanceID)
+		})
+	}
+}
+
+// A terminate whose unmount completed owes nothing, and an attachment that is
+// already some other instance's is not this terminate's to undo.
+func TestTerminate_NothingOwedRecordsNoObligation(t *testing.T) {
+	for name, tc := range map[string]struct {
+		attachedTo string
+		owed       ec2instance.OwedUnpublish
+	}{
+		"unmount completed":   {attachedTo: "i-term", owed: ec2instance.OwedUnpublish{}},
+		"attached to another": {attachedTo: "i-other", owed: ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc := newTestVolumeService("ap-southeast-2a")
+			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+			putAttachment(t, svc, "vol-term-none", "in-use", tc.attachedTo, time.Unix(1_700_000_000, 0).UTC())
+
+			require.NoError(t, svc.DetachVolumeOnTerminate(context.Background(), "vol-term-none", testVolAccountID, tc.owed))
+
+			assert.Empty(t, openObligations(t, svc))
+		})
+	}
+}
+
+func TestTerminate_ObligationWriteFailureClearsNothing(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	nc := wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+	putAttachment(t, svc, "vol-term-nowrite", "in-use", "i-term", time.Unix(1_700_000_000, 0).UTC())
+	nc.Close()
+
+	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-term-nowrite", testVolAccountID,
+		ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"})
+
+	require.Error(t, err)
+	assert.Equal(t, "i-term", readVolume(t, svc, "vol-term-nowrite").AttachedInstance)
+}
+
+// --- The reaper ---
+
+func recordObligation(t *testing.T, svc *VolumeServiceImpl, node, volumeID, instanceID string, attachedAt time.Time) ebsobligation.UnpublishObligation {
+	t.Helper()
+	o := ebsobligation.UnpublishObligation{
+		VolumeID: volumeID, AccountID: testVolAccountID, Handle: "handle-" + volumeID,
+		NodeID: node, InstanceID: instanceID, AttachedAt: attachedAt, RecordedAt: attachedAt.Add(time.Minute),
+	}
+	require.NoError(t, svc.obligations.Record(context.Background(), o))
+	return o
+}
+
+func sweepObligations(t *testing.T, svc *VolumeServiceImpl, node string, unpublisher ObligationUnpublisher) int {
+	t.Helper()
+	closed, err := svc.NewUnpublishObligationReaper(node, unpublisher).Sweep(context.Background())
+	require.NoError(t, err)
+	return closed
+}
+
+// TestS7_ObligationReaperNeverUnpublishesACurrentAttachment pins that the
+// reaper, which runs with no customer asking, never tears down an export a
+// live attachment uses and never removes or rewrites the volume.
+func TestS7_ObligationReaperNeverUnpublishesACurrentAttachment(t *testing.T) {
+	cleared := time.Unix(1_700_000_000, 0).UTC()
+	for name, tc := range map[string]struct {
+		state       string
+		attachedTo  string
+		attachedAt  time.Time
+		publishedAt time.Time // a publish this process made, zero if none
+		stillOpen   bool
+	}{
+		// The volume moved to another instance, possibly on this very node.
+		"attached to another instance": {state: "in-use", attachedTo: "i-new", attachedAt: cleared.Add(time.Hour)},
+		// The same instance was attached again after the clear.
+		"same instance attached again": {state: "in-use", attachedTo: "i-old", attachedAt: cleared.Add(time.Hour)},
+		// The clear has not landed: the obligation was written first.
+		"attachment not yet cleared": {state: "in-use", attachedTo: "i-old", attachedAt: cleared, stillOpen: true},
+		// A hot attach published here and has not yet written in-use.
+		"newer publish here, record not yet in-use": {state: "available", publishedAt: cleared.Add(time.Hour), stillOpen: true},
+		// Force detach, then attach again on this node: the new publish is here.
+		"attached again here after a newer publish": {state: "in-use", attachedTo: "i-new", attachedAt: cleared.Add(time.Hour), publishedAt: cleared.Add(time.Hour)},
+		"same instance attached again here":         {state: "in-use", attachedTo: "i-old", attachedAt: cleared.Add(time.Hour), publishedAt: cleared.Add(time.Hour)},
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc := newTestVolumeService("ap-southeast-2a")
+			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+			before := putAttachment(t, svc, "vol-s7-current", tc.state, tc.attachedTo, tc.attachedAt)
+			recordObligation(t, svc, "node-a", "vol-s7-current", "i-old", cleared)
+			unpublisher := &fakeUnpublisher{publishedAt: map[string]time.Time{}}
+			if !tc.publishedAt.IsZero() {
+				unpublisher.publishedAt["vol-s7-current"] = tc.publishedAt
+			}
+
+			sweepObligations(t, svc, "node-a", unpublisher)
+
+			assert.Empty(t, unpublisher.unpublishes(), "a current attachment's export must never be unpublished")
+			assert.Zero(t, unpublisher.held, "every hold is released")
+			after := readVolume(t, svc, "vol-s7-current")
+			assert.Equal(t, before.State, after.State, "the reaper never rewrites the volume")
+			assert.Equal(t, before.AttachedInstance, after.AttachedInstance)
+			if tc.stillOpen {
+				assert.Len(t, openObligations(t, svc), 1, "the obligation waits for its attachment to be gone")
+			} else {
+				assert.Empty(t, openObligations(t, svc), "a superseded obligation closes without a provider call")
+			}
+		})
+	}
+}
+
+// TestS7_ObligationReaperSealsLiveDataAndDropsOnlyDeletingVolumes pins that
+// the reaper keeps a surviving volume's writes and only drops them unsealed
+// for a volume that is being deleted or is gone.
+func TestS7_ObligationReaperSealsLiveDataAndDropsOnlyDeletingVolumes(t *testing.T) {
+	cleared := time.Unix(1_700_000_000, 0).UTC()
+	for name, tc := range map[string]struct {
+		state  string // empty for no document
+		noSeal bool
+	}{
+		"available keeps its writes": {state: "available", noSeal: false},
+		"deleting is dropped":        {state: volumeStateDeleting, noSeal: true},
+		"absent is dropped":          {state: "", noSeal: true},
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc := newTestVolumeService("ap-southeast-2a")
+			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+			if tc.state != "" {
+				putAttachment(t, svc, "vol-s7-settle", tc.state, "", time.Time{})
+			}
+			recordObligation(t, svc, "node-a", "vol-s7-settle", "i-old", cleared)
+			unpublisher := &fakeUnpublisher{}
+
+			assert.Equal(t, 1, sweepObligations(t, svc, "node-a", unpublisher))
+
+			assert.Equal(t, []unpublishCall{{volumeID: "vol-s7-settle", accountID: testVolAccountID, handle: "handle-vol-s7-settle", noSeal: tc.noSeal}},
+				unpublisher.unpublishes())
+			assert.Empty(t, openObligations(t, svc))
+			if tc.state != "" {
+				assert.Equal(t, tc.state, readVolume(t, svc, "vol-s7-settle").State, "the reaper never rewrites the volume")
+			}
+		})
+	}
+}
+
+// TestS7_ObligationReaperNeverTreatsAnUnreadableVolumeAsGone pins that a read
+// failing for a reason other than absence leaves the obligation open: taking
+// it for a deleted volume would drop live writes unsealed.
+func TestS7_ObligationReaperNeverTreatsAnUnreadableVolumeAsGone(t *testing.T) {
+	memory := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", memory)
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+	putAttachment(t, svc, "vol-s7-unread", "available", "", time.Time{})
+	recordObligation(t, svc, "node-a", "vol-s7-unread", "i-old", time.Unix(1_700_000_000, 0).UTC())
+	failing := &getFailingStore{MemoryObjectStore: memory, failKey: metadataKey(t, "vol-s7-unread")}
+	svc.metadata = ebsmetadata.NewStore(failing, "test-bucket")
+	unpublisher := &fakeUnpublisher{}
+
+	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
+
+	assert.Empty(t, unpublisher.unpublishes())
+	open := openObligations(t, svc)
+	require.Len(t, open, 1)
+	assert.Equal(t, 1, open[0].Attempts, "the failure is counted where an operator can see it")
+	assert.Contains(t, open[0].Error, "reconstruction failed")
+}
+
+// End to end through the service: a force detach followed by a new attachment
+// of the same volume on the same node must leave that new export alone.
+func TestS7_ForceDetachThenReattachOnTheSameNodeKeepsTheNewExport(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a", "i-new": "node-a"}}, "node-a")
+	putAttachment(t, svc, "vol-s7-reattach", "in-use", "i-old", time.Now().Add(-time.Hour).UTC())
+
+	_, err := forceDetach(t, svc, "vol-s7-reattach")
+	require.NoError(t, err)
+	require.NoError(t, svc.UpdateVolumeState(testVolAccountID, "vol-s7-reattach", "in-use", "i-new", "/dev/sdg"))
+	unpublisher := &fakeUnpublisher{}
+
+	assert.Equal(t, 1, sweepObligations(t, svc, "node-a", unpublisher))
+
+	assert.Empty(t, unpublisher.unpublishes(), "the new attachment's export must survive the old obligation")
+	assert.Equal(t, "i-new", readVolume(t, svc, "vol-s7-reattach").AttachedInstance)
+	assert.Empty(t, openObligations(t, svc))
+}
+
+func TestObligationReaper_SettlesOnlyItsOwnNode(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a", "node-b")
+	putAttachment(t, svc, "vol-own", "available", "", time.Time{})
+	recordObligation(t, svc, "node-b", "vol-own", "i-old", time.Unix(1_700_000_000, 0).UTC())
+	unpublisher := &fakeUnpublisher{}
+
+	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
+
+	assert.Empty(t, unpublisher.unpublishes(), "only the node holding the export can tear it down")
+	assert.Len(t, openObligations(t, svc), 1)
+}
+
+// A publish or unpublish of the volume in flight on this node holds its lock;
+// the sweep skips it rather than waiting behind a possibly wedged call.
+func TestObligationReaper_SkipsABusyVolume(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+	putAttachment(t, svc, "vol-busy", "available", "", time.Time{})
+	recordObligation(t, svc, "node-a", "vol-busy", "i-old", time.Unix(1_700_000_000, 0).UTC())
+	unpublisher := &fakeUnpublisher{busy: map[string]bool{"vol-busy": true}}
+
+	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
+
+	assert.Empty(t, unpublisher.unpublishes())
+	open := openObligations(t, svc)
+	require.Len(t, open, 1)
+	assert.Zero(t, open[0].Attempts, "a busy volume is not a failure")
+}
+
+func TestObligationReaper_FailedUnpublishStaysOpenAndCounts(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+	putAttachment(t, svc, "vol-fail", "available", "", time.Time{})
+	recordObligation(t, svc, "node-a", "vol-fail", "i-old", time.Unix(1_700_000_000, 0).UTC())
+	unpublisher := &fakeUnpublisher{err: errors.New("seal refused")}
+
+	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
+	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
+
+	open := openObligations(t, svc)
+	require.Len(t, open, 1)
+	assert.Equal(t, 2, open[0].Attempts)
+	assert.Equal(t, "seal refused", open[0].Error)
+	assert.Zero(t, unpublisher.held, "a failed unpublish still releases the hold")
+}
+
+// An obligation with no attachment time falls back to when it was recorded, so
+// a publish here after the clear still counts as newer.
+func TestObligationSince_FallsBackToRecordedAt(t *testing.T) {
+	recorded := time.Unix(1_700_000_000, 0)
+	assert.Equal(t, recorded, obligationSince(ebsobligation.UnpublishObligation{RecordedAt: recorded}))
+	attached := recorded.Add(-time.Hour)
+	assert.Equal(t, attached, obligationSince(ebsobligation.UnpublishObligation{AttachedAt: attached, RecordedAt: recorded}))
+}
+
+// --- The cluster-wide close ---
+
+func TestObligationCloser_ClosesOnlyDepartedNodes(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
+	at := time.Unix(1_700_000_000, 0).UTC()
+	recordObligation(t, svc, "node-gone", "vol-c1", "i-1", at)
+	recordObligation(t, svc, "node-gone", "vol-c2", "i-2", at)
+	recordObligation(t, svc, "node-down", "vol-c3", "i-3", at)
+	recordObligation(t, svc, "node-unknown", "vol-c4", "i-4", at)
+	asked := map[string]int{}
+	departed := func(_ context.Context, node string) (bool, error) {
+		asked[node]++
+		switch node {
+		case "node-gone":
+			return true, nil
+		case "node-unknown":
+			return false, errors.New("heartbeat bucket unreachable")
+		default:
+			return false, nil
+		}
+	}
+
+	closed, err := svc.NewUnpublishObligationCloser(departed).Sweep(context.Background())
+	require.NoError(t, err)
+
+	assert.Equal(t, 2, closed)
+	remaining := []string{}
+	for _, o := range openObligations(t, svc) {
+		remaining = append(remaining, o.NodeID)
+	}
+	assert.ElementsMatch(t, []string{"node-down", "node-unknown"}, remaining,
+		"a node that is down, or whose membership cannot be read, keeps its obligations")
+	assert.Equal(t, 1, asked["node-gone"], "membership is asked once per node per sweep")
+}

spinifex/domains/ecr/awsapi/batch_delete_image.go

@@ -33,19 +33,19 @@ func BatchDeleteImage(ctx context.Context, deleter ImageDeleter, accountID strin
 	if err := ValidateRepositoryScope(req.RepositoryName, req.RegistryID, accountID); err != nil {
 		return nil, err
 	}
-	if len(req.ImageIDs) > maxImageBatch {
-		return nil, MaxItemsError("imageIds", maxImageBatch)
+	if err := validateImageIDs(req.ImageIDs, true); err != nil {
+		return nil, err
 	}
 
 	deleted := make([]*ecr.ImageIdentifier, 0, len(req.ImageIDs))
 	failures := make([]*ecr.ImageFailure, 0)
 	for _, imageID := range req.ImageIDs {
-		if imageID.ImageDigest == "" && imageID.ImageTag == "" {
+		if imageID.ImageDigest == "" && imageID.ImageTag == nil {
 			failures = append(failures, imageFailure(imageID, ecr.ImageFailureCodeMissingDigestAndTag, "no digest or tag specified"))
 			continue
 		}
 
-		digest, err := deleter.DeleteImage(ctx, accountID, req.RepositoryName, imageID.ImageTag, imageID.ImageDigest)
+		digest, err := deleter.DeleteImage(ctx, accountID, req.RepositoryName, imageID.tag(), imageID.ImageDigest)
 		if errors.Is(err, ecrregistry.ErrImageNotFound) {
 			failures = append(failures, imageFailure(imageID, ecr.ImageFailureCodeImageNotFound, ImageNotFoundReason))
 			continue
@@ -56,8 +56,8 @@ func BatchDeleteImage(ctx context.Context, deleter ImageDeleter, accountID strin
 		}
 
 		deletedImage := &ecr.ImageIdentifier{ImageDigest: aws.String(digest)}
-		if imageID.ImageTag != "" {
-			deletedImage.ImageTag = aws.String(imageID.ImageTag)
+		if imageID.ImageTag != nil {
+			deletedImage.ImageTag = imageID.ImageTag
 		}
 		deleted = append(deleted, deletedImage)
 	}

spinifex/domains/ecr/awsapi/batch_get_image.go

@@ -36,8 +36,13 @@ func BatchGetImage(ctx context.Context, reader ManifestReader, accountID string,
 	if err := ValidateRepositoryScope(req.RepositoryName, req.RegistryID, accountID); err != nil {
 		return nil, err
 	}
-	if len(req.ImageIDs) > maxImageBatch {
-		return nil, MaxItemsError("imageIds", maxImageBatch)
+	if err := validateImageIDs(req.ImageIDs, true); err != nil {
+		return nil, err
+	}
+	if req.AcceptedMediaTypes != nil {
+		if err := validateListLength("acceptedMediaTypes", len(req.AcceptedMediaTypes), 1, maxMediaTypeListItems); err != nil {
+			return nil, err
+		}
 	}
 
 	images := make([]*ecr.Image, 0, len(req.ImageIDs))
@@ -45,7 +50,7 @@ func BatchGetImage(ctx context.Context, reader ManifestReader, accountID string,
 	for _, imageID := range req.ImageIDs {
 		reference := imageID.ImageDigest
 		if reference == "" {
-			reference = imageID.ImageTag
+			reference = imageID.tag()
 		}
 		if reference == "" {
 			failures = append(failures, imageFailure(imageID, ecr.ImageFailureCodeMissingDigestAndTag, "no digest or tag specified"))
@@ -69,8 +74,8 @@ func BatchGetImage(ctx context.Context, reader ManifestReader, accountID string,
 			ImageManifest:          aws.String(string(manifest)),
 			ImageManifestMediaType: aws.String(mediaType),
 		}
-		if imageID.ImageTag != "" {
-			image.ImageId.ImageTag = aws.String(imageID.ImageTag)
+		if imageID.ImageTag != nil {
+			image.ImageId.ImageTag = imageID.ImageTag
 		}
 		images = append(images, image)
 	}
@@ -87,8 +92,8 @@ func imageFailure(imageID imageIdentifier, code, reason string) *ecr.ImageFailur
 	if imageID.ImageDigest != "" {
 		failure.ImageId.ImageDigest = aws.String(imageID.ImageDigest)
 	}
-	if imageID.ImageTag != "" {
-		failure.ImageId.ImageTag = aws.String(imageID.ImageTag)
+	if imageID.ImageTag != nil {
+		failure.ImageId.ImageTag = imageID.ImageTag
 	}
 	return failure
 }

spinifex/domains/ecr/awsapi/describe_images.go

@@ -10,10 +10,15 @@ import (
 	ecrregistry "github.com/mulgadc/spinifex/spinifex/domains/ecr/registry"
 )
 
-// imageIdentifier is the AWS JSON 1.1 {imageDigest, imageTag} pair.
+// imageIdentifier is the AWS JSON 1.1 {imageDigest, imageTag} pair. ImageTag
+// is a pointer so an empty tag can be told apart from an omitted one.
 type imageIdentifier struct {
-	ImageDigest string `json:"imageDigest"`
-	ImageTag    string `json:"imageTag"`
+	ImageDigest string  `json:"imageDigest"`
+	ImageTag    *string `json:"imageTag"`
+}
+
+func (id imageIdentifier) tag() string {
+	return aws.StringValue(id.ImageTag)
 }
 
 type describeImagesRequest struct {
@@ -36,6 +41,9 @@ func DescribeImages(ctx context.Context, catalog ImageCatalog, accountID string,
 	if err := ValidateRepositoryScope(req.RepositoryName, req.RegistryID, accountID); err != nil {
 		return nil, err
 	}
+	if err := validateImageIDs(req.ImageIDs, false); err != nil {
+		return nil, err
+	}
 	if err := req.Filter.validate(); err != nil {
 		return nil, err
 	}
@@ -53,7 +61,7 @@ func DescribeImages(ctx context.Context, catalog ImageCatalog, accountID string,
 			if image.ImageDigest != "" && image.ImageDigest == record.Digest {
 				return true
 			}
-			if image.ImageTag != "" && slices.Contains(record.Tags, image.ImageTag) {
+			if image.ImageTag != nil && slices.Contains(record.Tags, *image.ImageTag) {
 				return true
 			}
 		}
@@ -82,7 +90,7 @@ func DescribeImages(ctx context.Context, catalog ImageCatalog, accountID string,
 	}
 	if len(req.ImageIDs) > 0 && len(details) == 0 {
 		first := req.ImageIDs[0]
-		return nil, ImageNotFoundError(accountID, req.RepositoryName, first.ImageDigest, first.ImageTag)
+		return nil, ImageNotFoundError(accountID, req.RepositoryName, first.ImageDigest, first.tag())
 	}
 
 	return &ecr.DescribeImagesOutput{ImageDetails: details}, nil

spinifex/domains/ecr/awsapi/describe_repositories.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"log/slog"
 
 	"github.com/aws/aws-sdk-go/service/ecr"
@@ -40,6 +41,13 @@ func DescribeRepositories(ctx context.Context, store RepositoryStore, endpoint R
 		return nil, errors.New(awserrors.ErrorAccessDenied)
 	}
 
+	switch {
+	case req.RepositoryNames != nil && len(req.RepositoryNames) == 0:
+		return nil, ConstraintError("RepositoryNameList", "Must not be empty")
+	case len(req.RepositoryNames) > maxRepositoryNameList:
+		return nil, ConstraintError("RepositoryNameList", fmt.Sprintf("Must have length less than or equal to %d", maxRepositoryNameList))
+	}
+
 	names := req.RepositoryNames
 	if len(names) == 0 {
 		var err error

spinifex/domains/ecr/awsapi/image_list_validation_test.go

@@ -0,0 +1,92 @@
+package awsapi
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ecr"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The messages below are AWS's, captured from real ECR refusals.
+const (
+	awsImageIDsNull     = "Invalid parameter at 'imageIds' failed to satisfy constraint: 'Member must not be null'"
+	awsImageIDsEmpty    = "Invalid parameter at 'imageIds' failed to satisfy constraint: 'Member must have length greater than or equal to 1'"
+	awsImageIDsTooMany  = "Invalid parameter at 'imageIds' failed to satisfy constraint: 'Member must have length less than or equal to 100'"
+	awsImageTagEmpty    = "Invalid parameter at 'imageIds.2.member.imageTag' failed to satisfy constraint: 'Member must have length greater than or equal to 1'"
+	awsImageTagTooLong  = "Invalid parameter at 'imageIds.1.member.imageTag' failed to satisfy constraint: 'Member must have length less than or equal to 300'"
+	awsMediaTypesEmpty  = "Invalid parameter at 'acceptedMediaTypes' failed to satisfy constraint: 'Member must have length greater than or equal to 1'"
+	awsMediaTypesTooMny = "Invalid parameter at 'acceptedMediaTypes' failed to satisfy constraint: 'Member must have length less than or equal to 100'"
+)
+
+func requireECRInvalidParameter(t *testing.T, err error, want string) {
+	t.Helper()
+	require.Error(t, err)
+	code, message, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorECRInvalidParameter, code)
+	assert.Equal(t, want, message)
+}
+
+func imageIDCases() map[string]struct{ ids, want string } {
+	long := strings.Repeat("a", 301)
+	return map[string]struct{ ids, want string }{
+		"omitted":      {"", awsImageIDsNull},
+		"empty":        {`,"imageIds":[]`, awsImageIDsEmpty},
+		"too many":     {`,"imageIds":[` + repeatImageIdentifier(101) + `]`, awsImageIDsTooMany},
+		"empty tag":    {`,"imageIds":[{"imageTag":"v1"},{"imageDigest":"sha256:x","imageTag":""}]`, awsImageTagEmpty},
+		"tag too long": {`,"imageIds":[{"imageTag":"` + long + `"}]`, awsImageTagTooLong},
+	}
+}
+
+func TestBatchImageActions_RefuseImageIDsOutsideModel(t *testing.T) {
+	for name, tc := range imageIDCases() {
+		body := []byte(`{"repositoryName":"team/app"` + tc.ids + `}`)
+		t.Run("BatchGetImage "+name, func(t *testing.T) {
+			_, err := BatchGetImage(context.Background(), fakeManifestReader{}, "123456789012", body)
+			requireECRInvalidParameter(t, err, tc.want)
+		})
+		t.Run("BatchDeleteImage "+name, func(t *testing.T) {
+			_, err := BatchDeleteImage(context.Background(), fakeImageDeleter{}, "123456789012", body)
+			requireECRInvalidParameter(t, err, tc.want)
+		})
+	}
+}
+
+func TestBatchGetImage_RefusesAcceptedMediaTypesOutsideModel(t *testing.T) {
+	many := strings.TrimSuffix(strings.Repeat(`"m",`, 101), ",")
+	for want, types := range map[string]string{awsMediaTypesEmpty: `[]`, awsMediaTypesTooMny: `[` + many + `]`} {
+		body := []byte(`{"repositoryName":"team/app","imageIds":[{"imageTag":"v1"}],"acceptedMediaTypes":` + types + `}`)
+		_, err := BatchGetImage(context.Background(), fakeManifestReader{}, "123456789012", body)
+		requireECRInvalidParameter(t, err, want)
+	}
+}
+
+// DescribeImages makes imageIds optional, so only an omitted list lists all.
+func TestDescribeImages_RefusesImageIDsOutsideModel(t *testing.T) {
+	for name, tc := range imageIDCases() {
+		if tc.want == awsImageIDsNull {
+			continue
+		}
+		t.Run(name, func(t *testing.T) {
+			_, err := DescribeImages(context.Background(), nil, "123456789012", []byte(`{"repositoryName":"team/app"`+tc.ids+`}`))
+			requireECRInvalidParameter(t, err, tc.want)
+		})
+	}
+}
+
+func TestDescribeRepositories_RefusesRepositoryNamesOutsideModel(t *testing.T) {
+	store := ecr.NewNATSMetaStore(newRepositoryActionTestConn(t))
+	many := strings.TrimSuffix(strings.Repeat(`"team/app",`, 101), ",")
+	cases := map[string]string{
+		`[]`:             "Invalid parameter at 'RepositoryNameList' failed to satisfy constraint: 'Must not be empty'",
+		`[` + many + `]`: "Invalid parameter at 'RepositoryNameList' failed to satisfy constraint: 'Must have length less than or equal to 100'",
+	}
+	for names, want := range cases {
+		_, err := DescribeRepositories(context.Background(), store, repositoryActionEndpoint(), repositoryActionTestAccount, []byte(`{"repositoryNames":`+names+`}`))
+		requireECRInvalidParameter(t, err, want)
+	}
+}

spinifex/domains/ecr/awsapi/validation.go

@@ -18,6 +18,11 @@ const (
 	minLifecyclePolicyTextLength  = 100
 	maxLifecyclePolicyTextLength  = 30720
 	maxRepositoryPolicyTextLength = 10240
+
+	minImageTagLength     = 1
+	maxImageTagLength     = 300
+	maxMediaTypeListItems = 100
+	maxRepositoryNameList = 100
 )
 
 // ValidateRepositoryName returns InvalidParameterException naming the first
@@ -52,6 +57,42 @@ func MaxItemsError(field string, limit int) error {
 	return ConstraintError(field, fmt.Sprintf("Member must have length less than or equal to %d", limit))
 }
 
+// validateListLength refuses a list field whose member count falls outside
+// [minLen, maxLen], the bounds of field's model shape.
+func validateListLength(field string, n, minLen, maxLen int) error {
+	if n < minLen {
+		return ConstraintError(field, fmt.Sprintf("Member must have length greater than or equal to %d", minLen))
+	}
+	if n > maxLen {
+		return MaxItemsError(field, maxLen)
+	}
+	return nil
+}
+
+// validateImageIDs checks imageIds against ImageIdentifierList: 1-100 members,
+// each tag 1-300 characters. An omitted list is refused only when required.
+func validateImageIDs(ids []imageIdentifier, required bool) error {
+	if ids == nil {
+		if required {
+			return RequiredParameterError("imageIds")
+		}
+		return nil
+	}
+	if err := validateListLength("imageIds", len(ids), 1, maxImageBatch); err != nil {
+		return err
+	}
+	for i, id := range ids {
+		if id.ImageTag == nil {
+			continue
+		}
+		field := fmt.Sprintf("imageIds.%d.member.imageTag", i+1)
+		if err := ValidateTextLength(field, *id.ImageTag, minImageTagLength, maxImageTagLength); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // ValidateTextLength refuses text whose length in characters falls outside
 // [minLen, maxLen], the bounds of field's model shape.
 func ValidateTextLength(field, text string, minLen, maxLen int) error {

spinifex/domains/ecs/awsapi/actions.go

@@ -76,6 +76,9 @@ func DescribeClusters(ctx context.Context, nc *nats.Conn, accountID string, body
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkClustersInclude(input.Include); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).DescribeClusters(ctx, input, accountID)
 }
 
@@ -134,6 +137,9 @@ func ListTaskDefinitions(ctx context.Context, nc *nats.Conn, accountID string, b
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkListTaskDefinitions(input); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).ListTaskDefinitions(ctx, input, accountID)
 }
 
@@ -190,6 +196,9 @@ func DescribeContainerInstances(ctx context.Context, nc *nats.Conn, accountID st
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkContainerInstancesInclude(input.Include); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).DescribeContainerInstances(ctx, input, accountID)
 }
 
@@ -313,6 +322,9 @@ func DescribeTasks(ctx context.Context, nc *nats.Conn, accountID string, body []
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkTasksInclude(input.Include); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).DescribeTasks(ctx, input, accountID)
 }
 
@@ -323,6 +335,9 @@ func ListTasks(ctx context.Context, nc *nats.Conn, accountID string, body []byte
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkListTasks(input); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).ListTasks(ctx, input, accountID)
 }
 
@@ -391,6 +406,9 @@ func DescribeServices(ctx context.Context, nc *nats.Conn, accountID string, body
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkServicesInclude(input.Include); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).DescribeServices(ctx, input, accountID)
 }
 
@@ -401,6 +419,9 @@ func ListServices(ctx context.Context, nc *nats.Conn, accountID string, body []b
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkListServices(body, input); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).ListServices(ctx, input, accountID)
 }
 
@@ -493,6 +514,9 @@ func CreateCapacityProvider(ctx context.Context, nc *nats.Conn, accountID string
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkAutoScalingGroupProvider(input.AutoScalingGroupProvider); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).CreateCapacityProvider(ctx, input, accountID)
 }
 
@@ -504,6 +528,9 @@ func DescribeCapacityProviders(ctx context.Context, nc *nats.Conn, accountID str
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
+	if err := checkCapacityProvidersInclude(input.Include); err != nil {
+		return nil, err
+	}
 	return ecsdomain.NewNATSECSService(nc).DescribeCapacityProviders(ctx, input, accountID)
 }
 

spinifex/domains/ecs/awsapi/validate_capacityprovider.go

@@ -0,0 +1,59 @@
+package awsapi
+
+import (
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// Messages AWS answers, as ClientException, for each autoScalingGroupProvider
+// break, before it looks up the Auto Scaling group.
+const (
+	msgManagedDraining              = "The specified managed draining setting is invalid. Specify ENABLED or DISABLED and try again."
+	msgManagedTerminationProtection = "The specified managed termination protection setting is invalid. Specify ENABLED or DISABLED and try again."
+	msgManagedScalingStatus         = "The specified managed scaling status setting is invalid. Specify ENABLED or DISABLED and try again."
+	msgTargetCapacity               = "The managed scaling target capacity value is invalid. Specify an integer greater than 0 and less than or equal to 100 and try again."
+	msgMinimumScalingStepSize       = "The specified minimum scaling step size is invalid. Specify an integer between 1 and 10000 and try again."
+	msgMaximumScalingStepSize       = "The specified maximum scaling step size is invalid. Specify an integer between 1 and 10000 and try again."
+	msgInstanceWarmupPeriod         = "The managed scaling instance warmup period value is invalid. Specify an integer greater than or equal to 0 and less than or equal to 9999 and try again."
+)
+
+func validEnabledDisabled(v *string) bool {
+	return v == nil || *v == "ENABLED" || *v == "DISABLED"
+}
+
+func validRange(v *int64, lo, hi int64) bool {
+	return v == nil || (*v >= lo && *v <= hi)
+}
+
+// checkAutoScalingGroupProvider refuses an enum or range break inside
+// autoScalingGroupProvider. AWS refuses an instanceWarmupPeriod of 10000 although
+// the model allows it, so the upper bound here is 9999.
+func checkAutoScalingGroupProvider(p *ecs.AutoScalingGroupProvider) error {
+	if p == nil {
+		return nil
+	}
+	fail := func(msg string) error { return awserrors.Errorf(awserrors.ErrorECSClientException, "%s", msg) }
+	if !validEnabledDisabled(p.ManagedDraining) {
+		return fail(msgManagedDraining)
+	}
+	if !validEnabledDisabled(p.ManagedTerminationProtection) {
+		return fail(msgManagedTerminationProtection)
+	}
+	ms := p.ManagedScaling
+	if ms == nil {
+		return nil
+	}
+	switch {
+	case !validEnabledDisabled(ms.Status):
+		return fail(msgManagedScalingStatus)
+	case !validRange(ms.TargetCapacity, 1, 100):
+		return fail(msgTargetCapacity)
+	case !validRange(ms.MinimumScalingStepSize, 1, 10000):
+		return fail(msgMinimumScalingStepSize)
+	case !validRange(ms.MaximumScalingStepSize, 1, 10000):
+		return fail(msgMaximumScalingStepSize)
+	case !validRange(ms.InstanceWarmupPeriod, 0, 9999):
+		return fail(msgInstanceWarmupPeriod)
+	}
+	return nil
+}

spinifex/domains/ecs/awsapi/validate_capacityprovider_test.go

@@ -0,0 +1,51 @@
+package awsapi
+
+import (
+	"context"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Each break answers AWS's ClientException and message before dispatch. A valid
+// request reaches dispatch, which fails here only because nc is nil.
+func TestCreateCapacityProviderAutoScalingGroupProvider(t *testing.T) {
+	body := func(asg string) []byte {
+		return []byte(`{"name":"cp","autoScalingGroupProvider":{"autoScalingGroupArn":"arn:aws:autoscaling:x"` + asg + `}}`)
+	}
+	refused := []struct {
+		name, asg, msg string
+	}{
+		{"managedDraining", `,"managedDraining":"BOGUS"`, msgManagedDraining},
+		{"managedTerminationProtection", `,"managedTerminationProtection":"enabled"`, msgManagedTerminationProtection},
+		{"status", `,"managedScaling":{"status":"BOGUS"}`, msgManagedScalingStatus},
+		{"targetCapacity/0", `,"managedScaling":{"targetCapacity":0}`, msgTargetCapacity},
+		{"targetCapacity/101", `,"managedScaling":{"targetCapacity":101}`, msgTargetCapacity},
+		{"minimumScalingStepSize/0", `,"managedScaling":{"minimumScalingStepSize":0}`, msgMinimumScalingStepSize},
+		{"maximumScalingStepSize/10001", `,"managedScaling":{"maximumScalingStepSize":10001}`, msgMaximumScalingStepSize},
+		{"instanceWarmupPeriod/-1", `,"managedScaling":{"instanceWarmupPeriod":-1}`, msgInstanceWarmupPeriod},
+		{"instanceWarmupPeriod/10000", `,"managedScaling":{"instanceWarmupPeriod":10000}`, msgInstanceWarmupPeriod},
+	}
+	for _, tc := range refused {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := CreateCapacityProvider(context.Background(), nil, "123456789012", body(tc.asg), nil)
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "err = %v", err)
+			assert.Equal(t, awserrors.ErrorECSClientException, code)
+			assert.Equal(t, tc.msg, msg)
+		})
+	}
+
+	valid := []string{
+		``,
+		`,"managedDraining":"ENABLED","managedTerminationProtection":"DISABLED","managedScaling":{"status":"ENABLED","targetCapacity":1,"minimumScalingStepSize":1,"maximumScalingStepSize":10000,"instanceWarmupPeriod":0}`,
+		`,"managedDraining":"DISABLED","managedTerminationProtection":"ENABLED","managedScaling":{"status":"DISABLED","targetCapacity":100,"minimumScalingStepSize":10000,"maximumScalingStepSize":1,"instanceWarmupPeriod":9999}`,
+	}
+	for _, asg := range valid {
+		_, err := CreateCapacityProvider(context.Background(), nil, "123456789012", body(asg), nil)
+		_, _, refusedErr := awserrors.ResolveErrorDetail(err)
+		assert.False(t, refusedErr, "asg %q: err = %v", asg, err)
+	}
+}

spinifex/domains/ecs/awsapi/validate_include.go

@@ -0,0 +1,68 @@
+package awsapi
+
+import (
+	"slices"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// Include values each Describe* operation accepts, in the order AWS lists them
+// in its refusal. Taken from the ECS model rather than the SDK's *_Values().
+var (
+	clusterIncludeValues           = []string{"ATTACHMENTS", "CONFIGURATIONS", "SETTINGS", "STATISTICS", "TAGS"}
+	tagsIncludeValues              = []string{"TAGS"}
+	containerInstanceIncludeValues = []string{"TAGS", "CONTAINER_INSTANCE_HEALTH"}
+)
+
+// firstInvalidInclude returns the first include value outside allowed.
+func firstInvalidInclude(include []*string, allowed []string) (string, bool) {
+	for _, v := range include {
+		if s := aws.StringValue(v); !slices.Contains(allowed, s) {
+			return s, true
+		}
+	}
+	return "", false
+}
+
+func includeOneOf(allowed []string) string {
+	return "[" + strings.Join(allowed, ",") + "]"
+}
+
+func checkClustersInclude(include []*string) error {
+	if _, bad := firstInvalidInclude(include, clusterIncludeValues); bad {
+		return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "include should be one of %s", includeOneOf(clusterIncludeValues))
+	}
+	return nil
+}
+
+func checkServicesInclude(include []*string) error {
+	if _, bad := firstInvalidInclude(include, tagsIncludeValues); bad {
+		return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "include should be one of %s", includeOneOf(tagsIncludeValues))
+	}
+	return nil
+}
+
+func checkTasksInclude(include []*string) error {
+	if _, bad := firstInvalidInclude(include, tagsIncludeValues); bad {
+		return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "include contains an invalid value, should be one of %s", includeOneOf(tagsIncludeValues))
+	}
+	return nil
+}
+
+func checkContainerInstancesInclude(include []*string) error {
+	if _, bad := firstInvalidInclude(include, containerInstanceIncludeValues); bad {
+		return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "include should be one of %s", includeOneOf(containerInstanceIncludeValues))
+	}
+	return nil
+}
+
+// checkCapacityProvidersInclude answers ClientException, as AWS does for this
+// operation alone, naming the refused value.
+func checkCapacityProvidersInclude(include []*string) error {
+	if v, bad := firstInvalidInclude(include, tagsIncludeValues); bad {
+		return awserrors.Errorf(awserrors.ErrorECSClientException, "%s is not a valid value for include. The value must be one of [%s]", v, includeOneOf(tagsIncludeValues))
+	}
+	return nil
+}

spinifex/domains/ecs/awsapi/validate_include_test.go

@@ -0,0 +1,53 @@
+package awsapi
+
+import (
+	"context"
+	"testing"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type describeHandler func(context.Context, *nats.Conn, string, []byte, PassRoleChecker) (any, error)
+
+// A refused include answers before dispatch with AWS's code and message. A
+// valid one reaches dispatch, which fails here only because nc is nil.
+func TestDescribeInclude(t *testing.T) {
+	cases := []struct {
+		name     string
+		handler  describeHandler
+		valid    []string
+		wantCode string
+		wantMsg  string
+	}{
+		{"DescribeClusters", DescribeClusters, []string{"ATTACHMENTS", "CONFIGURATIONS", "SETTINGS", "STATISTICS", "TAGS"},
+			awserrors.ErrorECSInvalidParameter, "include should be one of [ATTACHMENTS,CONFIGURATIONS,SETTINGS,STATISTICS,TAGS]"},
+		{"DescribeServices", DescribeServices, []string{"TAGS"},
+			awserrors.ErrorECSInvalidParameter, "include should be one of [TAGS]"},
+		{"DescribeTasks", DescribeTasks, []string{"TAGS"},
+			awserrors.ErrorECSInvalidParameter, "include contains an invalid value, should be one of [TAGS]"},
+		{"DescribeContainerInstances", DescribeContainerInstances, []string{"TAGS", "CONTAINER_INSTANCE_HEALTH"},
+			awserrors.ErrorECSInvalidParameter, "include should be one of [TAGS,CONTAINER_INSTANCE_HEALTH]"},
+		{"DescribeCapacityProviders", DescribeCapacityProviders, []string{"TAGS"},
+			awserrors.ErrorECSClientException, "BOGUS is not a valid value for include. The value must be one of [[TAGS]]"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name+"/invalid", func(t *testing.T) {
+			_, err := tc.handler(context.Background(), nil, "123456789012", []byte(`{"include":["TAGS","BOGUS"]}`), nil)
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "err = %v", err)
+			assert.Equal(t, tc.wantCode, code)
+			assert.Equal(t, tc.wantMsg, msg)
+		})
+		for _, v := range tc.valid {
+			t.Run(tc.name+"/"+v, func(t *testing.T) {
+				_, err := tc.handler(context.Background(), nil, "123456789012", []byte(`{"include":["`+v+`"]}`), nil)
+				_, _, refused := awserrors.ResolveErrorDetail(err)
+				assert.False(t, refused, "err = %v", err)
+			})
+		}
+	}
+}

spinifex/domains/ecs/awsapi/validate_list.go

@@ -0,0 +1,62 @@
+package awsapi
+
+import (
+	"encoding/json"
+	"slices"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// Enum sets the List* operations accept, in the order AWS lists them in its
+// refusal. launchType also accepts FMI, which AWS takes but the model omits.
+var (
+	listLaunchTypes             = []string{"EC2", "FARGATE", "EXTERNAL", "FMI", "MANAGED_INSTANCES"}
+	listSchedulingStrategies    = []string{"REPLICA", "DAEMON"}
+	listResourceManagementTypes = []string{"CUSTOMER", "ECS"}
+	listSortOrders              = []string{"ASC", "DESC"}
+	listTaskDefinitionStatuses  = []string{"ACTIVE", "INACTIVE", "DELETE_IN_PROGRESS"}
+	listDesiredStatuses         = []string{"RUNNING", "PENDING", "STOPPED"}
+)
+
+// checkListEnum refuses a set value outside allowed, naming the member as
+// AWS's message does. An omitted value is left to the operation's default.
+func checkListEnum(v *string, label string, allowed []string) error {
+	if v == nil || slices.Contains(allowed, *v) {
+		return nil
+	}
+	return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "%s should be one of [%s]", label, strings.Join(allowed, ","))
+}
+
+func checkListServices(body []byte, input *ecs.ListServicesInput) error {
+	if err := checkListEnum(input.LaunchType, "launch type", listLaunchTypes); err != nil {
+		return err
+	}
+	if err := checkListEnum(input.SchedulingStrategy, "Scheduling strategy", listSchedulingStrategies); err != nil {
+		return err
+	}
+	// The pinned SDK's ListServicesInput predates resourceManagementType, so it
+	// is read off the raw body; a body that fails to parse was already refused.
+	var raw struct {
+		ResourceManagementType *string `json:"resourceManagementType"`
+	}
+	if len(body) > 0 {
+		_ = json.Unmarshal(body, &raw)
+	}
+	return checkListEnum(raw.ResourceManagementType, "Resource management type", listResourceManagementTypes)
+}
+
+func checkListTaskDefinitions(input *ecs.ListTaskDefinitionsInput) error {
+	if err := checkListEnum(input.Sort, "Sort", listSortOrders); err != nil {
+		return err
+	}
+	return checkListEnum(input.Status, "Status", listTaskDefinitionStatuses)
+}
+
+func checkListTasks(input *ecs.ListTasksInput) error {
+	if err := checkListEnum(input.DesiredStatus, "Desired Status", listDesiredStatuses); err != nil {
+		return err
+	}
+	return checkListEnum(input.LaunchType, "launch type", listLaunchTypes)
+}

spinifex/domains/ecs/awsapi/validate_list_test.go

@@ -0,0 +1,53 @@
+package awsapi
+
+import (
+	"context"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A value outside a List* enum is refused before dispatch with AWS's message.
+// A valid one reaches dispatch, which fails here only because nc is nil.
+func TestListEnums(t *testing.T) {
+	cases := []struct {
+		name    string
+		handler describeHandler
+		member  string
+		valid   []string
+		wantMsg string
+	}{
+		{"ListServices", ListServices, "launchType", []string{"EC2", "FARGATE", "EXTERNAL", "FMI", "MANAGED_INSTANCES"},
+			"launch type should be one of [EC2,FARGATE,EXTERNAL,FMI,MANAGED_INSTANCES]"},
+		{"ListServices", ListServices, "schedulingStrategy", []string{"REPLICA", "DAEMON"},
+			"Scheduling strategy should be one of [REPLICA,DAEMON]"},
+		{"ListServices", ListServices, "resourceManagementType", []string{"CUSTOMER", "ECS"},
+			"Resource management type should be one of [CUSTOMER,ECS]"},
+		{"ListTaskDefinitions", ListTaskDefinitions, "sort", []string{"ASC", "DESC"},
+			"Sort should be one of [ASC,DESC]"},
+		{"ListTaskDefinitions", ListTaskDefinitions, "status", []string{"ACTIVE", "INACTIVE", "DELETE_IN_PROGRESS"},
+			"Status should be one of [ACTIVE,INACTIVE,DELETE_IN_PROGRESS]"},
+		{"ListTasks", ListTasks, "desiredStatus", []string{"RUNNING", "PENDING", "STOPPED"},
+			"Desired Status should be one of [RUNNING,PENDING,STOPPED]"},
+		{"ListTasks", ListTasks, "launchType", []string{"EC2", "FARGATE", "EXTERNAL", "FMI", "MANAGED_INSTANCES"},
+			"launch type should be one of [EC2,FARGATE,EXTERNAL,FMI,MANAGED_INSTANCES]"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name+"/"+tc.member+"/invalid", func(t *testing.T) {
+			_, err := tc.handler(context.Background(), nil, "123456789012", []byte(`{"`+tc.member+`":"BOGUS"}`), nil)
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "err = %v", err)
+			assert.Equal(t, awserrors.ErrorECSInvalidParameter, code)
+			assert.Equal(t, tc.wantMsg, msg)
+		})
+		for _, v := range tc.valid {
+			t.Run(tc.name+"/"+tc.member+"/"+v, func(t *testing.T) {
+				_, err := tc.handler(context.Background(), nil, "123456789012", []byte(`{"`+tc.member+`":"`+v+`"}`), nil)
+				_, _, refused := awserrors.ResolveErrorDetail(err)
+				assert.False(t, refused, "err = %v", err)
+			})
+		}
+	}
+}

spinifex/domains/elbv2/service_impl_attributes.go

@@ -32,8 +32,12 @@ type attrResource[R any] struct {
 	save        func(*R) error
 	accountIDOf func(*R) string
 	defaults    func(*R) map[string]string
-	attrsOf     func(*R) map[string]string
-	setAttrs    func(*R, map[string]string)
+	// accepts overrides the known-key check, which otherwise uses defaults.
+	accepts func(key string) bool
+	// unknownKey builds the error for a rejected key; nil means a bare ValidationError.
+	unknownKey func(key string) error
+	attrsOf    func(*R) map[string]string
+	setAttrs   func(*R, map[string]string)
 	// validate rejects a bad value for a known key before anything is stored.
 	validate func(key, value string) error
 	// afterChange runs once the changed keys are persisted; its error fails the call.
@@ -57,29 +61,36 @@ func (r attrResource[R]) load() (*R, error) {
 
 // modifyResourceAttributes validates and merges submitted attributes, persisting only
 // on change. rawCount is pre-nil-filter count, used to distinguish "nothing sent"
-// from "everything sent was invalid".
-func modifyResourceAttributes[R any](res attrResource[R], pairs []attrPair, rawCount int) ([]attrPair, error) {
+// from "everything sent was invalid". It returns the submitted pairs and the full merged set.
+func modifyResourceAttributes[R any](res attrResource[R], pairs []attrPair, rawCount int) (submitted, full []attrPair, err error) {
 	if res.arn == "" {
-		return nil, errors.New(awserrors.ErrorMissingParameter)
+		return nil, nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 	rec, err := res.load()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	known := res.defaults(rec)
+	accepts := res.accepts
+	if accepts == nil {
+		known := res.defaults(rec)
+		accepts = func(k string) bool { _, ok := known[k]; return ok }
+	}
 	attrs := res.attrsOf(rec)
-	submitted := make([]attrPair, 0, len(pairs))
+	submitted = make([]attrPair, 0, len(pairs))
 	var changed []string
 	for _, p := range pairs {
-		if _, ok := known[p.Key]; !ok {
+		if !accepts(p.Key) {
 			slog.Warn(res.opName+": rejecting unknown attribute key", "arn", res.arn, "key", p.Key)
-			return nil, errors.New(awserrors.ErrorValidationError)
+			if res.unknownKey != nil {
+				return nil, nil, res.unknownKey(p.Key)
+			}
+			return nil, nil, errors.New(awserrors.ErrorValidationError)
 		}
 		if res.validate != nil {
 			if err := res.validate(p.Key, p.Value); err != nil {
 				slog.Warn(res.opName+": rejecting attribute value", "arn", res.arn, "key", p.Key)
-				return nil, err
+				return nil, nil, err
 			}
 		}
 	}
@@ -102,7 +113,7 @@ func modifyResourceAttributes[R any](res attrResource[R], pairs []attrPair, rawC
 	// landed write.
 	if rawCount > 0 && len(submitted) == 0 {
 		slog.Warn(res.opName+": all submitted attributes were invalid", "arn", res.arn, "submitted_count", rawCount)
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+		return nil, nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}
 
 	// Skip the NATS/KV write when nothing changed. Terraform re-applies the same
@@ -112,16 +123,16 @@ func modifyResourceAttributes[R any](res attrResource[R], pairs []attrPair, rawC
 		res.setAttrs(rec, attrs)
 		if err := res.save(rec); err != nil {
 			slog.Error(res.opName+": failed to persist", "arn", res.arn, "err", err)
-			return nil, errors.New(awserrors.ErrorServerInternal)
+			return nil, nil, errors.New(awserrors.ErrorServerInternal)
 		}
 		if res.afterChange != nil {
 			if err := res.afterChange(rec, changed); err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 		}
 	}
 
-	return submitted, nil
+	return submitted, mergedAttributes(res, rec), nil
 }
 
 // describeResourceAttributes returns the record's attributes merged over the
@@ -136,6 +147,11 @@ func describeResourceAttributes[R any](res attrResource[R]) ([]attrPair, error)
 		return nil, err
 	}
 
+	return mergedAttributes(res, rec), nil
+}
+
+// mergedAttributes returns rec's stored attributes over its defaults, sorted by key.
+func mergedAttributes[R any](res attrResource[R], rec *R) []attrPair {
 	merged := res.defaults(rec)
 	maps.Copy(merged, res.attrsOf(rec))
 
@@ -144,7 +160,7 @@ func describeResourceAttributes[R any](res attrResource[R]) ([]attrPair, error)
 	for _, k := range keys {
 		out = append(out, attrPair{Key: k, Value: merged[k]})
 	}
-	return out, nil
+	return out
 }
 
 // sdkAttrsIn converts an SDK attribute slice into internal pairs, dropping nil
@@ -196,9 +212,15 @@ func (s *ELBv2ServiceImpl) tgAttrResource(ctx context.Context, arn, accountID, o
 			return err
 		},
 		accountIDOf: func(r *TargetGroupRecord) string { return r.AccountID },
-		defaults:    func(*TargetGroupRecord) map[string]string { return DefaultTargetGroupAttributes() },
-		attrsOf:     func(r *TargetGroupRecord) map[string]string { return r.Attributes },
-		setAttrs:    func(r *TargetGroupRecord, m map[string]string) { r.Attributes = m },
+		defaults: func(r *TargetGroupRecord) map[string]string {
+			return DefaultTargetGroupAttributes(r.Protocol, r.TargetType)
+		},
+		accepts: func(key string) bool { return targetGroupAttributeKeys[key] },
+		unknownKey: func(key string) error {
+			return awserrors.Errorf(awserrors.ErrorValidationError, "Target group attribute key '%s' is not recognized", key)
+		},
+		attrsOf:  func(r *TargetGroupRecord) map[string]string { return r.Attributes },
+		setAttrs: func(r *TargetGroupRecord, m map[string]string) { r.Attributes = m },
 	}
 }
 
@@ -299,11 +321,11 @@ func (s *ELBv2ServiceImpl) ModifyTargetGroupAttributes(ctx context.Context, inpu
 	const op = "ModifyTargetGroupAttributes"
 	arn := aws.StringValue(input.TargetGroupArn)
 	pairs, raw := sdkAttrsIn(input.Attributes, tgAttrKeyVal, arn, op)
-	submitted, err := modifyResourceAttributes(s.tgAttrResource(ctx, arn, accountID, op), pairs, raw)
+	_, full, err := modifyResourceAttributes(s.tgAttrResource(ctx, arn, accountID, op), pairs, raw)
 	if err != nil {
 		return nil, err
 	}
-	return &elbv2.ModifyTargetGroupAttributesOutput{Attributes: sdkAttrsOut(submitted, mkTGAttr)}, nil
+	return &elbv2.ModifyTargetGroupAttributesOutput{Attributes: sdkAttrsOut(full, mkTGAttr)}, nil
 }
 
 func (s *ELBv2ServiceImpl) DescribeTargetGroupAttributes(ctx context.Context, input *elbv2.DescribeTargetGroupAttributesInput, accountID string) (*elbv2.DescribeTargetGroupAttributesOutput, error) {
@@ -326,7 +348,7 @@ func (s *ELBv2ServiceImpl) ModifyLoadBalancerAttributes(ctx context.Context, inp
 	const op = "ModifyLoadBalancerAttributes"
 	arn := aws.StringValue(input.LoadBalancerArn)
 	pairs, raw := sdkAttrsIn(input.Attributes, lbAttrKeyVal, arn, op)
-	submitted, err := modifyResourceAttributes(s.lbAttrResource(ctx, arn, accountID, op), pairs, raw)
+	submitted, _, err := modifyResourceAttributes(s.lbAttrResource(ctx, arn, accountID, op), pairs, raw)
 	if err != nil {
 		return nil, err
 	}

spinifex/domains/elbv2/service_impl_test.go

@@ -5,7 +5,9 @@ import (
 	"encoding/xml"
 	"fmt"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"maps"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -2747,65 +2749,143 @@ func TestUpdateStoredConfig_MissingTargetGroup(t *testing.T) {
 
 // --- Attribute tests ---
 
-// TestDefaultTargetGroupAttributes_CoversTerraformKeys guards the full
-// TargetGroupAttribute key set from the ELBv2 API reference. A missing key
-// fails the whole ModifyTargetGroupAttributes call with ValidationError.
-func TestDefaultTargetGroupAttributes_CoversTerraformKeys(t *testing.T) {
-	t.Parallel()
-	attrs := DefaultTargetGroupAttributes()
-
-	expected := map[string]string{
+var (
+	wantHTTPTGAttrs = map[string]string{
 		"deregistration_delay.timeout_seconds":                                           "300",
-		"stickiness.enabled":                                                             "false",
-		"stickiness.type":                                                                "lb_cookie",
-		"stickiness.lb_cookie.duration_seconds":                                          "86400",
+		"load_balancing.algorithm.anomaly_mitigation":                                    "off",
+		"load_balancing.algorithm.type":                                                  "round_robin",
+		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
+		"slow_start.duration_seconds":                                                    "0",
 		"stickiness.app_cookie.cookie_name":                                              "",
 		"stickiness.app_cookie.duration_seconds":                                         "86400",
-		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
-		"load_balancing.algorithm.type":                                                  "round_robin",
-		"load_balancing.algorithm.anomaly_mitigation":                                    "off",
-		"slow_start.duration_seconds":                                                    "0",
-		"lambda.multi_value_headers.enabled":                                             "false",
+		"stickiness.enabled":                                                             "false",
+		"stickiness.lb_cookie.duration_seconds":                                          "86400",
+		"stickiness.type":                                                                "lb_cookie",
 		"target_group_health.dns_failover.minimum_healthy_targets.count":                 "1",
 		"target_group_health.dns_failover.minimum_healthy_targets.percentage":            "off",
 		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.count":      "1",
 		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.percentage": "off",
-		"proxy_protocol_v2.enabled":                                                      "false",
+	}
+	wantTCPTGAttrs = map[string]string{
 		"deregistration_delay.connection_termination.enabled":                            "false",
-		"preserve_client_ip.enabled":                                                     "false",
+		"deregistration_delay.timeout_seconds":                                           "300",
+		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
+		"proxy_protocol_v2.enabled":                                                      "false",
+		"stickiness.enabled":                                                             "false",
+		"stickiness.type":                                                                "source_ip",
+		"target_group_health.dns_failover.minimum_healthy_targets.count":                 "1",
+		"target_group_health.dns_failover.minimum_healthy_targets.percentage":            "off",
+		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.count":      "1",
+		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.percentage": "off",
 		"target_health_state.unhealthy.connection_termination.enabled":                   "true",
 		"target_health_state.unhealthy.draining_interval_seconds":                        "0",
-		"target_failover.on_deregistration":                                              "no_rebalance",
-		"target_failover.on_unhealthy":                                                   "no_rebalance",
 	}
+)
 
-	for k, v := range expected {
-		got, ok := attrs[k]
-		assert.True(t, ok, "target group default attributes missing key %q — terraform will hit ValidationError", k)
-		assert.Equal(t, v, got, "target group default value mismatch for key %s", k)
+// wantTGAttrs returns the attribute set AWS reports for a new target group.
+func wantTGAttrs(protocol, targetType string) map[string]string {
+	if protocol == "HTTP" {
+		return maps.Clone(wantHTTPTGAttrs)
 	}
+	m := maps.Clone(wantTCPTGAttrs)
+	m["preserve_client_ip.enabled"] = strconv.FormatBool(targetType == "instance")
+	return m
+}
+
+func tgAttrMap(attrs []*elbv2.TargetGroupAttribute) map[string]string {
+	m := make(map[string]string, len(attrs))
+	for _, a := range attrs {
+		m[*a.Key] = *a.Value
+	}
+	return m
+}
+
+var tgAttrCases = []struct{ protocol, targetType string }{
+	{"HTTP", "instance"}, {"HTTP", "ip"}, {"TCP", "instance"}, {"TCP", "ip"},
+}
+
+func createAttrTG(t *testing.T, svc *ELBv2ServiceImpl, protocol, targetType string) *string {
+	t.Helper()
+	out, err := svc.CreateTargetGroup(context.Background(), &elbv2.CreateTargetGroupInput{
+		Name:       aws.String("tg-attr-" + strings.ToLower(protocol) + "-" + targetType),
+		Protocol:   aws.String(protocol),
+		Port:       aws.Int64(80),
+		TargetType: aws.String(targetType),
+	}, testAccountID)
+	require.NoError(t, err)
+	return out.TargetGroups[0].TargetGroupArn
 }
 
 func TestDescribeTargetGroupAttributes_Defaults(t *testing.T) {
 	t.Parallel()
-	svc := setupTestService(t)
-	tgOut, err := svc.CreateTargetGroup(context.Background(), &elbv2.CreateTargetGroupInput{Name: aws.String("tg-attr-defaults")}, testAccountID)
-	require.NoError(t, err)
+	for _, tc := range tgAttrCases {
+		t.Run(tc.protocol+"/"+tc.targetType, func(t *testing.T) {
+			t.Parallel()
+			svc := setupTestService(t)
+			arn := createAttrTG(t, svc, tc.protocol, tc.targetType)
 
-	out, err := svc.DescribeTargetGroupAttributes(context.Background(), &elbv2.DescribeTargetGroupAttributesInput{
-		TargetGroupArn: tgOut.TargetGroups[0].TargetGroupArn,
+			out, err := svc.DescribeTargetGroupAttributes(context.Background(), &elbv2.DescribeTargetGroupAttributesInput{
+				TargetGroupArn: arn,
+			}, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, wantTGAttrs(tc.protocol, tc.targetType), tgAttrMap(out.Attributes))
+		})
+	}
+}
+
+func TestModifyTargetGroupAttributes_ReturnsFullSet(t *testing.T) {
+	t.Parallel()
+	for _, tc := range tgAttrCases {
+		t.Run(tc.protocol+"/"+tc.targetType, func(t *testing.T) {
+			t.Parallel()
+			svc := setupTestService(t)
+			arn := createAttrTG(t, svc, tc.protocol, tc.targetType)
+
+			out, err := svc.ModifyTargetGroupAttributes(context.Background(), &elbv2.ModifyTargetGroupAttributesInput{
+				TargetGroupArn: arn,
+				Attributes: []*elbv2.TargetGroupAttribute{
+					{Key: aws.String("deregistration_delay.timeout_seconds"), Value: aws.String("30")},
+				},
+			}, testAccountID)
+			require.NoError(t, err)
+			want := wantTGAttrs(tc.protocol, tc.targetType)
+			want["deregistration_delay.timeout_seconds"] = "30"
+			assert.Equal(t, want, tgAttrMap(out.Attributes))
+		})
+	}
+}
+
+// The load balancer controller writes NLB keys on every target group it manages.
+func TestModifyTargetGroupAttributes_AcceptsOtherProtocolKeys(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	arn := createAttrTG(t, svc, "HTTP", "instance")
+
+	_, err := svc.ModifyTargetGroupAttributes(context.Background(), &elbv2.ModifyTargetGroupAttributesInput{
+		TargetGroupArn: arn,
+		Attributes: []*elbv2.TargetGroupAttribute{
+			{Key: aws.String("proxy_protocol_v2.enabled"), Value: aws.String("false")},
+			{Key: aws.String("target_failover.on_unhealthy"), Value: aws.String("no_rebalance")},
+			{Key: aws.String("target_failover.on_deregistration"), Value: aws.String("no_rebalance")},
+			{Key: aws.String("lambda.multi_value_headers.enabled"), Value: aws.String("false")},
+		},
 	}, testAccountID)
 	require.NoError(t, err)
+}
 
-	defaults := DefaultTargetGroupAttributes()
-	assert.Len(t, out.Attributes, len(defaults))
-	attrMap := make(map[string]string)
-	for _, a := range out.Attributes {
-		attrMap[*a.Key] = *a.Value
-	}
-	for k, v := range defaults {
-		assert.Equal(t, v, attrMap[k], "default mismatch for key %s", k)
-	}
+func TestModifyTargetGroupAttributes_UnknownKeyMessage(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	arn := createAttrTG(t, svc, "TCP", "ip")
+
+	_, err := svc.ModifyTargetGroupAttributes(context.Background(), &elbv2.ModifyTargetGroupAttributesInput{
+		TargetGroupArn: arn,
+		Attributes:     []*elbv2.TargetGroupAttribute{{Key: aws.String("bogus.key"), Value: aws.String("1")}},
+	}, testAccountID)
+	require.Error(t, err)
+	code, msg, _ := awserrors.ResolveErrorDetail(err)
+	assert.Equal(t, awserrors.ErrorValidationError, code)
+	assert.Equal(t, "Target group attribute key 'bogus.key' is not recognized", msg)
 }
 
 func TestDescribeLoadBalancerAttributes_ALBDefaults(t *testing.T) {
@@ -3025,6 +3105,14 @@ func echoMap(pairs []echoPair) map[string]string {
 	return m
 }
 
+// modifyLen is the number of pairs k's Modify returns after n were submitted.
+func (k attrKind) modifyLen(n int) int {
+	if k.modifySetLen > 0 {
+		return k.modifySetLen
+	}
+	return n
+}
+
 // attrArnPtr returns nil for an empty ARN so the MissingArn cases exercise the
 // real "no ARN supplied" path rather than an empty-string ARN.
 func attrArnPtr(arn string) *string {
@@ -3046,6 +3134,7 @@ type attrKind struct {
 	defaultKey        string // a default present after create
 	defaultVal        string
 	unknownKey        string // a key the handler must reject
+	modifySetLen      int    // size of the full set Modify returns; 0 when it echoes the submitted pairs
 
 	create   func(t *testing.T, svc *ELBv2ServiceImpl, name string) string
 	modify   func(svc *ELBv2ServiceImpl, arn, accountID string, attrs []rawAttr) ([]echoPair, error)
@@ -3062,7 +3151,8 @@ func attrKinds() []attrKind {
 			keyA:       "deregistration_delay.timeout_seconds", valA: "45", valA2: "30",
 			keyB: "stickiness.enabled", valB: "true",
 			defaultKey: "stickiness.type", defaultVal: "lb_cookie",
-			unknownKey: "stickness.enabled", // typo of "stickiness"
+			unknownKey:   "stickness.enabled", // typo of "stickiness"
+			modifySetLen: len(wantHTTPTGAttrs),
 			create: func(t *testing.T, svc *ELBv2ServiceImpl, name string) string {
 				out, err := svc.CreateTargetGroup(context.Background(), &elbv2.CreateTargetGroupInput{Name: aws.String(name)}, testAccountID)
 				require.NoError(t, err)
@@ -3179,7 +3269,7 @@ func TestAttributeRoundTrip(t *testing.T) {
 			// pass a length check.
 			echoed, err := k.modify(svc, arn, testAccountID, kvAttrs([2]string{k.keyA, k.valA}, [2]string{k.keyB, k.valB}))
 			require.NoError(t, err)
-			require.Len(t, echoed, 2)
+			require.Len(t, echoed, k.modifyLen(2))
 			em := echoMap(echoed)
 			assert.Equal(t, k.valA, em[k.keyA])
 			assert.Equal(t, k.valB, em[k.keyB])
@@ -3298,10 +3388,9 @@ func TestAttributeSkipsInvalidEntries(t *testing.T) {
 				kvAttr(k.keyB, k.valB),
 			})
 			require.NoError(t, err)
-			// Only the one valid attribute should be returned.
-			require.Len(t, echoed, 1)
-			assert.Equal(t, k.keyB, echoed[0].Key)
-			assert.Equal(t, k.valB, echoed[0].Val)
+			// The invalid elements are dropped and the valid one applied.
+			require.Len(t, echoed, k.modifyLen(1))
+			assert.Equal(t, k.valB, echoMap(echoed)[k.keyB])
 
 			// The valid attribute should have been persisted.
 			desc, err := k.describe(svc, arn, testAccountID)
@@ -3387,7 +3476,7 @@ func TestAttributeNoopSkipsPersist(t *testing.T) {
 			// Second modify with identical values — must skip the Put.
 			echoed, err := k.modify(svc, arn, testAccountID, kvAttrs([2]string{k.keyA, k.valA}))
 			require.NoError(t, err)
-			require.Len(t, echoed, 1)
+			require.Len(t, echoed, k.modifyLen(1))
 			assert.Equal(t, revBefore, k.revision(t, svc, arn), "identical modify must not increment KV revision")
 
 			// Empty attribute list is also a no-op.
@@ -3410,7 +3499,8 @@ func TestAttributeRejectsUnknownKey(t *testing.T) {
 			arn := k.create(t, svc, "attr-unknown-key")
 
 			_, err := k.modify(svc, arn, testAccountID, kvAttrs([2]string{k.unknownKey, "true"}))
-			assert.EqualError(t, err, awserrors.ErrorValidationError)
+			code, _ := awserrors.ResolveErrorCode(err)
+			assert.Equal(t, awserrors.ErrorValidationError, code)
 
 			// The rejected key must not have been persisted.
 			desc, err := k.describe(svc, arn, testAccountID)

spinifex/domains/elbv2/types.go

@@ -342,9 +342,18 @@ func DefaultLoadBalancerAttributes(lbType string) map[string]string {
 	}
 }
 
-// DefaultTargetGroupAttributes returns the default attribute set for target groups.
-func DefaultTargetGroupAttributes() map[string]string {
-	return maps.Clone(targetGroupAttributeDefaults)
+// DefaultTargetGroupAttributes returns the attributes AWS reports for a new
+// target group of this protocol and target type. HTTPS uses the HTTP set and
+// every other protocol the TCP set.
+func DefaultTargetGroupAttributes(protocol, targetType string) map[string]string {
+	if protocol == ProtocolHTTP || protocol == ProtocolHTTPS {
+		return maps.Clone(httpTargetGroupAttributeDefaults)
+	}
+	attrs := maps.Clone(tcpTargetGroupAttributeDefaults)
+	if targetType == TargetTypeInstance {
+		attrs["preserve_client_ip.enabled"] = "true"
+	}
+	return attrs
 }
 
 // Default attribute sets are package-level vars; callers receive a clone so
@@ -395,32 +404,56 @@ var (
 		"secondary_ips.auto_assigned.per_subnet": "0",
 	}
 
-	targetGroupAttributeDefaults = map[string]string{
+	// HTTP and HTTPS target groups, as AWS reports them for instance and ip targets.
+	httpTargetGroupAttributeDefaults = map[string]string{
 		"deregistration_delay.timeout_seconds":                                           "300",
-		"stickiness.enabled":                                                             "false",
-		"stickiness.type":                                                                "lb_cookie",
-		"stickiness.lb_cookie.duration_seconds":                                          "86400",
-		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
-		"load_balancing.algorithm.type":                                                  "round_robin",
-		"slow_start.duration_seconds":                                                    "0",
 		"load_balancing.algorithm.anomaly_mitigation":                                    "off",
+		"load_balancing.algorithm.type":                                                  "round_robin",
+		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
+		"slow_start.duration_seconds":                                                    "0",
 		"stickiness.app_cookie.cookie_name":                                              "",
 		"stickiness.app_cookie.duration_seconds":                                         "86400",
-		"lambda.multi_value_headers.enabled":                                             "false",
+		"stickiness.enabled":                                                             "false",
+		"stickiness.lb_cookie.duration_seconds":                                          "86400",
+		"stickiness.type":                                                                "lb_cookie",
 		"target_group_health.dns_failover.minimum_healthy_targets.count":                 "1",
 		"target_group_health.dns_failover.minimum_healthy_targets.percentage":            "off",
 		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.count":      "1",
 		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.percentage": "off",
-		// NLB target-group defaults. The AWS Load Balancer Controller always
-		// writes these on NLB target groups (e.g. ingress-nginx); without them as
-		// known keys the modify is rejected and LBC loops on the Service forever.
-		"proxy_protocol_v2.enabled":                                    "false",
-		"deregistration_delay.connection_termination.enabled":          "false",
-		"preserve_client_ip.enabled":                                   "false",
-		"target_health_state.unhealthy.connection_termination.enabled": "true",
-		"target_health_state.unhealthy.draining_interval_seconds":      "0",
-		// GWLB-only; AWS requires both to hold the same value.
-		"target_failover.on_deregistration": "no_rebalance",
-		"target_failover.on_unhealthy":      "no_rebalance",
 	}
+
+	// TCP-family target groups; preserve_client_ip.enabled is set by target type.
+	tcpTargetGroupAttributeDefaults = map[string]string{
+		"deregistration_delay.connection_termination.enabled":                            "false",
+		"deregistration_delay.timeout_seconds":                                           "300",
+		"load_balancing.cross_zone.enabled":                                              "use_load_balancer_configuration",
+		"preserve_client_ip.enabled":                                                     "false",
+		"proxy_protocol_v2.enabled":                                                      "false",
+		"stickiness.enabled":                                                             "false",
+		"stickiness.type":                                                                "source_ip",
+		"target_group_health.dns_failover.minimum_healthy_targets.count":                 "1",
+		"target_group_health.dns_failover.minimum_healthy_targets.percentage":            "off",
+		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.count":      "1",
+		"target_group_health.unhealthy_state_routing.minimum_healthy_targets.percentage": "off",
+		"target_health_state.unhealthy.connection_termination.enabled":                   "true",
+		"target_health_state.unhealthy.draining_interval_seconds":                        "0",
+	}
+
+	// Keys ModifyTargetGroupAttributes accepts on any target group. The load
+	// balancer controller writes the NLB keys on every NLB group, so this is the
+	// union of the protocol sets plus the lambda and GWLB keys.
+	targetGroupAttributeKeys = func() map[string]bool {
+		keys := map[string]bool{
+			"lambda.multi_value_headers.enabled": true,
+			"target_failover.on_deregistration":  true,
+			"target_failover.on_unhealthy":       true,
+		}
+		for k := range httpTargetGroupAttributeDefaults {
+			keys[k] = true
+		}
+		for k := range tcpTargetGroupAttributeDefaults {
+			keys[k] = true
+		}
+		return keys
+	}()
 )

spinifex/domains/iam/awsapi/authz.go

@@ -167,6 +167,17 @@ func ResourceARNs(action, accountID string, input any, svc iamdomain.IAMService)
 	if resource == "" {
 		resource = anyResource
 	}
+	if action == "UpdateUser" && resource != anyResource {
+		newName, err := stringField(input, "NewUserName")
+		if err != nil {
+			return nil, err
+		}
+		if newName != "" {
+			// AWS also authorizes the rename target, at the user's current path.
+			prefix := resource[:strings.LastIndex(resource, "/")+1]
+			return []string{resource, prefix + newName}, nil
+		}
+	}
 	return []string{resource}, nil
 }
 

spinifex/domains/iam/awsapi/gateway_authz_test.go

@@ -32,6 +32,7 @@ type iamAuthzPolicyService struct {
 	canonicalErr  error
 	deleteRoleErr error
 	deleted       []string
+	renamed       []string
 }
 
 func (s *iamAuthzPolicyService) CanonicalResourceARN(_ string, _ arn.IAMResourceType, name string) (string, error) {
@@ -53,6 +54,11 @@ func (s *iamAuthzPolicyService) DeleteRole(_ string, input *iam.DeleteRoleInput)
 	return &iam.DeleteRoleOutput{}, nil
 }
 
+func (s *iamAuthzPolicyService) UpdateUser(_ string, input *iam.UpdateUserInput) (*iam.UpdateUserOutput, error) {
+	s.renamed = append(s.renamed, *input.UserName+"->"+*input.NewUserName)
+	return &iam.UpdateUserOutput{}, nil
+}
+
 // iamGateway pairs a caller holding statements with the service they act on.
 type iamGateway struct {
 	h   *gwtest.Harness
@@ -249,3 +255,34 @@ func TestAddRoleToInstanceProfile_MissingRoleStaysValidationFault(t *testing.T)
 	err := dispatchIAM(t, gw, "Action=AddRoleToInstanceProfile&InstanceProfileName=web")
 	require.EqualError(t, err, awserrors.ErrorMissingParameter)
 }
+
+// TestIAMRequest_UpdateUserRenameAuthorizesSourceAndTarget pins AWS's rule that
+// a rename needs iam:UpdateUser on both the user and user/<NewUserName>, the
+// target taking the user's current path.
+func TestIAMRequest_UpdateUserRenameAuthorizesSourceAndTarget(t *testing.T) {
+	users := map[string]string{"alice": "arn:aws:iam::123456789012:user/teams/alice"}
+	const (
+		source = "arn:aws:iam::*:user/teams/alice"
+		target = "arn:aws:iam::*:user/teams/bob"
+		body   = "Action=UpdateUser&UserName=alice&NewUserName=bob"
+	)
+
+	t.Run("source-only grant is denied", func(t *testing.T) {
+		gw, svc := iamScopedGateway(users, statement("Allow", "iam:UpdateUser", source))
+		assertDenied(t, dispatchIAM(t, gw, body))
+		assert.Empty(t, svc.renamed)
+	})
+
+	t.Run("target-only grant is denied", func(t *testing.T) {
+		gw, svc := iamScopedGateway(users, statement("Allow", "iam:UpdateUser", target))
+		assertDenied(t, dispatchIAM(t, gw, body))
+		assert.Empty(t, svc.renamed)
+	})
+
+	t.Run("grant on both succeeds", func(t *testing.T) {
+		gw, svc := iamScopedGateway(users, statement("Allow", "iam:UpdateUser", source),
+			statement("Allow", "iam:UpdateUser", target))
+		require.NoError(t, dispatchIAM(t, gw, body))
+		assert.Equal(t, []string{"alice->bob"}, svc.renamed)
+	})
+}

spinifex/domains/iam/awsapi/registration_test.go

@@ -281,7 +281,7 @@ func TestRegistration_UpdateUser(t *testing.T) {
 	})
 	require.NoError(t, err)
 	require.Len(t, calls, 1)
-	assert.Equal(t, []string{"arn:aws:iam::" + testAccountID + ":user/alice"}, calls[0].resources)
+	assert.Equal(t, []string{"arn:aws:iam::" + testAccountID + ":user/alice", "arn:aws:iam::" + testAccountID + ":user/alicia"}, calls[0].resources)
 	assert.Contains(t, w.Body.String(), "UpdateUserResponse")
 	require.NotNil(t, svc.updateUser)
 	assert.Equal(t, "alicia", *svc.updateUser.NewUserName)

spinifex/domains/lambda/awsapi/create.go

@@ -110,7 +110,7 @@ func (d Deps) createFunction(ctx context.Context, c *call) (reply, error) {
 
 // rollback makes one compare-and-swap attempt to free the name. It cannot undo
 // a create that completed, because the revision it holds has moved on. The code
-// object, if one landed, is left in place and recorded as a cleanup obligation.
+// object, if one landed, stays in place under its owed upload record.
 func (d Deps) rollback(ctx context.Context, c *call, name string, createRev uint64, ref codestore.Ref, why string, cause error) {
 	slog.ErrorContext(ctx, "Lambda: create did not complete", "reason", why, "function_name", name,
 		"function_id", ref.FunctionID, "err", cause)

spinifex/domains/lambda/awsapi/get.go

@@ -40,6 +40,12 @@ func (d Deps) lookup(ctx context.Context, c *call, query string) (function.Recor
 		}
 		q = query
 	}
+	return d.lookupTarget(ctx, c, name, q)
+}
+
+// lookupTarget reads the record for an already resolved name and qualifier;
+// only $LATEST exists.
+func (d Deps) lookupTarget(ctx context.Context, c *call, name, q string) (function.Record, uint64, error) {
 	if q != "" && q != "$LATEST" {
 		return function.Record{}, 0, notFound(c.Region, c.AccountID, name+":"+q)
 	}

spinifex/domains/lambda/awsapi/handler_test.go

@@ -45,6 +45,7 @@ type localClient struct {
 	svc *function.MetaService
 
 	mu             sync.Mutex
+	forceActive    bool
 	failCodeStored bool
 	passOwnerClock bool
 	rollbacks      int
@@ -68,6 +69,11 @@ func (c *localClient) Get(ctx context.Context, account, region, name string) (fu
 	if !r.Found {
 		return function.Record{}, 0, function.ErrNotFound
 	}
+	c.mu.Lock()
+	if c.forceActive {
+		r.Record.State = function.StateActive
+	}
+	c.mu.Unlock()
 	return r.Record, r.Revision, nil
 }
 
@@ -104,11 +110,20 @@ type memStore struct {
 	objects map[codestore.Ref][]byte
 	putErr  error
 	urlErr  error
+	// beforePut runs once, unlocked, before the next write lands.
+	beforePut func()
 }
 
 var _ awsapi.CodeStore = (*memStore)(nil)
 
 func (m *memStore) Put(_ context.Context, ref codestore.Ref, code []byte) (string, error) {
+	m.mu.Lock()
+	hook := m.beforePut
+	m.beforePut = nil
+	m.mu.Unlock()
+	if hook != nil {
+		hook()
+	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	if m.putErr != nil {
@@ -164,6 +179,40 @@ func newHarness(t *testing.T, statements ...iamdomain.Statement) (*gwtest.Harnes
 	return gwtest.Scoped(statements...), meta, store, kv, key
 }
 
+// uploads returns every upload record in the account bucket, by function ID.
+func (e *env) uploads(t *testing.T) map[string]function.Upload {
+	t.Helper()
+	kv := e.kv(t)
+	keys, err := kv.ListKeys(t.Context())
+	require.NoError(t, err)
+	out := map[string]function.Upload{}
+	for k := range keys.Keys() {
+		if !strings.HasPrefix(k, "upload.") {
+			continue
+		}
+		entry, err := kv.Get(t.Context(), k)
+		require.NoError(t, err)
+		var up function.Upload
+		require.NoError(t, json.Unmarshal(entry.Value(), &up))
+		require.Equal(t, function.KVUploadKey(up.Region, up.FunctionID, up.RevisionID), k)
+		out[up.FunctionID] = up
+	}
+	return out
+}
+
+// requireTracked asserts every code copy in the store has an upload record.
+func (e *env) requireTracked(t *testing.T) {
+	t.Helper()
+	ups := e.uploads(t)
+	e.store.mu.Lock()
+	defer e.store.mu.Unlock()
+	for ref := range e.store.objects {
+		up, ok := ups[ref.FunctionID]
+		require.True(t, ok, "code copy %s has no upload record", ref.FunctionID)
+		require.Equal(t, ref.RevisionID, up.RevisionID)
+	}
+}
+
 type env struct {
 	h     *gwtest.Harness
 	meta  *localClient
@@ -275,6 +324,15 @@ func TestCreateFunction_Success(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotContains(t, string(entry.Value()), "hunter2", "the environment is never stored in plaintext")
 	assert.Contains(t, string(entry.Value()), "envCipher")
+
+	e.requireTracked(t)
+	ups := e.uploads(t)
+	require.Len(t, ups, 1)
+	for _, up := range ups {
+		assert.Equal(t, function.UploadAttached, up.State)
+		assert.Equal(t, entry.Revision(), up.AttachedRevision)
+		assert.Equal(t, "hello", up.FunctionName)
+	}
 }
 
 func TestCreateFunction_RendersTheStoredTimestamps(t *testing.T) {
@@ -307,6 +365,8 @@ func TestCreateFunction_DuplicateNameLeavesFirstUntouched(t *testing.T) {
 	assert.Contains(t, err.Error(), "Function already exist: hello")
 
 	assert.Equal(t, 1, e.store.count(), "the loser wrote no code")
+	assert.Len(t, e.uploads(t), 1, "the refused create leaves no upload record")
+	e.requireTracked(t)
 	after, _, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.NoError(t, err)
 	assert.Equal(t, first, after)
@@ -321,6 +381,11 @@ func TestCreateFunction_CodePutFailureFreesTheName(t *testing.T) {
 	assert.Equal(t, 1, e.meta.rollbacks)
 	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.ErrorIs(t, err, function.ErrNotFound)
+	ups := e.uploads(t)
+	require.Len(t, ups, 1)
+	for _, up := range ups {
+		assert.Equal(t, function.UploadOwed, up.State, "a failed write may still have landed")
+	}
 
 	e.store.putErr = nil
 	rec, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
@@ -337,6 +402,51 @@ func TestCreateFunction_CodeStoredFailureRollsBackOnce(t *testing.T) {
 	assert.Equal(t, 1, e.store.count(), "the code object is left in place, never deleted here")
 	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.ErrorIs(t, err, function.ErrNotFound, "the still-storing claim was withdrawn")
+	e.requireTracked(t)
+	for _, up := range e.uploads(t) {
+		assert.Equal(t, function.UploadOwed, up.State, "the unnamed copy stays owed")
+	}
+}
+
+// The first create's write stalls past its claim, a second create takes the
+// name over, then the first write lands and its completion is refused.
+func TestCreateFunction_LateWriterAfterTakeoverStaysTracked(t *testing.T) {
+	e := newEnv(t)
+	key := function.KVFunctionKey(gwtest.Region, "hello")
+	e.store.beforePut = func() {
+		entry, err := e.kv(t).Get(t.Context(), key)
+		require.NoError(t, err)
+		var rec function.Record
+		require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+		rec.ClaimExpiresAt = time.Now().Add(-time.Second)
+		data, err := json.Marshal(rec)
+		require.NoError(t, err)
+		_, err = e.kv(t).Update(t.Context(), key, data, entry.Revision())
+		require.NoError(t, err)
+
+		rec2, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+			b["Code"] = map[string]any{"ZipFile": mustZip(t, "second")}
+		}))
+		require.NoError(t, err)
+		require.Equal(t, http.StatusCreated, rec2.Code)
+	}
+
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+	assert.Equal(t, 2, e.store.count(), "both copies landed")
+	e.requireTracked(t)
+
+	winner, _, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+	require.NoError(t, err)
+	ups := e.uploads(t)
+	require.Len(t, ups, 2)
+	for id, up := range ups {
+		if id == winner.FunctionID {
+			assert.Equal(t, function.UploadAttached, up.State)
+			continue
+		}
+		assert.Equal(t, function.UploadOwed, up.State, "the late writer's copy is never inferred complete")
+	}
 }
 
 func TestCreateFunction_RefusesUnsupportedInput(t *testing.T) {
@@ -655,7 +765,7 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "ListVersionsByFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListVersionsByFunction"}, reg.Inventory.Registered)
 	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
 	assert.Equal(t, "GetFunctionCodeSigningConfig", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2020-06-30/functions/x/code-signing-config", nil)))
 	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))

spinifex/domains/lambda/awsapi/invoke.go

@@ -0,0 +1,186 @@
+package awsapi
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// maxSyncPayload is the largest synchronous request payload AWS accepts.
+const maxSyncPayload = 6 * 1024 * 1024
+
+// waitOverhead is how much longer than the function timeout a request waits
+// for its result. It bounds the wait only, never the execution.
+const waitOverhead = 60 * time.Second
+
+// SyncInvocation is a RequestResponse request handed to the execution owner,
+// which decides admission.
+type SyncInvocation struct {
+	// Function and Revision are the one resolved configuration the attempt
+	// binds; the environment stays sealed in Function.EnvCipher.
+	Function function.Record
+	Revision uint64
+	// Qualifier is the validated qualifier the request named, or "".
+	Qualifier string
+	Payload   []byte
+	// WaitUntil is when the request stops waiting. Code may still be running
+	// after it; it proves nothing about stop and releases no slot.
+	WaitUntil time.Time
+}
+
+// SyncResult is what the function returned. FunctionError is "" on success;
+// ExecutedVersion is the version that ran.
+type SyncResult struct {
+	Payload         []byte
+	FunctionError   string
+	ExecutedVersion string
+}
+
+// Executor admits and runs one synchronous attempt. Its context ends only at
+// WaitUntil, never on caller disconnect. A *RefusalError means no code was
+// dispatched; any other error means the outcome is unknown.
+type Executor interface {
+	InvokeSync(ctx context.Context, inv SyncInvocation) (SyncResult, error)
+}
+
+// RefusalError is a known pre-dispatch refusal, such as a capacity throttle. Err
+// is the AWS error returned to the caller.
+type RefusalError struct{ Err error }
+
+func (r *RefusalError) Error() string { return "invocation refused before dispatch: " + r.Err.Error() }
+
+func (r *RefusalError) Unwrap() error { return r.Err }
+
+// invokeTarget is the function and qualifier an Invoke request names, once
+// every deterministic check on the request has passed.
+type invokeTarget struct{ Name, Qualifier string }
+
+// validateInvoke runs every check that needs no stored state: options, size,
+// payload JSON, function name and qualifier. Nothing here reads metadata.
+func validateInvoke(c *call) (*invokeTarget, error) {
+	if err := checkInvokeOptions(c.Header); err != nil {
+		return nil, err
+	}
+	if len(c.Body) > maxSyncPayload {
+		return nil, awserrors.Errorf(awserrors.ErrorRequestEntityTooLargeException,
+			"Request must be smaller than %d bytes for the InvokeFunction operation", maxSyncPayload)
+	}
+	// An empty body is dispatched as it is; anything else must be JSON.
+	if len(c.Body) > 0 && !json.Valid(c.Body) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidRequestContentException,
+			"Could not parse request body into json")
+	}
+	if len(c.Params[0]) > maxFunctionNameLen {
+		return nil, invalid("1 validation error detected: Value at 'functionName' failed to satisfy constraint: Member must have length less than or equal to 256")
+	}
+	name, q, err := resolveFunctionName(c.Region, c.AccountID, c.Params[0])
+	if err != nil {
+		return nil, err
+	}
+	if q != "" && !nameQualifierRE.MatchString(q) {
+		return nil, invalidFunctionName()
+	}
+	if query, ok := c.Query["Qualifier"]; ok {
+		if len(query) != 1 || query[0] == "" {
+			return nil, invalid("1 validation error detected: Value at 'qualifier' failed to satisfy constraint: Member must have length greater than or equal to 1")
+		}
+		if len(query[0]) > maxQualifierLen {
+			return nil, invalid("1 validation error detected: Value at 'qualifier' failed to satisfy constraint: Member must have length less than or equal to 128")
+		}
+		if !queryQualifierRE.MatchString(query[0]) {
+			return nil, invalid("1 validation error detected: Value at 'qualifier' failed to satisfy constraint: Member must satisfy regular expression pattern: \\$(LATEST(\\.PUBLISHED)?)|[a-zA-Z0-9-_$]+")
+		}
+		if q != "" && q != query[0] {
+			return nil, invalid("The derived qualifier from the function name does not match the specified qualifier.")
+		}
+		q = query[0]
+	}
+	return &invokeTarget{Name: name, Qualifier: q}, nil
+}
+
+// invoke implements Invoke for RequestResponse on $LATEST. serve has already
+// validated the request into c.Invoke and authorized it.
+func (d Deps) invoke(ctx context.Context, c *call) (reply, error) {
+	if c.Invoke == nil {
+		return reply{}, serviceError("invoke request was not validated")
+	}
+	rec, revision, err := d.lookupTarget(ctx, c, c.Invoke.Name, c.Invoke.Qualifier)
+	if err != nil {
+		return reply{}, err
+	}
+	if rec.State != function.StateActive {
+		return reply{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
+			"The operation cannot be performed at this time. The function is currently in the following state: %s", rec.State)
+	}
+	if d.Exec == nil || rec.Code.State != function.CodeStored {
+		return reply{}, serviceError("function execution is unavailable")
+	}
+
+	if err := ctx.Err(); err != nil {
+		return reply{}, err
+	}
+	// Once accepted, a disconnecting caller does not cancel the attempt; only
+	// the wait is bounded, and a late result is discarded.
+	waitUntil := time.Now().Add(time.Duration(rec.Timeout)*time.Second + waitOverhead)
+	actx, cancel := context.WithDeadline(context.WithoutCancel(ctx), waitUntil)
+	defer cancel()
+	out, err := d.Exec.InvokeSync(actx, SyncInvocation{
+		Function:  rec,
+		Revision:  revision,
+		Qualifier: c.Invoke.Qualifier,
+		Payload:   c.Body,
+		WaitUntil: waitUntil,
+	})
+	if refusal, ok := errors.AsType[*RefusalError](err); ok {
+		if _, coded := awserrors.ResolveErrorCode(refusal.Err); coded {
+			return reply{}, refusal.Err
+		}
+		err = refusal
+	}
+	if err != nil {
+		slog.ErrorContext(ctx, "Lambda: synchronous attempt outcome unknown", "function_id", rec.FunctionID, "err", err)
+		return reply{}, serviceError("the invocation outcome is unknown")
+	}
+	if out.ExecutedVersion == "" || len(out.Payload) > maxSyncPayload {
+		slog.ErrorContext(ctx, "Lambda: executor returned an invalid result", "function_id", rec.FunctionID,
+			"payload_bytes", len(out.Payload))
+		return reply{}, serviceError("the invocation outcome is unknown")
+	}
+	hdr := http.Header{}
+	hdr.Set("X-Amz-Executed-Version", out.ExecutedVersion)
+	if out.FunctionError != "" {
+		hdr.Set("X-Amz-Function-Error", out.FunctionError)
+	}
+	return reply{Status: http.StatusOK, Body: rawJSON(out.Payload), Header: hdr}, nil
+}
+
+// checkInvokeOptions refuses every request option this build does not
+// implement rather than ignoring it.
+func checkInvokeOptions(h http.Header) error {
+	switch t := h.Get("X-Amz-Invocation-Type"); t {
+	case "", "RequestResponse":
+	case "Event", "DryRun":
+		return unsupported("InvocationType " + t)
+	default:
+		return invalid("InvocationType must be RequestResponse, Event or DryRun")
+	}
+	switch l := h.Get("X-Amz-Log-Type"); l {
+	case "", "None":
+	case "Tail":
+		return unsupported("LogType Tail")
+	default:
+		return invalid("LogType must be None or Tail")
+	}
+	for _, name := range []string{"X-Amz-Client-Context", "X-Amz-Tenant-Id", "X-Amz-Durable-Execution-Name"} {
+		if _, ok := h[name]; ok {
+			return unsupported(name)
+		}
+	}
+	return nil
+}

spinifex/domains/lambda/awsapi/invoke_test.go

@@ -0,0 +1,455 @@
+package awsapi_test
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"maps"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeExec struct {
+	mu     sync.Mutex
+	calls  []awsapi.SyncInvocation
+	ctxs   []context.Context
+	result awsapi.SyncResult
+	err    error
+	// started, when set, is closed once the call begins and the call then
+	// blocks until release is closed.
+	started, release chan struct{}
+}
+
+func (f *fakeExec) InvokeSync(ctx context.Context, inv awsapi.SyncInvocation) (awsapi.SyncResult, error) {
+	f.mu.Lock()
+	f.calls = append(f.calls, inv)
+	f.ctxs = append(f.ctxs, ctx)
+	started, release := f.started, f.release
+	f.mu.Unlock()
+	if started != nil {
+		close(started)
+		<-release
+	}
+	return f.result, f.err
+}
+
+func (f *fakeExec) count() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.calls)
+}
+
+var okResult = awsapi.SyncResult{Payload: []byte(`{"sum":42}`), ExecutedVersion: "$LATEST"}
+
+// newInvokeEnv creates "hello" with its code stored and reports it Active.
+func newInvokeEnv(t *testing.T, statements ...iamdomain.Statement) (*env, *fakeExec) {
+	t.Helper()
+	e := newEnv(t, statements...)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+	e.meta.forceActive = true
+	ex := &fakeExec{result: okResult}
+	e.deps.Exec = ex
+	return e, ex
+}
+
+func (e *env) invoke(ctx context.Context, path string, header http.Header, payload []byte) (*httptest.ResponseRecorder, error) {
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(ctx, http.MethodPost, path, bytes.NewReader(payload))
+	maps.Copy(req.Header, header)
+	err := e.h.Dispatch(awsapi.NewRegistration(e.deps), awsapi.ServiceName, rec, req)
+	return rec, err
+}
+
+const invokePath = "/2015-03-31/functions/hello/invocations"
+
+func TestInvoke_ReturnsTheExecutorPayloadAndHeaders(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	payload := []byte(`{"a":40,"b":2,"note":"caf\u00e9"}`)
+	rec, err := e.invoke(t.Context(), invokePath, nil, payload)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, `{"sum":42}`, rec.Body.String())
+	assert.Equal(t, "$LATEST", rec.Header().Get("X-Amz-Executed-Version"))
+	assert.NotContains(t, rec.Header(), "X-Amz-Function-Error")
+	assert.NotContains(t, rec.Header(), "X-Amz-Log-Result")
+
+	require.Equal(t, 1, ex.count())
+	got := ex.calls[0]
+	assert.Equal(t, payload, got.Payload)
+	stored, revision, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+	require.NoError(t, err)
+	assert.Equal(t, revision, got.Revision)
+	assert.Equal(t, stored, got.Function)
+	assert.NotEmpty(t, got.Function.EnvCipher)
+	assert.NotContains(t, got.Function.EnvCipher, "prod")
+}
+
+func TestInvoke_FunctionErrorHeaderComesFromTheExecutor(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	ex.result = awsapi.SyncResult{Payload: []byte(`{"errorType":"x"}`), FunctionError: "Unhandled", ExecutedVersion: "$LATEST"}
+	rec, err := e.invoke(t.Context(), invokePath, nil, nil)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, "Unhandled", rec.Header().Get("X-Amz-Function-Error"))
+	assert.Equal(t, `{"errorType":"x"}`, rec.Body.String())
+}
+
+func TestInvoke_PendingFunctionIsAConflictAndNeverDispatches(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	e.meta.forceActive = false
+	_, err := e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	assert.Equal(t, 0, ex.count())
+}
+
+func TestInvoke_FailsClosedWithoutAnExecutor(t *testing.T) {
+	e, _ := newInvokeEnv(t)
+	e.deps.Exec = nil
+	_, err := e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+}
+
+func TestInvoke_DoesNotNeedTheCodeStore(t *testing.T) {
+	e, _ := newInvokeEnv(t)
+	e.deps.Codes = func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") }
+	_, err := e.invoke(t.Context(), invokePath, nil, nil)
+	require.NoError(t, err)
+	e.deps.Codes = nil
+	_, err = e.invoke(t.Context(), invokePath, nil, nil)
+	require.NoError(t, err)
+}
+
+func TestInvoke_RefusalBeforeDispatchIsKeptApartFromAnUnknownOutcome(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+
+	ex.err = &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorThrottling, "Rate exceeded")}
+	_, err := e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorThrottling)
+
+	ex.err = &awsapi.RefusalError{Err: errors.New("uncoded")}
+	_, err = e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+
+	ex.err = errors.New("transport lost")
+	_, err = e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+	assert.Contains(t, err.Error()+errMessage(err), "outcome is unknown")
+
+	ex.err = nil
+	ex.result = awsapi.SyncResult{Payload: []byte(`{}`)}
+	_, err = e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+}
+
+func errMessage(err error) string {
+	_, msg, _ := awserrors.ResolveErrorDetail(err)
+	return msg
+}
+
+func TestInvoke_RefusesOptionsItDoesNotImplement(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	for _, h := range []http.Header{
+		{"X-Amz-Invocation-Type": {"Event"}},
+		{"X-Amz-Invocation-Type": {"DryRun"}},
+		{"X-Amz-Invocation-Type": {"Bogus"}},
+		{"X-Amz-Log-Type": {"Tail"}},
+		{"X-Amz-Log-Type": {"Bogus"}},
+		{"X-Amz-Client-Context": {"e30="}},
+		{"X-Amz-Tenant-Id": {"t1"}},
+	} {
+		_, err := e.invoke(t.Context(), invokePath, h, nil)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+	assert.Equal(t, 0, ex.count())
+
+	for _, h := range []http.Header{
+		{"X-Amz-Invocation-Type": {"RequestResponse"}},
+		{"X-Amz-Log-Type": {"None"}},
+	} {
+		_, err := e.invoke(t.Context(), invokePath, h, nil)
+		require.NoError(t, err)
+	}
+}
+
+// jsonOfSize is a JSON string exactly n bytes long.
+func jsonOfSize(n int) []byte {
+	return []byte(`"` + strings.Repeat("a", n-2) + `"`)
+}
+
+func TestInvoke_PayloadLimit(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	_, err := e.invoke(t.Context(), invokePath, nil, jsonOfSize(6*1024*1024))
+	require.NoError(t, err)
+	_, err = e.invoke(t.Context(), invokePath, nil, jsonOfSize(6*1024*1024+1))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorRequestEntityTooLargeException)
+	assert.Equal(t, 1, ex.count())
+}
+
+func TestInvoke_PayloadMustBeJSONUnlessEmpty(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	for _, bad := range []string{"not json", `{"a":`, "{} {}", "\x00"} {
+		_, err := e.invoke(t.Context(), invokePath, nil, []byte(bad))
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidRequestContentException)
+	}
+	assert.Equal(t, 0, ex.count(), "a malformed payload reached the executor")
+
+	for _, good := range []string{"", `{"a":1}`, `[1,2]`, `"s"`, `42`, `null`, " true "} {
+		_, err := e.invoke(t.Context(), invokePath, nil, []byte(good))
+		require.NoError(t, err, "payload %q", good)
+	}
+	require.Equal(t, 7, ex.count())
+	assert.Empty(t, ex.calls[0].Payload, "an empty body is dispatched as it is")
+	assert.Equal(t, []byte(" true "), ex.calls[6].Payload)
+}
+
+func TestInvoke_MissingAndUnsupportedFunctions(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	_, err := e.invoke(t.Context(), "/2015-03-31/functions/ghost/invocations", nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, err = e.invoke(t.Context(), invokePath+"?Qualifier=1", nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, err = e.invoke(t.Context(), invokePath+"?Qualifier=%24LATEST", nil, nil)
+	require.NoError(t, err)
+	assert.Equal(t, 1, ex.count())
+}
+
+// Each form follows its own documented Invoke pattern; a documented value this
+// prototype does not serve passes validation and fails closed at lookup.
+func TestInvoke_QualifierFormsAndLengths(t *testing.T) {
+	fn := "/2015-03-31/functions/"
+	long := func(n int) string { return strings.Repeat("a", n) }
+	cases := []struct {
+		name, path, code string
+	}{
+		{"path $LATEST.PUBLISHED", fn + "hello:$LATEST.PUBLISHED/invocations", awserrors.ErrorResourceNotFoundException},
+		{"query $LATEST.PUBLISHED", invokePath + "?Qualifier=%24LATEST.PUBLISHED", awserrors.ErrorResourceNotFoundException},
+		{"query $foo", invokePath + "?Qualifier=%24foo", awserrors.ErrorResourceNotFoundException},
+		{"query of 128 characters", invokePath + "?Qualifier=" + long(128), awserrors.ErrorResourceNotFoundException},
+		{"query of 129 characters", invokePath + "?Qualifier=" + long(129), awserrors.ErrorInvalidParameterValueException},
+		{"empty query", invokePath + "?Qualifier=", awserrors.ErrorInvalidParameterValueException},
+		{"query with a dot", invokePath + "?Qualifier=a.b", awserrors.ErrorInvalidParameterValueException},
+		{"query $LATEST.OTHER", invokePath + "?Qualifier=%24LATEST.OTHER", awserrors.ErrorInvalidParameterValueException},
+		{"path $foo", fn + "hello:$foo/invocations", awserrors.ErrorInvalidParameterValueException},
+		{"path with a dot", fn + "hello:a.b/invocations", awserrors.ErrorInvalidParameterValueException},
+		{"function name of 256 characters", fn + "hello:" + long(250) + "/invocations", awserrors.ErrorResourceNotFoundException},
+		{"function name of 257 characters", fn + "hello:" + long(251) + "/invocations", awserrors.ErrorInvalidParameterValueException},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			e, ex := newInvokeEnv(t)
+			_, err := e.invoke(t.Context(), tc.path, nil, nil)
+			testutil.RequireAWSCode(t, err, tc.code)
+			assert.Equal(t, 0, ex.count())
+		})
+	}
+}
+
+func TestInvoke_CancelledBeforeDispatchNeverReachesTheExecutor(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	_, err := e.invoke(ctx, invokePath, nil, nil)
+	require.Error(t, err)
+	assert.Equal(t, 0, ex.count())
+}
+
+func TestInvoke_CallerDisconnectDoesNotCancelAnAcceptedAttempt(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	ex.started, ex.release = make(chan struct{}), make(chan struct{})
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan error, 1)
+	go func() {
+		_, err := e.invoke(ctx, invokePath, nil, nil)
+		done <- err
+	}()
+	<-ex.started
+	cancel()
+
+	actx := ex.ctxs[0]
+	assert.NoError(t, actx.Err(), "the caller's disconnect reached the attempt")
+	deadline, ok := actx.Deadline()
+	require.True(t, ok, "the wait must be bounded")
+	assert.Equal(t, ex.calls[0].WaitUntil, deadline)
+	assert.WithinDuration(t, time.Now().Add(3*time.Second+60*time.Second), deadline, 5*time.Second)
+
+	close(ex.release)
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		t.Fatal("the handler did not return after the attempt finished")
+	}
+}
+
+func TestInvoke_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	run := func(t *testing.T, statements ...iamdomain.Statement) (*fakeExec, error) {
+		return runPath(t, invokePath, statements...)
+	}
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		ex, err := run(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))
+		gwtest.AssertDenied(t, err)
+		assert.Equal(t, 0, ex.count())
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		ex, err := run(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:InvokeFunction", arn))
+		gwtest.AssertDenied(t, err)
+		assert.Equal(t, 0, ex.count())
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		_, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+"-other"))
+		gwtest.AssertDenied(t, err)
+	})
+	t.Run("an allow of lambda:InvokeFunction on the function ARN passes", func(t *testing.T) {
+		ex, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn))
+		require.NoError(t, err)
+		assert.Equal(t, 1, ex.count())
+	})
+	t.Run("an unqualified call does not match a qualified-only allow", func(t *testing.T) {
+		ex, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":$LATEST"))
+		gwtest.AssertDenied(t, err)
+		assert.Equal(t, 0, ex.count())
+	})
+	t.Run("an unqualified call does not match an allow on any qualified ARN", func(t *testing.T) {
+		_, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":*"))
+		gwtest.AssertDenied(t, err)
+	})
+	t.Run("an unqualified call matches a wildcard allow", func(t *testing.T) {
+		ex, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+"*"))
+		require.NoError(t, err)
+		assert.Equal(t, 1, ex.count())
+	})
+
+	// Lambda compares the policy resource with both the name's qualifier and
+	// the Qualifier parameter, so both forms are checked the same way.
+	qualified := map[string]string{
+		"path qualifier":  "/2015-03-31/functions/hello:$LATEST/invocations",
+		"query qualifier": invokePath + "?Qualifier=%24LATEST",
+	}
+	for form, path := range qualified {
+		t.Run(form, func(t *testing.T) {
+			t.Run("a qualified allow passes", func(t *testing.T) {
+				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":$LATEST"))
+				require.NoError(t, err)
+				assert.Equal(t, 1, ex.count())
+			})
+			t.Run("an explicit qualified deny overrides the matching qualified allow", func(t *testing.T) {
+				ex, err := runPath(t, path,
+					gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":$LATEST"),
+					gwtest.Statement("Deny", "lambda:InvokeFunction", arn+":$LATEST"))
+				gwtest.AssertDenied(t, err)
+				assert.Equal(t, 0, ex.count())
+			})
+			t.Run("an unqualified-only allow does not match", func(t *testing.T) {
+				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn))
+				gwtest.AssertDenied(t, err)
+				assert.Equal(t, 0, ex.count())
+			})
+			t.Run("an allow on any qualified ARN passes", func(t *testing.T) {
+				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":*"))
+				require.NoError(t, err)
+				assert.Equal(t, 1, ex.count())
+			})
+			t.Run("a wildcard allow passes", func(t *testing.T) {
+				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+"*"))
+				require.NoError(t, err)
+				assert.Equal(t, 1, ex.count())
+			})
+		})
+	}
+}
+
+func runPath(t *testing.T, path string, statements ...iamdomain.Statement) (*fakeExec, error) {
+	t.Helper()
+	e := newEnv(t, statements...)
+	sum := sha256.Sum256(testZip)
+	rev, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+		FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+		Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+	})
+	require.NoError(t, err)
+	_, _, err = e.meta.CodeStored(t.Context(), gwtest.AccountID, gwtest.Region, "hello", rev, hex.EncodeToString(sum[:]), int64(len(testZip)))
+	require.NoError(t, err)
+	e.meta.forceActive = true
+	ex := &fakeExec{result: okResult}
+	e.deps.Exec = ex
+	_, err = e.invoke(t.Context(), path, nil, nil)
+	return ex, err
+}
+
+// countingMeta counts metadata reads, to show a request never got that far.
+type countingMeta struct {
+	function.Client
+
+	gets atomic.Int32
+}
+
+func (m *countingMeta) Get(ctx context.Context, account, region, name string) (function.Record, uint64, error) {
+	m.gets.Add(1)
+	return m.Client.Get(ctx, account, region, name)
+}
+
+// Every deterministic check runs before authorization, so a caller with no
+// Invoke access still gets the validation error, and neither the metadata
+// store nor the executor is reached. Whether AWS orders these the same way
+// is not verified.
+func TestInvoke_ValidationPrecedesAuthorization(t *testing.T) {
+	big := append([]byte(`"`), bytes.Repeat([]byte("a"), 6*1024*1024)...)
+	cases := []struct {
+		name    string
+		path    string
+		header  http.Header
+		payload []byte
+		code    string
+	}{
+		{"unsupported invocation type", invokePath, http.Header{"X-Amz-Invocation-Type": {"Event"}}, nil, awserrors.ErrorInvalidParameterValueException},
+		{"invalid invocation type", invokePath, http.Header{"X-Amz-Invocation-Type": {"Later"}}, nil, awserrors.ErrorInvalidParameterValueException},
+		{"payload too large", invokePath, nil, append(big, '"'), awserrors.ErrorRequestEntityTooLargeException},
+		{"payload not JSON", invokePath, nil, []byte(`{`), awserrors.ErrorInvalidRequestContentException},
+		{"malformed function name", "/2015-03-31/functions/bad%20name/invocations", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"malformed path qualifier", "/2015-03-31/functions/hello:a.b/invocations", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"malformed query qualifier", invokePath + "?Qualifier=a.b", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"empty query qualifier", invokePath + "?Qualifier=", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"conflicting qualifiers", "/2015-03-31/functions/hello:$LATEST/invocations?Qualifier=2", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"malformed query encoding", invokePath + "?Qualifier=%ZZ", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"malformed pair beside a valid qualifier", invokePath + "?Qualifier=%24LATEST&x=%ZZ", nil, nil, awserrors.ErrorInvalidParameterValueException},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			for access, statement := range map[string]iamdomain.Statement{
+				"denied":  gwtest.Statement("Allow", "lambda:GetFunction", "*"),
+				"allowed": gwtest.Statement("Allow", "lambda:InvokeFunction", "*"),
+			} {
+				e := newEnv(t, statement)
+				meta := &countingMeta{Client: e.meta}
+				e.deps.Meta = meta
+				ex := &fakeExec{result: okResult}
+				e.deps.Exec = ex
+				_, err := e.invoke(t.Context(), tc.path, tc.header, tc.payload)
+				testutil.RequireAWSCode(t, err, tc.code)
+				assert.Zero(t, meta.gets.Load(), "%s: metadata was read", access)
+				assert.Equal(t, 0, ex.count(), "%s: the executor ran", access)
+			}
+		})
+	}
+}

spinifex/domains/lambda/awsapi/registration.go

@@ -8,7 +8,9 @@ import (
 	"errors"
 	"io"
 	"log/slog"
+	"maps"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -50,6 +52,8 @@ type Deps struct {
 	Trusts func(accountID, roleARN string) (bool, error)
 	// AEAD seals function environment variables; it is the master key's cipher.
 	AEAD cipher.AEAD
+	// Exec runs synchronous attempts; nil refuses every Invoke.
+	Exec Executor
 }
 
 // NewRegistration returns the Lambda REST-JSON registration.
@@ -89,6 +93,17 @@ func functionARN(region, accountID, name string) string {
 	return arn.FormatLambdaFunction(region, accountID, name)
 }
 
+// qualifiedFunctionARN names the version or alias a validated request selects,
+// since Lambda matches a qualified request only against a qualified policy
+// resource.
+func qualifiedFunctionARN(region, accountID, name, qualifier string) string {
+	base := functionARN(region, accountID, name)
+	if base == "*" || qualifier == "" {
+		return base
+	}
+	return base + ":" + qualifier
+}
+
 // serve resolves the route, authorizes the request, then runs the action.
 func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
@@ -104,7 +119,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 	if err != nil {
 		return err
 	}
-	c := &call{AccountID: inv.AccountID, Region: inv.Region, Params: params, Query: r.URL.Query(), Body: body}
+	c := &call{AccountID: inv.AccountID, Region: inv.Region, Params: params, Query: r.URL.Query(), Body: body, Header: r.Header}
 
 	// The region is the gateway's own, never the caller-supplied credential
 	// scope, so a caller cannot sign for another region to slide out from
@@ -121,8 +136,19 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		if name, _, err := resolveFunctionName(inv.Region, inv.AccountID, params[0]); err == nil {
 			resource = functionARN(inv.Region, inv.AccountID, name)
 		}
+	case "Invoke":
+		// Invoke validates deterministically before authorizing, and its
+		// resource comes from that same validated target. URL.Query drops
+		// malformed pairs, so the raw query is parsed strictly here.
+		if c.Query, err = url.ParseQuery(r.URL.RawQuery); err != nil {
+			return invalid("The request query string could not be parsed.")
+		}
+		if c.Invoke, err = validateInvoke(c); err != nil {
+			return err
+		}
+		resource = qualifiedFunctionARN(inv.Region, inv.AccountID, c.Invoke.Name, c.Invoke.Qualifier)
 	}
-	if err := inv.Authorize(ServiceName, action, []string{resource}, nil); err != nil {
+	if err := inv.Authorize(ServiceName, iamAction(action), []string{resource}, nil); err != nil {
 		return err
 	}
 	if decodeErr != nil {
@@ -133,9 +159,9 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 			return err
 		}
 	}
-	// Only GetFunction presigns code; the other reads are metadata only.
+	// Only CreateFunction and GetFunction touch the code store.
 	needsCodes := action != "GetFunctionConfiguration" && action != "GetFunctionCodeSigningConfig" &&
-		action != "ListVersionsByFunction"
+		action != "ListVersionsByFunction" && action != "Invoke"
 	if d.Meta == nil || (d.Codes == nil && needsCodes) {
 		return errors.New(awserrors.ErrorServiceException)
 	}
@@ -148,6 +174,14 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 	return nil
 }
 
+// iamAction is the IAM action an API operation is authorized as.
+func iamAction(action string) string {
+	if action == "Invoke" {
+		return "InvokeFunction"
+	}
+	return action
+}
+
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	return d.serve(w, inv)
 }
@@ -197,6 +231,7 @@ func writeJSON(w http.ResponseWriter, out reply) {
 		_, _ = w.Write(envelope.JSONBody(awserrors.ErrorServiceException, "failed to marshal response"))
 		return
 	}
+	maps.Copy(w.Header(), out.Header)
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(out.Status)
 	if _, err := w.Write(body); err != nil {

spinifex/domains/lambda/awsapi/routes.go

@@ -2,6 +2,7 @@ package awsapi
 
 import (
 	"context"
+	"net/http"
 	"net/url"
 
 	"github.com/aws/aws-sdk-go/service/lambda"
@@ -15,9 +16,13 @@ type call struct {
 	Params    []string
 	Query     url.Values
 	Body      []byte
+	Header    http.Header
 	// Create is the decoded CreateFunction body, decoded once ahead of the
 	// authorization gate and the PassRole check that both read it.
 	Create *lambda.CreateFunctionInput
+	// Invoke is Invoke's validated target, resolved ahead of the
+	// authorization gate whose resource it names.
+	Invoke *invokeTarget
 }
 
 // rawJSON is a response body written as is, for a shape the SDK encoder
@@ -28,6 +33,7 @@ type rawJSON []byte
 type reply struct {
 	Status int
 	Body   any
+	Header http.Header
 }
 
 type route = rest.Route[routeHandler]
@@ -56,6 +62,10 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.getFunctionCodeSigningConfig(ctx, c)
 		}},
+	{Method: "POST", Pattern: "/2015-03-31/functions/{FunctionName}/invocations", Action: "Invoke",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.invoke(ctx, c)
+		}},
 }
 
 var router = rest.NewRouter("lambda", routes)

spinifex/domains/lambda/awsapi/validate.go

@@ -30,6 +30,20 @@ const (
 
 var nameRE = regexp.MustCompile(`^[a-zA-Z0-9-_]{1,64}$`)
 
+// nameQualifierRE is the qualifier suffix of Invoke's documented FunctionName
+// pattern; queryQualifierRE is its Qualifier pattern, bounded by maxQualifierLen.
+var (
+	nameQualifierRE  = regexp.MustCompile(`^(\$LATEST(\.PUBLISHED)?|[a-zA-Z0-9-_]+)$`)
+	queryQualifierRE = regexp.MustCompile(`^(\$LATEST(\.PUBLISHED)?|[a-zA-Z0-9-_$]+)$`)
+)
+
+// maxQualifierLen bounds the Qualifier query; maxFunctionNameLen bounds the
+// whole FunctionName, qualifier suffix included.
+const (
+	maxQualifierLen    = 128
+	maxFunctionNameLen = 256
+)
+
 func invalid(format string, args ...any) error {
 	return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, format, args...)
 }

spinifex/domains/lambda/execution/grant.go

@@ -0,0 +1,396 @@
+// Package execution owns Lambda attempt grants: who may run an attempt and
+// report its result. This is a provisional single-node owner; it is not the
+// ADR-0009 M1 mechanism, which is still open.
+package execution
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// KVBucket holds one record per node: its grant sequence and current attempt.
+const KVBucket = "lambda-execution"
+
+const formatVersion = 1
+
+// Attempt states. The attempt lives in its node's record, and every change is
+// a conditional write on that one key, so a start, a withdrawal and the next
+// grant can never all succeed.
+const (
+	// StateGranted: recorded; nothing may have started.
+	StateGranted = "granted"
+	// StateRunning: the daemon may have launched the attempt.
+	StateRunning = "running"
+	// StateAccepted: a result was accepted under the grant; stop not yet proven.
+	StateAccepted = "accepted"
+	// StateWithdrawn: the attempt never started, and now never can.
+	StateWithdrawn = "withdrawn"
+	// StateReleased: the VM stop is proven and its resources released, but the
+	// daemon's own record of it is not yet gone; the node stays held.
+	StateReleased = "released"
+	// StateEnded: the stop is proven and nothing of the attempt remains.
+	StateEnded = "ended"
+	// StateAmbiguous: the end is unproven; the attempt keeps the node's slot.
+	StateAmbiguous = "ambiguous"
+)
+
+var (
+	// ErrNodeBusy: the node holds an attempt that has not ended.
+	ErrNodeBusy = errors.New("execution: node holds an unended attempt")
+	// ErrNotCurrent: the grant is not the node's current grant.
+	ErrNotCurrent = errors.New("execution: grant is not current")
+	// ErrWrongState: the attempt is not in a state that allows the change.
+	ErrWrongState = errors.New("execution: attempt is in the wrong state")
+	// ErrExpired: the grant's launch deadline has passed.
+	ErrExpired = errors.New("execution: grant expired before launch")
+	// ErrUnknownRecord: a record this owner cannot read or did not write.
+	ErrUnknownRecord = errors.New("execution: unrecognised record")
+	// ErrMismatch: the request or code differs from what the grant authorises.
+	ErrMismatch = errors.New("execution: request does not match the grant")
+	// ErrOutcomeUnknown: a write's outcome is unknown and may have landed.
+	ErrOutcomeUnknown = errors.New("execution: write outcome unknown")
+)
+
+var segment = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`)
+
+// Grant is the identity that authorises one attempt on one environment
+// generation. Seq orders grants on a node and never changes for the attempt.
+type Grant struct {
+	GrantID       string `json:"grantId"`
+	Seq           uint64 `json:"seq"`
+	AttemptID     string `json:"attemptId"`
+	EnvironmentID string `json:"environmentId"`
+	Generation    uint64 `json:"generation"`
+	Node          string `json:"node"`
+}
+
+// Identity is the exact request and code a grant authorises.
+type Identity struct {
+	RequestID  string `json:"requestId"`
+	AccountID  string `json:"accountId"`
+	Region     string `json:"region"`
+	FunctionID string `json:"functionId"`
+	RevisionID string `json:"revisionId"`
+	CodeSHA256 string `json:"codeSha256"`
+}
+
+// Attempt is the owner's record of one attempt and its grant.
+type Attempt struct {
+	Grant     Grant     `json:"grant"`
+	Identity  Identity  `json:"identity"`
+	State     string    `json:"state"`
+	LaunchBy  time.Time `json:"launchBy"`
+	CreatedAt time.Time `json:"createdAt"`
+	Launch    *Launch   `json:"launch,omitempty"`
+}
+
+// Launch identifies an attempt's VM process, its launcher and what it holds,
+// so a restarted daemon can prove the stop. The pidfs inodes are comparable
+// only within BootID on MachineID; they do not identify the host.
+type Launch struct {
+	MachineID     string `json:"machineId"`
+	BootID        string `json:"bootId"`
+	PIDNamespace  uint64 `json:"pidNamespace"`
+	LauncherPID   int    `json:"launcherPid"`
+	LauncherInode uint64 `json:"launcherInode"`
+	PID           int    `json:"pid"`
+	Inode         uint64 `json:"inode"`
+	TapName       string `json:"tapName"`
+	TapIndex      int    `json:"tapIndex"`
+	TapOwner      string `json:"tapOwner"`
+}
+
+func (l Launch) valid() bool {
+	return l.MachineID != "" && l.BootID != "" && l.PIDNamespace != 0 && l.LauncherPID > 0 &&
+		l.LauncherInode != 0 && l.PID > 0 && l.Inode != 0
+}
+
+// nodeRecord is a node's one slot. Seq survives every attempt so each later
+// grant orders after each earlier one.
+type nodeRecord struct {
+	FormatVersion int      `json:"formatVersion"`
+	Seq           uint64   `json:"seq"`
+	Attempt       *Attempt `json:"attempt,omitempty"`
+}
+
+// Store is the only writer of node records.
+type Store struct {
+	kv  jetstream.KeyValue
+	now func() time.Time
+	// beforeWrite runs between a read and its conditional write; tests use it.
+	beforeWrite func()
+}
+
+// Open opens (or creates) the execution bucket.
+func Open(ctx context.Context, js jetstream.KeyValueManager) (*Store, error) {
+	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucket, 1)
+	if err != nil {
+		return nil, fmt.Errorf("execution: open bucket: %w", err)
+	}
+	return &Store{kv: kv, now: time.Now}, nil
+}
+
+func nodeKey(node string) string { return "node." + node }
+
+// Issue records a new attempt and its grant before anything is dispatched.
+// It succeeds only if the node's current attempt ended, was withdrawn, or
+// provably never started before its launch deadline.
+func (s *Store) Issue(ctx context.Context, a Attempt) (Attempt, error) {
+	g := a.Grant
+	for _, seg := range []string{g.AttemptID, g.EnvironmentID, g.Node, a.Identity.FunctionID} {
+		if !segment.MatchString(seg) {
+			return Attempt{}, fmt.Errorf("execution: invalid identifier %q", seg)
+		}
+	}
+	id := a.Identity
+	if id.RequestID == "" || id.AccountID == "" || id.Region == "" || id.RevisionID == "" || id.CodeSHA256 == "" {
+		return Attempt{}, errors.New("execution: request and code identity are required")
+	}
+	if g.Generation == 0 {
+		return Attempt{}, errors.New("execution: environment generation is required")
+	}
+	grantID, err := randomID()
+	if err != nil {
+		return Attempt{}, err
+	}
+	a.Grant.GrantID, a.State, a.CreatedAt = grantID, StateGranted, s.now()
+
+	rec, rev, err := s.read(ctx, g.Node)
+	missing := errors.Is(err, jetstream.ErrKeyNotFound)
+	if err != nil && !missing {
+		return Attempt{}, err
+	}
+	if !missing && rec.Attempt != nil {
+		if rec.Attempt.Grant.AttemptID == g.AttemptID {
+			return Attempt{}, fmt.Errorf("%w: attempt %s already granted", ErrWrongState, g.AttemptID)
+		}
+		if !s.supersedable(*rec.Attempt) {
+			return Attempt{}, ErrNodeBusy
+		}
+	}
+	a.Grant.Seq = rec.Seq + 1
+	next := nodeRecord{FormatVersion: formatVersion, Seq: a.Grant.Seq, Attempt: &a}
+	if missing {
+		err = s.create(ctx, g.Node, next)
+	} else {
+		err = s.write(ctx, g.Node, next, rev)
+	}
+	if errors.Is(err, ErrNotCurrent) {
+		return Attempt{}, ErrNodeBusy
+	}
+	if err != nil {
+		return Attempt{}, err
+	}
+	return a, nil
+}
+
+// supersedable reports whether a new grant may replace cur. A granted attempt
+// past its deadline qualifies because Start refuses it then, and both writes
+// are conditional on the same record revision.
+func (s *Store) supersedable(cur Attempt) bool {
+	switch cur.State {
+	case StateEnded, StateWithdrawn:
+		return true
+	case StateGranted:
+		return !s.now().Before(cur.LaunchBy)
+	}
+	return false
+}
+
+// Start moves a current, unexpired grant from granted to running, only for
+// the exact request and code it authorises. The daemon calls it before
+// allocating anything; a repeat, replay or mismatch is refused unchanged.
+func (s *Store) Start(ctx context.Context, g Grant, want Identity) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		if a.Identity != want {
+			return ErrMismatch
+		}
+		if a.State != StateGranted {
+			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+		}
+		if !s.now().Before(a.LaunchBy) {
+			return ErrExpired
+		}
+		a.State = StateRunning
+		return nil
+	})
+}
+
+// Accept records that a result under g is accepted. Only one result is ever
+// accepted, and none once the attempt has ended or been held as ambiguous.
+func (s *Store) Accept(ctx context.Context, g Grant) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		if a.State != StateRunning {
+			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+		}
+		a.State = StateAccepted
+		return nil
+	})
+}
+
+// Launched records the running attempt's launch identity, once, before the
+// daemon sends the guest anything.
+func (s *Store) Launched(ctx context.Context, g Grant, l Launch) error {
+	if !l.valid() {
+		return errors.New("execution: incomplete launch identity")
+	}
+	return s.transition(ctx, g, func(a *Attempt) error {
+		if a.State != StateRunning || a.Launch != nil {
+			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+		}
+		a.Launch = &l
+		return nil
+	})
+}
+
+// Withdraw ends a grant that never started, for a caller that knows it sent
+// nothing. It fails once the daemon has started the attempt.
+func (s *Store) Withdraw(ctx context.Context, g Grant) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		if a.State != StateGranted {
+			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+		}
+		a.State = StateWithdrawn
+		return nil
+	})
+}
+
+// Released records the caller's proof that the attempt's VM has stopped and
+// everything it held is released. The node stays held until Stopped.
+func (s *Store) Released(ctx context.Context, g Grant) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		switch a.State {
+		case StateRunning, StateAccepted, StateAmbiguous:
+			a.State = StateReleased
+			return nil
+		}
+		return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+	})
+}
+
+// Stopped records that nothing of a released attempt remains, which frees the
+// node for the next grant.
+func (s *Store) Stopped(ctx context.Context, g Grant) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		if a.State != StateReleased {
+			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+		}
+		a.State = StateEnded
+		return nil
+	})
+}
+
+// Ambiguous records that the attempt's end is unproven. The node stays held
+// until Stopped; no amount of time releases it.
+func (s *Store) Ambiguous(ctx context.Context, g Grant) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		switch a.State {
+		case StateRunning, StateAccepted, StateAmbiguous:
+			a.State = StateAmbiguous
+			return nil
+		}
+		return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+	})
+}
+
+// Current returns the node's current attempt, read-only.
+func (s *Store) Current(ctx context.Context, node string) (Attempt, error) {
+	rec, _, err := s.read(ctx, node)
+	if errors.Is(err, jetstream.ErrKeyNotFound) || err == nil && rec.Attempt == nil {
+		return Attempt{}, ErrNotCurrent
+	}
+	if err != nil {
+		return Attempt{}, err
+	}
+	return *rec.Attempt, nil
+}
+
+// transition applies change to the attempt under g, written only at the
+// revision it was read, so a concurrent change refuses it.
+func (s *Store) transition(ctx context.Context, g Grant, change func(*Attempt) error) error {
+	rec, rev, err := s.read(ctx, g.Node)
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return ErrNotCurrent
+	}
+	if err != nil {
+		return err
+	}
+	if rec.Attempt == nil || rec.Attempt.Grant != g {
+		return ErrNotCurrent
+	}
+	a := *rec.Attempt
+	if err := change(&a); err != nil {
+		return err
+	}
+	rec.Attempt = &a
+	return s.write(ctx, g.Node, rec, rev)
+}
+
+func (s *Store) read(ctx context.Context, node string) (nodeRecord, uint64, error) {
+	e, err := s.kv.Get(ctx, nodeKey(node))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return nodeRecord{}, 0, err
+	}
+	if err != nil {
+		return nodeRecord{}, 0, fmt.Errorf("execution: read node record: %w", err)
+	}
+	var rec nodeRecord
+	if err := json.Unmarshal(e.Value(), &rec); err != nil || rec.FormatVersion != formatVersion || rec.Seq == 0 ||
+		(rec.Attempt != nil && (rec.Attempt.Grant.Node != node || rec.Attempt.Grant.Seq != rec.Seq)) {
+		return nodeRecord{}, 0, ErrUnknownRecord
+	}
+	return rec, e.Revision(), nil
+}
+
+func (s *Store) create(ctx context.Context, node string, rec nodeRecord) error {
+	body, err := json.Marshal(rec)
+	if err != nil {
+		return err
+	}
+	if s.beforeWrite != nil {
+		s.beforeWrite()
+	}
+	if _, err := s.kv.Create(ctx, nodeKey(node), body); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return ErrNodeBusy
+		}
+		return fmt.Errorf("%w: %w", ErrOutcomeUnknown, err)
+	}
+	return nil
+}
+
+// write replaces the node record only at rev. A revision conflict means
+// another change won; any other error may have landed.
+func (s *Store) write(ctx context.Context, node string, rec nodeRecord, rev uint64) error {
+	body, err := json.Marshal(rec)
+	if err != nil {
+		return err
+	}
+	if s.beforeWrite != nil {
+		s.beforeWrite()
+	}
+	if _, err := s.kv.Update(ctx, nodeKey(node), body, rev); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return fmt.Errorf("%w: record changed concurrently", ErrNotCurrent)
+		}
+		return fmt.Errorf("%w: %w", ErrOutcomeUnknown, err)
+	}
+	return nil
+}
+
+func randomID() (string, error) {
+	b := make([]byte, 16)
+	if _, err := rand.Read(b); err != nil {
+		return "", fmt.Errorf("execution: grant ID: %w", err)
+	}
+	return hex.EncodeToString(b), nil
+}

spinifex/domains/lambda/execution/grant_test.go

@@ -0,0 +1,376 @@
+package execution
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/require"
+)
+
+type clock struct {
+	mu sync.Mutex
+	t  time.Time
+}
+
+func (c *clock) now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t }
+
+func (c *clock) add(d time.Duration) { c.mu.Lock(); c.t = c.t.Add(d); c.mu.Unlock() }
+
+func newStore(t *testing.T) (*Store, *clock) {
+	t.Helper()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	s, err := Open(context.Background(), testutil.NewJetStream(t, nc))
+	require.NoError(t, err)
+	c := &clock{t: time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC)}
+	s.now = c.now
+	return s, c
+}
+
+func attempt(c *clock, id string) Attempt {
+	return Attempt{
+		Grant:    Grant{AttemptID: id, EnvironmentID: "env-" + id, Generation: 1, Node: "node1"},
+		Identity: identity(id), LaunchBy: c.now().Add(time.Minute),
+	}
+}
+
+func identity(id string) Identity {
+	return Identity{RequestID: "req-" + id, AccountID: "123456789012", Region: "ap-southeast-2",
+		FunctionID: "fn1", RevisionID: "rev1", CodeSHA256: "aa"}
+}
+
+func state(t *testing.T, s *Store, id string) string {
+	t.Helper()
+	rec, _, err := s.read(context.Background(), "node1")
+	require.NoError(t, err)
+	require.NotNil(t, rec.Attempt)
+	require.Equal(t, id, rec.Attempt.Grant.AttemptID)
+	return rec.Attempt.State
+}
+
+func started(t *testing.T, s *Store, c *clock, id string) Attempt {
+	t.Helper()
+	a, err := s.Issue(context.Background(), attempt(c, id))
+	require.NoError(t, err)
+	require.NoError(t, s.Start(context.Background(), a.Grant, a.Identity))
+	return a
+}
+
+func TestGrant_LifecycleFreesTheSlotOnlyOnProvenStop(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a := started(t, s, c, "a1")
+	require.Equal(t, uint64(1), a.Grant.Seq)
+	require.NotEmpty(t, a.Grant.GrantID)
+
+	_, err := s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy, "no second grant while the first runs")
+
+	require.NoError(t, s.Accept(ctx, a.Grant))
+	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrWrongState, "one result per attempt")
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy, "an accepted result does not free the slot")
+
+	require.NoError(t, s.Released(ctx, a.Grant))
+	require.NoError(t, s.Stopped(ctx, a.Grant))
+	require.Equal(t, StateEnded, state(t, s, "a1"))
+	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrWrongState, "no result accepted after the end")
+
+	b, err := s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err)
+	require.Equal(t, uint64(2), b.Grant.Seq, "later grants order after earlier ones")
+}
+
+func TestGrant_AmbiguousHoldsTheSlotUntilStopIsProven(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a := started(t, s, c, "a1")
+	require.NoError(t, s.Ambiguous(ctx, a.Grant))
+	require.NoError(t, s.Ambiguous(ctx, a.Grant), "holding again is a no-op")
+
+	c.add(24 * time.Hour)
+	_, err := s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy, "time alone never releases an ambiguous attempt")
+	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrWrongState, "no result accepted while ambiguous")
+	require.ErrorIs(t, s.Withdraw(ctx, a.Grant), ErrWrongState, "a started attempt cannot be withdrawn")
+
+	require.NoError(t, s.Released(ctx, a.Grant))
+	require.NoError(t, s.Stopped(ctx, a.Grant))
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err)
+}
+
+func TestGrant_StartRefusesReplayAndEveryMismatchedIdentity(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+
+	mismatched := map[string]func(g *Grant){
+		"grant ID":    func(g *Grant) { g.GrantID = "other" },
+		"older seq":   func(g *Grant) { g.Seq-- },
+		"environment": func(g *Grant) { g.EnvironmentID = "env-other" },
+		"generation":  func(g *Grant) { g.Generation++ },
+		"attempt":     func(g *Grant) { g.AttemptID = "a9" },
+		"node":        func(g *Grant) { g.Node = "node2" },
+	}
+	for name, mutate := range mismatched {
+		g := a.Grant
+		mutate(&g)
+		require.ErrorIs(t, s.Start(ctx, g, a.Identity), ErrNotCurrent, name)
+		require.ErrorIs(t, s.Stopped(ctx, g), ErrNotCurrent, name)
+	}
+
+	require.NoError(t, s.Start(ctx, a.Grant, a.Identity))
+	require.ErrorIs(t, s.Start(ctx, a.Grant, a.Identity), ErrWrongState, "a replayed start is refused")
+}
+
+func TestGrant_ExpiredGrantIsSupersededOnlyIfItNeverStarted(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	old, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+
+	c.add(2 * time.Minute)
+	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrExpired)
+
+	next, err := s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err, "a holder that can never start does not block the node")
+	require.Greater(t, next.Grant.Seq, old.Grant.Seq)
+	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrNotCurrent, "the superseded grant stays refused")
+
+	// A started holder is never superseded, however late.
+	require.NoError(t, s.Start(ctx, next.Grant, next.Identity))
+	c.add(24 * time.Hour)
+	_, err = s.Issue(ctx, attempt(c, "a3"))
+	require.ErrorIs(t, err, ErrNodeBusy)
+	require.Equal(t, StateRunning, state(t, s, "a2"))
+}
+
+func TestGrant_WithdrawFreesOnlyANeverStartedGrant(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+	require.NoError(t, s.Withdraw(ctx, a.Grant))
+	require.ErrorIs(t, s.Start(ctx, a.Grant, a.Identity), ErrWrongState, "a withdrawn grant can never start")
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err)
+}
+
+func TestGrant_UnexpiredGrantedHolderBlocks(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	_, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy)
+}
+
+func TestGrant_ConcurrentIssueGrantsExactlyOne(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	// Seed the slot so racers contend on the conditional update, not Create.
+	seed, err := s.Issue(ctx, attempt(c, "seed"))
+	require.NoError(t, err)
+	require.NoError(t, s.Withdraw(ctx, seed.Grant))
+
+	const racers = 16
+	var wg sync.WaitGroup
+	results := make(chan error, racers)
+	for i := range racers {
+		wg.Go(func() {
+			_, err := s.Issue(ctx, attempt(c, fmt.Sprintf("r%d", i)))
+			results <- err
+		})
+	}
+	wg.Wait()
+	close(results)
+	granted := 0
+	for err := range results {
+		if err == nil {
+			granted++
+		} else {
+			require.ErrorIs(t, err, ErrNodeBusy)
+		}
+	}
+	require.Equal(t, 1, granted)
+}
+
+// TestGrant_StartLosesToASupersedingGrant forces the interleaving where a
+// Start has read an expiring grant and a new grant replaces it before Start
+// writes: Start must fail, leaving only the new grant able to run.
+func TestGrant_StartLosesToASupersedingGrant(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	old, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+
+	// Start reads at a time the old grant is still valid; the new grant is
+	// issued after expiry, between Start's read and its write.
+	var next Attempt
+	s.beforeWrite = func() {
+		s.beforeWrite = nil
+		c.add(2 * time.Minute)
+		next, err = s.Issue(ctx, attempt(c, "a2"))
+		require.NoError(t, err)
+	}
+	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrNotCurrent)
+	c.add(-2 * time.Minute)
+	require.NoError(t, s.Start(ctx, next.Grant, next.Identity))
+	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrNotCurrent)
+	require.Equal(t, StateRunning, state(t, s, "a2"))
+}
+
+func TestGrant_AcceptLosesToAConcurrentHold(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a := started(t, s, c, "a1")
+	s.beforeWrite = func() {
+		s.beforeWrite = nil
+		require.NoError(t, s.Ambiguous(ctx, a.Grant))
+	}
+	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrNotCurrent, "the result was not accepted")
+	require.Equal(t, StateAmbiguous, state(t, s, "a1"))
+}
+
+func TestGrant_DuplicateAttemptIsRefused(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a := started(t, s, c, "a1")
+	require.NoError(t, s.Released(ctx, a.Grant))
+	require.NoError(t, s.Stopped(ctx, a.Grant))
+
+	_, err := s.Issue(ctx, attempt(c, "a1"))
+	require.ErrorIs(t, err, ErrWrongState, "an attempt ID is not granted twice in a row")
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err, "the refused duplicate does not take the slot")
+}
+
+// lostAck lands attempt Creates and then reports a failure, as a lost
+// acknowledgement would.
+type lostAck struct{ jetstream.KeyValue }
+
+func (l lostAck) Create(ctx context.Context, key string, v []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
+	if _, err := l.KeyValue.Create(ctx, key, v, opts...); err != nil {
+		return 0, err
+	}
+	return 0, errors.New("ack lost")
+}
+
+func TestGrant_AmbiguousRecordWriteKeepsTheSlot(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	inner := s.kv
+	s.kv = lostAck{inner}
+	_, err := s.Issue(ctx, attempt(c, "a1"))
+	require.ErrorIs(t, err, ErrOutcomeUnknown)
+	s.kv = inner
+
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy, "a grant that may be recorded keeps the slot")
+	require.Equal(t, StateGranted, state(t, s, "a1"))
+	c.add(2 * time.Minute)
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err, "past its deadline the landed grant can never start")
+}
+
+func TestGrant_UnrecognisedRecordIsRefusedUnchanged(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+	require.NoError(t, s.Withdraw(ctx, a.Grant))
+
+	e, err := s.kv.Get(ctx, nodeKey("node1"))
+	require.NoError(t, err)
+	var raw map[string]any
+	require.NoError(t, json.Unmarshal(e.Value(), &raw))
+	raw["formatVersion"] = formatVersion + 1
+	body, _ := json.Marshal(raw)
+	rev, err := s.kv.Update(ctx, nodeKey("node1"), body, e.Revision())
+	require.NoError(t, err)
+
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrUnknownRecord)
+	require.ErrorIs(t, s.Start(ctx, a.Grant, a.Identity), ErrUnknownRecord)
+	after, err := s.kv.Get(ctx, nodeKey("node1"))
+	require.NoError(t, err)
+	require.Equal(t, rev, after.Revision())
+	require.Equal(t, body, after.Value())
+}
+
+func TestGrant_StartRefusesAMismatchedRequestUnchanged(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+	mismatched := map[string]func(id *Identity){
+		"request":  func(id *Identity) { id.RequestID = "req-other" },
+		"account":  func(id *Identity) { id.AccountID = "210987654321" },
+		"region":   func(id *Identity) { id.Region = "us-east-1" },
+		"function": func(id *Identity) { id.FunctionID = "fn2" },
+		"revision": func(id *Identity) { id.RevisionID = "rev2" },
+		"code":     func(id *Identity) { id.CodeSHA256 = "bb" },
+	}
+	for name, mutate := range mismatched {
+		want := a.Identity
+		mutate(&want)
+		require.ErrorIs(t, s.Start(ctx, a.Grant, want), ErrMismatch, name)
+		require.Equal(t, StateGranted, state(t, s, "a1"), name)
+	}
+	require.NoError(t, s.Start(ctx, a.Grant, a.Identity))
+}
+
+// A released attempt still holds the node; only Stopped, and only after
+// Released, frees it for the next grant.
+func TestReleasedHoldsTheNodeUntilStopped(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+	require.NoError(t, s.Start(ctx, a.Grant, a.Identity))
+	require.ErrorIs(t, s.Stopped(ctx, a.Grant), ErrWrongState)
+
+	require.NoError(t, s.Released(ctx, a.Grant))
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy)
+
+	require.NoError(t, s.Stopped(ctx, a.Grant))
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err)
+}
+
+// The launch identity is recorded once, complete, and only while running, so
+// recovery never acts on a partial or replaced identity.
+func TestGrant_LaunchedRecordsACompleteIdentityOnce(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	l := Launch{MachineID: "m", BootID: "b", PIDNamespace: 4026531836, LauncherPID: 10,
+		LauncherInode: 11, PID: 20, Inode: 21}
+
+	granted, err := s.Issue(ctx, attempt(c, "a0"))
+	require.NoError(t, err)
+	require.ErrorIs(t, s.Launched(ctx, granted.Grant, l), ErrWrongState, "not yet running")
+	require.NoError(t, s.Withdraw(ctx, granted.Grant))
+
+	a := started(t, s, c, "a1")
+	partial := l
+	partial.Inode = 0
+	require.Error(t, s.Launched(ctx, a.Grant, partial))
+	require.NoError(t, s.Launched(ctx, a.Grant, l))
+	other := l
+	other.PID = 99
+	require.ErrorIs(t, s.Launched(ctx, a.Grant, other), ErrWrongState, "recorded once")
+
+	rec, err := s.Current(ctx, "node1")
+	require.NoError(t, err)
+	require.NotNil(t, rec.Launch)
+	require.Equal(t, l, *rec.Launch)
+}

spinifex/domains/lambda/function/meta.go

@@ -26,6 +26,7 @@ var (
 	nameRE   = regexp.MustCompile(`^[a-zA-Z0-9-_]{1,64}$`)
 	regionRE = regexp.MustCompile(`^[a-z0-9-]{1,32}$`)
 	digestRE = regexp.MustCompile(`^[0-9a-f]{64}$`)
+	idRE     = regexp.MustCompile(`^[a-zA-Z0-9_-]{1,64}$`)
 )
 
 // ValidName reports whether name is a legal function name.
@@ -81,17 +82,15 @@ func decode(data []byte) (Record, error) {
 	return rec, nil
 }
 
-// Create binds the name by writing the record with kv.Create. The record starts
-// Pending with its code storing and a claim that expires after ClaimTTL. A name
-// held only by an expired, still-storing claim is taken over once.
-//
-// Orphaned code objects from rolled-back or taken-over claims are not cleaned up yet.
+// Create records the code upload, then claims the name with kv.Create: Pending,
+// code storing, claim expiring after ClaimTTL. A name held only by an expired,
+// still-storing claim is taken over once; the taken-over upload stays owed.
 func (s *MetaService) Create(ctx context.Context, req *CreateRequest, accountID string) (*CreateResponse, error) {
 	rec := req.Record
 	if !regionRE.MatchString(req.Region) {
 		return nil, badRegion()
 	}
-	if !ValidName(rec.FunctionName) || rec.FunctionID == "" || rec.Code.RevisionID == "" || accountID == "" {
+	if !ValidName(rec.FunctionName) || !idRE.MatchString(rec.FunctionID) || !idRE.MatchString(rec.Code.RevisionID) || accountID == "" {
 		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "function record is incomplete")
 	}
 	kv, err := s.bucket(ctx, accountID)
@@ -112,6 +111,21 @@ func (s *MetaService) Create(ctx context.Context, req *CreateRequest, accountID
 	}
 	key := KVFunctionKey(req.Region, rec.FunctionName)
 
+	// The upload is recorded before the claim exists, so no copy can be written
+	// for a claim that has no upload record.
+	upKey := KVUploadKey(req.Region, rec.FunctionID, rec.Code.RevisionID)
+	upData, err := json.Marshal(Upload{
+		FormatVersion: FormatVersion, AccountID: accountID, Region: req.Region, FunctionName: rec.FunctionName,
+		FunctionID: rec.FunctionID, RevisionID: rec.Code.RevisionID, State: UploadOwed, CreatedAt: now,
+	})
+	if err != nil {
+		return nil, err
+	}
+	upRev, err := kv.Create(ctx, upKey, upData)
+	if err != nil {
+		return nil, fmt.Errorf("lambda: record code upload: %w", err)
+	}
+
 	rev, err := kv.Create(ctx, key, data)
 	if err == nil {
 		return &CreateResponse{Revision: rev}, nil
@@ -123,27 +137,85 @@ func (s *MetaService) Create(ctx context.Context, req *CreateRequest, accountID
 	exists := conflict("Function already exist: %s", rec.FunctionName)
 	entry, err := kv.Get(ctx, key)
 	if err != nil {
-		return nil, exists
+		return nil, s.refused(ctx, kv, upKey, upRev, exists)
 	}
 	old, err := decode(entry.Value())
 	if err != nil || old.Code.State != CodeStoring || !s.now().After(old.ClaimExpiresAt) {
-		return nil, exists
+		return nil, s.refused(ctx, kv, upKey, upRev, exists)
 	}
 	if err := kv.Delete(ctx, key, jetstream.LastRevision(entry.Revision())); err != nil {
-		return nil, exists
+		return nil, s.refused(ctx, kv, upKey, upRev, exists)
 	}
 	slog.WarnContext(ctx, "Lambda: took over an expired function name claim",
 		"function_name", rec.FunctionName, "old_function_id", old.FunctionID)
 	rev, err = kv.Create(ctx, key, data)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, exists
+			return nil, s.refused(ctx, kv, upKey, upRev, exists)
 		}
 		return nil, fmt.Errorf("lambda: create function record: %w", err)
 	}
 	return &CreateResponse{Revision: rev}, nil
 }
 
+// refused drops the upload record of a create that certainly holds no claim:
+// the gateway writes code only after Create succeeds. A failed drop leaves the
+// upload owed, which is safe.
+func (s *MetaService) refused(ctx context.Context, kv jetstream.KeyValue, upKey string, upRev uint64, err error) error {
+	if derr := kv.Delete(ctx, upKey, jetstream.LastRevision(upRev)); derr != nil {
+		slog.WarnContext(ctx, "Lambda: upload record of a refused create was not dropped", "key", upKey, "err", derr)
+	}
+	return err
+}
+
+// checkUpload returns the owed upload record for rec. Any other record, or
+// none, refuses the caller; nothing is rewritten, because the state of a record
+// this owner cannot vouch for is unknown.
+func checkUpload(ctx context.Context, kv jetstream.KeyValue, rec Record) (Upload, uint64, error) {
+	refuse := func(why string, args ...any) (Upload, uint64, error) {
+		slog.WarnContext(ctx, "Lambda: code upload record refused; its state is unknown",
+			"function_id", rec.FunctionID, "revision_id", rec.Code.RevisionID, "reason", fmt.Sprintf(why, args...))
+		return Upload{}, 0, awserrors.Errorf(awserrors.ErrorServiceException, "function code upload is not tracked")
+	}
+	entry, err := kv.Get(ctx, KVUploadKey(rec.Region, rec.FunctionID, rec.Code.RevisionID))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return refuse("missing")
+	}
+	if err != nil {
+		return Upload{}, 0, fmt.Errorf("lambda: get code upload record: %w", err)
+	}
+	var up Upload
+	if err := json.Unmarshal(entry.Value(), &up); err != nil {
+		return refuse("unreadable")
+	}
+	if up.FormatVersion != FormatVersion {
+		return refuse("format version %d", up.FormatVersion)
+	}
+	if up.AccountID != rec.AccountID || up.Region != rec.Region || up.FunctionName != rec.FunctionName ||
+		up.FunctionID != rec.FunctionID || up.RevisionID != rec.Code.RevisionID {
+		return refuse("identity mismatch")
+	}
+	if up.State != UploadOwed {
+		return refuse("state %s", up.State)
+	}
+	return up, entry.Revision(), nil
+}
+
+// attach marks the checked upload as named by the function record at revision.
+// KV has no multi-key write, so a change since the check fails this CAS, and
+// the record is then left as found, its state unknown.
+func attach(ctx context.Context, kv jetstream.KeyValue, up Upload, upRev, revision uint64) {
+	up.State, up.AttachedRevision = UploadAttached, revision
+	data, err := json.Marshal(up)
+	if err == nil {
+		_, err = kv.Update(ctx, KVUploadKey(up.Region, up.FunctionID, up.RevisionID), data, upRev)
+	}
+	if err != nil {
+		slog.WarnContext(ctx, "Lambda: upload record not marked attached; its state is unknown",
+			"function_id", up.FunctionID, "revision_id", up.RevisionID, "err", err)
+	}
+}
+
 // Get returns the record, or Found false when the name is unbound.
 func (s *MetaService) Get(ctx context.Context, req *GetRequest, accountID string) (*GetResponse, error) {
 	if !regionRE.MatchString(req.Region) {
@@ -190,8 +262,9 @@ func (s *MetaService) readStoring(ctx context.Context, kv jetstream.KeyValue, re
 	return rec, nil
 }
 
-// CodeStored marks the code stored. It is refused, leaving the record unchanged,
-// when the revision moved, the code is not storing, or the claim has expired.
+// CodeStored marks the code stored. It is refused, leaving both records unchanged,
+// when the revision moved, the code is not storing, the claim has expired, or
+// the upload record is not the owed one this create wrote.
 func (s *MetaService) CodeStored(ctx context.Context, req *CodeStoredRequest, accountID string) (*CodeStoredResponse, error) {
 	if !regionRE.MatchString(req.Region) {
 		return nil, badRegion()
@@ -214,6 +287,10 @@ func (s *MetaService) CodeStored(ctx context.Context, req *CodeStoredRequest, ac
 	if now.After(rec.ClaimExpiresAt) {
 		return nil, conflict("Function creation claim expired: %s", req.FunctionName)
 	}
+	up, upRev, err := checkUpload(ctx, kv, rec)
+	if err != nil {
+		return nil, err
+	}
 	rec.Code.State, rec.Code.SHA256, rec.Code.Size = CodeStored, req.SHA256, req.Size
 	rec.State, rec.StateReasonCode, rec.StateReason = StatePending, ReasonCodeCreating, ReasonCreating
 	rec.LastModified = now.UTC()
@@ -228,12 +305,13 @@ func (s *MetaService) CodeStored(ctx context.Context, req *CodeStoredRequest, ac
 		}
 		return nil, fmt.Errorf("lambda: update function record: %w", err)
 	}
+	attach(ctx, kv, up, upRev, rev)
 	return &CodeStoredResponse{Record: rec, Revision: rev}, nil
 }
 
 // RollbackCreate frees the name of a create whose own code write failed. It
 // deletes only a record still storing at exactly the revision create returned;
-// an already-absent name is success.
+// an already-absent name is success. The upload stays owed: the write may have landed.
 func (s *MetaService) RollbackCreate(ctx context.Context, req *RollbackCreateRequest, accountID string) (*RollbackCreateResponse, error) {
 	if !regionRE.MatchString(req.Region) {
 		return nil, badRegion()

spinifex/domains/lambda/function/record.go

@@ -62,8 +62,37 @@ const (
 	KVBucketAccountPrefix  = "lambda-account-"
 	KVBucketAccountHistory = 1
 	kvFunctionKeyPrefix    = "fn."
+	kvUploadKeyPrefix      = "upload."
 )
 
+// KVUploadKey returns the key of the record that tracks one code upload. The
+// bucket is shared across Regions, so the Region is part of the key.
+func KVUploadKey(region, functionID, revisionID string) string {
+	return kvUploadKeyPrefix + region + "." + functionID + "." + revisionID
+}
+
+// Upload states. An owed upload may have a code copy that no function record
+// names; it stays owed until the owner records otherwise.
+const (
+	UploadOwed     = "owed"
+	UploadAttached = "attached"
+)
+
+// Upload tracks one code upload. A create begun through this owner writes it
+// before the copy; records and copies from older owners have none.
+type Upload struct {
+	FormatVersion int       `json:"formatVersion"`
+	AccountID     string    `json:"accountId"`
+	Region        string    `json:"region"`
+	FunctionName  string    `json:"functionName"`
+	FunctionID    string    `json:"functionId"`
+	RevisionID    string    `json:"revisionId"`
+	State         string    `json:"state"`
+	CreatedAt     time.Time `json:"createdAt"`
+	// AttachedRevision is the function record revision that names the copy.
+	AttachedRevision uint64 `json:"attachedRevision,omitempty"`
+}
+
 // KVAccountBucket returns the per-account JetStream KV bucket name.
 func KVAccountBucket(accountID string) string { return KVBucketAccountPrefix + accountID }
 

spinifex/domains/lambda/function/upload_test.go

@@ -0,0 +1,212 @@
+package function
+
+import (
+	"encoding/json"
+	"errors"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// upload reads the tracking record for functionID, or reports it absent.
+func upload(t *testing.T, svc *MetaService, functionID string) (Upload, bool) {
+	t.Helper()
+	return uploadIn(t, svc, testRegion, functionID)
+}
+
+func uploadIn(t *testing.T, svc *MetaService, region, functionID string) (Upload, bool) {
+	t.Helper()
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), KVUploadKey(region, functionID, "rev-"+functionID))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return Upload{}, false
+	}
+	require.NoError(t, err)
+	var up Upload
+	require.NoError(t, json.Unmarshal(entry.Value(), &up))
+	return up, true
+}
+
+func requireOwed(t *testing.T, svc *MetaService, functionID string) {
+	t.Helper()
+	up, ok := upload(t, svc, functionID)
+	require.True(t, ok, "upload record for %s is missing", functionID)
+	assert.Equal(t, UploadOwed, up.State)
+}
+
+func TestUpload_CreateRecordsItOwedAndCodeStoredAttachesIt(t *testing.T) {
+	svc, c := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+	up, ok := upload(t, svc, "f1")
+	require.True(t, ok)
+	assert.Equal(t, Upload{FormatVersion: FormatVersion, AccountID: testAccount, Region: testRegion, FunctionName: "hello",
+		FunctionID: "f1", RevisionID: "rev-f1", State: UploadOwed, CreatedAt: c.t}, up)
+
+	stored, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+	require.NoError(t, err)
+	up, ok = upload(t, svc, "f1")
+	require.True(t, ok)
+	assert.Equal(t, UploadAttached, up.State)
+	assert.Equal(t, stored.Revision, up.AttachedRevision)
+}
+
+func TestUpload_IsRecordedBeforeTheNameIsClaimed(t *testing.T) {
+	svc, _ := newTestService(t)
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	_, err = kv.Create(t.Context(), KVUploadKey(testRegion, "f1", "rev-f1"), []byte(`{}`))
+	require.NoError(t, err)
+
+	_, err = svc.Create(t.Context(), createReq("hello", "f1"), testAccount)
+	require.Error(t, err)
+	assert.False(t, mustGet(t, svc, "hello").Found, "no claim exists when the upload could not be recorded")
+}
+
+func TestUpload_RefusedDuplicateLeavesNoRecord(t *testing.T) {
+	svc, _ := newTestService(t)
+	mustCreate(t, svc, "hello", "f1")
+	_, err := svc.Create(t.Context(), createReq("hello", "f2"), testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	_, ok := upload(t, svc, "f2")
+	assert.False(t, ok, "a create refused before any claim can never be written to")
+	requireOwed(t, svc, "f1")
+}
+
+func TestUpload_RollbackKeepsItOwed(t *testing.T) {
+	svc, _ := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+	_, err := svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: rev}, testAccount)
+	require.NoError(t, err)
+	assert.False(t, mustGet(t, svc, "hello").Found)
+	requireOwed(t, svc, "f1")
+}
+
+func TestUpload_ExpiredClaimIsNotTakenAsCompletion(t *testing.T) {
+	svc, c := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+	c.t = c.t.Add(ClaimTTL + time.Second)
+	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	requireOwed(t, svc, "f1")
+}
+
+// The original writer stalls past its claim, a second create takes the name,
+// and the original then finishes its write and reports it.
+func TestUpload_TakeoverAndLateWriterLeaveTheOriginalOwed(t *testing.T) {
+	svc, c := newTestService(t)
+	oldRev := mustCreate(t, svc, "hello", "f1")
+	c.t = c.t.Add(ClaimTTL + time.Second)
+	newRev := mustCreate(t, svc, "hello", "f2")
+	requireOwed(t, svc, "f1")
+	requireOwed(t, svc, "f2")
+
+	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: oldRev, SHA256: testDigest, Size: 9}, testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: oldRev}, testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	requireOwed(t, svc, "f1")
+
+	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: newRev, SHA256: testDigest, Size: 9}, testAccount)
+	require.NoError(t, err)
+	up, _ := upload(t, svc, "f2")
+	assert.Equal(t, UploadAttached, up.State)
+	requireOwed(t, svc, "f1")
+}
+
+func TestUpload_RefusesIdentifiersThatCannotBeKeys(t *testing.T) {
+	svc, _ := newTestService(t)
+	for _, id := range []string{"a.b", "a b", "a*"} {
+		_, err := svc.Create(t.Context(), createReq("hello", id), testAccount)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+	assert.False(t, mustGet(t, svc, "hello").Found)
+}
+
+func TestUpload_SameIdentifiersInTwoRegionsAreTrackedApart(t *testing.T) {
+	svc, _ := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+	other := createReq("hello", "f1")
+	other.Region = "us-east-1"
+	_, err := svc.Create(t.Context(), other, testAccount)
+	require.NoError(t, err)
+
+	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+	require.NoError(t, err)
+	here, ok := uploadIn(t, svc, testRegion, "f1")
+	require.True(t, ok)
+	assert.Equal(t, UploadAttached, here.State)
+	there, ok := uploadIn(t, svc, "us-east-1", "f1")
+	require.True(t, ok)
+	assert.Equal(t, UploadOwed, there.State)
+	assert.Equal(t, "us-east-1", there.Region)
+}
+
+// An upload record the owner cannot vouch for refuses CodeStored, and neither
+// it nor the function record changes.
+func TestUpload_CodeStoredRefusesAnUnrecognisedRecord(t *testing.T) {
+	asJSON := func(mutate func(*Upload)) func([]byte) []byte {
+		return func(b []byte) []byte {
+			var up Upload
+			if err := json.Unmarshal(b, &up); err != nil {
+				panic(err)
+			}
+			mutate(&up)
+			out, err := json.Marshal(up)
+			if err != nil {
+				panic(err)
+			}
+			return out
+		}
+	}
+	for name, rewrite := range map[string]func([]byte) []byte{
+		"missing":          nil,
+		"unreadable":       func([]byte) []byte { return []byte("not json") },
+		"future format":    asJSON(func(u *Upload) { u.FormatVersion = FormatVersion + 1 }),
+		"other function":   asJSON(func(u *Upload) { u.FunctionName = "other" }),
+		"other account":    asJSON(func(u *Upload) { u.AccountID = "000000000002" }),
+		"already attached": asJSON(func(u *Upload) { u.State, u.AttachedRevision = UploadAttached, 7 }),
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc, _ := newTestService(t)
+			rev := mustCreate(t, svc, "hello", "f1")
+			kv, err := svc.bucket(t.Context(), testAccount)
+			require.NoError(t, err)
+			key := KVUploadKey(testRegion, "f1", "rev-f1")
+			entry, err := kv.Get(t.Context(), key)
+			require.NoError(t, err)
+			if rewrite == nil {
+				require.NoError(t, kv.Delete(t.Context(), key))
+			} else {
+				_, err = kv.Update(t.Context(), key, rewrite(entry.Value()), entry.Revision())
+				require.NoError(t, err)
+			}
+			before, err := kv.Get(t.Context(), key)
+			if rewrite == nil {
+				require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
+			} else {
+				require.NoError(t, err)
+			}
+
+			_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+			testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+
+			fn := mustGet(t, svc, "hello")
+			assert.Equal(t, rev, fn.Revision, "the function record is unchanged")
+			assert.Equal(t, CodeStoring, fn.Record.Code.State)
+			after, err := kv.Get(t.Context(), key)
+			if rewrite == nil {
+				require.ErrorIs(t, err, jetstream.ErrKeyNotFound, "no record is invented")
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, before.Revision(), after.Revision())
+			assert.Equal(t, before.Value(), after.Value())
+		})
+	}
+}

spinifex/domains/network/host/owned_tap.go

@@ -0,0 +1,128 @@
+//go:build lambdatestlink
+
+package host
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	hostcommand "github.com/mulgadc/spinifex/spinifex/runtime/host/command"
+)
+
+// ErrTapExists means a link with the requested name already exists; nothing
+// was changed.
+var ErrTapExists = errors.New("tap name already in use")
+
+// ErrTapNotCreated wraps a CreateOwnedTap failure that left no link behind.
+// Any other failure may have left one.
+var ErrTapNotCreated = errors.New("no tap created")
+
+// ownedTapSysfs is a var so tests can stand in a fake /sys/class/net.
+var ownedTapSysfs = "/sys/class/net"
+
+const ownerExternalID = "spinifex-owner"
+
+// OwnedTap identifies a tap CreateOwnedTap made: a later remove acts only on a
+// link with the same ifindex and alias, and an OVS port with the same owner.
+type OwnedTap struct {
+	Name    string
+	IfIndex int
+	Owner   string
+}
+
+// CreateOwnedTap creates and attaches a new tap, refusing to replace a link or
+// OVS port of the same name. The check-then-create gap remains, so callers use
+// names no other creator would pick.
+func (p *OVSPlumber) CreateOwnedTap(spec vm.TapSpec, owner string) (OwnedTap, error) {
+	if owner == "" || strings.ContainsAny(owner, " \t\n\"'") {
+		return OwnedTap{}, fmt.Errorf("%w: invalid owner", ErrTapNotCreated)
+	}
+	if _, err := os.Lstat(filepath.Join(ownedTapSysfs, spec.Name)); err == nil {
+		return OwnedTap{}, fmt.Errorf("%w: %w: %s", ErrTapNotCreated, ErrTapExists, spec.Name)
+	}
+	addArgs := []string{"tuntap", "add", "dev", spec.Name, "mode", "tap"}
+	if uid := os.Geteuid(); uid != 0 {
+		addArgs = append(addArgs, "user", strconv.Itoa(uid), "group", strconv.Itoa(os.Getegid()))
+	}
+	if out, err := hostcommand.SudoCommand("ip", addArgs...).CombinedOutput(); err != nil {
+		// The add may have taken effect before failing, so this is not ErrTapNotCreated.
+		return OwnedTap{}, fmt.Errorf("create tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err)
+	}
+	idx, err := readIfIndex(spec.Name)
+	if err != nil {
+		// Without its ifindex the new link cannot be told apart, so it stays.
+		return OwnedTap{}, fmt.Errorf("created tap %s but cannot read its ifindex; left in place: %w", spec.Name, err)
+	}
+	tap := OwnedTap{Name: spec.Name, IfIndex: idx, Owner: owner}
+	upArgs := []string{"link", "set", "dev", spec.Name, "alias", owner, "up"}
+	if spec.MTU > 0 {
+		upArgs = append(upArgs, "mtu", strconv.Itoa(spec.MTU))
+	}
+	if out, err := hostcommand.SudoCommand("ip", upArgs...).CombinedOutput(); err != nil {
+		return tap, errors.Join(fmt.Errorf("bring up tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err), p.RemoveOwnedTap(tap))
+	}
+	// No --may-exist: a port record of the same name fails the add.
+	addPort := []string{"add-port", spec.Bridge, spec.Name, "--", "set", "Interface", spec.Name,
+		fmt.Sprintf("external_ids:%s=%s", ownerExternalID, owner)}
+	if out, err := hostcommand.SudoCommand("ovs-vsctl", addPort...).CombinedOutput(); err != nil {
+		return tap, errors.Join(fmt.Errorf("add tap %s to %s: %s: %w", spec.Name, spec.Bridge, strings.TrimSpace(string(out)), err), p.removeOwnedLink(tap))
+	}
+	return tap, nil
+}
+
+// RemoveOwnedTap removes the OVS port and kernel link only where each still
+// carries tap's owner; anything else under that name is left alone. A link
+// already gone is success.
+func (p *OVSPlumber) RemoveOwnedTap(tap OwnedTap) error {
+	out, err := hostcommand.SudoCommand("ovs-vsctl", "--if-exists", "get", "Interface", tap.Name,
+		"external_ids:"+ownerExternalID).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("read OVS owner of %s: %s: %w", tap.Name, strings.TrimSpace(string(out)), err)
+	}
+	switch got := strings.Trim(strings.TrimSpace(string(out)), `"`); got {
+	case tap.Owner:
+		if out, err := hostcommand.SudoCommand("ovs-vsctl", "--if-exists", "del-port", tap.Name).CombinedOutput(); err != nil {
+			return fmt.Errorf("remove OVS port %s: %s: %w", tap.Name, strings.TrimSpace(string(out)), err)
+		}
+	case "":
+	default:
+		return fmt.Errorf("OVS port %s belongs to another owner; left in place", tap.Name)
+	}
+	return p.removeOwnedLink(tap)
+}
+
+func (p *OVSPlumber) removeOwnedLink(tap OwnedTap) error {
+	idx, err := readIfIndex(tap.Name)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("read ifindex of %s: %w", tap.Name, err)
+	}
+	alias, err := os.ReadFile(filepath.Join(ownedTapSysfs, tap.Name, "ifalias"))
+	if err != nil {
+		return fmt.Errorf("read alias of %s: %w", tap.Name, err)
+	}
+	if idx != tap.IfIndex || strings.TrimSpace(string(alias)) != tap.Owner {
+		return fmt.Errorf("link %s is not the tap this owner created; left in place", tap.Name)
+	}
+	if out, err := deleteTapLink(tap.Name).CombinedOutput(); err != nil {
+		return fmt.Errorf("delete tap %s: %s: %w", tap.Name, strings.TrimSpace(string(out)), err)
+	}
+	slog.Info("Owned tap removed", "tap", tap.Name)
+	return nil
+}
+
+func readIfIndex(name string) (int, error) {
+	b, err := os.ReadFile(filepath.Join(ownedTapSysfs, name, "ifindex"))
+	if err != nil {
+		return 0, err
+	}
+	return strconv.Atoi(strings.TrimSpace(string(b)))
+}

spinifex/domains/network/host/owned_tap_test.go

@@ -0,0 +1,164 @@
+//go:build lambdatestlink
+
+package host
+
+import (
+	"errors"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	hostcommand "github.com/mulgadc/spinifex/spinifex/runtime/host/command"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeHost stands in for sysfs and the privileged commands: ip creates and
+// deletes link directories, ovs-vsctl reports a port owner.
+type fakeHost struct {
+	sysfs     string
+	calls     []string
+	ovsOwner  string
+	failCmd   string
+	nextIndex string
+}
+
+func newFakeHost(t *testing.T) *fakeHost {
+	f := &fakeHost{sysfs: t.TempDir(), nextIndex: "41"}
+	orig := ownedTapSysfs
+	ownedTapSysfs = f.sysfs
+	t.Cleanup(func() { ownedTapSysfs = orig })
+	t.Cleanup(hostcommand.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd {
+		call := name + " " + strings.Join(args, " ")
+		f.calls = append(f.calls, call)
+		if f.failCmd != "" && strings.Contains(call, f.failCmd) {
+			return exec.Command("/bin/false")
+		}
+		switch {
+		case name == "ip" && args[0] == "tuntap":
+			f.link(t, args[3], f.nextIndex, "")
+		case name == "ip" && slices.Contains(args, "alias"):
+			i := slices.Index(args, "alias")
+			require.NoError(t, os.WriteFile(filepath.Join(f.sysfs, args[3], "ifalias"), []byte(args[i+1]+"\n"), 0o600))
+		case name == "ip" && args[0] == "link" && args[1] == "del":
+			require.NoError(t, os.RemoveAll(filepath.Join(f.sysfs, args[3])))
+		case name == "ovs-vsctl" && slices.Contains(args, "get"):
+			return exec.Command("/bin/echo", f.ovsOwner)
+		}
+		return exec.Command("/bin/true")
+	}))
+	return f
+}
+
+func (f *fakeHost) link(t *testing.T, name, index, alias string) {
+	dir := filepath.Join(f.sysfs, name)
+	require.NoError(t, os.MkdirAll(dir, 0o700))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "ifindex"), []byte(index+"\n"), 0o600))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "ifalias"), []byte(alias+"\n"), 0o600))
+}
+
+func (f *fakeHost) ran(sub string) bool {
+	return slices.ContainsFunc(f.calls, func(c string) bool { return strings.Contains(c, sub) })
+}
+
+func TestCreateOwnedTap_RefusesAnExistingLinkWithoutTouchingIt(t *testing.T) {
+	f := newFakeHost(t)
+	f.link(t, "lm1", "7", "someone-else")
+	_, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "lambda-a")
+	require.ErrorIs(t, err, ErrTapExists)
+	require.ErrorIs(t, err, ErrTapNotCreated)
+	require.Empty(t, f.calls, "an existing link must not be deleted, replaced or detached")
+}
+
+func TestCreateOwnedTap_CreatesWithoutReplacing(t *testing.T) {
+	f := newFakeHost(t)
+	tap, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt", MTU: 1400}, "lambda-a")
+	require.NoError(t, err)
+	require.Equal(t, OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"}, tap)
+	require.True(t, f.ran("add-port br-mgmt lm1 -- set Interface lm1 external_ids:spinifex-owner=lambda-a"))
+	require.True(t, f.ran("alias lambda-a up mtu 1400"))
+	require.False(t, f.ran("del-port"), "creation never removes an existing port")
+	require.False(t, f.ran("--may-exist"))
+	require.False(t, f.ran("link del"))
+}
+
+func TestCreateOwnedTap_OnlyARefusalClaimsNothingCreated(t *testing.T) {
+	f := newFakeHost(t)
+	_, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "bad owner")
+	require.ErrorIs(t, err, ErrTapNotCreated)
+	require.Empty(t, f.calls)
+
+	f = newFakeHost(t)
+	f.failCmd = "tuntap add"
+	_, err = NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "lambda-a")
+	require.Error(t, err)
+	require.NotErrorIs(t, err, ErrTapNotCreated, "a failed add may still have created the link")
+
+	f = newFakeHost(t)
+	f.nextIndex = "not-a-number"
+	tap, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm2", Bridge: "br-mgmt"}, "lambda-a")
+	require.Error(t, err)
+	require.NotErrorIs(t, err, ErrTapNotCreated, "a link exists that the caller cannot identify")
+	require.Empty(t, tap.Name)
+	require.False(t, f.ran("link del"))
+}
+
+func TestCreateOwnedTap_FailedAttachRemovesOnlyItsOwnLink(t *testing.T) {
+	f := newFakeHost(t)
+	f.failCmd = "add-port"
+	_, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "lambda-a")
+	require.Error(t, err)
+	require.True(t, f.ran("link del dev lm1"))
+	require.False(t, f.ran("del-port"), "a port this call failed to add is never removed")
+}
+
+func TestRemoveOwnedTap_LeavesAnythingItDidNotCreate(t *testing.T) {
+	cases := map[string]struct {
+		index, alias, ovsOwner string
+		wantDelPort            bool
+	}{
+		"other OVS owner":  {index: "41", alias: "lambda-a", ovsOwner: `"other"`},
+		"replaced link":    {index: "99", alias: "lambda-a", ovsOwner: `"lambda-a"`, wantDelPort: true},
+		"other link alias": {index: "41", alias: "other", ovsOwner: ""},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			f := newFakeHost(t)
+			f.link(t, "lm1", tc.index, tc.alias)
+			f.ovsOwner = tc.ovsOwner
+			err := NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"})
+			require.Error(t, err)
+			require.False(t, f.ran("link del"))
+			require.Equal(t, tc.wantDelPort, f.ran("del-port"))
+		})
+	}
+}
+
+func TestRemoveOwnedTap_RemovesItsOwnTap(t *testing.T) {
+	f := newFakeHost(t)
+	f.link(t, "lm1", "41", "lambda-a")
+	f.ovsOwner = `"lambda-a"`
+	require.NoError(t, NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"}))
+	require.True(t, f.ran("del-port lm1"))
+	require.True(t, f.ran("link del dev lm1"))
+
+	// Already gone is success, and touches nothing.
+	f.calls, f.ovsOwner = nil, ""
+	require.NoError(t, NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"}))
+	require.False(t, f.ran("del-port"))
+	require.False(t, f.ran("link del"))
+}
+
+func TestRemoveOwnedTap_OVSReadFailureRemovesNothing(t *testing.T) {
+	f := newFakeHost(t)
+	f.link(t, "lm1", "41", "lambda-a")
+	f.failCmd = "get Interface"
+	err := NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"})
+	require.Error(t, err)
+	require.False(t, f.ran("del-port"))
+	require.False(t, f.ran("link del"))
+	require.False(t, errors.Is(err, ErrTapExists))
+}

spinifex/domains/network/policy/acl.go

@@ -47,8 +47,8 @@ const denyACLSeverity = "info"
 // MUST be validated upstream; values are interpolated verbatim.
 type Rule struct {
 	IPProtocol string // "tcp", "udp", "icmp", or "-1" (all)
-	FromPort   int64
-	ToPort     int64
+	FromPort   int64  // ICMP type for "icmp", -1 for any
+	ToPort     int64  // ICMP code for "icmp", -1 for any
 	CIDR       string
 	SourceSG   string
 }
@@ -181,7 +181,15 @@ func appendProtocolMatch(parts []string, rule Rule) ([]string, error) {
 	case "udp":
 		return appendPortMatch(parts, "udp", rule.FromPort, rule.ToPort), nil
 	case "icmp":
-		return append(parts, "icmp4"), nil
+		// AWS reads an ICMP rule's FromPort as the type and ToPort as the code.
+		parts = append(parts, "icmp4")
+		if rule.FromPort != -1 {
+			parts = append(parts, fmt.Sprintf("icmp4.type == %d", rule.FromPort))
+		}
+		if rule.ToPort != -1 {
+			parts = append(parts, fmt.Sprintf("icmp4.code == %d", rule.ToPort))
+		}
+		return parts, nil
 	case "-1", "":
 		return parts, nil
 	default:

spinifex/domains/network/policy/acl_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestACL_TCPPortFromCIDR(t *testing.T) {
@@ -45,6 +46,8 @@ func TestACL_PortRange(t *testing.T) {
 func TestACL_ICMP(t *testing.T) {
 	match, err := BuildIngressACLMatch("sg_test", Rule{
 		IPProtocol: "icmp",
+		FromPort:   -1,
+		ToPort:     -1,
 		CIDR:       "0.0.0.0/0",
 	})
 	assert.NoError(t, err)
@@ -53,6 +56,37 @@ func TestACL_ICMP(t *testing.T) {
 	assert.NotContains(t, match, "udp.dst")
 }
 
+// An ICMP rule opens only the type and code it names, -1 meaning any; the
+// console's ping rule (type 8, code -1) must not open echo reply or redirects.
+func TestACL_ICMPTypeAndCode(t *testing.T) {
+	cases := []struct {
+		name           string
+		fromPort       int64
+		toPort         int64
+		want, wantNone []string
+	}{
+		{"echo request any code", 8, -1, []string{"icmp4", "icmp4.type == 8"}, []string{"icmp4.code"}},
+		{"type and code", 3, 4, []string{"icmp4.type == 3", "icmp4.code == 4"}, nil},
+		{"echo reply", 0, 0, []string{"icmp4.type == 0", "icmp4.code == 0"}, nil},
+		{"all icmp", -1, -1, []string{"icmp4"}, []string{"icmp4.type", "icmp4.code"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			rule := Rule{IPProtocol: "icmp", FromPort: tc.fromPort, ToPort: tc.toPort, CIDR: "0.0.0.0/0"}
+			for _, build := range []func(string, Rule) (string, error){BuildIngressACLMatch, BuildEgressACLMatch} {
+				match, err := build("sg_test", rule)
+				require.NoError(t, err)
+				for _, w := range tc.want {
+					assert.Contains(t, match, w)
+				}
+				for _, n := range tc.wantNone {
+					assert.NotContains(t, match, n)
+				}
+			}
+		})
+	}
+}
+
 func TestACL_AllProtocols(t *testing.T) {
 	match, err := BuildIngressACLMatch("sg_test", Rule{
 		IPProtocol: "-1",

spinifex/domains/sqs/message/jetstream/attributes_test.go

@@ -27,6 +27,7 @@ func md5Hex(b []byte) string {
 }
 
 func TestSendWithAttributesStoresVersionedRecordAndReceivesThem(t *testing.T) {
+	t.Parallel()
 	e := newReceiveEnv(t)
 	q := e.create(t, acct, "orders", nil)
 	in := input("orders", "hello")
@@ -56,6 +57,7 @@ func TestSendWithAttributesStoresVersionedRecordAndReceivesThem(t *testing.T) {
 }
 
 func TestSendWithoutAttributesKeepsPlainRecord(t *testing.T) {
+	t.Parallel()
 	e := newReceiveEnv(t)
 	q := e.create(t, acct, "orders", nil)
 	res := e.send(t, "orders", "plain", nil)
@@ -70,6 +72,7 @@ func TestSendWithoutAttributesKeepsPlainRecord(t *testing.T) {
 
 // A record as an earlier release wrote it, appended directly, still reads.
 func TestReceiveReadsEarlierPlainRecords(t *testing.T) {
+	t.Parallel()
 	e := newReceiveEnv(t)
 	q := e.create(t, acct, "orders", nil)
 	enq := strconv.FormatInt(fixedNow.UnixMilli(), 10)

spinifex/domains/sqs/message/jetstream/body_enum_test.go

@@ -32,6 +32,7 @@ func messageStreamRequests(subjects []string, stream string) (leaderGets, infos
 // for bodies: one leader message get per stored body through the bound, with
 // no direct get and no consumer on the message stream.
 func TestBodyEnumerationIsLeaderServed(t *testing.T) {
+	t.Parallel()
 	srv, js := startServer(t, t.TempDir())
 	s, err := sqsjs.New(context.Background(), js, testConfig)
 	require.NoError(t, err)
@@ -85,6 +86,7 @@ func TestBodyEnumerationIsLeaderServed(t *testing.T) {
 // and neighbouring namespaces in stream order, and stops at the first page's
 // bound even when later bodies exist.
 func TestScanStepsOverGapsInOrderWithinTheBound(t *testing.T) {
+	t.Parallel()
 	s, _ := newStore(t)
 	ctx := context.Background()
 	a, b := ns(0, 0, "gap-a"), ns(0, 0, "gap-b")
@@ -119,6 +121,7 @@ func TestScanStepsOverGapsInOrderWithinTheBound(t *testing.T) {
 }
 
 func TestBodyEnumerationCancelled(t *testing.T) {
+	t.Parallel()
 	s, _ := newStore(t)
 	appendTo(t, s, ns(0, 0, "cancel"), "b")
 	ctx, cancel := context.WithCancel(context.Background())

spinifex/domains/sqs/message/jetstream/claim_test.go

@@ -47,6 +47,7 @@ func claimPos(pos message.Position, vt time.Duration) message.ClaimInput {
 }
 
 func TestClaimLifecycleOnJetStream(t *testing.T) {
+	t.Parallel()
 	s := claimStore(t)
 	ctx := context.Background()
 	pos := appendAt(t, s, fixedNow, fixedNow.Add(10*time.Second))
@@ -80,6 +81,7 @@ func TestClaimLifecycleOnJetStream(t *testing.T) {
 
 // Competing receivers on one message: exactly one claim is acknowledged.
 func TestClaimCompetingReceiversOnJetStream(t *testing.T) {
+	t.Parallel()
 	s := claimStore(t)
 	ctx := context.Background()
 	for _, prior := range []bool{false, true} {
@@ -140,6 +142,7 @@ func (l *lostClaimReply) PutState(ctx context.Context, ns message.Namespace, pos
 }
 
 func TestClaimAppliedButLostReplyOnJetStream(t *testing.T) {
+	t.Parallel()
 	s := claimStore(t)
 	ctx := context.Background()
 	pos := appendAt(t, s, fixedNow, fixedNow)
@@ -208,6 +211,7 @@ func (h *hookedStore) PutState(ctx context.Context, ns message.Namespace, pos me
 // tombstone between the claim's body read and its state read leaves the
 // record absent again; the claim must not resurrect the deleted message.
 func TestClaimDoesNotResurrectRemovedMessage(t *testing.T) {
+	t.Parallel()
 	s := claimStore(t)
 	ctx := context.Background()
 	pos := appendAt(t, s, fixedNow, fixedNow)
@@ -255,6 +259,7 @@ func (c *steppingClock) claimer(s message.Store) *message.Claimer {
 // A slow body or state read does not spend the visibility: the deadline is
 // measured from the claim attempt, after both reads.
 func TestClaimDeadlineFromAttemptNotRequestStart(t *testing.T) {
+	t.Parallel()
 	s := claimStore(t)
 	ctx := context.Background()
 	pos := appendAt(t, s, fixedNow, fixedNow)

spinifex/domains/sqs/message/jetstream/conformance_test.go

@@ -36,12 +36,14 @@ func r1Harness(t *testing.T) conformance.Harness {
 }
 
 func TestR1Conformance(t *testing.T) {
+	t.Parallel()
 	conformance.Run(t, r1Harness)
 }
 
 // TestR1SharedInstanceConformance runs every case on one long-lived store,
 // so each must pass beside other queues' data on the shared streams.
 func TestR1SharedInstanceConformance(t *testing.T) {
+	t.Parallel()
 	s, _ := newStore(t)
 	conformance.Run(t, func(*testing.T) conformance.Harness {
 		return conformance.Harness{Store: s, MessageShards: testConfig.MessageStreams, StateShards: testConfig.StateBuckets}

spinifex/domains/sqs/message/jetstream/delete_test.go

@@ -50,6 +50,7 @@ func (e receiveEnv) state(t *testing.T, q queue.Record, h string) (map[string]an
 }
 
 func TestDeleteCurrentHandleWritesTombstone(t *testing.T) {
+	t.Parallel()
 	e := newReceiveEnv(t)
 	q := e.create(t, acct, "orders", nil)
 	m := e.receiveOne(t, "orders")
@@ -73,6 +74,7 @@ func TestDeleteCurrentHandleWritesTombstone(t *testing.T) {
 
 // An older handle succeeds without deleting the newer receive.
 func TestDeleteOlderHandleKeepsNewerReceive(t *testing.T) {
+	t.Parallel()
 	e := newReceiveEnv(t)
 	q := e.create(t, acct, "orders", nil)
 	first := e.receiveOne(t, "orders")
@@ -94,6 +96,7 @@ func TestDeleteOlderHandleKeepsNewerReceive(t *testing.T) {
 }
 
 func TestDeleteIsIdempotent(t *testing.T) {
+	t.Parallel()
 	e := newReceiveEnv(t)
 	q := e.create(t, acct, "orders", nil)
 	m := e.receiveOne(t, "orders")
@@ -113,6 +116,7 @@ func TestDeleteIsIdempotent(t *testing.T) {
 // A delete racing a re-receive either tombstones before it, so the receive
 // gets nothing, or loses to it and leaves the newer receive in flight.
 func TestDeleteRacingReReceive(t *testing.T) {
+	t.Parallel()
 	for range 10 {
 		e := newReceiveEnv(t)
 		q := e.create(t, acct, "orders", nil)
@@ -139,6 +143,7 @@ func TestDeleteRacingReReceive(t *testing.T) {
 }
 
 func TestDeleteRefusesInvalidHandles(t *testing.T) {
+	t.Parallel()
 	e := newReceiveEnv(t)
 	q := e.create(t, acct, "orders", nil)
 	e.create(t, acct, "audit", nil)
@@ -170,7 +175,9 @@ func TestDeleteRefusesInvalidHandles(t *testing.T) {
 }
 
 func TestDeleteUnsettledOutcomes(t *testing.T) {
+	t.Parallel()
 	t.Run("lost reply, tombstone applied", func(t *testing.T) {
+		t.Parallel()
 		e := newReceiveEnv(t)
 		q := e.create(t, acct, "orders", nil)
 		m := e.receiveOne(t, "orders")
@@ -180,6 +187,7 @@ func TestDeleteUnsettledOutcomes(t *testing.T) {
 		assert.Equal(t, message.StatusDeleted, st["s"])
 	})
 	t.Run("lost reply, nothing applied", func(t *testing.T) {
+		t.Parallel()
 		e := newReceiveEnv(t)
 		q := e.create(t, acct, "orders", nil)
 		m := e.receiveOne(t, "orders")
@@ -190,6 +198,7 @@ func TestDeleteUnsettledOutcomes(t *testing.T) {
 		assert.Equal(t, before, after)
 	})
 	t.Run("state read fails", func(t *testing.T) {
+		t.Parallel()
 		e := newReceiveEnv(t)
 		e.create(t, acct, "orders", nil)
 		m := e.receiveOne(t, "orders")
@@ -203,6 +212,7 @@ func TestDeleteUnsettledOutcomes(t *testing.T) {
 		"older than key": `{"v":1,"s":"inflight","c":1,"g":0,"d":1,"r":1,"f":1}`,
 	} {
 		t.Run(name, func(t *testing.T) {
+			t.Parallel()
 			e := newReceiveEnv(t)
 			q := e.create(t, acct, "orders", nil)
 			m := e.receiveOne(t, "orders")
@@ -222,6 +232,7 @@ func TestDeleteUnsettledOutcomes(t *testing.T) {
 
 // Neither handles nor bodies appear in errors or logs on any refusal.
 func TestDeleteKeepsHandlesOutOfErrorsAndLogs(t *testing.T) {
+	// Not parallel: it captures the process-wide default logger.
 	var logs bytes.Buffer
 	prev := slog.Default()
 	slog.SetDefault(slog.New(slog.NewJSONHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))